1stages: 2 - test 3 4noether-rocm: 5 stage: test 6 tags: 7 - rocm 8 interruptible: true 9 image: jedbrown/rocm:latest 10 script: 11 - rm -f .SUCCESS 12# Compilers 13 - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc 14 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 15 - echo "-------------- CC ------------------" && $CC --version 16 - echo "-------------- CXX -----------------" && $CXX --version 17 - echo "-------------- FC ------------------" && $FC --version 18 - echo "-------------- HIPCC ---------------" && $HIPCC --version 19 - echo "-------------- GCOV ----------------" && gcov --version 20# Libraries for backends 21# -- MAGMA from dev branch 22 - echo "-------------- MAGMA ---------------" 23 - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR describe 24# -- LIBXSMM v1.16.1 25 - cd .. && export XSMM_VERSION=libxsmm-1.16.1 && { [[ -d $XSMM_VERSION ]] || { git clone --depth 1 --branch 1.16.1 https://github.com/hfp/libxsmm.git $XSMM_VERSION && make -C $XSMM_VERSION -j$(nproc); }; } && export XSMM_DIR=$PWD/$XSMM_VERSION && cd libCEED 26 - echo "-------------- LIBXSMM -------------" && git -C $XSMM_DIR describe --tags 27# -- OCCA v1.1.0 28 - cd .. && export OCCA_VERSION=occa-1.1.0 OCCA_OPENCL_ENABLED=0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.1.0 https://github.com/libocca/occa.git $OCCA_VERSION && make -C $OCCA_VERSION -j$(nproc); }; } && export OCCA_DIR=$PWD/$OCCA_VERSION && cd libCEED 29 - echo "-------------- OCCA ----------------" && make -C $OCCA_DIR info 30# libCEED 31 - make configure HIP_DIR=/opt/rocm OPT='-O -march=native -ffp-contract=fast' 32# Note: OCCA CPU backends currently disabled in CI 33 - BACKENDS_CPU=$(OCCA_DIR= make info-backends-all | grep -o '/cpu[^ ]*') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*') 34 - echo "-------------- libCEED -------------" && make info 35 - echo "-------------- BACKENDS_CPU---------" && echo $BACKENDS_CPU 36 - echo "-------------- BACKENDS_GPU---------" && echo $BACKENDS_GPU 37 - make -j$NPROC_CPU 38# -- libCEED only tests 39 - echo "-------------- core tests ----------" 40 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 41# Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests 42 - export PETSC_DIR= PETSC_ARCH= 43 - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit realsearch=% 44 - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit realsearch=% 45# Libraries for examples 46# -- PETSc with HIP (minimal) 47 - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR describe 48 - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info 49 - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit search="petsc fluids solids" 50 - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit search="petsc fluids solids" 51# -- MFEM v4.2 52 - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED 53 - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info 54 - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit search=mfem 55 - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit search=mfem 56# -- Nek5000 v19.0 57 - export COVERAGE=0 58 - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED 59 - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags 60 - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit search=nek 61 - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit search=nek 62# Clang-tidy 63 - echo "-------------- clang-tidy ----------" && clang-tidy --version 64 - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --exit-code 65# Report status 66 - touch .SUCCESS 67 after_script: 68 - | 69 if [ -f .SUCCESS ]; then 70 lcov --directory . --capture --output-file coverage.info; 71 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface; 72 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery; 73 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends; 74 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests; 75 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples; 76 fi 77 artifacts: 78 paths: 79 - build/*.junit 80 reports: 81 junit: build/*.junit 82 performance: performance.json 83 84noether-float: 85 stage: test 86 tags: 87 - rocm 88 interruptible: true 89 image: jedbrown/rocm:latest 90 script: 91# Compilers 92 - export COVERAGE=1 CC=gcc CXX=g++ FC= HIPCC=hipcc 93 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 94 - echo "-------------- CC ------------------" && $CC --version 95 - echo "-------------- CXX -----------------" && $CXX --version 96 - echo "-------------- HIPCC ---------------" && $HIPCC --version 97 - echo "-------------- GCOV ----------------" && gcov --version 98# Libraries for backends 99# -- MAGMA from dev branch 100 - echo "-------------- MAGMA ---------------" 101 - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR describe 102# -- LIBXSMM v1.16.1 103 - cd .. && export XSMM_VERSION=libxsmm-1.16.1 && { [[ -d $XSMM_VERSION ]] || { git clone --depth 1 --branch 1.16.1 https://github.com/hfp/libxsmm.git $XSMM_VERSION && make -C $XSMM_VERSION -j$(nproc); }; } && export XSMM_DIR=$PWD/$XSMM_VERSION && cd libCEED 104 - echo "-------------- LIBXSMM -------------" && git -C $XSMM_DIR describe --tags 105# libCEED 106# First change to single precision 107 - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/ceed.h 108# Build libCEED 109 - make configure HIP_DIR=/opt/rocm OPT='-O -march=native -ffp-contract=fast' 110 - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*') 111 - echo "-------------- libCEED -------------" && make info 112 - echo "-------------- BACKENDS_CPU---------" && echo $BACKENDS_CPU 113 - echo "-------------- BACKENDS_GPU---------" && echo $BACKENDS_GPU 114 - make -j$NPROC_CPU 115# -- libCEED only tests 116 - echo "-------------- core tests ----------" 117 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 118# Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests 119 - export PETSC_DIR= PETSC_ARCH= 120 - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit realsearch=% 121 - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit realsearch=% 122# Report status 123 - echo "SUCCESS" > .job_status 124 after_script: 125 - | 126 if [ $(cat .job_status) == "SUCCESS" ]; then 127 lcov --directory . --capture --output-file coverage.info; 128 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface; 129 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery; 130 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends; 131 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests; 132 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples; 133 fi 134 - sed -i 's/ceed-f32/ceed-f64/1' include/ceed/ceed.h 135 artifacts: 136 paths: 137 - build/*.junit 138 reports: 139 junit: build/*.junit 140 performance: performance.json 141 142lv-cuda: 143 stage: test 144 tags: 145 - cuda 146 interruptible: true 147 before_script: 148# Compilers 149 - . /opt/rh/gcc-toolset-10/enable 150 - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran 151 - export CUDA_DIR=/usr/local/cuda-11.4 152 - export CUDA_VISIBLE_DEVICES=GPU-c4529365-8229-f689-b43d-ccd7f1677079 # our RTX 2080 Super via nvidia-smi -L 153 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=1 && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 154 - echo "-------------- CC ------------------" && $CC --version 155 - echo "-------------- CXX -----------------" && $CXX --version 156 - echo "-------------- FC ------------------" && $FC --version 157 - echo "-------------- NVCC ----------------" && $CUDA_DIR/bin/nvcc --version 158# libCEED 159 - make configure OPT='-O -march=native -ffp-contract=fast' 160 - echo "-------------- libCEED -------------" && make info 161 - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*') 162 - echo "-------------- BACKENDS_GPU---------" && echo $BACKENDS_GPU 163 - export PETSC_DIR=/home/jeth8984/petsc PETSC_ARCH=cuda-O && git -C $PETSC_DIR describe 164 script: 165 - rm -f .SUCCESS 166 - nice make -k -j$NPROC_CPU -l$NPROC_CPU 167 - echo "-------------- quick CPU tests to get everything built (no junit) --------------" 168 - nice make -k -j$NPROC_CPU -l$NPROC_CPU test BACKENDS=/cpu/self/avx/blocked realsearch=% 169 - echo "-------------- CUDA tests without PETSc (makes *.junit) --------------" 170 - nice make -k -j$NPROC_GPU junit PETSC_DIR= BACKENDS="$BACKENDS_GPU" realsearch=% 171 - echo "-------------- CUDA tests with PETSc (makes *.junit) --------------" 172# skip fluids and solids due to CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES on RTX 2080 Super 173 - nice make -k -j$NPROC_GPU junit BACKENDS="$BACKENDS_GPU" search="petsc" 174# Report status 175 - touch .SUCCESS 176 after_script: 177 - | 178 if [ -f .SUCCESS ]; then 179 lcov --directory . --capture --output-file coverage.info; 180 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface; 181 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery; 182 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends; 183 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests; 184 bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples; 185 fi 186 artifacts: 187 paths: 188 - build/*.junit 189 reports: 190 junit: build/*.junit 191