1# ---------------------------------------------------------------------------------------- 2# HONEE GitLab CI 3# ---------------------------------------------------------------------------------------- 4stages: 5 - test:stage-lint 6 - test:stage-full 7 - test:docs 8 - deploy 9 10.test-basic: 11 interruptible: true 12 only: 13 refs: 14 - web 15 - merge_requests 16 17.test: 18 extends: .test-basic 19 only: 20 refs: 21 - web 22 - merge_requests 23 - main 24 - release 25 except: 26 variables: 27 # Skip if the No-Code label is attached to a merge request (i.e., documentation only) 28 - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/ 29 needs: 30 - job: docker-build 31 optional: true 32 33.docs: 34 image: python:3.10 35 before_script: 36 - pip install -r doc/requirements.txt 37 - apt-get update 38 - apt-get install -y doxygen librsvg2-bin 39 40 41# ---------------------------------------------------------------------------------------- 42# Test formatting and static analysis 43# ---------------------------------------------------------------------------------------- 44noether-lint: 45 stage: test:stage-lint 46 extends: .test 47 tags: 48 - noether 49 - shell 50 needs: [] 51 script: 52 - rm -f .SUCCESS 53 # Environment 54 - export CC=gcc HIPCC=hipcc 55 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 56 - echo "-------------- CC ------------------" && $CC --version 57 - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 58 # Libraries 59 # -- libCEED 60 - echo "-------------- libCEED -------------" 61 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 62 # -- PETSc 63 - echo "-------------- PETSc ---------------" 64 - export PETSC_DIR=/projects/honee/petsc 65 - export PETSC_ARCH=arch-serial-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 66 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 67 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 68 # make with Werror, Wall, supress loop vectorization warnings 69 - echo "-------------- make Werror ---------" 70 - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU 71 # make format 72 - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version 73 - make -j$NPROC_CPU format && git diff --color=always --exit-code 74 # Clang-tidy 75 - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version 76 - PETSC_ARCH=arch-serial-hip make -j$NPROC_CPU tidy 77 # Report status 78 - touch .SUCCESS 79 80 81# ---------------------------------------------------------------------------------------- 82# Test memory access assumptions 83# ---------------------------------------------------------------------------------------- 84noether-memcheck: 85 stage: test:stage-lint 86 extends: .test 87 tags: 88 - noether 89 - shell 90 needs: [] 91 script: 92 - rm -f .SUCCESS 93 # Environment 94 # -- NOTE: Coverage disabled because it doesn't play nice with the ASAN options 95 - export CC=clang-15 96 - export NPROC_POOL=8 97 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 98 - echo "-------------- CC ------------------" && $CC --version 99 # Libraries 100 # -- libCEED 101 - echo "-------------- libCEED -------------" 102 - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 103 # -- PETSc 104 - echo "-------------- PETSc ---------------" 105 - export PETSC_DIR=/projects/honee/petsc 106 - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 107 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 108 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 109 # ASAN 110 - echo "-------------- ASAN ----------------" 111 - echo "ASAN intentionlly skipped, known leaks" 112 # - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" 113 - echo $AFLAGS 114 # HONEE 115 - echo "-------------- HONEE ---------------" && make info 116 - make clean 117 - make -j$NPROC_CPU 118 # Test suite 119 - echo "-------------- HONEE tests ---------" 120 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 121 # -- Memcheck libCEED CPU backend, serial 122 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes 123 # Report status 124 - touch .SUCCESS 125 artifacts: 126 paths: 127 - build/*.junit 128 reports: 129 junit: build/*.junit 130 performance: performance.json 131 expire_in: 28 days 132 133 134# ---------------------------------------------------------------------------------------- 135# CPU testing on Noether 136# ---------------------------------------------------------------------------------------- 137noether-cpu: 138 stage: test:stage-full 139 extends: .test 140 tags: 141 - noether 142 - shell 143 script: 144 - rm -f .SUCCESS 145 # Environment 146 - export COVERAGE=1 CC=gcc 147 - export NPROC_POOL=4 148 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 149 - echo "-------------- CC ------------------" && $CC --version 150 - echo "-------------- GCOV ----------------" && gcov --version 151 # Libraries 152 # -- libCEED 153 - echo "-------------- libCEED -------------" 154 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 155 # -- PETSc 156 - echo "-------------- PETSc ---------------" 157 - export PETSC_DIR=/projects/honee/petsc 158 - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 159 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 160 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 161 # HONEE 162 - echo "-------------- HONEE ---------------" && make info 163 - make clean 164 - make -j$NPROC_CPU 165 # Test suite 166 - echo "-------------- HONEE tests ---------" 167 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 168 # -- Fastest libCEED CPU backend, parallel 169 - echo "Parallel tests skipped for now" 170 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 171 - export SMARTREDIS_DIR=/home/phypid/SmartSimTestingSoftware/smartredis/install 172 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes 173 - spack unload py-torch@2.3+cuda && export USE_TORCH=0 174 - source /home/phypid/SmartSimTestingSoftware/bin/activate 175 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim" 176 # Report status 177 - touch .SUCCESS 178 after_script: 179 - | 180 if [ -f .SUCCESS ]; then 181 gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml; 182 fi 183 coverage: '/^lines:\s+(\d+.\d\%)/' 184 artifacts: 185 paths: 186 - coverage.xml 187 - build/*.junit 188 reports: 189 coverage_report: 190 coverage_format: cobertura 191 path: coverage.xml 192 junit: build/*.junit 193 performance: performance.json 194 expire_in: 28 days 195 196 197# ---------------------------------------------------------------------------------------- 198# CPU Int64 testing on Noether 199# ---------------------------------------------------------------------------------------- 200noether-cpu-int64: 201 stage: test:stage-full 202 extends: .test 203 tags: 204 - noether 205 - shell 206 script: 207 - rm -f .SUCCESS 208 # Environment 209 - export COVERAGE=1 CC=gcc 210 - export NPROC_POOL=4 211 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 212 - echo "-------------- CC ------------------" && $CC --version 213 - echo "-------------- GCOV ----------------" && gcov --version 214 # Libraries 215 # -- libCEED 216 - echo "-------------- libCEED -------------" 217 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 218 # -- PETSc 219 - echo "-------------- PETSc ---------------" 220 - export PETSC_DIR=/projects/honee/petsc 221 - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 222 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 223 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 224 # HONEE 225 - echo "-------------- HONEE ---------------" && make info 226 - make clean 227 - make -j$NPROC_CPU 228 # Test suite 229 - echo "-------------- HONEE tests ---------" 230 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 231 # -- Fastest libCEED CPU backend, serial 232 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 233 - export SMARTREDIS_DIR=/home/phypid/SmartSimTestingSoftware/smartredis/install 234 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes 235 - spack unload py-torch@2.3+cuda && export USE_TORCH=0 236 - source /home/phypid/SmartSimTestingSoftware/bin/activate 237 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-py-smartsim_regression_framework" 238 # Report status 239 - touch .SUCCESS 240 after_script: 241 - | 242 if [ -f .SUCCESS ]; then 243 gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml; 244 fi 245 coverage: '/^lines:\s+(\d+.\d\%)/' 246 artifacts: 247 paths: 248 - coverage.xml 249 - build/*.junit 250 reports: 251 coverage_report: 252 coverage_format: cobertura 253 path: coverage.xml 254 junit: build/*.junit 255 performance: performance.json 256 expire_in: 28 days 257 258 259# ---------------------------------------------------------------------------------------- 260# GPU testing on Noether 261# ---------------------------------------------------------------------------------------- 262noether-hip: 263 stage: test:stage-full 264 extends: .test 265 tags: 266 - noether 267 - shell 268 script: 269 - rm -f .SUCCESS 270 # Environment 271 - export COVERAGE=1 CC=gcc HIPCC=hipcc 272 - export NPROC_POOL=4 273 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 274 - echo "-------------- CC ------------------" && $CC --version 275 - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 276 - echo "-------------- GCOV ----------------" && gcov --version 277 # Libraries 278 # -- libCEED 279 - echo "-------------- libCEED -------------" 280 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 281 # -- PETSc 282 - echo "-------------- PETSc ---------------" 283 - export PETSC_DIR=/projects/honee/petsc 284 - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 285 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 286 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 287 # HONEE 288 - echo "-------------- HONEE ---------------" && make info 289 - make clean 290 - make -j$NPROC_CPU 291 # Test suite 292 - echo "-------------- HONEE tests ---------" 293 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 294 # -- Fastest libCEED HIP backend, serial 295 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 296 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes 297 # Report status 298 - touch .SUCCESS 299 after_script: 300 - | 301 if [ -f .SUCCESS ]; then 302 gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml; 303 fi 304 coverage: '/^lines:\s+(\d+.\d\%)/' 305 artifacts: 306 paths: 307 - coverage.xml 308 - build/*.junit 309 reports: 310 coverage_report: 311 coverage_format: cobertura 312 path: coverage.xml 313 junit: build/*.junit 314 performance: performance.json 315 expire_in: 28 days 316 317 318noether-cuda: 319 stage: test:stage-full 320 extends: .test 321 tags: 322 - noether 323 - shell 324 script: 325 - rm -f .SUCCESS 326 # Environment 327 - export COVERAGE=1 CC=gcc NVCC=nvcc 328 - export NPROC_POOL=4 329 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 330 - echo "-------------- CC ------------------" && $CC --version 331 - echo "-------------- NVCC ----------------" && $NVCC --version 332 - echo "-------------- GCOV ----------------" && gcov --version 333 # Libraries 334 # -- libCEED 335 - echo "-------------- libCEED -------------" 336 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 337 # -- PETSc 338 - echo "-------------- PETSc ---------------" 339 - export PETSC_DIR=/projects/honee/petsc 340 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 341 - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests 342 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 343 # HONEE 344 - echo "-------------- HONEE ---------------" && make info 345 - make clean 346 - make -j$NPROC_CPU 347 # Test suite 348 - echo "-------------- HONEE tests ---------" 349 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 350 # -- Fastest libCEED CUDA backend, serial 351 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 352 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 353 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes 354 # Report status 355 - touch .SUCCESS 356 after_script: 357 - | 358 if [ -f .SUCCESS ]; then 359 gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml; 360 fi 361 coverage: '/^lines:\s+(\d+.\d\%)/' 362 artifacts: 363 paths: 364 - coverage.xml 365 - build/*.junit 366 reports: 367 coverage_report: 368 coverage_format: cobertura 369 path: coverage.xml 370 junit: build/*.junit 371 performance: performance.json 372 expire_in: 28 days 373 374 375# ---------------------------------------------------------------------------------------- 376# Build documentation 377# ---------------------------------------------------------------------------------------- 378docs-review: 379 stage: test:docs 380 tags: 381 - noether 382 - docker 383 extends: 384 - .docs 385 - .test-basic 386 interruptible: true 387 script: 388 - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-hip CEED_DIR=/projects/honee/libCEED 389 - git submodule update --init 390 - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W 391 - mv doc/build/html public 392 artifacts: 393 paths: 394 - public 395 expire_in: 28 days 396 environment: 397 name: review/$CI_COMMIT_REF_NAME 398 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html 399 400 401# ---------------------------------------------------------------------------------------- 402# Deploy documentation using GitLab pages 403# ---------------------------------------------------------------------------------------- 404pages: # this job name has special meaning to GitLab 405 stage: deploy 406 tags: 407 - noether 408 - docker 409 extends: .docs 410 interruptible: false 411 script: 412 - git submodule update --init 413 - make doc-dirhtml pkgconf=true DOXYGENOPTS= 414 - mv doc/build/dirhtml public 415 only: 416 - main 417 artifacts: 418 paths: 419 - public 420