1#!/usr/bin/python3 2 3# Use GNU compilers: 4# 5# module load cudatoolkit-standalone PrgEnv-gnu cray-libsci 6# 7# Note cray-libsci provides BLAS etc. In summary, we have 8# 9# module load cudatoolkit-standalone/11.8.0 PrgEnv-gnu gcc/10.3.0 cray-libsci 10# 11# $ module list 12# Currently Loaded Modules: 13# 1) craype-x86-rome 5) craype-accel-nvidia80 9) cray-dsmml/0.2.2 13) PrgEnv-gnu/8.3.3 14# 2) libfabric/1.15.2.0 6) cmake/3.23.2 10) cray-pmi/6.1.10 14) cray-libsci/23.02.1.1 15# 3) craype-network-ofi 7) cudatoolkit-standalone/11.8.0 11) cray-pals/1.2.11 15) gcc/10.3.0 16# 4) perftools-base/23.03.0 8) craype/2.7.20 12) cray-libpals/1.2.11 16) cray-mpich/8.1.25 17 18if __name__ == '__main__': 19 import sys 20 import os 21 sys.path.insert(0, os.path.abspath('config')) 22 import configure 23 configure_options = [ 24 '--with-cc=cc', 25 '--with-cxx=CC', 26 '--with-fc=ftn', 27 '--with-debugging=0', 28 '--with-cuda', 29 '--with-cudac=nvcc', 30 '--with-cuda-arch=80', # Since there is no easy way to auto-detect the cuda arch on the gpu-less Polaris login nodes, we explicitly set it. 31 '--download-kokkos', 32 '--download-kokkos-kernels', 33 ] 34 configure.petsc_configure(configure_options) 35 36# Use NVHPC compilers 37# 38# Unset so that cray won't add -gpu to nvc even when craype-accel-nvidia80 is loaded 39# unset CRAY_ACCEL_TARGET 40# module load nvhpc/22.11 PrgEnv-nvhpc 41# 42# I met two problems with nvhpc and Kokkos (and Kokkos-Kernels) 4.2.0. 43# 1) Kokkos-Kernles failed at configuration to find TPL cublas and cusparse from NVHPC. 44# As a workaround, I just load cudatoolkit-standalone/11.8.0 to let KK use cublas and cusparse from cudatoolkit-standalone. 45# 2) KK failed at compilation 46# "/home/jczhang/petsc/arch-kokkos-dbg/externalpackages/git.kokkos-kernels/batched/dense/impl/KokkosBatched_Gemm_Serial_Internal.hpp", line 94: error: expression must have a constant value 47# constexpr int nbAlgo = Algo::Gemm::Blocked::mb(); 48# ^ 49# "/home/jczhang/petsc/arch-kokkos-dbg/externalpackages/git.kokkos-kernels/blas/impl/KokkosBlas_util.hpp", line 58: note: cannot call non-constexpr function "__builtin_is_device_code" (declared implicitly) 50# KOKKOS_IF_ON_HOST((return 4;)) 51# ^ 52# detected during: 53# 54# It is a KK problem and I have to wait for their fix. 55