1*2e294c49SRichard Tran Mills#!/usr/bin/env python3 2*2e294c49SRichard Tran Mills 3*2e294c49SRichard Tran Mills# Example configure script for Perlmutter, the HPE Cray EX system at NERSC/LBNL equipped with 4*2e294c49SRichard Tran Mills# wth AMD EPYC CPUS and NVIDIA A100 GPUS. Here we target the GPU compute nodes and builds with 5*2e294c49SRichard Tran Mills# support for the CUDA/cuSPARSE, Kokkos, and ViennaCL back-ends. 6*2e294c49SRichard Tran Mills# 7*2e294c49SRichard Tran Mills# Currently, configuring PETSc on the system does not require loading many , if any, non-default modules. 8*2e294c49SRichard Tran Mills# As documented at https://docs.nersc.gov/systems/perlmutter/software/#mpi, typical settings might be 9*2e294c49SRichard Tran Mills# 10*2e294c49SRichard Tran Mills# export MPICH_GPU_SUPPORT_ENABLED=1 11*2e294c49SRichard Tran Mills# module load cudatoolkit 12*2e294c49SRichard Tran Mills# module load PrgEnv-gnu 13*2e294c49SRichard Tran Mills# module load craype-accel-nvidia80 14*2e294c49SRichard Tran Mills# 15*2e294c49SRichard Tran Mills# The above are currently present in the default environment. Users may wish to 'module load' a 16*2e294c49SRichard Tran Mills# different programming environment (which will generally force a reload of certain related modules, 17*2e294c49SRichard Tran Mills# such as the one corresponding to the MPI implementation). 18*2e294c49SRichard Tran Mills 19*2e294c49SRichard Tran Millsif __name__ == '__main__': 20*2e294c49SRichard Tran Mills import sys 21*2e294c49SRichard Tran Mills import os 22*2e294c49SRichard Tran Mills sys.path.insert(0, os.path.abspath('config')) 23*2e294c49SRichard Tran Mills import configure 24*2e294c49SRichard Tran Mills configure_options = [ 25*2e294c49SRichard Tran Mills '--with-make-np=8', # Must limit size of parallel build to stay within resource limitations imposed by the center 26*2e294c49SRichard Tran Mills '--with-mpiexec=srun -G4', # '-G4' requests all four GPUs present on a Perlmutter GPU compute node. 27*2e294c49SRichard Tran Mills '--with-batch=0', 28*2e294c49SRichard Tran Mills 29*2e294c49SRichard Tran Mills # Use the Cray compiler wrappers, regardless of the underlying compilers loaded by the programming environment module: 30*2e294c49SRichard Tran Mills '--with-cc=cc', 31*2e294c49SRichard Tran Mills '--with-cxx=CC', 32*2e294c49SRichard Tran Mills '--with-fc=ftn', 33*2e294c49SRichard Tran Mills 34*2e294c49SRichard Tran Mills # Build with aggressive optimization ('-O3') but also include debugging symbols ('-g') to support detailed profiling. 35*2e294c49SRichard Tran Mills # If you are doing development, using no optimization ('-O0') can be a good idea. Also note that some compilers (GNU 36*2e294c49SRichard Tran Mills # is one) support the '-g3' debug flag, which allows macro expansion in some debuggers; this can be very useful when 37*2e294c49SRichard Tran Mills # debugging PETSc code, as PETSc makes extensive use of macros. 38*2e294c49SRichard Tran Mills '--COPTFLAGS= -g -O3', 39*2e294c49SRichard Tran Mills '--CXXOPTFLAGS= -g -O3', 40*2e294c49SRichard Tran Mills '--FOPTFLAGS= -g -O3', 41*2e294c49SRichard Tran Mills '--CUDAFLAGS= -g -O3', 42*2e294c49SRichard Tran Mills '--with-debugging=0', # Disable debugging for production builds; use '--with-debugging=1' for development work. 43*2e294c49SRichard Tran Mills 44*2e294c49SRichard Tran Mills # Set sowing-cc and sowing-cxx explicitly, as this prevents errors caused by compiling sowing with GCC when a 45*2e294c49SRichard Tran Mills # programming environment other than PrgEnv-gnu has been loaded. If there is this compiler mismatch, we will see 46*2e294c49SRichard Tran Mills # errors like 47*2e294c49SRichard Tran Mills # 48*2e294c49SRichard Tran Mills # /opt/nvidia/hpc_sdk/Linux_x86_64/22.5/compilers/include/bits/floatn.h:60:17: error: two or more data types in declaration specifiers 49*2e294c49SRichard Tran Mills # typedef float _Float32; 50*2e294c49SRichard Tran Mills # ^~~~~~~~ 51*2e294c49SRichard Tran Mills '--download-sowing-cc=cc', # Note that sowing is only needed when Fortran bindings are required. 52*2e294c49SRichard Tran Mills '--download-sowing-cc=CC', 53*2e294c49SRichard Tran Mills 54*2e294c49SRichard Tran Mills 55*2e294c49SRichard Tran Mills # Build with support for CUDA/cuSPARSE, Kokkos/Kokkos Kernels, and ViennaCL back-ends: 56*2e294c49SRichard Tran Mills '--with-cuda=1', 57*2e294c49SRichard Tran Mills '--with-cuda-arch=80', 58*2e294c49SRichard Tran Mills '--download-viennacl', 59*2e294c49SRichard Tran Mills '--download-kokkos', 60*2e294c49SRichard Tran Mills '--download-kokkos-kernels', 61*2e294c49SRichard Tran Mills '--with-kokkos-kernels-tpl=0', # Use native Kokkos kernels, rather than NVIDIA-provided ones. 62*2e294c49SRichard Tran Mills 63*2e294c49SRichard Tran Mills # Download and build a few commonly-used packages: 64*2e294c49SRichard Tran Mills '--download-hypre', 65*2e294c49SRichard Tran Mills '--download-metis', 66*2e294c49SRichard Tran Mills '--download-parmetis', 67*2e294c49SRichard Tran Mills '--download-hdf5', # Note that NERSC does provide an HDF5 module, but using our own is generally reliable. 68*2e294c49SRichard Tran Mills '--download-hdf5-fortran-bindings', 69*2e294c49SRichard Tran Mills ] 70*2e294c49SRichard Tran Mills configure.petsc_configure(configure_options) 71