1869551cdSJunchao Zhang#!/usr/bin/python3 2869551cdSJunchao Zhang 3869551cdSJunchao Zhang# Modules loaded by default (on login to Crusher): 4869551cdSJunchao Zhang# 5*57411e1fSSatish Balay# 1) craype-x86-trento 8) cce/13.0.0 6*57411e1fSSatish Balay# 2) libfabric/1.13.0.0 9) craype/2.7.13 7*57411e1fSSatish Balay# 3) craype-network-ofi 10) cray-dsmml/0.2.2 8*57411e1fSSatish Balay# 4) perftools-base/21.12.0 11) cray-mpich/8.1.12 9*57411e1fSSatish Balay# 5) xpmem/2.3.2-2.2_1.16__g9ea452c.shasta 12) cray-libsci/21.08.1.2 10*57411e1fSSatish Balay# 6) cray-pmi/6.0.16 13) PrgEnv-cray/8.2.0 11*57411e1fSSatish Balay# 7) cray-pmi-lib/6.0.16 14) DefApps/default 12*57411e1fSSatish Balay# 13*57411e1fSSatish Balay# Need to load additional rocm module to build with hip 14*57411e1fSSatish Balay# 15*57411e1fSSatish Balay# module load rocm/4.5.0 16869551cdSJunchao Zhang# 17869551cdSJunchao Zhang# We use Cray Programming Environment, Cray compilers, Cray-mpich. 18869551cdSJunchao Zhang# To enable GPU-aware MPI, one has to also set this runtime environment variable 19869551cdSJunchao Zhang# 20869551cdSJunchao Zhang# export MPICH_GPU_SUPPORT_ENABLED=1 21869551cdSJunchao Zhang# 22869551cdSJunchao Zhang# Additional note: If "craype-accel-amd-gfx90a" module is loaded (that is 23869551cdSJunchao Zhang# needed for "OpenMP offload") - it causes link errors when using 'cc or hipcc' 24869551cdSJunchao Zhang# with fortran objs, hence not used 25869551cdSJunchao Zhang# 26869551cdSJunchao Zhang 27869551cdSJunchao Zhangif __name__ == '__main__': 28869551cdSJunchao Zhang import sys 29869551cdSJunchao Zhang import os 30869551cdSJunchao Zhang sys.path.insert(0, os.path.abspath('config')) 31869551cdSJunchao Zhang import configure 32869551cdSJunchao Zhang configure_options = [ 33869551cdSJunchao Zhang '--with-cc=cc', 34869551cdSJunchao Zhang '--with-cxx=CC', 35869551cdSJunchao Zhang '--with-fc=ftn', 36869551cdSJunchao Zhang 'LIBS=-L{x}/gtl/lib -lmpi_gtl_hsa'.format(x=os.environ['CRAY_MPICH_ROOTDIR']), 37869551cdSJunchao Zhang '--with-debugging=0', 38869551cdSJunchao Zhang '--with-mpiexec=srun -p batch -N 1 -A csc314_crusher -t 00:10:00', 39869551cdSJunchao Zhang '--with-hip', 40869551cdSJunchao Zhang '--with-hipc=hipcc', 41869551cdSJunchao Zhang '--download-kokkos', 42869551cdSJunchao Zhang '--download-kokkos-kernels', 43869551cdSJunchao Zhang ] 44869551cdSJunchao Zhang configure.petsc_configure(configure_options) 45