1*a8c70ff5SJunchao Zhang#!/usr/bin/python3 2*a8c70ff5SJunchao Zhang 3*a8c70ff5SJunchao Zhang# Use GNU compilers 4*a8c70ff5SJunchao Zhang# module load PrgEnv-gnu 5*a8c70ff5SJunchao Zhang# module load cray-mpich 6*a8c70ff5SJunchao Zhang# module load amd-mixed/5.4.0 7*a8c70ff5SJunchao Zhang# 8*a8c70ff5SJunchao Zhang# To enable GPU-aware MPI, one has to also set this runtime environment variable 9*a8c70ff5SJunchao Zhang# 10*a8c70ff5SJunchao Zhang# export MPICH_GPU_SUPPORT_ENABLED=1 11*a8c70ff5SJunchao Zhang# 12*a8c70ff5SJunchao Zhangif __name__ == '__main__': 13*a8c70ff5SJunchao Zhang import sys 14*a8c70ff5SJunchao Zhang import os 15*a8c70ff5SJunchao Zhang sys.path.insert(0, os.path.abspath('config')) 16*a8c70ff5SJunchao Zhang import configure 17*a8c70ff5SJunchao Zhang configure_options = [ 18*a8c70ff5SJunchao Zhang '--with-debugging=0', 19*a8c70ff5SJunchao Zhang '--with-cc=cc', 20*a8c70ff5SJunchao Zhang '--with-cxx=CC', 21*a8c70ff5SJunchao Zhang '--with-fc=ftn', 22*a8c70ff5SJunchao Zhang '--with-mpiexec=srun -p batch -N 1 -A csc314 -t 00:10:00', 23*a8c70ff5SJunchao Zhang '--with-hip', 24*a8c70ff5SJunchao Zhang '--with-hipc=hipcc', 25*a8c70ff5SJunchao Zhang '--download-kokkos', 26*a8c70ff5SJunchao Zhang '--download-kokkos-kernels', 27*a8c70ff5SJunchao Zhang ] 28*a8c70ff5SJunchao Zhang configure.petsc_configure(configure_options) 29*a8c70ff5SJunchao Zhang 30*a8c70ff5SJunchao Zhang# Use Cray compilers 31*a8c70ff5SJunchao Zhang# module load PrgEnv-cray 32*a8c70ff5SJunchao Zhang# module load cray-mpich 33*a8c70ff5SJunchao Zhang# module load amd-mixed/5.4.0 34*a8c70ff5SJunchao Zhang 35*a8c70ff5SJunchao Zhang# To enable GPU-aware MPI, one has to also set this runtime environment variable 36*a8c70ff5SJunchao Zhang# 37*a8c70ff5SJunchao Zhang# export MPICH_GPU_SUPPORT_ENABLED=1 38*a8c70ff5SJunchao Zhang# 39*a8c70ff5SJunchao Zhang# Additional note: "craype-accel-amd-gfx90a" module is recommended for 40*a8c70ff5SJunchao Zhang# "OpenMP offload" or "GPU enabled MPI". It requires "--with-openmp" option. 41*a8c70ff5SJunchao Zhang# [otherwise building c examples gives link errors (when fortran bindings are enabled)] 42*a8c70ff5SJunchao Zhang# Alternative is to use "-lmpi_gtl_hsa" as shown below. 43*a8c70ff5SJunchao Zhang# 44*a8c70ff5SJunchao Zhang# ld.lld: error: lib/libpetsc.so: undefined reference to .omp_offloading.img_start.cray_amdgcn-amd-amdhsa [--no-allow-shlib-undefined] 45*a8c70ff5SJunchao Zhang# 46*a8c70ff5SJunchao Zhang# Also, please ignore warnings like this. If you don't use Fortran, use '--with-fc=0' to get rid of them. 47*a8c70ff5SJunchao Zhang# 48*a8c70ff5SJunchao Zhang# ftn-878 ftn: WARNING PETSC, File = ../../../autofs/nccs-svm1_home1/jczhang/petsc/src/tao/f90-mod/petsctaomod.F90, Line = 37, Column = 13 49*a8c70ff5SJunchao Zhang# A module named "PETSCVECDEFDUMMY" has already been directly or indirectly use associated into this scope. 50*a8c70ff5SJunchao Zhang 51*a8c70ff5SJunchao Zhang 52*a8c70ff5SJunchao Zhang# if __name__ == '__main__': 53*a8c70ff5SJunchao Zhang# import sys 54*a8c70ff5SJunchao Zhang# import os 55*a8c70ff5SJunchao Zhang# sys.path.insert(0, os.path.abspath('config')) 56*a8c70ff5SJunchao Zhang# import configure 57*a8c70ff5SJunchao Zhang# configure_options = [ 58*a8c70ff5SJunchao Zhang# '--with-debugging=0', 59*a8c70ff5SJunchao Zhang# '--with-cc=cc', 60*a8c70ff5SJunchao Zhang# '--with-cxx=CC', 61*a8c70ff5SJunchao Zhang# '--with-fc=ftn', 62*a8c70ff5SJunchao Zhang# # -std=c2x is a workaround for this hipsparse problem 63*a8c70ff5SJunchao Zhang# # /opt/rocm-5.4.0/include/hipsparse/hipsparse.h:8741:28: error: expected '= constant-expression' or end of enumerator definition 64*a8c70ff5SJunchao Zhang# # HIPSPARSE_ORDER_COLUMN [[deprecated("Please use HIPSPARSE_ORDER_COL instead")]] = 1, 65*a8c70ff5SJunchao Zhang# # -Wno-constant-logical-operand is a workaround to supress excessive warnings caused by -std=c2x in petsc source which we don't want to address, see MR !6287 66*a8c70ff5SJunchao Zhang# '--CFLAGS=-std=c2x -Wno-constant-logical-operand', 67*a8c70ff5SJunchao Zhang# 'LIBS={GTLDIR} {GTLLIBS}'.format(GTLDIR=os.environ['PE_MPICH_GTL_DIR_amd_gfx90a'], GTLLIBS=os.environ['PE_MPICH_GTL_LIBS_amd_gfx90a']), 68*a8c70ff5SJunchao Zhang# #'--with-openmp=1', # enable if using "craype-accel-amd-gfx90a" module 69*a8c70ff5SJunchao Zhang# '--with-mpiexec=srun -p batch -N 1 -A csc314 -t 00:10:00', 70*a8c70ff5SJunchao Zhang# '--with-hip', 71*a8c70ff5SJunchao Zhang# '--with-hipc=hipcc', 72*a8c70ff5SJunchao Zhang# '--download-kokkos', 73*a8c70ff5SJunchao Zhang# '--download-kokkos-kernels', 74*a8c70ff5SJunchao Zhang# ] 75*a8c70ff5SJunchao Zhang# configure.petsc_configure(configure_options)