1#!/usr/bin/python3 2 3# Use GNU compilers 4# module load PrgEnv-gnu 5# module load cray-mpich 6# module load amd-mixed/5.4.0 7# 8# To enable GPU-aware MPI, one has to also set this runtime environment variable 9# 10# export MPICH_GPU_SUPPORT_ENABLED=1 11# 12if __name__ == '__main__': 13 import sys 14 import os 15 sys.path.insert(0, os.path.abspath('config')) 16 import configure 17 configure_options = [ 18 '--with-debugging=0', 19 '--with-cc=cc', 20 '--with-cxx=CC', 21 '--with-fc=ftn', 22 '--with-mpiexec=srun -p batch -N 1 -A csc314 -t 00:10:00', 23 '--with-hip', 24 '--with-hipc=hipcc', 25 '--download-kokkos', 26 '--download-kokkos-kernels', 27 ] 28 configure.petsc_configure(configure_options) 29 30# Use Cray compilers 31# module load PrgEnv-cray 32# module load cray-mpich 33# module load amd-mixed/5.4.0 34 35# To enable GPU-aware MPI, one has to also set this runtime environment variable 36# 37# export MPICH_GPU_SUPPORT_ENABLED=1 38# 39# Additional note: "craype-accel-amd-gfx90a" module is recommended for 40# "OpenMP offload" or "GPU enabled MPI". It requires "--with-openmp" option. 41# [otherwise building c examples gives link errors (when fortran bindings are enabled)] 42# Alternative is to use "-lmpi_gtl_hsa" as shown below. 43# 44# ld.lld: error: lib/libpetsc.so: undefined reference to .omp_offloading.img_start.cray_amdgcn-amd-amdhsa [--no-allow-shlib-undefined] 45# 46# Also, please ignore warnings like this. If you don't use Fortran, use '--with-fc=0' to get rid of them. 47# 48# ftn-878 ftn: WARNING PETSC, File = ../../../autofs/nccs-svm1_home1/jczhang/petsc/src/tao/f90-mod/petsctaomod.F90, Line = 37, Column = 13 49# A module named "PETSCVECDEFDUMMY" has already been directly or indirectly use associated into this scope. 50 51 52# if __name__ == '__main__': 53# import sys 54# import os 55# sys.path.insert(0, os.path.abspath('config')) 56# import configure 57# configure_options = [ 58# '--with-debugging=0', 59# '--with-cc=cc', 60# '--with-cxx=CC', 61# '--with-fc=ftn', 62# # -std=c2x is a workaround for this hipsparse problem 63# # /opt/rocm-5.4.0/include/hipsparse/hipsparse.h:8741:28: error: expected '= constant-expression' or end of enumerator definition 64# # HIPSPARSE_ORDER_COLUMN [[deprecated("Please use HIPSPARSE_ORDER_COL instead")]] = 1, 65# # -Wno-constant-logical-operand is a workaround to supress excessive warnings caused by -std=c2x in petsc source which we don't want to address, see MR !6287 66# '--CFLAGS=-std=c2x -Wno-constant-logical-operand', 67# 'LIBS={GTLDIR} {GTLLIBS}'.format(GTLDIR=os.environ['PE_MPICH_GTL_DIR_amd_gfx90a'], GTLLIBS=os.environ['PE_MPICH_GTL_LIBS_amd_gfx90a']), 68# #'--with-openmp=1', # enable if using "craype-accel-amd-gfx90a" module 69# '--with-mpiexec=srun -p batch -N 1 -A csc314 -t 00:10:00', 70# '--with-hip', 71# '--with-hipc=hipcc', 72# '--download-kokkos', 73# '--download-kokkos-kernels', 74# ] 75# configure.petsc_configure(configure_options)