1#!/usr/bin/env python3 2 3import os 4from sysconfig import _parse_makefile as parse_makefile 5import sys 6import logging 7sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 8from collections import defaultdict 9 10AUTODIRS = set('ftn-auto ftn-custom f90-custom ftn-auto-interfaces'.split()) # Automatically recurse into these, if they exist 11SKIPDIRS = set('benchmarks build'.split()) # Skip these during the build 12NOWARNDIRS = set('tests tutorials'.split()) # Do not warn about mismatch in these 13 14def pathsplit(path): 15 """Recursively split a path, returns a tuple""" 16 stem, basename = os.path.split(path) 17 if stem == '': 18 return (basename,) 19 if stem == path: # fixed point, likely '/' 20 return (path,) 21 return pathsplit(stem) + (basename,) 22 23def getlangext(name): 24 """Returns everything after the first . in the filename, including the .""" 25 file = os.path.basename(name) 26 loc = file.find('.') 27 if loc > -1: return file[loc:] 28 else: return '' 29 30def getlangsplit(name): 31 """Returns everything before the first . in the filename, excluding the .""" 32 file = os.path.basename(name) 33 loc = file.find('.') 34 if loc > -1: return os.path.join(os.path.dirname(name),file[:loc]) 35 raise RuntimeError("No . in filename") 36 37class Mistakes(object): 38 def __init__(self, log, verbose=False): 39 self.mistakes = [] 40 self.verbose = verbose 41 self.log = log 42 43 def compareDirLists(self,root, mdirs, dirs): 44 if NOWARNDIRS.intersection(pathsplit(root)): 45 return 46 smdirs = set(mdirs) 47 sdirs = set(dirs).difference(AUTODIRS) 48 if not smdirs.issubset(sdirs): 49 self.mistakes.append('%s/makefile contains a directory not on the filesystem: %r' % (root, sorted(smdirs - sdirs))) 50 if not self.verbose: return 51 if smdirs != sdirs: 52 from sys import stderr 53 stderr.write('Directory mismatch at %s:\n\t%s: %r\n\t%s: %r\n\t%s: %r\n' 54 % (root, 55 'in makefile ',sorted(smdirs), 56 'on filesystem ',sorted(sdirs), 57 'symmetric diff',sorted(smdirs.symmetric_difference(sdirs)))) 58 59 def summary(self): 60 for m in self.mistakes: 61 self.log.write(m + '\n') 62 if self.mistakes: 63 raise RuntimeError('\n\nThe PETSc makefiles contain mistakes or files are missing on the filesystem.\n%s\nPossible reasons:\n\t1. Files were deleted locally, try "git checkout filename", where "filename" is the missing file.\n\t2. Files were deleted from the repository, but were not removed from the makefile. Send mail to petsc-maint@mcs.anl.gov.\n\t3. Someone forgot to "add" new files to the repository. Send mail to petsc-maint@mcs.anl.gov.\n\n' % ('\n'.join(self.mistakes))) 64 65def stripsplit(line): 66 return line[len('#requires'):].replace("'","").split() 67 68PetscPKGS = 'sys vec mat dm ksp snes ts tao'.split() 69# the key is actually the language suffix, it won't work for suffixes such as 'kokkos.cxx' so use an _ and replace the _ as needed with . 70LANGS = dict(kokkos_cxx='KOKKOS', hip_cpp='HIP', sycl_cxx='SYCL', raja_cxx='RAJA', c='C', cxx='CXX', cpp='CPP', cu='CU', F='F', F90='F90') 71 72class debuglogger(object): 73 def __init__(self, log): 74 self._log = log 75 76 def write(self, string): 77 self._log.debug(string) 78 79class Petsc(object): 80 def __init__(self, petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, verbose=False): 81 if petsc_dir is None: 82 petsc_dir = os.environ.get('PETSC_DIR') 83 if petsc_dir is None: 84 try: 85 petsc_dir = parse_makefile(os.path.join('lib','petsc','conf', 'petscvariables')).get('PETSC_DIR') 86 finally: 87 if petsc_dir is None: 88 raise RuntimeError('Could not determine PETSC_DIR, please set in environment') 89 if petsc_arch is None: 90 petsc_arch = os.environ.get('PETSC_ARCH') 91 if petsc_arch is None: 92 try: 93 petsc_arch = parse_makefile(os.path.join(petsc_dir, 'lib','petsc','conf', 'petscvariables')).get('PETSC_ARCH') 94 finally: 95 if petsc_arch is None: 96 raise RuntimeError('Could not determine PETSC_ARCH, please set in environment') 97 self.petsc_dir = os.path.normpath(petsc_dir) 98 self.petsc_arch = petsc_arch.rstrip(os.sep) 99 self.pkg_dir = pkg_dir 100 self.pkg_name = pkg_name 101 self.pkg_arch = pkg_arch 102 if self.pkg_dir is None: 103 self.pkg_dir = petsc_dir 104 self.pkg_name = 'petsc' 105 self.pkg_arch = self.petsc_arch 106 if self.pkg_name is None: 107 self.pkg_name = os.path.basename(os.path.normpath(self.pkg_dir)) 108 if self.pkg_arch is None: 109 self.pkg_arch = self.petsc_arch 110 self.pkg_pkgs = PetscPKGS 111 if pkg_pkgs is not None: 112 self.pkg_pkgs += list(set(pkg_pkgs.split(','))-set(self.pkg_pkgs)) 113 self.read_conf() 114 try: 115 logging.basicConfig(filename=self.pkg_arch_path('lib',self.pkg_name,'conf', 'gmake.log'), level=logging.DEBUG) 116 except IOError: 117 # Disable logging if path is not writeable (e.g., prefix install) 118 logging.basicConfig(filename='/dev/null', level=logging.DEBUG) 119 self.log = logging.getLogger('gmakegen') 120 self.mistakes = Mistakes(debuglogger(self.log), verbose=verbose) 121 self.gendeps = [] 122 123 def arch_path(self, *args): 124 return os.path.join(self.petsc_dir, self.petsc_arch, *args) 125 126 def pkg_arch_path(self, *args): 127 return os.path.join(self.pkg_dir, self.pkg_arch, *args) 128 129 def read_conf(self): 130 self.conf = dict() 131 with open(self.arch_path('include', 'petscconf.h')) as petscconf_h: 132 for line in petscconf_h: 133 if line.startswith('#define '): 134 define = line[len('#define '):] 135 space = define.find(' ') 136 key = define[:space] 137 val = define[space+1:] 138 self.conf[key] = val 139 self.conf.update(parse_makefile(self.arch_path('lib','petsc','conf', 'petscvariables'))) 140 # allow parsing package additional configurations (if any) 141 if self.pkg_name != 'petsc' : 142 f = self.pkg_arch_path('include', self.pkg_name + 'conf.h') 143 if os.path.isfile(f): 144 with open(self.pkg_arch_path('include', self.pkg_name + 'conf.h')) as pkg_conf_h: 145 for line in pkg_conf_h: 146 if line.startswith('#define '): 147 define = line[len('#define '):] 148 space = define.find(' ') 149 key = define[:space] 150 val = define[space+1:] 151 self.conf[key] = val 152 f = self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables') 153 if os.path.isfile(f): 154 self.conf.update(parse_makefile(self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables'))) 155 self.have_fortran = int(self.conf.get('PETSC_HAVE_FORTRAN', '0')) 156 157 def inconf(self, key, val): 158 if key in ['package', 'function', 'define']: 159 return self.conf.get(val) 160 elif key == 'precision': 161 return val == self.conf['PETSC_PRECISION'] 162 elif key == 'scalar': 163 return val == self.conf['PETSC_SCALAR'] 164 elif key == 'language': 165 return val == self.conf['PETSC_LANGUAGE'] 166 raise RuntimeError('Unknown conf check: %s %s' % (key, val)) 167 168 def relpath(self, root, src): 169 return os.path.relpath(os.path.join(root, src), self.pkg_dir) 170 171 def get_sources_from_files(self, files): 172 """Return dict {lang: list_of_source_files}""" 173 source = dict() 174 for lang, sourcelang in LANGS.items(): 175 source[lang] = [f for f in files if f.endswith('.'+lang.replace('_','.'))] 176 files = [f for f in files if not f.endswith('.'+lang.replace('_','.'))] 177 return source 178 179 def gen_pkg(self, pkg): 180 pkgsrcs = dict() 181 for lang in LANGS: 182 pkgsrcs[lang] = [] 183 for root, dirs, files in os.walk(os.path.join(self.pkg_dir, 'src', pkg)): 184 if NOWARNDIRS.intersection(pathsplit(root)): continue 185 dirs.sort() 186 files.sort() 187 makefile = os.path.join(root,'makefile') 188 if not os.path.exists(makefile): 189 dirs[:] = [] 190 continue 191 with open(makefile) as mklines: 192 conditions = set(tuple(stripsplit(line)) for line in mklines if line.startswith('#requires')) 193 if not all(self.inconf(key, val) for key, val in conditions): 194 dirs[:] = [] 195 continue 196 makevars = parse_makefile(makefile) 197 mdirs = makevars.get('DIRS','').split() # Directories specified in the makefile 198 self.mistakes.compareDirLists(root, mdirs, dirs) # diagnostic output to find unused directories 199 candidates = set(mdirs).union(AUTODIRS).difference(SKIPDIRS) 200 dirs[:] = list(candidates.intersection(dirs)) 201 allsource = [] 202 def mkrel(src): 203 return self.relpath(root, src) 204 source = self.get_sources_from_files(files) 205 for lang, s in source.items(): 206 pkgsrcs[lang] += [mkrel(t) for t in s] 207 self.gendeps.append(self.relpath(root, 'makefile')) 208 return pkgsrcs 209 210 def gen_gnumake(self, fd): 211 def write(stem, srcs): 212 for lang in LANGS: 213 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]))) 214 for pkg in self.pkg_pkgs: 215 srcs = self.gen_pkg(pkg) 216 write('srcs-' + pkg, srcs) 217 return self.gendeps 218 219 def gen_ninja(self, fd): 220 libobjs = [] 221 for pkg in self.pkg_pkgs: 222 srcs = self.gen_pkg(pkg) 223 for lang in LANGS: 224 for src in srcs[lang]: 225 obj = '$objdir/%s.o' % src 226 fd.write('build %(obj)s : %(lang)s_COMPILE %(src)s\n' % dict(obj=obj, lang=lang.upper(), src=os.path.join(self.pkg_dir,src))) 227 libobjs.append(obj) 228 fd.write('\n') 229 fd.write('build $libdir/libpetsc.so : %s_LINK_SHARED %s\n\n' % ('CF'[self.have_fortran], ' '.join(libobjs))) 230 fd.write('build petsc : phony || $libdir/libpetsc.so\n\n') 231 232 def summary(self): 233 self.mistakes.summary() 234 235def WriteGnuMake(petsc): 236 arch_files = petsc.pkg_arch_path('lib',petsc.pkg_name,'conf', 'files') 237 with open(arch_files, 'w') as fd: 238 gendeps = petsc.gen_gnumake(fd) 239 fd.write('\n') 240 fd.write('# Dependency to regenerate this file\n') 241 fd.write('%s : %s %s\n' % (os.path.relpath(arch_files, petsc.pkg_dir), 242 os.path.relpath(__file__, os.path.realpath(petsc.pkg_dir)), 243 ' '.join(gendeps))) 244 fd.write('\n') 245 fd.write('# Dummy dependencies in case makefiles are removed\n') 246 fd.write(''.join([dep + ':\n' for dep in gendeps])) 247 248def WriteNinja(petsc): 249 conf = dict() 250 parse_makefile(os.path.join(petsc.petsc_dir, 'lib', 'petsc','conf', 'variables'), conf) 251 parse_makefile(petsc.arch_path('lib','petsc','conf', 'petscvariables'), conf) 252 build_ninja = petsc.arch_path('build.ninja') 253 with open(build_ninja, 'w') as fd: 254 fd.write('objdir = obj-ninja\n') 255 fd.write('libdir = lib\n') 256 fd.write('c_compile = %(PCC)s\n' % conf) 257 fd.write('c_flags = %(PETSC_CC_INCLUDES)s %(PCC_FLAGS)s %(CCPPFLAGS)s\n' % conf) 258 fd.write('c_link = %(PCC_LINKER)s\n' % conf) 259 fd.write('c_link_flags = %(PCC_LINKER_FLAGS)s\n' % conf) 260 if petsc.have_fortran: 261 fd.write('f_compile = %(FC)s\n' % conf) 262 fd.write('f_flags = %(PETSC_FC_INCLUDES)s %(FC_FLAGS)s %(FCPPFLAGS)s\n' % conf) 263 fd.write('f_link = %(FC_LINKER)s\n' % conf) 264 fd.write('f_link_flags = %(FC_LINKER_FLAGS)s\n' % conf) 265 fd.write('petsc_external_lib = %(PETSC_EXTERNAL_LIB_BASIC)s\n' % conf) 266 fd.write('python = %(PYTHON)s\n' % conf) 267 fd.write('\n') 268 fd.write('rule C_COMPILE\n' 269 ' command = $c_compile -MMD -MF $out.d $c_flags -c $in -o $out\n' 270 ' description = CC $out\n' 271 ' depfile = $out.d\n' 272 # ' deps = gcc\n') # 'gcc' is default, 'msvc' only recognized by newer versions of ninja 273 '\n') 274 fd.write('rule C_LINK_SHARED\n' 275 ' command = $c_link $c_link_flags -shared -o $out $in $petsc_external_lib\n' 276 ' description = CLINK_SHARED $out\n' 277 '\n') 278 if petsc.have_fortran: 279 fd.write('rule F_COMPILE\n' 280 ' command = $f_compile -MMD -MF $out.d $f_flags -c $in -o $out\n' 281 ' description = FC $out\n' 282 ' depfile = $out.d\n' 283 '\n') 284 fd.write('rule F_LINK_SHARED\n' 285 ' command = $f_link $f_link_flags -shared -o $out $in $petsc_external_lib\n' 286 ' description = FLINK_SHARED $out\n' 287 '\n') 288 fd.write('rule GEN_NINJA\n' 289 ' command = $python $in --output=ninja\n' 290 ' generator = 1\n' 291 '\n') 292 petsc.gen_ninja(fd) 293 fd.write('\n') 294 fd.write('build %s : GEN_NINJA | %s %s %s %s\n' % (build_ninja, 295 os.path.abspath(__file__), 296 os.path.join(petsc.petsc_dir, 'lib','petsc','conf', 'variables'), 297 petsc.arch_path('lib','petsc','conf', 'petscvariables'), 298 ' '.join(os.path.join(petsc.pkg_dir, dep) for dep in petsc.gendeps))) 299 300def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, output=None, verbose=False): 301 if output is None: 302 output = 'gnumake' 303 writer = dict(gnumake=WriteGnuMake, ninja=WriteNinja) 304 petsc = Petsc(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_name=pkg_name, pkg_arch=pkg_arch, pkg_pkgs=pkg_pkgs, verbose=verbose) 305 writer[output](petsc) 306 petsc.summary() 307 308if __name__ == '__main__': 309 import optparse 310 parser = optparse.OptionParser() 311 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 312 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 313 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 314 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 315 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 316 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 317 parser.add_option('--output', help='Location to write output file', default=None) 318 opts, extra_args = parser.parse_args() 319 if extra_args: 320 import sys 321 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 322 exit(1) 323 main(petsc_arch=opts.petsc_arch, pkg_dir=opts.pkg_dir, pkg_name=opts.pkg_name, pkg_arch=opts.pkg_arch, pkg_pkgs=opts.pkg_pkgs, output=opts.output, verbose=opts.verbose) 324