1#!/usr/bin/env python3 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9import shlex 10sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25Case 1. Normal builds: 26 27 +---------------------+----------------------------------+ 28 | PETSC_DIR | <git dir> | 29 +---------------------+----------------------------------+ 30 | PETSC_ARCH | arch-foo | 31 +---------------------+----------------------------------+ 32 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 33 +---------------------+----------------------------------+ 34 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 35 +---------------------+----------------------------------+ 36 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 37 +---------------------+----------------------------------+ 38 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 41 +---------------------+----------------------------------+ 42 43Case 2. From install dir: 44 45 +---------------------+-------------------------------------------------------+ 46 | PETSC_DIR | <prefix dir> | 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_ARCH | '' | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 59 +---------------------+-------------------------------------------------------+ 60 61""" 62 63def install_files(source, destdir): 64 """Install file or directory 'source' to 'destdir'. Does not preserve 65 mode (permissions). 66 """ 67 if not os.path.isdir(destdir): 68 os.makedirs(destdir) 69 if os.path.isdir(source): 70 for name in os.listdir(source): 71 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 72 else: 73 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 74 75def nameSpace(srcfile,srcdir): 76 """ 77 Because the scripts have a non-unique naming, the pretty-printing 78 needs to convey the srcdir and srcfile. There are two ways of doing this. 79 """ 80 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 81 prefix=srcdir.replace("/","_")+"-" 82 nameString=prefix+srcfile 83 return nameString 84 85class generateExamples(Petsc): 86 """ 87 gmakegen.py has basic structure for finding the files, writing out 88 the dependencies, etc. 89 """ 90 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 91 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs) 92 93 self.single_ex=single_ex 94 self.srcdir=srcdir 95 self.check_output=check 96 97 # Set locations to handle movement 98 self.inInstallDir=self.getInInstallDir(thisscriptdir) 99 100 # Special configuration for CI testing 101 if self.petsc_arch.find('valgrind') >= 0: 102 self.conf['PETSCTEST_VALGRIND']=1 103 104 if self.inInstallDir: 105 # Case 2 discussed above 106 # set PETSC_ARCH to install directory to allow script to work in both 107 dirlist=thisscriptdir.split(os.path.sep) 108 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 109 self.arch_dir=installdir 110 if self.srcdir is None: 111 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 112 else: 113 if petsc_arch == '': 114 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 115 # Case 1 discussed above 116 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 117 if self.srcdir is None: 118 self.srcdir=os.path.join(self.petsc_dir,'src') 119 120 self.testroot_dir=os.path.abspath(testdir) 121 122 self.verbose=verbose 123 # Whether to write out a useful debugging 124 self.summarize=True if verbose else False 125 126 # For help in setting the requirements 127 self.precision_types="__fp16 single double __float128".split() 128 self.integer_types="int32 int64 long32 long64".split() 129 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 130 131 # Things that are not test 132 self.buildkeys=testparse.buildkeys 133 134 # Adding a dictionary for storing sources, objects, and tests 135 # to make building the dependency tree easier 136 self.sources={} 137 self.objects={} 138 self.tests={} 139 for pkg in self.pkg_pkgs: 140 self.sources[pkg]={} 141 self.objects[pkg]=[] 142 self.tests[pkg]={} 143 for lang in LANGS: 144 self.sources[pkg][lang]={} 145 self.sources[pkg][lang]['srcs']=[] 146 self.tests[pkg][lang]={} 147 148 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 149 150 self.indent=" " 151 if self.verbose: print('Finishing the constructor') 152 return 153 154 def srcrelpath(self,rdir): 155 """ 156 Get relative path to source directory 157 """ 158 return os.path.relpath(rdir,self.srcdir) 159 160 def getInInstallDir(self,thisscriptdir): 161 """ 162 When PETSc is installed then this file in installed in: 163 <PREFIX>/share/petsc/examples/config/gmakegentest.py 164 otherwise the path is: 165 <PETSC_DIR>/config/gmakegentest.py 166 We use this difference to determine if we are in installdir 167 """ 168 dirlist=thisscriptdir.split(os.path.sep) 169 if len(dirlist)>4: 170 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 171 if lastfour==os.path.join('share','petsc','examples','config'): 172 return True 173 else: 174 return False 175 else: 176 return False 177 178 def getLanguage(self,srcfile): 179 """ 180 Based on the source, determine associated language as found in gmakegen.LANGS 181 Can we just return srcext[1:] now? 182 """ 183 langReq=None 184 srcext = getlangext(srcfile) 185 if srcext in ".F90".split(): langReq="F90" 186 if srcext in ".F".split(): langReq="F" 187 if srcext in ".cxx".split(): langReq="cxx" 188 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 189 if srcext in ".hip.cpp".split(): langReq="hip_cpp" 190 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 191 if srcext in ".cpp".split(): langReq="cpp" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getAltList(self,output_file,srcdir): 198 ''' Calculate AltList based on output file-- see 199 src/snes/tutorials/output/ex22*.out 200 ''' 201 altlist=[output_file] 202 basefile = getlangsplit(output_file) 203 for i in range(1,9): 204 altroot=basefile+"_alt" 205 if i > 1: altroot=altroot+"_"+str(i) 206 af=altroot+".out" 207 srcaf=os.path.join(srcdir,af) 208 fullaf=os.path.join(self.petsc_dir,srcaf) 209 if os.path.isfile(fullaf): altlist.append(srcaf) 210 211 return altlist 212 213 def _getLoopVars(self,inDict,testname, isSubtest=False): 214 """ 215 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 216 Return: 217 inDict['args']: -ksp_monitor 218 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 219 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 220 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 221 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 222 subst should be passed in instead of inDict 223 """ 224 loopVars={}; newargs=[] 225 lsuffix='+' 226 argregex = re.compile(' (?=-[a-zA-Z])') 227 from testparse import parseLoopArgs 228 for key in inDict: 229 if key in ('SKIP', 'regexes'): 230 continue 231 akey=('subargs' if key=='args' else key) # what to assign 232 if akey not in inDict: inDict[akey]='' 233 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 234 # Always generate a loop over nsize, even if there is only one value 235 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 236 keystr = str(inDict[key]) 237 varlist = [] 238 for varset in argregex.split(keystr): 239 if not varset.strip(): continue 240 if '{{' in varset: 241 keyvar,lvars,ftype=parseLoopArgs(varset) 242 if akey not in loopVars: loopVars[akey]={} 243 varlist.append(keyvar) 244 loopVars[akey][keyvar]=[keyvar,lvars] 245 if akey=='nsize': 246 if len(lvars.split()) > 1: 247 lsuffix += akey +'-${i' + keyvar + '}' 248 else: 249 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 250 lsuffix+=keyvar+'-${i' + keyvar + '}_' 251 else: 252 if key=='args': 253 newargs.append(varset.strip()) 254 if varlist: 255 loopVars[akey]['varlist']=varlist 256 257 # For subtests, args are always substituted in (not top level) 258 if isSubtest: 259 inDict['subargs'] += " "+" ".join(newargs) 260 inDict['args']='' 261 if 'label_suffix' in inDict: 262 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 263 else: 264 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 265 else: 266 if loopVars: 267 inDict['args'] = ' '.join(newargs) 268 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 269 return loopVars 270 271 def getArgLabel(self,testDict): 272 """ 273 In all of the arguments in the test dictionary, create a simple 274 string for searching within the makefile system. For simplicity in 275 search, remove "-", for strings, etc. 276 Also, concatenate the arg commands 277 For now, ignore nsize -- seems hard to search for anyway 278 """ 279 # Collect all of the args associated with a test 280 argStr=("" if 'args' not in testDict else testDict['args']) 281 if 'subtests' in testDict: 282 for stest in testDict["subtests"]: 283 sd=testDict[stest] 284 argStr=argStr+("" if 'args' not in sd else sd['args']) 285 286 # Now go through and cleanup 287 argStr=re.sub('{{(.*?)}}',"",argStr) 288 argStr=re.sub('-'," ",argStr) 289 for digit in string.digits: argStr=re.sub(digit," ",argStr) 290 argStr=re.sub(r"\.","",argStr) 291 argStr=re.sub(",","",argStr) 292 argStr=re.sub(r'\+',' ',argStr) 293 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 294 return argStr.strip() 295 296 def addToSources(self,exfile,rpath,srcDict): 297 """ 298 Put into data structure that allows easy generation of makefile 299 """ 300 pkg=rpath.split(os.path.sep)[0] 301 relpfile=os.path.join(rpath,exfile) 302 lang=self.getLanguage(exfile) 303 if not lang: return 304 if pkg not in self.sources: return 305 self.sources[pkg][lang]['srcs'].append(relpfile) 306 self.sources[pkg][lang][relpfile] = [] 307 if 'depends' in srcDict: 308 depSrcList=srcDict['depends'].split() 309 for depSrc in depSrcList: 310 depObj = getlangsplit(depSrc)+'.o' 311 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 312 313 # In gmakefile, ${TESTDIR} var specifies the object compilation 314 testsdir=rpath+"/" 315 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 316 self.objects[pkg].append(objfile) 317 return 318 319 def addToTests(self,test,rpath,exfile,execname,testDict): 320 """ 321 Put into data structure that allows easy generation of makefile 322 Organized by languages to allow testing of languages 323 """ 324 pkg=rpath.split("/")[0] 325 nmtest=os.path.join(rpath,test) 326 lang=self.getLanguage(exfile) 327 if not lang: return 328 if pkg not in self.tests: return 329 self.tests[pkg][lang][nmtest]={} 330 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 331 self.tests[pkg][lang][nmtest]['exec']=execname 332 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 333 return 334 335 def getExecname(self,exfile,rpath): 336 """ 337 Generate bash script using template found next to this file. 338 This file is read in at constructor time to avoid file I/O 339 """ 340 if self.single_ex: 341 execname=rpath.split("/")[1]+"-ex" 342 else: 343 execname=getlangsplit(exfile) 344 return execname 345 346 def getSubstVars(self,testDict,rpath,testname): 347 """ 348 Create a dictionary with all of the variables that get substituted 349 into the template commands found in example_template.py 350 """ 351 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 352 if 'nsize' not in testDict: testDict['nsize'] = '1' 353 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 354 subst = {key : testDict.get(key, '') for key in testparse.acceptedkeys if key != 'test'} 355 356 # Now do other variables 357 subst['env'] = '\n'.join('export '+cmd for cmd in shlex.split(subst['env'])) 358 subst['execname']=testDict['execname'] 359 subst['error']='' 360 if 'filter' in testDict: 361 if testDict['filter'].startswith("Error:"): 362 subst['error']="Error" 363 subst['filter']=testDict['filter'].lstrip("Error:") 364 else: 365 subst['filter']=testDict['filter'] 366 367 # Others 368 subst['subargs']='' # Default. For variables override 369 subst['srcdir']=os.path.join(self.srcdir, rpath) 370 subst['label_suffix']='' 371 subst['comments']="\n#".join(subst['comments'].split("\n")) 372 if subst['comments']: subst['comments']="#"+subst['comments'] 373 subst['executable']="../"+subst['execname'] 374 subst['testroot']=self.testroot_dir 375 subst['testname']=testname 376 dp = self.conf.get('DATAFILESPATH','') 377 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 378 379 # This is used to label some matrices 380 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 381 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 382 383 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 384 385 #Conf vars 386 if self.petsc_arch.find('valgrind')>=0: 387 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 388 else: 389 subst['mpiexec']=self.conf['MPIEXEC'] 390 subst['mpiexec_tail']=self.conf['MPIEXEC_TAIL'] 391 subst['pkg_name']=self.pkg_name 392 subst['pkg_dir']=self.pkg_dir 393 subst['pkg_arch']=self.pkg_arch 394 subst['CONFIG_DIR']=thisscriptdir 395 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 396 subst['diff']=self.conf['DIFF'] 397 subst['rm']=self.conf['RM'] 398 subst['grep']=self.conf['GREP'] 399 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 400 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 401 402 # Output file is special because of subtests override 403 defroot = testparse.getDefaultOutputFileRoot(testname) 404 if 'output_file' not in testDict: 405 subst['output_file']="output/"+defroot+".out" 406 subst['redirect_file']=defroot+".tmp" 407 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 408 409 # Add in the full path here. 410 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 411 412 subst['regexes']={} 413 for subkey in subst: 414 if subkey=='regexes': continue 415 if not isinstance(subst[subkey],str): continue 416 patt="@"+subkey.upper()+"@" 417 subst['regexes'][subkey]=re.compile(patt) 418 419 return subst 420 421 def _substVars(self,subst,origStr): 422 """ 423 Substitute variables 424 """ 425 Str=origStr 426 for subkey, subvalue in subst.items(): 427 if subkey=='regexes': continue 428 if not isinstance(subvalue,str): continue 429 if subkey.upper() not in Str: continue 430 Str=subst['regexes'][subkey].sub(lambda x: subvalue,Str) 431 return Str 432 433 def getCmds(self,subst,i, debug=False): 434 """ 435 Generate bash script using template found next to this file. 436 This file is read in at constructor time to avoid file I/O 437 """ 438 nindnt=i # the start and has to be consistent with below 439 cmdindnt=self.indent*nindnt 440 cmdLines="" 441 442 # MPI is the default -- but we have a few odd commands 443 if subst['temporaries']: 444 if '*' in subst['temporaries']: 445 raise RuntimeError('{}/{}: list of temporary files to remove may not include wildcards'.format(subst['srcdir'], subst['execname'])) 446 cmd=cmdindnt+self._substVars(subst,example_template.preclean) 447 cmdLines+=cmd+"\n" 448 if not subst['command']: 449 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 450 else: 451 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 452 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 453 454 cmdLines+=cmdindnt+'if test $res = 0; then\n' 455 diffindnt=self.indent*(nindnt+1) 456 457 # Do some checks on existence of output_file and alt files 458 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 459 if not subst['TODO']: 460 print("Warning: "+subst['output_file']+" not found.") 461 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 462 463 # altlist always has output_file 464 if len(altlist)==1: 465 cmd=diffindnt+self._substVars(subst,example_template.difftest) 466 else: 467 if debug: print("Found alt files: ",altlist) 468 # Have to do it by hand a bit because of variable number of alt files 469 rf=subst['redirect_file'] 470 cmd=diffindnt+example_template.difftest.split('@')[0] 471 for i in range(len(altlist)): 472 af=altlist[i] 473 cmd+=af+' '+rf 474 if i!=len(altlist)-1: 475 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 476 cmd+=' || ${diff_exe} ' 477 else: 478 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 479 cmd+=subst['label_suffix']+' ""' # Quotes are painful 480 cmdLines+=cmd+"\n" 481 cmdLines+=cmdindnt+'else\n' 482 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 483 cmdLines+=cmdindnt+'fi\n' 484 return cmdLines 485 486 def _writeTodoSkip(self,fh,tors,reasons,footer): 487 """ 488 Write out the TODO and SKIP lines in the file 489 The TODO or SKIP variable, tors, should be lower case 490 """ 491 TORS=tors.upper() 492 template=eval("example_template."+tors+"line") 493 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 494 tab = '' 495 if reasons: 496 fh.write('if ! $force; then\n') 497 tab = tab + ' ' 498 if reasons == ["Requires DATAFILESPATH"]: 499 # The only reason not to run is DATAFILESPATH, which we check at run-time 500 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 501 tab = tab + ' ' 502 if reasons: 503 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 504 fh.write(tab+footer+"\n") 505 fh.write(tab+"exit\n") 506 if reasons == ["Requires DATAFILESPATH"]: 507 fh.write(' fi\n') 508 if reasons: 509 fh.write('fi\n') 510 fh.write('\n\n') 511 return 512 513 def getLoopVarsHead(self,loopVars,i,usedVars={}): 514 """ 515 Generate a nicely indented string with the format loops 516 Here is what the data structure looks like 517 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 518 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 519 loopVars['subargs']['pc_type']=["j","cholesky sor"] 520 """ 521 outstr=''; indnt=self.indent 522 523 for key in loopVars: 524 for var in loopVars[key]['varlist']: 525 varval=loopVars[key][var] 526 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 527 outstr += "\n\n" 528 529 for key in loopVars: 530 for var in loopVars[key]['varlist']: 531 varval=loopVars[key][var] 532 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 533 i = i + 1 534 return (outstr,i) 535 536 def getLoopVarsFoot(self,loopVars,i): 537 outstr=''; indnt=self.indent 538 for key in loopVars: 539 for var in loopVars[key]['varlist']: 540 i = i - 1 541 outstr += indnt * i + "done\n" 542 return (outstr,i) 543 544 def genRunScript(self,testname,root,isRun,srcDict): 545 """ 546 Generate bash script using template found next to this file. 547 This file is read in at constructor time to avoid file I/O 548 """ 549 def opener(path,flags,*args,**kwargs): 550 kwargs.setdefault('mode',0o755) 551 return os.open(path,flags,*args,**kwargs) 552 553 # runscript_dir directory has to be consistent with gmakefile 554 testDict=srcDict[testname] 555 rpath=self.srcrelpath(root) 556 runscript_dir=os.path.join(self.testroot_dir,rpath) 557 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 558 with open(os.path.join(runscript_dir,testname+".sh"),"w",opener=opener) as fh: 559 560 # Get variables to go into shell scripts. last time testDict used 561 subst=self.getSubstVars(testDict,rpath,testname) 562 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 563 if 'subtests' in testDict: 564 # The subtests inherit inDict, so we don't need top-level loops. 565 loopVars = {} 566 567 #Handle runfiles 568 for lfile in subst.get('localrunfiles','').split(): 569 install_files(os.path.join(root, lfile), 570 os.path.join(runscript_dir, os.path.dirname(lfile))) 571 # Check subtests for local runfiles 572 for stest in subst.get("subtests",[]): 573 for lfile in testDict[stest].get('localrunfiles','').split(): 574 install_files(os.path.join(root, lfile), 575 os.path.join(runscript_dir, os.path.dirname(lfile))) 576 577 # Now substitute the key variables into the header and footer 578 header=self._substVars(subst,example_template.header) 579 # The header is done twice to enable @...@ in header 580 header=self._substVars(subst,header) 581 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 582 583 # Start writing the file 584 fh.write(header+"\n") 585 586 # If there is a TODO or a SKIP then we do it before writing out the 587 # rest of the command (which is useful for working on the test) 588 # SKIP and TODO can be for the source file or for the runs 589 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 590 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 591 592 j=0 # for indentation 593 594 if loopVars: 595 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 596 if (loopHead): fh.write(loopHead+"\n") 597 598 # Subtests are special 599 allLoopVars=list(loopVars.keys()) 600 if 'subtests' in testDict: 601 substP=subst # Subtests can inherit args but be careful 602 k=0 # for label suffixes 603 for stest in testDict["subtests"]: 604 subst=substP.copy() 605 subst.update(testDict[stest]) 606 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 607 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 608 if sLoopVars: 609 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 610 allLoopVars+=list(sLoopVars.keys()) 611 fh.write(sLoopHead+"\n") 612 fh.write(self.getCmds(subst,j)+"\n") 613 if sLoopVars: 614 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 615 fh.write(sLoopFoot+"\n") 616 else: 617 fh.write(self.getCmds(subst,j)+"\n") 618 619 if loopVars: 620 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 621 fh.write(loopFoot+"\n") 622 623 fh.write(footer+"\n") 624 return 625 626 def genScriptsAndInfo(self,exfile,root,srcDict): 627 """ 628 Generate scripts from the source file, determine if built, etc. 629 For every test in the exfile with info in the srcDict: 630 1. Determine if it needs to be run for this arch 631 2. Generate the script 632 3. Generate the data needed to write out the makefile in a 633 convenient way 634 All tests are *always* run, but some may be SKIP'd per the TAP standard 635 """ 636 debug=False 637 rpath=self.srcrelpath(root) 638 execname=self.getExecname(exfile,rpath) 639 isBuilt=self._isBuilt(exfile,srcDict) 640 for test in srcDict.copy(): 641 if test in self.buildkeys: continue 642 isRun=self._isRun(srcDict[test]) 643 # if the next two lines are dropped all scripts are generating included the unneeded 644 # if the unneeded are generated when run they will skip their tests automatically 645 # not generating them saves setup time 646 allow = False 647 if 'SKIP' in srcDict[test]: 648 allow = srcDict[test]['SKIP'] in [['Requires DATAFILESPATH'], ['PETSC_HAVE_PYVISTA requirement not met']] 649 if not isRun and not allow: 650 del srcDict[test] 651 continue 652 if 'TODO' in srcDict[test]: 653 del srcDict[test] 654 continue 655 srcDict[test]['execname']=execname # Convenience in generating scripts 656 self.genRunScript(test,root,isRun,srcDict) 657 srcDict[test]['isrun']=isRun 658 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 659 660 # This adds to datastructure for building deps 661 if isBuilt: self.addToSources(exfile,rpath,srcDict) 662 return 663 664 def _isBuilt(self,exfile,srcDict): 665 """ 666 Determine if this file should be built. 667 """ 668 # Get the language based on file extension 669 srcDict['SKIP'] = [] 670 lang=self.getLanguage(exfile) 671 if (lang=="F" or lang=="F90"): 672 if not self.have_fortran: 673 srcDict["SKIP"].append("Fortran required for this test") 674 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 675 srcDict["SKIP"].append("Fortran f90freeform required for this test") 676 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 677 srcDict["SKIP"].append("CUDA required for this test") 678 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 679 srcDict["SKIP"].append("HIP required for this test") 680 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 681 srcDict["SKIP"].append("SYCL required for this test") 682 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 683 srcDict["SKIP"].append("KOKKOS required for this test") 684 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 685 srcDict["SKIP"].append("RAJA required for this test") 686 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 687 srcDict["SKIP"].append("C++ required for this test") 688 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 689 srcDict["SKIP"].append("C++ required for this test") 690 691 # Deprecated source files 692 if srcDict.get("TODO"): 693 return False 694 695 # isRun can work with srcDict to handle the requires 696 if "requires" in srcDict: 697 if srcDict["requires"]: 698 return self._isRun(srcDict) 699 700 return srcDict['SKIP'] == [] 701 702 def _isRun(self,testDict, debug=False): 703 """ 704 Based on the requirements listed in the src file and the petscconf.h 705 info, determine whether this test should be run or not. 706 """ 707 indent=" " 708 709 if 'SKIP' not in testDict: 710 testDict['SKIP'] = [] 711 # MPI requirements 712 if 'MPI_IS_MPIUNI' in self.conf: 713 if testDict.get('nsize', '1') != '1': 714 testDict['SKIP'].append("Parallel test with serial build") 715 716 # The requirements for the test are the sum of all the run subtests 717 if 'subtests' in testDict: 718 if 'requires' not in testDict: testDict['requires']="" 719 for stest in testDict['subtests']: 720 if 'requires' in testDict[stest]: 721 testDict['requires']+=" "+testDict[stest]['requires'] 722 if testDict[stest].get('nsize', '1') != '1': 723 testDict['SKIP'].append("Parallel test with serial build") 724 break 725 726 # Now go through all requirements 727 if 'requires' in testDict: 728 for requirement in testDict['requires'].split(): 729 requirement=requirement.strip() 730 if not requirement: continue 731 if debug: print(indent+"Requirement: ", requirement) 732 isNull=False 733 if requirement.startswith("!"): 734 requirement=requirement[1:]; isNull=True 735 # 32-bit vs 64-bit pointers 736 if requirement == "64bitptr": 737 if self.conf['PETSC_SIZEOF_VOID_P']==8: 738 if isNull: 739 testDict['SKIP'].append("not 64bit-ptr required") 740 continue 741 continue # Success 742 elif not isNull: 743 testDict['SKIP'].append("64bit-ptr required") 744 continue 745 # Precision requirement for reals 746 if requirement in self.precision_types: 747 if self.conf['PETSC_PRECISION']==requirement: 748 if isNull: 749 testDict['SKIP'].append("not "+requirement+" required") 750 continue 751 continue # Success 752 elif not isNull: 753 testDict['SKIP'].append(requirement+" required") 754 continue 755 # Precision requirement for ints 756 if requirement in self.integer_types: 757 if requirement=="int32": 758 if self.conf['PETSC_SIZEOF_INT']==4: 759 if isNull: 760 testDict['SKIP'].append("not int32 required") 761 continue 762 continue # Success 763 elif not isNull: 764 testDict['SKIP'].append("int32 required") 765 continue 766 if requirement=="int64": 767 if self.conf['PETSC_SIZEOF_INT']==8: 768 if isNull: 769 testDict['SKIP'].append("NOT int64 required") 770 continue 771 continue # Success 772 elif not isNull: 773 testDict['SKIP'].append("int64 required") 774 continue 775 if requirement.startswith("long"): 776 reqsize = int(requirement[4:])//8 777 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 778 if longsize==reqsize: 779 if isNull: 780 testDict['SKIP'].append("not %s required" % requirement) 781 continue 782 continue # Success 783 elif not isNull: 784 testDict['SKIP'].append("%s required" % requirement) 785 continue 786 # Datafilespath 787 if requirement=="datafilespath" and not isNull: 788 testDict['SKIP'].append("Requires DATAFILESPATH") 789 continue 790 # Defines -- not sure I have comments matching 791 if "defined(" in requirement.lower(): 792 reqdef=requirement.split("(")[1].split(")")[0] 793 if reqdef in self.conf: 794 if isNull: 795 testDict['SKIP'].append("Null requirement not met: "+requirement) 796 continue 797 continue # Success 798 elif not isNull: 799 testDict['SKIP'].append("Required: "+requirement) 800 continue 801 802 # Rest should be packages that we can just get from conf 803 if requirement in ["complex","debug"]: 804 petscconfvar="PETSC_USE_"+requirement.upper() 805 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 806 else: 807 petscconfvar="PETSC_HAVE_"+requirement.upper() 808 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 809 petsccv = self.conf.get(petscconfvar) 810 pkgcv = self.conf.get(pkgconfvar) 811 812 if petsccv or pkgcv: 813 if isNull: 814 if petsccv: 815 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 816 continue 817 else: 818 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 819 continue 820 continue # Success 821 elif not isNull: 822 if not petsccv and not pkgcv: 823 if debug: print("requirement not found: ", requirement) 824 if self.pkg_name == 'petsc': 825 testDict['SKIP'].append(petscconfvar+" requirement not met") 826 else: 827 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 828 continue 829 return testDict['SKIP'] == [] 830 831 def checkOutput(self,exfile,root,srcDict): 832 """ 833 Check and make sure the output files are in the output directory 834 """ 835 debug=False 836 rpath=self.srcrelpath(root) 837 for test in srcDict: 838 if test in self.buildkeys: continue 839 if debug: print(rpath, exfile, test) 840 if 'output_file' in srcDict[test]: 841 output_file=srcDict[test]['output_file'] 842 else: 843 defroot = testparse.getDefaultOutputFileRoot(test) 844 if 'TODO' in srcDict[test]: continue 845 output_file="output/"+defroot+".out" 846 847 fullout=os.path.join(root,output_file) 848 if debug: print("---> ",fullout) 849 if not os.path.exists(fullout): 850 self.missing_files.append(fullout) 851 852 return 853 854 def genPetscTests_summarize(self,dataDict): 855 """ 856 Required method to state what happened 857 """ 858 if not self.summarize: return 859 indent=" " 860 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 861 with open(fhname, "w") as fh: 862 for root in dataDict: 863 relroot=self.srcrelpath(root) 864 pkg=relroot.split("/")[1] 865 if not pkg in self.sources: continue 866 fh.write(relroot+"\n") 867 allSrcs=[] 868 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 869 for exfile in dataDict[root]: 870 # Basic information 871 rfile=os.path.join(relroot,exfile) 872 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 873 fh.write(indent+exfile+indent*4+builtStatus+"\n") 874 for test in dataDict[root][exfile]: 875 if test in self.buildkeys: continue 876 line=indent*2+test 877 fh.write(line+"\n") 878 # Looks nice to have the keys in order 879 #for key in dataDict[root][exfile][test]: 880 for key in "isrun abstracted nsize args requires script".split(): 881 if key not in dataDict[root][exfile][test]: continue 882 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 883 fh.write(line+"\n") 884 fh.write("\n") 885 fh.write("\n") 886 fh.write("\n") 887 return 888 889 def genPetscTests(self,root,dirs,files,dataDict): 890 """ 891 Go through and parse the source files in the directory to generate 892 the examples based on the metadata contained in the source files 893 """ 894 debug=False 895 896 data = {} 897 for exfile in files: 898 #TST: Until we replace files, still leaving the originals as is 899 #if not exfile.startswith("new_"+"ex"): continue 900 #if not exfile.startswith("ex"): continue 901 902 # Ignore emacs and other temporary files 903 if exfile.startswith((".", "#")) or exfile.endswith("~"): continue 904 # Only parse source files 905 ext=getlangext(exfile).lstrip('.').replace('.','_') 906 if ext not in LANGS: continue 907 908 # Convenience 909 fullex=os.path.join(root,exfile) 910 if self.verbose: print(' --> '+fullex) 911 data.update(testparse.parseTestFile(fullex,0)) 912 if exfile in data: 913 if self.check_output: 914 self.checkOutput(exfile,root,data[exfile]) 915 else: 916 self.genScriptsAndInfo(exfile,root,data[exfile]) 917 918 dataDict[root] = data 919 return 920 921 def walktree(self,top): 922 """ 923 Walk a directory tree, starting from 'top' 924 """ 925 if self.check_output: 926 print("Checking for missing output files") 927 self.missing_files=[] 928 929 # Goal of action is to fill this dictionary 930 dataDict={} 931 for root, dirs, files in os.walk(top, topdown=True): 932 dirs.sort() 933 files.sort() 934 if "/tests" not in root and "/tutorials" not in root: continue 935 if "dSYM" in root: continue 936 if "tutorials"+os.sep+"build" in root: continue 937 if os.path.basename(root.rstrip("/")) == 'output': continue 938 if self.verbose: print(root) 939 self.genPetscTests(root,dirs,files,dataDict) 940 941 # If checking output, report results 942 if self.check_output: 943 if self.missing_files: 944 for file in set(self.missing_files): # set uniqifies 945 print(file) 946 sys.exit(1) 947 948 # Now summarize this dictionary 949 if self.verbose: self.genPetscTests_summarize(dataDict) 950 return dataDict 951 952 def gen_gnumake(self, fd): 953 """ 954 Overwrite of the method in the base PETSc class 955 """ 956 def write(stem, srcs): 957 for lang in LANGS: 958 if srcs[lang]['srcs']: 959 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 960 for pkg in self.pkg_pkgs: 961 srcs = self.gen_pkg(pkg) 962 write('testsrcs-' + pkg, srcs) 963 # Handle dependencies 964 for lang in LANGS: 965 for exfile in srcs[lang]['srcs']: 966 if exfile in srcs[lang]: 967 ex='$(TESTDIR)/'+getlangsplit(exfile) 968 exfo=ex+'.o' 969 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 970 if deps: 971 # The executable literally depends on the object file because it is linked 972 fd.write(ex +": " + " ".join(deps) +'\n') 973 # The object file containing 'main' does not normally depend on other object 974 # files, but it does when it includes their modules. This dependency is 975 # overly blunt and could be reduced to only depend on object files for 976 # modules that are used, like "*f90aux.o". 977 fd.write(exfo +": " + " ".join(deps) +'\n') 978 979 return self.gendeps 980 981 def gen_pkg(self, pkg): 982 """ 983 Overwrite of the method in the base PETSc class 984 """ 985 return self.sources[pkg] 986 987 def write_gnumake(self, dataDict, output=None): 988 """ 989 Write out something similar to files from gmakegen.py 990 991 Test depends on script which also depends on source 992 file, but since I don't have a good way generating 993 acting on a single file (oops) just depend on 994 executable which in turn will depend on src file 995 """ 996 # Different options for how to set up the targets 997 compileExecsFirst=False 998 999 # Open file 1000 with open(output, 'w') as fd: 1001 # Write out the sources 1002 gendeps = self.gen_gnumake(fd) 1003 1004 # Write out the tests and execname targets 1005 fd.write("\n#Tests and executables\n") # Delimiter 1006 1007 for pkg in self.pkg_pkgs: 1008 # These grab the ones that are built 1009 for lang in LANGS: 1010 testdeps=[] 1011 for ftest in self.tests[pkg][lang]: 1012 test=os.path.basename(ftest) 1013 basedir=os.path.dirname(ftest) 1014 testdeps.append(nameSpace(test,basedir)) 1015 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 1016 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 1017 1018 # test targets 1019 for ftest in self.tests[pkg][lang]: 1020 test=os.path.basename(ftest) 1021 basedir=os.path.dirname(ftest) 1022 testdir="${TESTDIR}/"+basedir+"/" 1023 nmtest=nameSpace(test,basedir) 1024 rundir=os.path.join(testdir,test) 1025 script=test+".sh" 1026 1027 # Deps 1028 exfile=self.tests[pkg][lang][ftest]['exfile'] 1029 fullex=os.path.join(self.srcdir,exfile) 1030 localexec=self.tests[pkg][lang][ftest]['exec'] 1031 execname=os.path.join(testdir,localexec) 1032 fullscript=os.path.join(testdir,script) 1033 tmpfile=os.path.join(testdir,test,test+".tmp") 1034 1035 # *.counts depends on the script and either executable (will 1036 # be run) or the example source file (SKIP or TODO) 1037 fd.write('%s.counts : %s %s' 1038 % (os.path.join('$(TESTDIR)/counts', nmtest), 1039 fullscript, 1040 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1041 ) 1042 if exfile in self.sources[pkg][lang]: 1043 for dep in self.sources[pkg][lang][exfile]: 1044 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1045 fd.write('\n') 1046 1047 # Now write the args: 1048 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1049 1050 return 1051 1052 def write_db(self, dataDict, testdir): 1053 """ 1054 Write out the dataDict into a pickle file 1055 """ 1056 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1057 pickle.dump(dataDict,fd) 1058 return 1059 1060def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1061 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1062 srcdir=None, testdir=None, check=False): 1063 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1064 testdir=os.path.normpath(testdir) 1065 if petsc_arch: 1066 petsc_arch=petsc_arch.rstrip(os.path.sep) 1067 if len(petsc_arch.split(os.path.sep))>1: 1068 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1069 output = os.path.join(testdir, 'testfiles') 1070 1071 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1072 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1073 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1074 testdir=testdir,check=check) 1075 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1076 if not pEx.check_output: 1077 pEx.write_gnumake(dataDict, output) 1078 pEx.write_db(dataDict, testdir) 1079 1080if __name__ == '__main__': 1081 import optparse 1082 parser = optparse.OptionParser() 1083 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1084 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1085 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1086 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1087 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1088 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1089 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1090 help='Check whether output files are in output directory') 1091 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1092 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1093 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1094 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1095 1096 opts, extra_args = parser.parse_args() 1097 if extra_args: 1098 import sys 1099 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1100 exit(1) 1101 if opts.testdir is None: 1102 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1103 1104 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1105 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1106 verbose=opts.verbose, 1107 single_ex=opts.single_executable, srcdir=opts.srcdir, 1108 testdir=opts.testdir, check=opts.check_output) 1109