1#!/usr/bin/env python3 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9import shlex 10sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78def nameSpace(srcfile,srcdir): 79 """ 80 Because the scripts have a non-unique naming, the pretty-printing 81 needs to convey the srcdir and srcfile. There are two ways of doing this. 82 """ 83 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 84 prefix=srcdir.replace("/","_")+"-" 85 nameString=prefix+srcfile 86 return nameString 87 88class generateExamples(Petsc): 89 """ 90 gmakegen.py has basic structure for finding the files, writing out 91 the dependencies, etc. 92 """ 93 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 94 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs) 95 96 self.single_ex=single_ex 97 self.srcdir=srcdir 98 self.check_output=check 99 100 # Set locations to handle movement 101 self.inInstallDir=self.getInInstallDir(thisscriptdir) 102 103 # Special configuration for CI testing 104 if self.petsc_arch.find('valgrind') >= 0: 105 self.conf['PETSCTEST_VALGRIND']=1 106 107 if self.inInstallDir: 108 # Case 2 discussed above 109 # set PETSC_ARCH to install directory to allow script to work in both 110 dirlist=thisscriptdir.split(os.path.sep) 111 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 112 self.arch_dir=installdir 113 if self.srcdir is None: 114 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 115 else: 116 if petsc_arch == '': 117 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 118 # Case 1 discussed above 119 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 120 if self.srcdir is None: 121 self.srcdir=os.path.join(self.petsc_dir,'src') 122 123 self.testroot_dir=os.path.abspath(testdir) 124 125 self.verbose=verbose 126 # Whether to write out a useful debugging 127 self.summarize=True if verbose else False 128 129 # For help in setting the requirements 130 self.precision_types="__fp16 single double __float128".split() 131 self.integer_types="int32 int64 long32 long64".split() 132 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 133 134 # Things that are not test 135 self.buildkeys=testparse.buildkeys 136 137 # Adding a dictionary for storing sources, objects, and tests 138 # to make building the dependency tree easier 139 self.sources={} 140 self.objects={} 141 self.tests={} 142 for pkg in self.pkg_pkgs: 143 self.sources[pkg]={} 144 self.objects[pkg]=[] 145 self.tests[pkg]={} 146 for lang in LANGS: 147 self.sources[pkg][lang]={} 148 self.sources[pkg][lang]['srcs']=[] 149 self.tests[pkg][lang]={} 150 151 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 152 153 self.indent=" " 154 if self.verbose: print('Finishing the constructor') 155 return 156 157 def srcrelpath(self,rdir): 158 """ 159 Get relative path to source directory 160 """ 161 return os.path.relpath(rdir,self.srcdir) 162 163 def getInInstallDir(self,thisscriptdir): 164 """ 165 When petsc is installed then this file in installed in: 166 <PREFIX>/share/petsc/examples/config/gmakegentest.py 167 otherwise the path is: 168 <PETSC_DIR>/config/gmakegentest.py 169 We use this difference to determine if we are in installdir 170 """ 171 dirlist=thisscriptdir.split(os.path.sep) 172 if len(dirlist)>4: 173 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 174 if lastfour==os.path.join('share','petsc','examples','config'): 175 return True 176 else: 177 return False 178 else: 179 return False 180 181 def getLanguage(self,srcfile): 182 """ 183 Based on the source, determine associated language as found in gmakegen.LANGS 184 Can we just return srcext[1:] now? 185 """ 186 langReq=None 187 srcext = getlangext(srcfile) 188 if srcext in ".F90".split(): langReq="F90" 189 if srcext in ".F".split(): langReq="F" 190 if srcext in ".cxx".split(): langReq="cxx" 191 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 192 if srcext in ".hip.cpp".split(): langReq="hip_cpp" 193 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 194 if srcext in ".cpp".split(): langReq="cpp" 195 if srcext == ".cu": langReq="cu" 196 if srcext == ".c": langReq="c" 197 #if not langReq: print("ERROR: ", srcext, srcfile) 198 return langReq 199 200 def _getAltList(self,output_file,srcdir): 201 ''' Calculate AltList based on output file-- see 202 src/snes/tutorials/output/ex22*.out 203 ''' 204 altlist=[output_file] 205 basefile = getlangsplit(output_file) 206 for i in range(1,9): 207 altroot=basefile+"_alt" 208 if i > 1: altroot=altroot+"_"+str(i) 209 af=altroot+".out" 210 srcaf=os.path.join(srcdir,af) 211 fullaf=os.path.join(self.petsc_dir,srcaf) 212 if os.path.isfile(fullaf): altlist.append(srcaf) 213 214 return altlist 215 216 217 def _getLoopVars(self,inDict,testname, isSubtest=False): 218 """ 219 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 220 Return: 221 inDict['args']: -ksp_monitor 222 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 223 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 224 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 225 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 226 subst should be passed in instead of inDict 227 """ 228 loopVars={}; newargs=[] 229 lsuffix='+' 230 argregex = re.compile(' (?=-[a-zA-Z])') 231 from testparse import parseLoopArgs 232 for key in inDict: 233 if key in ('SKIP', 'regexes'): 234 continue 235 akey=('subargs' if key=='args' else key) # what to assign 236 if akey not in inDict: inDict[akey]='' 237 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 238 # Always generate a loop over nsize, even if there is only one value 239 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 240 keystr = str(inDict[key]) 241 varlist = [] 242 for varset in argregex.split(keystr): 243 if not varset.strip(): continue 244 if '{{' in varset: 245 keyvar,lvars,ftype=parseLoopArgs(varset) 246 if akey not in loopVars: loopVars[akey]={} 247 varlist.append(keyvar) 248 loopVars[akey][keyvar]=[keyvar,lvars] 249 if akey=='nsize': 250 if len(lvars.split()) > 1: 251 lsuffix += akey +'-${i' + keyvar + '}' 252 else: 253 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 254 lsuffix+=keyvar+'-${i' + keyvar + '}_' 255 else: 256 if key=='args': 257 newargs.append(varset.strip()) 258 if varlist: 259 loopVars[akey]['varlist']=varlist 260 261 # For subtests, args are always substituted in (not top level) 262 if isSubtest: 263 inDict['subargs'] += " "+" ".join(newargs) 264 inDict['args']='' 265 if 'label_suffix' in inDict: 266 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 267 else: 268 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 269 else: 270 if loopVars: 271 inDict['args'] = ' '.join(newargs) 272 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 273 return loopVars 274 275 def getArgLabel(self,testDict): 276 """ 277 In all of the arguments in the test dictionary, create a simple 278 string for searching within the makefile system. For simplicity in 279 search, remove "-", for strings, etc. 280 Also, concatenate the arg commands 281 For now, ignore nsize -- seems hard to search for anyway 282 """ 283 # Collect all of the args associated with a test 284 argStr=("" if 'args' not in testDict else testDict['args']) 285 if 'subtests' in testDict: 286 for stest in testDict["subtests"]: 287 sd=testDict[stest] 288 argStr=argStr+("" if 'args' not in sd else sd['args']) 289 290 # Now go through and cleanup 291 argStr=re.sub('{{(.*?)}}',"",argStr) 292 argStr=re.sub('-'," ",argStr) 293 for digit in string.digits: argStr=re.sub(digit," ",argStr) 294 argStr=re.sub(r"\.","",argStr) 295 argStr=re.sub(",","",argStr) 296 argStr=re.sub(r'\+',' ',argStr) 297 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 298 return argStr.strip() 299 300 def addToSources(self,exfile,rpath,srcDict): 301 """ 302 Put into data structure that allows easy generation of makefile 303 """ 304 pkg=rpath.split(os.path.sep)[0] 305 relpfile=os.path.join(rpath,exfile) 306 lang=self.getLanguage(exfile) 307 if not lang: return 308 if pkg not in self.sources: return 309 self.sources[pkg][lang]['srcs'].append(relpfile) 310 self.sources[pkg][lang][relpfile] = [] 311 if 'depends' in srcDict: 312 depSrcList=srcDict['depends'].split() 313 for depSrc in depSrcList: 314 depObj = getlangsplit(depSrc)+'.o' 315 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 316 317 # In gmakefile, ${TESTDIR} var specifies the object compilation 318 testsdir=rpath+"/" 319 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 320 self.objects[pkg].append(objfile) 321 return 322 323 def addToTests(self,test,rpath,exfile,execname,testDict): 324 """ 325 Put into data structure that allows easy generation of makefile 326 Organized by languages to allow testing of languages 327 """ 328 pkg=rpath.split("/")[0] 329 nmtest=os.path.join(rpath,test) 330 lang=self.getLanguage(exfile) 331 if not lang: return 332 if pkg not in self.tests: return 333 self.tests[pkg][lang][nmtest]={} 334 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 335 self.tests[pkg][lang][nmtest]['exec']=execname 336 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 337 return 338 339 def getExecname(self,exfile,rpath): 340 """ 341 Generate bash script using template found next to this file. 342 This file is read in at constructor time to avoid file I/O 343 """ 344 if self.single_ex: 345 execname=rpath.split("/")[1]+"-ex" 346 else: 347 execname=getlangsplit(exfile) 348 return execname 349 350 def getSubstVars(self,testDict,rpath,testname,runscript_dir): 351 """ 352 Create a dictionary with all of the variables that get substituted 353 into the template commands found in example_template.py 354 """ 355 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 356 if 'nsize' not in testDict: testDict['nsize'] = '1' 357 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 358 subst = {key : testDict.get(key, '') for key in testparse.acceptedkeys if key != 'test'} 359 360 # Now do other variables 361 subst['env'] = '\n'.join('export '+cmd for cmd in shlex.split(subst['env'])) 362 subst['execname']=testDict['execname'] 363 subst['error']='' 364 if 'filter' in testDict: 365 if testDict['filter'].startswith("Error:"): 366 subst['error']="Error" 367 subst['filter']=testDict['filter'].lstrip("Error:") 368 else: 369 subst['filter']=testDict['filter'] 370 371 # Others 372 subst['subargs']='' # Default. For variables override 373 subst['srcdir']=os.path.join(self.srcdir, rpath) 374 subst['label_suffix']='' 375 subst['comments']="\n#".join(subst['comments'].split("\n")) 376 if subst['comments']: subst['comments']="#"+subst['comments'] 377 if os.path.exists('/usr/bin/cygcheck.exe'): 378 subst['exec']="../"+subst['execname'] 379 else: 380 subst['exec']=os.path.join(runscript_dir,subst['execname']) 381 subst['testroot']=self.testroot_dir 382 subst['testname']=testname 383 dp = self.conf.get('DATAFILESPATH','') 384 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 385 386 # This is used to label some matrices 387 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 388 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 389 390 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 391 392 #Conf vars 393 if self.petsc_arch.find('valgrind')>=0: 394 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 395 else: 396 subst['mpiexec']=self.conf['MPIEXEC'] 397 subst['mpiexec_tail']=self.conf['MPIEXEC_TAIL'] 398 subst['pkg_name']=self.pkg_name 399 subst['pkg_dir']=self.pkg_dir 400 subst['pkg_arch']=self.petsc_arch 401 subst['CONFIG_DIR']=thisscriptdir 402 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 403 subst['diff']=self.conf['DIFF'] 404 subst['rm']=self.conf['RM'] 405 subst['grep']=self.conf['GREP'] 406 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 407 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 408 409 # Output file is special because of subtests override 410 defroot = testparse.getDefaultOutputFileRoot(testname) 411 if 'output_file' not in testDict: 412 subst['output_file']="output/"+defroot+".out" 413 subst['redirect_file']=defroot+".tmp" 414 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 415 416 # Add in the full path here. 417 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 418 419 subst['regexes']={} 420 for subkey in subst: 421 if subkey=='regexes': continue 422 if not isinstance(subst[subkey],str): continue 423 patt="@"+subkey.upper()+"@" 424 subst['regexes'][subkey]=re.compile(patt) 425 426 return subst 427 428 def _substVars(self,subst,origStr): 429 """ 430 Substitute variables 431 """ 432 Str=origStr 433 for subkey, subvalue in subst.items(): 434 if subkey=='regexes': continue 435 if not isinstance(subvalue,str): continue 436 if subkey.upper() not in Str: continue 437 Str=subst['regexes'][subkey].sub(lambda x: subvalue,Str) 438 return Str 439 440 def getCmds(self,subst,i, debug=False): 441 """ 442 Generate bash script using template found next to this file. 443 This file is read in at constructor time to avoid file I/O 444 """ 445 nindnt=i # the start and has to be consistent with below 446 cmdindnt=self.indent*nindnt 447 cmdLines="" 448 449 # MPI is the default -- but we have a few odd commands 450 if subst['temporaries']: 451 if '*' in subst['temporaries']: 452 raise RuntimeError('{}/{}: list of temporary files to remove may not include wildcards'.format(subst['srcdir'], subst['execname'])) 453 cmd=cmdindnt+self._substVars(subst,example_template.preclean) 454 cmdLines+=cmd+"\n" 455 if not subst['command']: 456 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 457 else: 458 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 459 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 460 461 cmdLines+=cmdindnt+'if test $res = 0; then\n' 462 diffindnt=self.indent*(nindnt+1) 463 464 # Do some checks on existence of output_file and alt files 465 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 466 if not subst['TODO']: 467 print("Warning: "+subst['output_file']+" not found.") 468 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 469 470 # altlist always has output_file 471 if len(altlist)==1: 472 cmd=diffindnt+self._substVars(subst,example_template.difftest) 473 else: 474 if debug: print("Found alt files: ",altlist) 475 # Have to do it by hand a bit because of variable number of alt files 476 rf=subst['redirect_file'] 477 cmd=diffindnt+example_template.difftest.split('@')[0] 478 for i in range(len(altlist)): 479 af=altlist[i] 480 cmd+=af+' '+rf 481 if i!=len(altlist)-1: 482 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 483 cmd+=' || ${diff_exe} ' 484 else: 485 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 486 cmd+=subst['label_suffix']+' ""' # Quotes are painful 487 cmdLines+=cmd+"\n" 488 cmdLines+=cmdindnt+'else\n' 489 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 490 cmdLines+=cmdindnt+'fi\n' 491 return cmdLines 492 493 def _writeTodoSkip(self,fh,tors,reasons,footer): 494 """ 495 Write out the TODO and SKIP lines in the file 496 The TODO or SKIP variable, tors, should be lower case 497 """ 498 TORS=tors.upper() 499 template=eval("example_template."+tors+"line") 500 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 501 tab = '' 502 if reasons: 503 fh.write('if ! $force; then\n') 504 tab = tab + ' ' 505 if reasons == ["Requires DATAFILESPATH"]: 506 # The only reason not to run is DATAFILESPATH, which we check at run-time 507 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 508 tab = tab + ' ' 509 if reasons: 510 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 511 fh.write(tab+footer+"\n") 512 fh.write(tab+"exit\n") 513 if reasons == ["Requires DATAFILESPATH"]: 514 fh.write(' fi\n') 515 if reasons: 516 fh.write('fi\n') 517 fh.write('\n\n') 518 return 519 520 def getLoopVarsHead(self,loopVars,i,usedVars={}): 521 """ 522 Generate a nicely indented string with the format loops 523 Here is what the data structure looks like 524 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 525 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 526 loopVars['subargs']['pc_type']=["j","cholesky sor"] 527 """ 528 outstr=''; indnt=self.indent 529 530 for key in loopVars: 531 for var in loopVars[key]['varlist']: 532 varval=loopVars[key][var] 533 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 534 outstr += "\n\n" 535 536 for key in loopVars: 537 for var in loopVars[key]['varlist']: 538 varval=loopVars[key][var] 539 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 540 i = i + 1 541 return (outstr,i) 542 543 def getLoopVarsFoot(self,loopVars,i): 544 outstr=''; indnt=self.indent 545 for key in loopVars: 546 for var in loopVars[key]['varlist']: 547 i = i - 1 548 outstr += indnt * i + "done\n" 549 return (outstr,i) 550 551 def genRunScript(self,testname,root,isRun,srcDict): 552 """ 553 Generate bash script using template found next to this file. 554 This file is read in at constructor time to avoid file I/O 555 """ 556 def opener(path,flags,*args,**kwargs): 557 kwargs.setdefault('mode',0o755) 558 return os.open(path,flags,*args,**kwargs) 559 560 # runscript_dir directory has to be consistent with gmakefile 561 testDict=srcDict[testname] 562 rpath=self.srcrelpath(root) 563 runscript_dir=os.path.join(self.testroot_dir,rpath) 564 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 565 with open(os.path.join(runscript_dir,testname+".sh"),"w",opener=opener) as fh: 566 567 # Get variables to go into shell scripts. last time testDict used 568 subst=self.getSubstVars(testDict,rpath,testname,runscript_dir) 569 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 570 if 'subtests' in testDict: 571 # The subtests inherit inDict, so we don't need top-level loops. 572 loopVars = {} 573 574 #Handle runfiles 575 for lfile in subst.get('localrunfiles','').split(): 576 install_files(os.path.join(root, lfile), 577 os.path.join(runscript_dir, os.path.dirname(lfile))) 578 # Check subtests for local runfiles 579 for stest in subst.get("subtests",[]): 580 for lfile in testDict[stest].get('localrunfiles','').split(): 581 install_files(os.path.join(root, lfile), 582 os.path.join(runscript_dir, os.path.dirname(lfile))) 583 584 # Now substitute the key variables into the header and footer 585 header=self._substVars(subst,example_template.header) 586 # The header is done twice to enable @...@ in header 587 header=self._substVars(subst,header) 588 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 589 590 # Start writing the file 591 fh.write(header+"\n") 592 593 # If there is a TODO or a SKIP then we do it before writing out the 594 # rest of the command (which is useful for working on the test) 595 # SKIP and TODO can be for the source file or for the runs 596 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 597 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 598 599 j=0 # for indentation 600 601 if loopVars: 602 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 603 if (loopHead): fh.write(loopHead+"\n") 604 605 # Subtests are special 606 allLoopVars=list(loopVars.keys()) 607 if 'subtests' in testDict: 608 substP=subst # Subtests can inherit args but be careful 609 k=0 # for label suffixes 610 for stest in testDict["subtests"]: 611 subst=substP.copy() 612 subst.update(testDict[stest]) 613 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 614 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 615 if sLoopVars: 616 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 617 allLoopVars+=list(sLoopVars.keys()) 618 fh.write(sLoopHead+"\n") 619 fh.write(self.getCmds(subst,j)+"\n") 620 if sLoopVars: 621 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 622 fh.write(sLoopFoot+"\n") 623 else: 624 fh.write(self.getCmds(subst,j)+"\n") 625 626 if loopVars: 627 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 628 fh.write(loopFoot+"\n") 629 630 fh.write(footer+"\n") 631 return 632 633 def genScriptsAndInfo(self,exfile,root,srcDict): 634 """ 635 Generate scripts from the source file, determine if built, etc. 636 For every test in the exfile with info in the srcDict: 637 1. Determine if it needs to be run for this arch 638 2. Generate the script 639 3. Generate the data needed to write out the makefile in a 640 convenient way 641 All tests are *always* run, but some may be SKIP'd per the TAP standard 642 """ 643 debug=False 644 rpath=self.srcrelpath(root) 645 execname=self.getExecname(exfile,rpath) 646 isBuilt=self._isBuilt(exfile,srcDict) 647 for test in srcDict: 648 if test in self.buildkeys: continue 649 if debug: print(nameSpace(exfile,root), test) 650 srcDict[test]['execname']=execname # Convenience in generating scripts 651 isRun=self._isRun(srcDict[test]) 652 self.genRunScript(test,root,isRun,srcDict) 653 srcDict[test]['isrun']=isRun 654 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 655 656 # This adds to datastructure for building deps 657 if isBuilt: self.addToSources(exfile,rpath,srcDict) 658 return 659 660 def _isBuilt(self,exfile,srcDict): 661 """ 662 Determine if this file should be built. 663 """ 664 # Get the language based on file extension 665 srcDict['SKIP'] = [] 666 lang=self.getLanguage(exfile) 667 if (lang=="F" or lang=="F90"): 668 if not self.have_fortran: 669 srcDict["SKIP"].append("Fortran required for this test") 670 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 671 srcDict["SKIP"].append("Fortran f90freeform required for this test") 672 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 673 srcDict["SKIP"].append("CUDA required for this test") 674 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 675 srcDict["SKIP"].append("HIP required for this test") 676 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 677 srcDict["SKIP"].append("SYCL required for this test") 678 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 679 srcDict["SKIP"].append("KOKKOS required for this test") 680 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 681 srcDict["SKIP"].append("RAJA required for this test") 682 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 683 srcDict["SKIP"].append("C++ required for this test") 684 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 685 srcDict["SKIP"].append("C++ required for this test") 686 687 # Deprecated source files 688 if srcDict.get("TODO"): 689 return False 690 691 # isRun can work with srcDict to handle the requires 692 if "requires" in srcDict: 693 if srcDict["requires"]: 694 return self._isRun(srcDict) 695 696 return srcDict['SKIP'] == [] 697 698 699 def _isRun(self,testDict, debug=False): 700 """ 701 Based on the requirements listed in the src file and the petscconf.h 702 info, determine whether this test should be run or not. 703 """ 704 indent=" " 705 706 if 'SKIP' not in testDict: 707 testDict['SKIP'] = [] 708 # MPI requirements 709 if 'MPI_IS_MPIUNI' in self.conf: 710 if testDict.get('nsize', '1') != '1': 711 testDict['SKIP'].append("Parallel test with serial build") 712 713 # The requirements for the test are the sum of all the run subtests 714 if 'subtests' in testDict: 715 if 'requires' not in testDict: testDict['requires']="" 716 for stest in testDict['subtests']: 717 if 'requires' in testDict[stest]: 718 testDict['requires']+=" "+testDict[stest]['requires'] 719 if testDict[stest].get('nsize', '1') != '1': 720 testDict['SKIP'].append("Parallel test with serial build") 721 break 722 723 # Now go through all requirements 724 if 'requires' in testDict: 725 for requirement in testDict['requires'].split(): 726 requirement=requirement.strip() 727 if not requirement: continue 728 if debug: print(indent+"Requirement: ", requirement) 729 isNull=False 730 if requirement.startswith("!"): 731 requirement=requirement[1:]; isNull=True 732 # 32-bit vs 64-bit pointers 733 if requirement == "64bitptr": 734 if self.conf['PETSC_SIZEOF_VOID_P']==8: 735 if isNull: 736 testDict['SKIP'].append("not 64bit-ptr required") 737 continue 738 continue # Success 739 elif not isNull: 740 testDict['SKIP'].append("64bit-ptr required") 741 continue 742 # Precision requirement for reals 743 if requirement in self.precision_types: 744 if self.conf['PETSC_PRECISION']==requirement: 745 if isNull: 746 testDict['SKIP'].append("not "+requirement+" required") 747 continue 748 continue # Success 749 elif not isNull: 750 testDict['SKIP'].append(requirement+" required") 751 continue 752 # Precision requirement for ints 753 if requirement in self.integer_types: 754 if requirement=="int32": 755 if self.conf['PETSC_SIZEOF_INT']==4: 756 if isNull: 757 testDict['SKIP'].append("not int32 required") 758 continue 759 continue # Success 760 elif not isNull: 761 testDict['SKIP'].append("int32 required") 762 continue 763 if requirement=="int64": 764 if self.conf['PETSC_SIZEOF_INT']==8: 765 if isNull: 766 testDict['SKIP'].append("NOT int64 required") 767 continue 768 continue # Success 769 elif not isNull: 770 testDict['SKIP'].append("int64 required") 771 continue 772 if requirement.startswith("long"): 773 reqsize = int(requirement[4:])//8 774 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 775 if longsize==reqsize: 776 if isNull: 777 testDict['SKIP'].append("not %s required" % requirement) 778 continue 779 continue # Success 780 elif not isNull: 781 testDict['SKIP'].append("%s required" % requirement) 782 continue 783 # Datafilespath 784 if requirement=="datafilespath" and not isNull: 785 testDict['SKIP'].append("Requires DATAFILESPATH") 786 continue 787 # Defines -- not sure I have comments matching 788 if "defined(" in requirement.lower(): 789 reqdef=requirement.split("(")[1].split(")")[0] 790 if reqdef in self.conf: 791 if isNull: 792 testDict['SKIP'].append("Null requirement not met: "+requirement) 793 continue 794 continue # Success 795 elif not isNull: 796 testDict['SKIP'].append("Required: "+requirement) 797 continue 798 799 # Rest should be packages that we can just get from conf 800 if requirement in ["complex","debug"]: 801 petscconfvar="PETSC_USE_"+requirement.upper() 802 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 803 else: 804 petscconfvar="PETSC_HAVE_"+requirement.upper() 805 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 806 petsccv = self.conf.get(petscconfvar) 807 pkgcv = self.conf.get(pkgconfvar) 808 809 if petsccv or pkgcv: 810 if isNull: 811 if petsccv: 812 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 813 continue 814 else: 815 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 816 continue 817 continue # Success 818 elif not isNull: 819 if not petsccv and not pkgcv: 820 if debug: print("requirement not found: ", requirement) 821 if self.pkg_name == 'petsc': 822 testDict['SKIP'].append(petscconfvar+" requirement not met") 823 else: 824 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 825 continue 826 return testDict['SKIP'] == [] 827 828 def checkOutput(self,exfile,root,srcDict): 829 """ 830 Check and make sure the output files are in the output directory 831 """ 832 debug=False 833 rpath=self.srcrelpath(root) 834 for test in srcDict: 835 if test in self.buildkeys: continue 836 if debug: print(rpath, exfile, test) 837 if 'output_file' in srcDict[test]: 838 output_file=srcDict[test]['output_file'] 839 else: 840 defroot = testparse.getDefaultOutputFileRoot(test) 841 if 'TODO' in srcDict[test]: continue 842 output_file="output/"+defroot+".out" 843 844 fullout=os.path.join(root,output_file) 845 if debug: print("---> ",fullout) 846 if not os.path.exists(fullout): 847 self.missing_files.append(fullout) 848 849 return 850 851 def genPetscTests_summarize(self,dataDict): 852 """ 853 Required method to state what happened 854 """ 855 if not self.summarize: return 856 indent=" " 857 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 858 with open(fhname, "w") as fh: 859 for root in dataDict: 860 relroot=self.srcrelpath(root) 861 pkg=relroot.split("/")[1] 862 if not pkg in self.sources: continue 863 fh.write(relroot+"\n") 864 allSrcs=[] 865 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 866 for exfile in dataDict[root]: 867 # Basic information 868 rfile=os.path.join(relroot,exfile) 869 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 870 fh.write(indent+exfile+indent*4+builtStatus+"\n") 871 for test in dataDict[root][exfile]: 872 if test in self.buildkeys: continue 873 line=indent*2+test 874 fh.write(line+"\n") 875 # Looks nice to have the keys in order 876 #for key in dataDict[root][exfile][test]: 877 for key in "isrun abstracted nsize args requires script".split(): 878 if key not in dataDict[root][exfile][test]: continue 879 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 880 fh.write(line+"\n") 881 fh.write("\n") 882 fh.write("\n") 883 fh.write("\n") 884 return 885 886 def genPetscTests(self,root,dirs,files,dataDict): 887 """ 888 Go through and parse the source files in the directory to generate 889 the examples based on the metadata contained in the source files 890 """ 891 debug=False 892 893 data = {} 894 for exfile in files: 895 #TST: Until we replace files, still leaving the originals as is 896 #if not exfile.startswith("new_"+"ex"): continue 897 #if not exfile.startswith("ex"): continue 898 899 # Ignore emacs and other temporary files 900 if exfile.startswith((".", "#")) or exfile.endswith("~"): continue 901 # Only parse source files 902 ext=getlangext(exfile).lstrip('.').replace('.','_') 903 if ext not in LANGS: continue 904 905 # Convenience 906 fullex=os.path.join(root,exfile) 907 if self.verbose: print(' --> '+fullex) 908 data.update(testparse.parseTestFile(fullex,0)) 909 if exfile in data: 910 if self.check_output: 911 self.checkOutput(exfile,root,data[exfile]) 912 else: 913 self.genScriptsAndInfo(exfile,root,data[exfile]) 914 915 dataDict[root] = data 916 return 917 918 def walktree(self,top): 919 """ 920 Walk a directory tree, starting from 'top' 921 """ 922 if self.check_output: 923 print("Checking for missing output files") 924 self.missing_files=[] 925 926 # Goal of action is to fill this dictionary 927 dataDict={} 928 for root, dirs, files in os.walk(top, topdown=True): 929 dirs.sort() 930 files.sort() 931 if "/tests" not in root and "/tutorials" not in root: continue 932 if "dSYM" in root: continue 933 if "tutorials"+os.sep+"build" in root: continue 934 if os.path.basename(root.rstrip("/")) == 'output': continue 935 if self.verbose: print(root) 936 self.genPetscTests(root,dirs,files,dataDict) 937 938 # If checking output, report results 939 if self.check_output: 940 if self.missing_files: 941 for file in set(self.missing_files): # set uniqifies 942 print(file) 943 sys.exit(1) 944 945 # Now summarize this dictionary 946 if self.verbose: self.genPetscTests_summarize(dataDict) 947 return dataDict 948 949 def gen_gnumake(self, fd): 950 """ 951 Overwrite of the method in the base PETSc class 952 """ 953 def write(stem, srcs): 954 for lang in LANGS: 955 if srcs[lang]['srcs']: 956 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 957 for pkg in self.pkg_pkgs: 958 srcs = self.gen_pkg(pkg) 959 write('testsrcs-' + pkg, srcs) 960 # Handle dependencies 961 for lang in LANGS: 962 for exfile in srcs[lang]['srcs']: 963 if exfile in srcs[lang]: 964 ex='$(TESTDIR)/'+getlangsplit(exfile) 965 exfo=ex+'.o' 966 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 967 if deps: 968 # The executable literally depends on the object file because it is linked 969 fd.write(ex +": " + " ".join(deps) +'\n') 970 # The object file containing 'main' does not normally depend on other object 971 # files, but it does when it includes their modules. This dependency is 972 # overly blunt and could be reduced to only depend on object files for 973 # modules that are used, like "*f90aux.o". 974 fd.write(exfo +": " + " ".join(deps) +'\n') 975 976 return self.gendeps 977 978 def gen_pkg(self, pkg): 979 """ 980 Overwrite of the method in the base PETSc class 981 """ 982 return self.sources[pkg] 983 984 def write_gnumake(self, dataDict, output=None): 985 """ 986 Write out something similar to files from gmakegen.py 987 988 Test depends on script which also depends on source 989 file, but since I don't have a good way generating 990 acting on a single file (oops) just depend on 991 executable which in turn will depend on src file 992 """ 993 # Different options for how to set up the targets 994 compileExecsFirst=False 995 996 # Open file 997 with open(output, 'w') as fd: 998 # Write out the sources 999 gendeps = self.gen_gnumake(fd) 1000 1001 # Write out the tests and execname targets 1002 fd.write("\n#Tests and executables\n") # Delimiter 1003 1004 for pkg in self.pkg_pkgs: 1005 # These grab the ones that are built 1006 for lang in LANGS: 1007 testdeps=[] 1008 for ftest in self.tests[pkg][lang]: 1009 test=os.path.basename(ftest) 1010 basedir=os.path.dirname(ftest) 1011 testdeps.append(nameSpace(test,basedir)) 1012 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 1013 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 1014 1015 # test targets 1016 for ftest in self.tests[pkg][lang]: 1017 test=os.path.basename(ftest) 1018 basedir=os.path.dirname(ftest) 1019 testdir="${TESTDIR}/"+basedir+"/" 1020 nmtest=nameSpace(test,basedir) 1021 rundir=os.path.join(testdir,test) 1022 script=test+".sh" 1023 1024 # Deps 1025 exfile=self.tests[pkg][lang][ftest]['exfile'] 1026 fullex=os.path.join(self.srcdir,exfile) 1027 localexec=self.tests[pkg][lang][ftest]['exec'] 1028 execname=os.path.join(testdir,localexec) 1029 fullscript=os.path.join(testdir,script) 1030 tmpfile=os.path.join(testdir,test,test+".tmp") 1031 1032 # *.counts depends on the script and either executable (will 1033 # be run) or the example source file (SKIP or TODO) 1034 fd.write('%s.counts : %s %s' 1035 % (os.path.join('$(TESTDIR)/counts', nmtest), 1036 fullscript, 1037 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1038 ) 1039 if exfile in self.sources[pkg][lang]: 1040 for dep in self.sources[pkg][lang][exfile]: 1041 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1042 fd.write('\n') 1043 1044 # Now write the args: 1045 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1046 1047 return 1048 1049 def write_db(self, dataDict, testdir): 1050 """ 1051 Write out the dataDict into a pickle file 1052 """ 1053 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1054 pickle.dump(dataDict,fd) 1055 return 1056 1057def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1058 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1059 srcdir=None, testdir=None, check=False): 1060 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1061 testdir=os.path.normpath(testdir) 1062 if petsc_arch: 1063 petsc_arch=petsc_arch.rstrip(os.path.sep) 1064 if len(petsc_arch.split(os.path.sep))>1: 1065 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1066 output = os.path.join(testdir, 'testfiles') 1067 1068 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1069 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1070 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1071 testdir=testdir,check=check) 1072 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1073 if not pEx.check_output: 1074 pEx.write_gnumake(dataDict, output) 1075 pEx.write_db(dataDict, testdir) 1076 1077if __name__ == '__main__': 1078 import optparse 1079 parser = optparse.OptionParser() 1080 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1081 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1082 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1083 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1084 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1085 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1086 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1087 help='Check whether output files are in output director') 1088 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1089 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1090 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1091 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1092 1093 opts, extra_args = parser.parse_args() 1094 if extra_args: 1095 import sys 1096 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1097 exit(1) 1098 if opts.testdir is None: 1099 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1100 1101 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1102 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1103 verbose=opts.verbose, 1104 single_ex=opts.single_executable, srcdir=opts.srcdir, 1105 testdir=opts.testdir, check=opts.check_output) 1106