1#!/usr/bin/env python3 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9import shlex 10sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78def nameSpace(srcfile,srcdir): 79 """ 80 Because the scripts have a non-unique naming, the pretty-printing 81 needs to convey the srcdir and srcfile. There are two ways of doing this. 82 """ 83 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 84 prefix=srcdir.replace("/","_")+"-" 85 nameString=prefix+srcfile 86 return nameString 87 88class generateExamples(Petsc): 89 """ 90 gmakegen.py has basic structure for finding the files, writing out 91 the dependencies, etc. 92 """ 93 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 94 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 95 96 self.single_ex=single_ex 97 self.srcdir=srcdir 98 self.check_output=check 99 100 # Set locations to handle movement 101 self.inInstallDir=self.getInInstallDir(thisscriptdir) 102 103 # Special configuration for CI testing 104 if self.petsc_arch.find('valgrind') >= 0: 105 self.conf['PETSCTEST_VALGRIND']=1 106 107 if self.inInstallDir: 108 # Case 2 discussed above 109 # set PETSC_ARCH to install directory to allow script to work in both 110 dirlist=thisscriptdir.split(os.path.sep) 111 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 112 self.arch_dir=installdir 113 if self.srcdir is None: 114 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 115 else: 116 if petsc_arch == '': 117 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 118 # Case 1 discussed above 119 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 120 if self.srcdir is None: 121 self.srcdir=os.path.join(self.petsc_dir,'src') 122 123 self.testroot_dir=os.path.abspath(testdir) 124 125 self.verbose=verbose 126 # Whether to write out a useful debugging 127 self.summarize=True if verbose else False 128 129 # For help in setting the requirements 130 self.precision_types="__fp16 single double __float128".split() 131 self.integer_types="int32 int64 long32 long64".split() 132 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 133 134 # Things that are not test 135 self.buildkeys=testparse.buildkeys 136 137 # Adding a dictionary for storing sources, objects, and tests 138 # to make building the dependency tree easier 139 self.sources={} 140 self.objects={} 141 self.tests={} 142 for pkg in self.pkg_pkgs: 143 self.sources[pkg]={} 144 self.objects[pkg]=[] 145 self.tests[pkg]={} 146 for lang in LANGS: 147 self.sources[pkg][lang]={} 148 self.sources[pkg][lang]['srcs']=[] 149 self.tests[pkg][lang]={} 150 151 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 152 153 self.indent=" " 154 if self.verbose: print('Finishing the constructor') 155 return 156 157 def srcrelpath(self,rdir): 158 """ 159 Get relative path to source directory 160 """ 161 return os.path.relpath(rdir,self.srcdir) 162 163 def getInInstallDir(self,thisscriptdir): 164 """ 165 When petsc is installed then this file in installed in: 166 <PREFIX>/share/petsc/examples/config/gmakegentest.py 167 otherwise the path is: 168 <PETSC_DIR>/config/gmakegentest.py 169 We use this difference to determine if we are in installdir 170 """ 171 dirlist=thisscriptdir.split(os.path.sep) 172 if len(dirlist)>4: 173 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 174 if lastfour==os.path.join('share','petsc','examples','config'): 175 return True 176 else: 177 return False 178 else: 179 return False 180 181 def getLanguage(self,srcfile): 182 """ 183 Based on the source, determine associated language as found in gmakegen.LANGS 184 Can we just return srcext[1:] now? 185 """ 186 langReq=None 187 srcext = getlangext(srcfile) 188 if srcext in ".F90".split(): langReq="F90" 189 if srcext in ".F".split(): langReq="F" 190 if srcext in ".cxx".split(): langReq="cxx" 191 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 192 if srcext in ".hip.cpp".split(): langReq="hip_cpp" 193 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 194 if srcext in ".cpp".split(): langReq="cpp" 195 if srcext == ".cu": langReq="cu" 196 if srcext == ".c": langReq="c" 197 #if not langReq: print("ERROR: ", srcext, srcfile) 198 return langReq 199 200 def _getAltList(self,output_file,srcdir): 201 ''' Calculate AltList based on output file-- see 202 src/snes/tutorials/output/ex22*.out 203 ''' 204 altlist=[output_file] 205 basefile = getlangsplit(output_file) 206 for i in range(1,9): 207 altroot=basefile+"_alt" 208 if i > 1: altroot=altroot+"_"+str(i) 209 af=altroot+".out" 210 srcaf=os.path.join(srcdir,af) 211 fullaf=os.path.join(self.petsc_dir,srcaf) 212 if os.path.isfile(fullaf): altlist.append(srcaf) 213 214 return altlist 215 216 217 def _getLoopVars(self,inDict,testname, isSubtest=False): 218 """ 219 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 220 Return: 221 inDict['args']: -ksp_monitor 222 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 223 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 224 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 225 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 226 subst should be passed in instead of inDict 227 """ 228 loopVars={}; newargs=[] 229 lsuffix='+' 230 argregex = re.compile(' (?=-[a-zA-Z])') 231 from testparse import parseLoopArgs 232 for key in inDict: 233 if key in ('SKIP', 'regexes'): 234 continue 235 akey=('subargs' if key=='args' else key) # what to assign 236 if akey not in inDict: inDict[akey]='' 237 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 238 # Always generate a loop over nsize, even if there is only one value 239 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 240 keystr = str(inDict[key]) 241 varlist = [] 242 for varset in argregex.split(keystr): 243 if not varset.strip(): continue 244 if '{{' in varset: 245 keyvar,lvars,ftype=parseLoopArgs(varset) 246 if akey not in loopVars: loopVars[akey]={} 247 varlist.append(keyvar) 248 loopVars[akey][keyvar]=[keyvar,lvars] 249 if akey=='nsize': 250 if len(lvars.split()) > 1: 251 lsuffix += akey +'-${i' + keyvar + '}' 252 else: 253 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 254 lsuffix+=keyvar+'-${i' + keyvar + '}_' 255 else: 256 if key=='args': 257 newargs.append(varset.strip()) 258 if varlist: 259 loopVars[akey]['varlist']=varlist 260 261 # For subtests, args are always substituted in (not top level) 262 if isSubtest: 263 inDict['subargs'] += " "+" ".join(newargs) 264 inDict['args']='' 265 if 'label_suffix' in inDict: 266 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 267 else: 268 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 269 else: 270 if loopVars: 271 inDict['args'] = ' '.join(newargs) 272 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 273 return loopVars 274 275 def getArgLabel(self,testDict): 276 """ 277 In all of the arguments in the test dictionary, create a simple 278 string for searching within the makefile system. For simplicity in 279 search, remove "-", for strings, etc. 280 Also, concatenate the arg commands 281 For now, ignore nsize -- seems hard to search for anyway 282 """ 283 # Collect all of the args associated with a test 284 argStr=("" if 'args' not in testDict else testDict['args']) 285 if 'subtests' in testDict: 286 for stest in testDict["subtests"]: 287 sd=testDict[stest] 288 argStr=argStr+("" if 'args' not in sd else sd['args']) 289 290 # Now go through and cleanup 291 argStr=re.sub('{{(.*?)}}',"",argStr) 292 argStr=re.sub('-'," ",argStr) 293 for digit in string.digits: argStr=re.sub(digit," ",argStr) 294 argStr=re.sub(r"\.","",argStr) 295 argStr=re.sub(",","",argStr) 296 argStr=re.sub(r'\+',' ',argStr) 297 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 298 return argStr.strip() 299 300 def addToSources(self,exfile,rpath,srcDict): 301 """ 302 Put into data structure that allows easy generation of makefile 303 """ 304 pkg=rpath.split(os.path.sep)[0] 305 relpfile=os.path.join(rpath,exfile) 306 lang=self.getLanguage(exfile) 307 if not lang: return 308 if pkg not in self.sources: return 309 self.sources[pkg][lang]['srcs'].append(relpfile) 310 self.sources[pkg][lang][relpfile] = [] 311 if 'depends' in srcDict: 312 depSrcList=srcDict['depends'].split() 313 for depSrc in depSrcList: 314 depObj = getlangsplit(depSrc)+'.o' 315 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 316 317 # In gmakefile, ${TESTDIR} var specifies the object compilation 318 testsdir=rpath+"/" 319 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 320 self.objects[pkg].append(objfile) 321 return 322 323 def addToTests(self,test,rpath,exfile,execname,testDict): 324 """ 325 Put into data structure that allows easy generation of makefile 326 Organized by languages to allow testing of languages 327 """ 328 pkg=rpath.split("/")[0] 329 nmtest=os.path.join(rpath,test) 330 lang=self.getLanguage(exfile) 331 if not lang: return 332 if pkg not in self.tests: return 333 self.tests[pkg][lang][nmtest]={} 334 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 335 self.tests[pkg][lang][nmtest]['exec']=execname 336 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 337 return 338 339 def getExecname(self,exfile,rpath): 340 """ 341 Generate bash script using template found next to this file. 342 This file is read in at constructor time to avoid file I/O 343 """ 344 if self.single_ex: 345 execname=rpath.split("/")[1]+"-ex" 346 else: 347 execname=getlangsplit(exfile) 348 return execname 349 350 def getSubstVars(self,testDict,rpath,testname): 351 """ 352 Create a dictionary with all of the variables that get substituted 353 into the template commands found in example_template.py 354 """ 355 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 356 if 'nsize' not in testDict: testDict['nsize'] = '1' 357 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 358 subst = {key : testDict.get(key, '') for key in testparse.acceptedkeys if key != 'test'} 359 360 # Now do other variables 361 subst['env'] = '\n'.join('export '+cmd for cmd in shlex.split(subst['env'])) 362 subst['execname']=testDict['execname'] 363 subst['error']='' 364 if 'filter' in testDict: 365 if testDict['filter'].startswith("Error:"): 366 subst['error']="Error" 367 subst['filter']=testDict['filter'].lstrip("Error:") 368 else: 369 subst['filter']=testDict['filter'] 370 371 # Others 372 subst['subargs']='' # Default. For variables override 373 subst['srcdir']=os.path.join(self.srcdir, rpath) 374 subst['label_suffix']='' 375 subst['comments']="\n#".join(subst['comments'].split("\n")) 376 if subst['comments']: subst['comments']="#"+subst['comments'] 377 subst['exec']="../"+subst['execname'] 378 subst['testroot']=self.testroot_dir 379 subst['testname']=testname 380 dp = self.conf.get('DATAFILESPATH','') 381 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 382 383 # This is used to label some matrices 384 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 385 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 386 387 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 388 389 #Conf vars 390 if self.petsc_arch.find('valgrind')>=0: 391 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 392 else: 393 subst['mpiexec']=self.conf['MPIEXEC'] 394 subst['mpiexec_tail']=self.conf['MPIEXEC_TAIL'] 395 subst['pkg_name']=self.pkg_name 396 subst['pkg_dir']=self.pkg_dir 397 subst['pkg_arch']=self.petsc_arch 398 subst['CONFIG_DIR']=thisscriptdir 399 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 400 subst['diff']=self.conf['DIFF'] 401 subst['rm']=self.conf['RM'] 402 subst['grep']=self.conf['GREP'] 403 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 404 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 405 406 # Output file is special because of subtests override 407 defroot = testparse.getDefaultOutputFileRoot(testname) 408 if 'output_file' not in testDict: 409 subst['output_file']="output/"+defroot+".out" 410 subst['redirect_file']=defroot+".tmp" 411 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 412 413 # Add in the full path here. 414 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 415 416 subst['regexes']={} 417 for subkey in subst: 418 if subkey=='regexes': continue 419 if not isinstance(subst[subkey],str): continue 420 patt="@"+subkey.upper()+"@" 421 subst['regexes'][subkey]=re.compile(patt) 422 423 return subst 424 425 def _substVars(self,subst,origStr): 426 """ 427 Substitute variables 428 """ 429 Str=origStr 430 for subkey, subvalue in subst.items(): 431 if subkey=='regexes': continue 432 if not isinstance(subvalue,str): continue 433 if subkey.upper() not in Str: continue 434 Str=subst['regexes'][subkey].sub(lambda x: subvalue,Str) 435 return Str 436 437 def getCmds(self,subst,i, debug=False): 438 """ 439 Generate bash script using template found next to this file. 440 This file is read in at constructor time to avoid file I/O 441 """ 442 nindnt=i # the start and has to be consistent with below 443 cmdindnt=self.indent*nindnt 444 cmdLines="" 445 446 # MPI is the default -- but we have a few odd commands 447 if not subst['command']: 448 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 449 else: 450 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 451 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 452 453 cmdLines+=cmdindnt+'if test $res = 0; then\n' 454 diffindnt=self.indent*(nindnt+1) 455 456 # Do some checks on existence of output_file and alt files 457 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 458 if not subst['TODO']: 459 print("Warning: "+subst['output_file']+" not found.") 460 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 461 462 # altlist always has output_file 463 if len(altlist)==1: 464 cmd=diffindnt+self._substVars(subst,example_template.difftest) 465 else: 466 if debug: print("Found alt files: ",altlist) 467 # Have to do it by hand a bit because of variable number of alt files 468 rf=subst['redirect_file'] 469 cmd=diffindnt+example_template.difftest.split('@')[0] 470 for i in range(len(altlist)): 471 af=altlist[i] 472 cmd+=af+' '+rf 473 if i!=len(altlist)-1: 474 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 475 cmd+=' || ${diff_exe} ' 476 else: 477 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 478 cmd+=subst['label_suffix']+' ""' # Quotes are painful 479 cmdLines+=cmd+"\n" 480 cmdLines+=cmdindnt+'else\n' 481 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 482 cmdLines+=cmdindnt+'fi\n' 483 return cmdLines 484 485 def _writeTodoSkip(self,fh,tors,reasons,footer): 486 """ 487 Write out the TODO and SKIP lines in the file 488 The TODO or SKIP variable, tors, should be lower case 489 """ 490 TORS=tors.upper() 491 template=eval("example_template."+tors+"line") 492 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 493 tab = '' 494 if reasons: 495 fh.write('if ! $force; then\n') 496 tab = tab + ' ' 497 if reasons == ["Requires DATAFILESPATH"]: 498 # The only reason not to run is DATAFILESPATH, which we check at run-time 499 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 500 tab = tab + ' ' 501 if reasons: 502 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 503 fh.write(tab+footer+"\n") 504 fh.write(tab+"exit\n") 505 if reasons == ["Requires DATAFILESPATH"]: 506 fh.write(' fi\n') 507 if reasons: 508 fh.write('fi\n') 509 fh.write('\n\n') 510 return 511 512 def getLoopVarsHead(self,loopVars,i,usedVars={}): 513 """ 514 Generate a nicely indented string with the format loops 515 Here is what the data structure looks like 516 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 517 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 518 loopVars['subargs']['pc_type']=["j","cholesky sor"] 519 """ 520 outstr=''; indnt=self.indent 521 522 for key in loopVars: 523 for var in loopVars[key]['varlist']: 524 varval=loopVars[key][var] 525 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 526 outstr += "\n\n" 527 528 for key in loopVars: 529 for var in loopVars[key]['varlist']: 530 varval=loopVars[key][var] 531 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 532 i = i + 1 533 return (outstr,i) 534 535 def getLoopVarsFoot(self,loopVars,i): 536 outstr=''; indnt=self.indent 537 for key in loopVars: 538 for var in loopVars[key]['varlist']: 539 i = i - 1 540 outstr += indnt * i + "done\n" 541 return (outstr,i) 542 543 def genRunScript(self,testname,root,isRun,srcDict): 544 """ 545 Generate bash script using template found next to this file. 546 This file is read in at constructor time to avoid file I/O 547 """ 548 def opener(path,flags,*args,**kwargs): 549 kwargs.setdefault('mode',0o755) 550 return os.open(path,flags,*args,**kwargs) 551 552 # runscript_dir directory has to be consistent with gmakefile 553 testDict=srcDict[testname] 554 rpath=self.srcrelpath(root) 555 runscript_dir=os.path.join(self.testroot_dir,rpath) 556 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 557 with open(os.path.join(runscript_dir,testname+".sh"),"w",opener=opener) as fh: 558 559 # Get variables to go into shell scripts. last time testDict used 560 subst=self.getSubstVars(testDict,rpath,testname) 561 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 562 if 'subtests' in testDict: 563 # The subtests inherit inDict, so we don't need top-level loops. 564 loopVars = {} 565 566 #Handle runfiles 567 for lfile in subst.get('localrunfiles','').split(): 568 install_files(os.path.join(root, lfile), 569 os.path.join(runscript_dir, os.path.dirname(lfile))) 570 # Check subtests for local runfiles 571 for stest in subst.get("subtests",[]): 572 for lfile in testDict[stest].get('localrunfiles','').split(): 573 install_files(os.path.join(root, lfile), 574 os.path.join(runscript_dir, os.path.dirname(lfile))) 575 576 # Now substitute the key variables into the header and footer 577 header=self._substVars(subst,example_template.header) 578 # The header is done twice to enable @...@ in header 579 header=self._substVars(subst,header) 580 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 581 582 # Start writing the file 583 fh.write(header+"\n") 584 585 # If there is a TODO or a SKIP then we do it before writing out the 586 # rest of the command (which is useful for working on the test) 587 # SKIP and TODO can be for the source file or for the runs 588 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 589 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 590 591 j=0 # for indentation 592 593 if loopVars: 594 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 595 if (loopHead): fh.write(loopHead+"\n") 596 597 # Subtests are special 598 allLoopVars=list(loopVars.keys()) 599 if 'subtests' in testDict: 600 substP=subst # Subtests can inherit args but be careful 601 k=0 # for label suffixes 602 for stest in testDict["subtests"]: 603 subst=substP.copy() 604 subst.update(testDict[stest]) 605 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 606 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 607 if sLoopVars: 608 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 609 allLoopVars+=list(sLoopVars.keys()) 610 fh.write(sLoopHead+"\n") 611 fh.write(self.getCmds(subst,j)+"\n") 612 if sLoopVars: 613 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 614 fh.write(sLoopFoot+"\n") 615 else: 616 fh.write(self.getCmds(subst,j)+"\n") 617 618 if loopVars: 619 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 620 fh.write(loopFoot+"\n") 621 622 fh.write(footer+"\n") 623 return 624 625 def genScriptsAndInfo(self,exfile,root,srcDict): 626 """ 627 Generate scripts from the source file, determine if built, etc. 628 For every test in the exfile with info in the srcDict: 629 1. Determine if it needs to be run for this arch 630 2. Generate the script 631 3. Generate the data needed to write out the makefile in a 632 convenient way 633 All tests are *always* run, but some may be SKIP'd per the TAP standard 634 """ 635 debug=False 636 rpath=self.srcrelpath(root) 637 execname=self.getExecname(exfile,rpath) 638 isBuilt=self._isBuilt(exfile,srcDict) 639 for test in srcDict: 640 if test in self.buildkeys: continue 641 if debug: print(nameSpace(exfile,root), test) 642 srcDict[test]['execname']=execname # Convenience in generating scripts 643 isRun=self._isRun(srcDict[test]) 644 self.genRunScript(test,root,isRun,srcDict) 645 srcDict[test]['isrun']=isRun 646 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 647 648 # This adds to datastructure for building deps 649 if isBuilt: self.addToSources(exfile,rpath,srcDict) 650 return 651 652 def _isBuilt(self,exfile,srcDict): 653 """ 654 Determine if this file should be built. 655 """ 656 # Get the language based on file extension 657 srcDict['SKIP'] = [] 658 lang=self.getLanguage(exfile) 659 if (lang=="F" or lang=="F90"): 660 if not self.have_fortran: 661 srcDict["SKIP"].append("Fortran required for this test") 662 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 663 srcDict["SKIP"].append("Fortran f90freeform required for this test") 664 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 665 srcDict["SKIP"].append("CUDA required for this test") 666 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 667 srcDict["SKIP"].append("HIP required for this test") 668 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 669 srcDict["SKIP"].append("SYCL required for this test") 670 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 671 srcDict["SKIP"].append("KOKKOS required for this test") 672 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 673 srcDict["SKIP"].append("RAJA required for this test") 674 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 675 srcDict["SKIP"].append("C++ required for this test") 676 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 677 srcDict["SKIP"].append("C++ required for this test") 678 679 # Deprecated source files 680 if srcDict.get("TODO"): 681 return False 682 683 # isRun can work with srcDict to handle the requires 684 if "requires" in srcDict: 685 if srcDict["requires"]: 686 return self._isRun(srcDict) 687 688 return srcDict['SKIP'] == [] 689 690 691 def _isRun(self,testDict, debug=False): 692 """ 693 Based on the requirements listed in the src file and the petscconf.h 694 info, determine whether this test should be run or not. 695 """ 696 indent=" " 697 698 if 'SKIP' not in testDict: 699 testDict['SKIP'] = [] 700 # MPI requirements 701 if 'MPI_IS_MPIUNI' in self.conf: 702 if testDict.get('nsize', '1') != '1': 703 testDict['SKIP'].append("Parallel test with serial build") 704 705 # The requirements for the test are the sum of all the run subtests 706 if 'subtests' in testDict: 707 if 'requires' not in testDict: testDict['requires']="" 708 for stest in testDict['subtests']: 709 if 'requires' in testDict[stest]: 710 testDict['requires']+=" "+testDict[stest]['requires'] 711 if testDict[stest].get('nsize', '1') != '1': 712 testDict['SKIP'].append("Parallel test with serial build") 713 break 714 715 # Now go through all requirements 716 if 'requires' in testDict: 717 for requirement in testDict['requires'].split(): 718 requirement=requirement.strip() 719 if not requirement: continue 720 if debug: print(indent+"Requirement: ", requirement) 721 isNull=False 722 if requirement.startswith("!"): 723 requirement=requirement[1:]; isNull=True 724 # 32-bit vs 64-bit pointers 725 if requirement == "64bitptr": 726 if self.conf['PETSC_SIZEOF_VOID_P']==8: 727 if isNull: 728 testDict['SKIP'].append("not 64bit-ptr required") 729 continue 730 continue # Success 731 elif not isNull: 732 testDict['SKIP'].append("64bit-ptr required") 733 continue 734 # Precision requirement for reals 735 if requirement in self.precision_types: 736 if self.conf['PETSC_PRECISION']==requirement: 737 if isNull: 738 testDict['SKIP'].append("not "+requirement+" required") 739 continue 740 continue # Success 741 elif not isNull: 742 testDict['SKIP'].append(requirement+" required") 743 continue 744 # Precision requirement for ints 745 if requirement in self.integer_types: 746 if requirement=="int32": 747 if self.conf['PETSC_SIZEOF_INT']==4: 748 if isNull: 749 testDict['SKIP'].append("not int32 required") 750 continue 751 continue # Success 752 elif not isNull: 753 testDict['SKIP'].append("int32 required") 754 continue 755 if requirement=="int64": 756 if self.conf['PETSC_SIZEOF_INT']==8: 757 if isNull: 758 testDict['SKIP'].append("NOT int64 required") 759 continue 760 continue # Success 761 elif not isNull: 762 testDict['SKIP'].append("int64 required") 763 continue 764 if requirement.startswith("long"): 765 reqsize = int(requirement[4:])//8 766 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 767 if longsize==reqsize: 768 if isNull: 769 testDict['SKIP'].append("not %s required" % requirement) 770 continue 771 continue # Success 772 elif not isNull: 773 testDict['SKIP'].append("%s required" % requirement) 774 continue 775 # Datafilespath 776 if requirement=="datafilespath" and not isNull: 777 testDict['SKIP'].append("Requires DATAFILESPATH") 778 continue 779 # Defines -- not sure I have comments matching 780 if "defined(" in requirement.lower(): 781 reqdef=requirement.split("(")[1].split(")")[0] 782 if reqdef in self.conf: 783 if isNull: 784 testDict['SKIP'].append("Null requirement not met: "+requirement) 785 continue 786 continue # Success 787 elif not isNull: 788 testDict['SKIP'].append("Required: "+requirement) 789 continue 790 791 # Rest should be packages that we can just get from conf 792 if requirement in ["complex","debug"]: 793 petscconfvar="PETSC_USE_"+requirement.upper() 794 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 795 else: 796 petscconfvar="PETSC_HAVE_"+requirement.upper() 797 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 798 petsccv = self.conf.get(petscconfvar) 799 pkgcv = self.conf.get(pkgconfvar) 800 801 if petsccv or pkgcv: 802 if isNull: 803 if petsccv: 804 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 805 continue 806 else: 807 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 808 continue 809 continue # Success 810 elif not isNull: 811 if not petsccv and not pkgcv: 812 if debug: print("requirement not found: ", requirement) 813 if self.pkg_name == 'petsc': 814 testDict['SKIP'].append(petscconfvar+" requirement not met") 815 else: 816 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 817 continue 818 return testDict['SKIP'] == [] 819 820 def checkOutput(self,exfile,root,srcDict): 821 """ 822 Check and make sure the output files are in the output directory 823 """ 824 debug=False 825 rpath=self.srcrelpath(root) 826 for test in srcDict: 827 if test in self.buildkeys: continue 828 if debug: print(rpath, exfile, test) 829 if 'output_file' in srcDict[test]: 830 output_file=srcDict[test]['output_file'] 831 else: 832 defroot = testparse.getDefaultOutputFileRoot(test) 833 if 'TODO' in srcDict[test]: continue 834 output_file="output/"+defroot+".out" 835 836 fullout=os.path.join(root,output_file) 837 if debug: print("---> ",fullout) 838 if not os.path.exists(fullout): 839 self.missing_files.append(fullout) 840 841 return 842 843 def genPetscTests_summarize(self,dataDict): 844 """ 845 Required method to state what happened 846 """ 847 if not self.summarize: return 848 indent=" " 849 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 850 with open(fhname, "w") as fh: 851 for root in dataDict: 852 relroot=self.srcrelpath(root) 853 pkg=relroot.split("/")[1] 854 if not pkg in self.sources: continue 855 fh.write(relroot+"\n") 856 allSrcs=[] 857 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 858 for exfile in dataDict[root]: 859 # Basic information 860 rfile=os.path.join(relroot,exfile) 861 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 862 fh.write(indent+exfile+indent*4+builtStatus+"\n") 863 for test in dataDict[root][exfile]: 864 if test in self.buildkeys: continue 865 line=indent*2+test 866 fh.write(line+"\n") 867 # Looks nice to have the keys in order 868 #for key in dataDict[root][exfile][test]: 869 for key in "isrun abstracted nsize args requires script".split(): 870 if key not in dataDict[root][exfile][test]: continue 871 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 872 fh.write(line+"\n") 873 fh.write("\n") 874 fh.write("\n") 875 fh.write("\n") 876 return 877 878 def genPetscTests(self,root,dirs,files,dataDict): 879 """ 880 Go through and parse the source files in the directory to generate 881 the examples based on the metadata contained in the source files 882 """ 883 debug=False 884 # Use examplesAnalyze to get what the makefles think are sources 885 #self.examplesAnalyze(root,dirs,files,anlzDict) 886 887 data = {} 888 for exfile in files: 889 #TST: Until we replace files, still leaving the originals as is 890 #if not exfile.startswith("new_"+"ex"): continue 891 #if not exfile.startswith("ex"): continue 892 893 # Ignore emacs and other temporary files 894 if exfile.startswith((".", "#")) or exfile.endswith("~"): continue 895 # Only parse source files 896 ext=getlangext(exfile).lstrip('.').replace('.','_') 897 if ext not in LANGS: continue 898 899 # Convenience 900 fullex=os.path.join(root,exfile) 901 if self.verbose: print(' --> '+fullex) 902 data.update(testparse.parseTestFile(fullex,0)) 903 if exfile in data: 904 if self.check_output: 905 self.checkOutput(exfile,root,data[exfile]) 906 else: 907 self.genScriptsAndInfo(exfile,root,data[exfile]) 908 909 dataDict[root] = data 910 return 911 912 def walktree(self,top): 913 """ 914 Walk a directory tree, starting from 'top' 915 """ 916 if self.check_output: 917 print("Checking for missing output files") 918 self.missing_files=[] 919 920 # Goal of action is to fill this dictionary 921 dataDict={} 922 for root, dirs, files in os.walk(top, topdown=True): 923 dirs.sort() 924 files.sort() 925 if "/tests" not in root and "/tutorials" not in root: continue 926 if "dSYM" in root: continue 927 if "tutorials"+os.sep+"build" in root: continue 928 if os.path.basename(root.rstrip("/")) == 'output': continue 929 if self.verbose: print(root) 930 self.genPetscTests(root,dirs,files,dataDict) 931 932 # If checking output, report results 933 if self.check_output: 934 if self.missing_files: 935 for file in set(self.missing_files): # set uniqifies 936 print(file) 937 sys.exit(1) 938 939 # Now summarize this dictionary 940 if self.verbose: self.genPetscTests_summarize(dataDict) 941 return dataDict 942 943 def gen_gnumake(self, fd): 944 """ 945 Overwrite of the method in the base PETSc class 946 """ 947 def write(stem, srcs): 948 for lang in LANGS: 949 if srcs[lang]['srcs']: 950 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 951 for pkg in self.pkg_pkgs: 952 srcs = self.gen_pkg(pkg) 953 write('testsrcs-' + pkg, srcs) 954 # Handle dependencies 955 for lang in LANGS: 956 for exfile in srcs[lang]['srcs']: 957 if exfile in srcs[lang]: 958 ex='$(TESTDIR)/'+getlangsplit(exfile) 959 exfo=ex+'.o' 960 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 961 if deps: 962 # The executable literally depends on the object file because it is linked 963 fd.write(ex +": " + " ".join(deps) +'\n') 964 # The object file containing 'main' does not normally depend on other object 965 # files, but it does when it includes their modules. This dependency is 966 # overly blunt and could be reduced to only depend on object files for 967 # modules that are used, like "*f90aux.o". 968 fd.write(exfo +": " + " ".join(deps) +'\n') 969 970 return self.gendeps 971 972 def gen_pkg(self, pkg): 973 """ 974 Overwrite of the method in the base PETSc class 975 """ 976 return self.sources[pkg] 977 978 def write_gnumake(self, dataDict, output=None): 979 """ 980 Write out something similar to files from gmakegen.py 981 982 Test depends on script which also depends on source 983 file, but since I don't have a good way generating 984 acting on a single file (oops) just depend on 985 executable which in turn will depend on src file 986 """ 987 # Different options for how to set up the targets 988 compileExecsFirst=False 989 990 # Open file 991 with open(output, 'w') as fd: 992 # Write out the sources 993 gendeps = self.gen_gnumake(fd) 994 995 # Write out the tests and execname targets 996 fd.write("\n#Tests and executables\n") # Delimiter 997 998 for pkg in self.pkg_pkgs: 999 # These grab the ones that are built 1000 for lang in LANGS: 1001 testdeps=[] 1002 for ftest in self.tests[pkg][lang]: 1003 test=os.path.basename(ftest) 1004 basedir=os.path.dirname(ftest) 1005 testdeps.append(nameSpace(test,basedir)) 1006 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 1007 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 1008 1009 # test targets 1010 for ftest in self.tests[pkg][lang]: 1011 test=os.path.basename(ftest) 1012 basedir=os.path.dirname(ftest) 1013 testdir="${TESTDIR}/"+basedir+"/" 1014 nmtest=nameSpace(test,basedir) 1015 rundir=os.path.join(testdir,test) 1016 script=test+".sh" 1017 1018 # Deps 1019 exfile=self.tests[pkg][lang][ftest]['exfile'] 1020 fullex=os.path.join(self.srcdir,exfile) 1021 localexec=self.tests[pkg][lang][ftest]['exec'] 1022 execname=os.path.join(testdir,localexec) 1023 fullscript=os.path.join(testdir,script) 1024 tmpfile=os.path.join(testdir,test,test+".tmp") 1025 1026 # *.counts depends on the script and either executable (will 1027 # be run) or the example source file (SKIP or TODO) 1028 fd.write('%s.counts : %s %s' 1029 % (os.path.join('$(TESTDIR)/counts', nmtest), 1030 fullscript, 1031 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1032 ) 1033 if exfile in self.sources[pkg][lang]: 1034 for dep in self.sources[pkg][lang][exfile]: 1035 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1036 fd.write('\n') 1037 1038 # Now write the args: 1039 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1040 1041 return 1042 1043 def write_db(self, dataDict, testdir): 1044 """ 1045 Write out the dataDict into a pickle file 1046 """ 1047 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1048 pickle.dump(dataDict,fd) 1049 return 1050 1051def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1052 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1053 srcdir=None, testdir=None, check=False): 1054 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1055 testdir=os.path.normpath(testdir) 1056 if petsc_arch: 1057 petsc_arch=petsc_arch.rstrip(os.path.sep) 1058 if len(petsc_arch.split(os.path.sep))>1: 1059 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1060 output = os.path.join(testdir, 'testfiles') 1061 1062 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1063 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1064 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1065 testdir=testdir,check=check) 1066 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1067 if not pEx.check_output: 1068 pEx.write_gnumake(dataDict, output) 1069 pEx.write_db(dataDict, testdir) 1070 1071if __name__ == '__main__': 1072 import optparse 1073 parser = optparse.OptionParser() 1074 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1075 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1076 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1077 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1078 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1079 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1080 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1081 help='Check whether output files are in output director') 1082 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1083 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1084 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1085 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1086 1087 opts, extra_args = parser.parse_args() 1088 if extra_args: 1089 import sys 1090 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1091 exit(1) 1092 if opts.testdir is None: 1093 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1094 1095 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1096 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1097 verbose=opts.verbose, 1098 single_ex=opts.single_executable, srcdir=opts.srcdir, 1099 testdir=opts.testdir, check=opts.check_output) 1100