1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 # Special configuration for CI testing 103 if self.petsc_arch.find('valgrind') >= 0: 104 self.conf['PETSCTEST_VALGRIND']=1 105 106 if self.inInstallDir: 107 # Case 2 discussed above 108 # set PETSC_ARCH to install directory to allow script to work in both 109 dirlist=thisscriptdir.split(os.path.sep) 110 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 111 self.arch_dir=installdir 112 if self.srcdir is None: 113 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 114 else: 115 if petsc_arch == '': 116 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 117 # Case 1 discussed above 118 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 119 if self.srcdir is None: 120 self.srcdir=os.path.join(self.petsc_dir,'src') 121 122 self.testroot_dir=os.path.abspath(testdir) 123 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64 long32 long64".split() 131 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in self.pkg_pkgs: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.relpath(rdir,self.srcdir) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def getLanguage(self,srcfile): 181 """ 182 Based on the source, determine associated language as found in gmakegen.LANGS 183 Can we just return srcext[1:\] now? 184 """ 185 langReq=None 186 srcext = getlangext(srcfile) 187 if srcext in ".F90".split(): langReq="F90" 188 if srcext in ".F".split(): langReq="F" 189 if srcext in ".cxx".split(): langReq="cxx" 190 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 191 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 192 if srcext in ".cpp".split(): langReq="cpp" 193 if srcext == ".cu": langReq="cu" 194 if srcext == ".c": langReq="c" 195 #if not langReq: print("ERROR: ", srcext, srcfile) 196 return langReq 197 198 def _getAltList(self,output_file,srcdir): 199 ''' Calculate AltList based on output file-- see 200 src/snes/tutorials/output/ex22*.out 201 ''' 202 altlist=[output_file] 203 basefile = getlangsplit(output_file) 204 for i in range(1,9): 205 altroot=basefile+"_alt" 206 if i > 1: altroot=altroot+"_"+str(i) 207 af=altroot+".out" 208 srcaf=os.path.join(srcdir,af) 209 fullaf=os.path.join(self.petsc_dir,srcaf) 210 if os.path.isfile(fullaf): altlist.append(srcaf) 211 212 return altlist 213 214 215 def _getLoopVars(self,inDict,testname, isSubtest=False): 216 """ 217 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 218 Return: 219 inDict['args']: -ksp_monitor 220 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 221 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 222 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 223 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 224 subst should be passed in instead of inDict 225 """ 226 loopVars={}; newargs=[] 227 lsuffix='+' 228 argregex = re.compile(' (?=-[a-zA-Z])') 229 from testparse import parseLoopArgs 230 for key in inDict: 231 if key in ('SKIP', 'regexes'): 232 continue 233 akey=('subargs' if key=='args' else key) # what to assign 234 if akey not in inDict: inDict[akey]='' 235 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 236 # Always generate a loop over nsize, even if there is only one value 237 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 238 keystr = str(inDict[key]) 239 varlist = [] 240 for varset in argregex.split(keystr): 241 if not varset.strip(): continue 242 if '{{' in varset: 243 keyvar,lvars,ftype=parseLoopArgs(varset) 244 if akey not in loopVars: loopVars[akey]={} 245 varlist.append(keyvar) 246 loopVars[akey][keyvar]=[keyvar,lvars] 247 if akey=='nsize': 248 if len(lvars.split()) > 1: 249 lsuffix += akey +'-${i' + keyvar + '}' 250 else: 251 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 252 lsuffix+=keyvar+'-${i' + keyvar + '}_' 253 else: 254 if key=='args': 255 newargs.append(varset.strip()) 256 if varlist: 257 loopVars[akey]['varlist']=varlist 258 259 # For subtests, args are always substituted in (not top level) 260 if isSubtest: 261 inDict['subargs'] += " "+" ".join(newargs) 262 inDict['args']='' 263 if 'label_suffix' in inDict: 264 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 265 else: 266 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 267 else: 268 if loopVars: 269 inDict['args'] = ' '.join(newargs) 270 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 271 return loopVars 272 273 def getArgLabel(self,testDict): 274 """ 275 In all of the arguments in the test dictionary, create a simple 276 string for searching within the makefile system. For simplicity in 277 search, remove "-", for strings, etc. 278 Also, concatenate the arg commands 279 For now, ignore nsize -- seems hard to search for anyway 280 """ 281 # Collect all of the args associated with a test 282 argStr=("" if 'args' not in testDict else testDict['args']) 283 if 'subtests' in testDict: 284 for stest in testDict["subtests"]: 285 sd=testDict[stest] 286 argStr=argStr+("" if 'args' not in sd else sd['args']) 287 288 # Now go through and cleanup 289 argStr=re.sub('{{(.*?)}}',"",argStr) 290 argStr=re.sub('-'," ",argStr) 291 for digit in string.digits: argStr=re.sub(digit," ",argStr) 292 argStr=re.sub("\.","",argStr) 293 argStr=re.sub(",","",argStr) 294 argStr=re.sub('\+',' ',argStr) 295 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 296 return argStr.strip() 297 298 def addToSources(self,exfile,rpath,srcDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 """ 302 pkg=rpath.split(os.path.sep)[0] 303 relpfile=os.path.join(rpath,exfile) 304 lang=self.getLanguage(exfile) 305 if not lang: return 306 if pkg not in self.sources: return 307 self.sources[pkg][lang]['srcs'].append(relpfile) 308 self.sources[pkg][lang][relpfile] = [] 309 if 'depends' in srcDict: 310 depSrcList=srcDict['depends'].split() 311 for depSrc in depSrcList: 312 depObj = getlangsplit(depSrc)+'.o' 313 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 314 315 # In gmakefile, ${TESTDIR} var specifies the object compilation 316 testsdir=rpath+"/" 317 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 318 self.objects[pkg].append(objfile) 319 return 320 321 def addToTests(self,test,rpath,exfile,execname,testDict): 322 """ 323 Put into data structure that allows easy generation of makefile 324 Organized by languages to allow testing of languages 325 """ 326 pkg=rpath.split("/")[0] 327 nmtest=os.path.join(rpath,test) 328 lang=self.getLanguage(exfile) 329 if not lang: return 330 if pkg not in self.tests: return 331 self.tests[pkg][lang][nmtest]={} 332 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 333 self.tests[pkg][lang][nmtest]['exec']=execname 334 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 335 return 336 337 def getExecname(self,exfile,rpath): 338 """ 339 Generate bash script using template found next to this file. 340 This file is read in at constructor time to avoid file I/O 341 """ 342 if self.single_ex: 343 execname=rpath.split("/")[1]+"-ex" 344 else: 345 execname=getlangsplit(exfile) 346 return execname 347 348 def getSubstVars(self,testDict,rpath,testname): 349 """ 350 Create a dictionary with all of the variables that get substituted 351 into the template commands found in example_template.py 352 """ 353 subst={} 354 355 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 356 if 'nsize' not in testDict: testDict['nsize'] = '1' 357 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 358 for ak in testparse.acceptedkeys: 359 if ak=='test': continue 360 subst[ak]=(testDict[ak] if ak in testDict else '') 361 362 # Now do other variables 363 subst['execname']=testDict['execname'] 364 subst['error']='' 365 if 'filter' in testDict: 366 if testDict['filter'].startswith("Error:"): 367 subst['error']="Error" 368 subst['filter']=testDict['filter'].lstrip("Error:") 369 else: 370 subst['filter']=testDict['filter'] 371 372 # Others 373 subst['subargs']='' # Default. For variables override 374 subst['srcdir']=os.path.join(self.srcdir, rpath) 375 subst['label_suffix']='' 376 subst['comments']="\n#".join(subst['comments'].split("\n")) 377 if subst['comments']: subst['comments']="#"+subst['comments'] 378 subst['exec']="../"+subst['execname'] 379 subst['testroot']=self.testroot_dir 380 subst['testname']=testname 381 dp = self.conf.get('DATAFILESPATH','') 382 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 383 384 # This is used to label some matrices 385 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 386 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 387 388 #Conf vars 389 if self.petsc_arch.find('valgrind')>=0: 390 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 391 else: 392 subst['mpiexec']=self.conf['MPIEXEC'] 393 subst['pkg_name']=self.pkg_name 394 subst['pkg_dir']=self.pkg_dir 395 subst['pkg_arch']=self.petsc_arch 396 subst['CONFIG_DIR']=thisscriptdir 397 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 398 subst['diff']=self.conf['DIFF'] 399 subst['rm']=self.conf['RM'] 400 subst['grep']=self.conf['GREP'] 401 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 402 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 403 404 # Output file is special because of subtests override 405 defroot = testparse.getDefaultOutputFileRoot(testname) 406 if 'output_file' not in testDict: 407 subst['output_file']="output/"+defroot+".out" 408 subst['redirect_file']=defroot+".tmp" 409 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 410 411 # Add in the full path here. 412 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 413 414 subst['regexes']={} 415 for subkey in subst: 416 if subkey=='regexes': continue 417 if not isinstance(subst[subkey],str): continue 418 patt="@"+subkey.upper()+"@" 419 subst['regexes'][subkey]=re.compile(patt) 420 421 return subst 422 423 def _substVars(self,subst,origStr): 424 """ 425 Substitute variables 426 """ 427 Str=origStr 428 for subkey in subst: 429 if subkey=='regexes': continue 430 if not isinstance(subst[subkey],str): continue 431 if subkey.upper() not in Str: continue 432 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 433 return Str 434 435 def getCmds(self,subst,i, debug=False): 436 """ 437 Generate bash script using template found next to this file. 438 This file is read in at constructor time to avoid file I/O 439 """ 440 nindnt=i # the start and has to be consistent with below 441 cmdindnt=self.indent*nindnt 442 cmdLines="" 443 444 # MPI is the default -- but we have a few odd commands 445 if not subst['command']: 446 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 447 else: 448 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 449 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 450 451 cmdLines+=cmdindnt+'if test $res = 0; then\n' 452 diffindnt=self.indent*(nindnt+1) 453 454 # Do some checks on existence of output_file and alt files 455 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 456 if not subst['TODO']: 457 print("Warning: "+subst['output_file']+" not found.") 458 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 459 460 # altlist always has output_file 461 if len(altlist)==1: 462 cmd=diffindnt+self._substVars(subst,example_template.difftest) 463 else: 464 if debug: print("Found alt files: ",altlist) 465 # Have to do it by hand a bit because of variable number of alt files 466 rf=subst['redirect_file'] 467 cmd=diffindnt+example_template.difftest.split('@')[0] 468 for i in range(len(altlist)): 469 af=altlist[i] 470 cmd+=af+' '+rf 471 if i!=len(altlist)-1: 472 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 473 cmd+=' || ${diff_exe} ' 474 else: 475 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 476 cmd+=subst['label_suffix']+' ""' # Quotes are painful 477 cmdLines+=cmd+"\n" 478 cmdLines+=cmdindnt+'else\n' 479 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 480 cmdLines+=cmdindnt+'fi\n' 481 return cmdLines 482 483 def _writeTodoSkip(self,fh,tors,reasons,footer): 484 """ 485 Write out the TODO and SKIP lines in the file 486 The TODO or SKIP variable, tors, should be lower case 487 """ 488 TORS=tors.upper() 489 template=eval("example_template."+tors+"line") 490 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 491 tab = '' 492 if reasons: 493 fh.write('if ! $force; then\n') 494 tab = tab + ' ' 495 if reasons == ["Requires DATAFILESPATH"]: 496 # The only reason not to run is DATAFILESPATH, which we check at run-time 497 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 498 tab = tab + ' ' 499 if reasons: 500 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 501 fh.write(tab+footer+"\n") 502 fh.write(tab+"exit\n") 503 if reasons == ["Requires DATAFILESPATH"]: 504 fh.write(' fi\n') 505 if reasons: 506 fh.write('fi\n') 507 fh.write('\n\n') 508 return 509 510 def getLoopVarsHead(self,loopVars,i,usedVars={}): 511 """ 512 Generate a nicely indented string with the format loops 513 Here is what the data structure looks like 514 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 515 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 516 loopVars['subargs']['pc_type']=["j","cholesky sor"] 517 """ 518 outstr=''; indnt=self.indent 519 520 for key in loopVars: 521 for var in loopVars[key]['varlist']: 522 varval=loopVars[key][var] 523 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 524 outstr += "\n\n" 525 526 for key in loopVars: 527 for var in loopVars[key]['varlist']: 528 varval=loopVars[key][var] 529 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 530 i = i + 1 531 return (outstr,i) 532 533 def getLoopVarsFoot(self,loopVars,i): 534 outstr=''; indnt=self.indent 535 for key in loopVars: 536 for var in loopVars[key]['varlist']: 537 i = i - 1 538 outstr += indnt * i + "done\n" 539 return (outstr,i) 540 541 def genRunScript(self,testname,root,isRun,srcDict): 542 """ 543 Generate bash script using template found next to this file. 544 This file is read in at constructor time to avoid file I/O 545 """ 546 # runscript_dir directory has to be consistent with gmakefile 547 testDict=srcDict[testname] 548 rpath=self.srcrelpath(root) 549 runscript_dir=os.path.join(self.testroot_dir,rpath) 550 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 551 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 552 553 # Get variables to go into shell scripts. last time testDict used 554 subst=self.getSubstVars(testDict,rpath,testname) 555 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 556 if 'subtests' in testDict: 557 # The subtests inherit inDict, so we don't need top-level loops. 558 loopVars = {} 559 560 #Handle runfiles 561 for lfile in subst.get('localrunfiles','').split(): 562 install_files(os.path.join(root, lfile), 563 os.path.join(runscript_dir, os.path.dirname(lfile))) 564 # Check subtests for local runfiles 565 for stest in subst.get("subtests",[]): 566 for lfile in testDict[stest].get('localrunfiles','').split(): 567 install_files(os.path.join(root, lfile), 568 os.path.join(runscript_dir, os.path.dirname(lfile))) 569 570 # Now substitute the key variables into the header and footer 571 header=self._substVars(subst,example_template.header) 572 # The header is done twice to enable @...@ in header 573 header=self._substVars(subst,header) 574 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 575 576 # Start writing the file 577 fh.write(header+"\n") 578 579 # If there is a TODO or a SKIP then we do it before writing out the 580 # rest of the command (which is useful for working on the test) 581 # SKIP and TODO can be for the source file or for the runs 582 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 583 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 584 585 j=0 # for indentation 586 587 if loopVars: 588 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 589 if (loopHead): fh.write(loopHead+"\n") 590 591 # Subtests are special 592 allLoopVars=list(loopVars.keys()) 593 if 'subtests' in testDict: 594 substP=subst # Subtests can inherit args but be careful 595 k=0 # for label suffixes 596 for stest in testDict["subtests"]: 597 subst=substP.copy() 598 subst.update(testDict[stest]) 599 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 600 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 601 if sLoopVars: 602 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 603 allLoopVars+=list(sLoopVars.keys()) 604 fh.write(sLoopHead+"\n") 605 fh.write(self.getCmds(subst,j)+"\n") 606 if sLoopVars: 607 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 608 fh.write(sLoopFoot+"\n") 609 else: 610 fh.write(self.getCmds(subst,j)+"\n") 611 612 if loopVars: 613 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 614 fh.write(loopFoot+"\n") 615 616 fh.write(footer+"\n") 617 618 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 619 #if '10_9' in testname: sys.exit() 620 return 621 622 def genScriptsAndInfo(self,exfile,root,srcDict): 623 """ 624 Generate scripts from the source file, determine if built, etc. 625 For every test in the exfile with info in the srcDict: 626 1. Determine if it needs to be run for this arch 627 2. Generate the script 628 3. Generate the data needed to write out the makefile in a 629 convenient way 630 All tests are *always* run, but some may be SKIP'd per the TAP standard 631 """ 632 debug=False 633 rpath=self.srcrelpath(root) 634 execname=self.getExecname(exfile,rpath) 635 isBuilt=self._isBuilt(exfile,srcDict) 636 for test in srcDict: 637 if test in self.buildkeys: continue 638 if debug: print(nameSpace(exfile,root), test) 639 srcDict[test]['execname']=execname # Convenience in generating scripts 640 isRun=self._isRun(srcDict[test]) 641 self.genRunScript(test,root,isRun,srcDict) 642 srcDict[test]['isrun']=isRun 643 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 644 645 # This adds to datastructure for building deps 646 if isBuilt: self.addToSources(exfile,rpath,srcDict) 647 return 648 649 def _isBuilt(self,exfile,srcDict): 650 """ 651 Determine if this file should be built. 652 """ 653 # Get the language based on file extension 654 srcDict['SKIP'] = [] 655 lang=self.getLanguage(exfile) 656 if (lang=="F" or lang=="F90"): 657 if not self.have_fortran: 658 srcDict["SKIP"].append("Fortran required for this test") 659 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 660 srcDict["SKIP"].append("Fortran f90freeform required for this test") 661 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 662 srcDict["SKIP"].append("CUDA required for this test") 663 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 664 srcDict["SKIP"].append("HIP required for this test") 665 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 666 srcDict["SKIP"].append("SYCL required for this test") 667 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 668 srcDict["SKIP"].append("KOKKOS required for this test") 669 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 670 srcDict["SKIP"].append("RAJA required for this test") 671 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 672 srcDict["SKIP"].append("C++ required for this test") 673 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 674 srcDict["SKIP"].append("C++ required for this test") 675 676 # Deprecated source files 677 if srcDict.get("TODO"): 678 return False 679 680 # isRun can work with srcDict to handle the requires 681 if "requires" in srcDict: 682 if srcDict["requires"]: 683 return self._isRun(srcDict) 684 685 return srcDict['SKIP'] == [] 686 687 688 def _isRun(self,testDict, debug=False): 689 """ 690 Based on the requirements listed in the src file and the petscconf.h 691 info, determine whether this test should be run or not. 692 """ 693 indent=" " 694 695 if 'SKIP' not in testDict: 696 testDict['SKIP'] = [] 697 # MPI requirements 698 if 'MPI_IS_MPIUNI' in self.conf: 699 if testDict.get('nsize', '1') != '1': 700 testDict['SKIP'].append("Parallel test with serial build") 701 702 # The requirements for the test are the sum of all the run subtests 703 if 'subtests' in testDict: 704 if 'requires' not in testDict: testDict['requires']="" 705 for stest in testDict['subtests']: 706 if 'requires' in testDict[stest]: 707 testDict['requires']+=" "+testDict[stest]['requires'] 708 if testDict[stest].get('nsize', '1') != '1': 709 testDict['SKIP'].append("Parallel test with serial build") 710 break 711 712 # Now go through all requirements 713 if 'requires' in testDict: 714 for requirement in testDict['requires'].split(): 715 requirement=requirement.strip() 716 if not requirement: continue 717 if debug: print(indent+"Requirement: ", requirement) 718 isNull=False 719 if requirement.startswith("!"): 720 requirement=requirement[1:]; isNull=True 721 # Precision requirement for reals 722 if requirement in self.precision_types: 723 if self.conf['PETSC_PRECISION']==requirement: 724 if isNull: 725 testDict['SKIP'].append("not "+requirement+" required") 726 continue 727 continue # Success 728 elif not isNull: 729 testDict['SKIP'].append(requirement+" required") 730 continue 731 # Precision requirement for ints 732 if requirement in self.integer_types: 733 if requirement=="int32": 734 if self.conf['PETSC_SIZEOF_INT']==4: 735 if isNull: 736 testDict['SKIP'].append("not int32 required") 737 continue 738 continue # Success 739 elif not isNull: 740 testDict['SKIP'].append("int32 required") 741 continue 742 if requirement=="int64": 743 if self.conf['PETSC_SIZEOF_INT']==8: 744 if isNull: 745 testDict['SKIP'].append("NOT int64 required") 746 continue 747 continue # Success 748 elif not isNull: 749 testDict['SKIP'].append("int64 required") 750 continue 751 if requirement.startswith("long"): 752 reqsize = int(requirement[4:])//8 753 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 754 if longsize==reqsize: 755 if isNull: 756 testDict['SKIP'].append("not %s required" % requirement) 757 continue 758 continue # Success 759 elif not isNull: 760 testDict['SKIP'].append("%s required" % requirement) 761 continue 762 # Datafilespath 763 if requirement=="datafilespath" and not isNull: 764 testDict['SKIP'].append("Requires DATAFILESPATH") 765 continue 766 # Defines -- not sure I have comments matching 767 if "defined(" in requirement.lower(): 768 reqdef=requirement.split("(")[1].split(")")[0] 769 if reqdef in self.conf: 770 if isNull: 771 testDict['SKIP'].append("Null requirement not met: "+requirement) 772 continue 773 continue # Success 774 elif not isNull: 775 testDict['SKIP'].append("Required: "+requirement) 776 continue 777 778 # Rest should be packages that we can just get from conf 779 if requirement in ["complex","debug"]: 780 petscconfvar="PETSC_USE_"+requirement.upper() 781 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 782 else: 783 petscconfvar="PETSC_HAVE_"+requirement.upper() 784 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 785 petsccv = self.conf.get(petscconfvar) 786 pkgcv = self.conf.get(pkgconfvar) 787 788 if petsccv or pkgcv: 789 if isNull: 790 if petsccv: 791 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 792 continue 793 else: 794 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 795 continue 796 continue # Success 797 elif not isNull: 798 if not petsccv and not pkgcv: 799 if debug: print("requirement not found: ", requirement) 800 if self.pkg_name == 'petsc': 801 testDict['SKIP'].append(petscconfvar+" requirement not met") 802 else: 803 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 804 continue 805 return testDict['SKIP'] == [] 806 807 def checkOutput(self,exfile,root,srcDict): 808 """ 809 Check and make sure the output files are in the output directory 810 """ 811 debug=False 812 rpath=self.srcrelpath(root) 813 for test in srcDict: 814 if test in self.buildkeys: continue 815 if debug: print(rpath, exfile, test) 816 if 'output_file' in srcDict[test]: 817 output_file=srcDict[test]['output_file'] 818 else: 819 defroot = testparse.getDefaultOutputFileRoot(test) 820 if 'TODO' in srcDict[test]: continue 821 output_file="output/"+defroot+".out" 822 823 fullout=os.path.join(root,output_file) 824 if debug: print("---> ",fullout) 825 if not os.path.exists(fullout): 826 self.missing_files.append(fullout) 827 828 return 829 830 def genPetscTests_summarize(self,dataDict): 831 """ 832 Required method to state what happened 833 """ 834 if not self.summarize: return 835 indent=" " 836 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 837 with open(fhname, "w") as fh: 838 for root in dataDict: 839 relroot=self.srcrelpath(root) 840 pkg=relroot.split("/")[1] 841 fh.write(relroot+"\n") 842 allSrcs=[] 843 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 844 for exfile in dataDict[root]: 845 # Basic information 846 rfile=os.path.join(relroot,exfile) 847 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 848 fh.write(indent+exfile+indent*4+builtStatus+"\n") 849 for test in dataDict[root][exfile]: 850 if test in self.buildkeys: continue 851 line=indent*2+test 852 fh.write(line+"\n") 853 # Looks nice to have the keys in order 854 #for key in dataDict[root][exfile][test]: 855 for key in "isrun abstracted nsize args requires script".split(): 856 if key not in dataDict[root][exfile][test]: continue 857 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 858 fh.write(line+"\n") 859 fh.write("\n") 860 fh.write("\n") 861 fh.write("\n") 862 return 863 864 def genPetscTests(self,root,dirs,files,dataDict): 865 """ 866 Go through and parse the source files in the directory to generate 867 the examples based on the metadata contained in the source files 868 """ 869 debug=False 870 # Use examplesAnalyze to get what the makefles think are sources 871 #self.examplesAnalyze(root,dirs,files,anlzDict) 872 873 dataDict[root]={} 874 875 for exfile in files: 876 #TST: Until we replace files, still leaving the orginals as is 877 #if not exfile.startswith("new_"+"ex"): continue 878 #if not exfile.startswith("ex"): continue 879 880 # Ignore emacs and other temporary files 881 if exfile.startswith("."): continue 882 if exfile.startswith("#"): continue 883 if exfile.endswith("~"): continue 884 # Only parse source files 885 ext=getlangext(exfile).lstrip('.').replace('.','_') 886 if ext not in LANGS: continue 887 888 # Convenience 889 fullex=os.path.join(root,exfile) 890 if self.verbose: print(' --> '+fullex) 891 dataDict[root].update(testparse.parseTestFile(fullex,0)) 892 if exfile in dataDict[root]: 893 if not self.check_output: 894 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 895 else: 896 self.checkOutput(exfile,root,dataDict[root][exfile]) 897 898 return 899 900 def walktree(self,top): 901 """ 902 Walk a directory tree, starting from 'top' 903 """ 904 if self.check_output: 905 print("Checking for missing output files") 906 self.missing_files=[] 907 908 # Goal of action is to fill this dictionary 909 dataDict={} 910 for root, dirs, files in os.walk(top, topdown=True): 911 dirs.sort() 912 files.sort() 913 if "/tests" not in root and "/tutorials" not in root: continue 914 if "dSYM" in root: continue 915 if "tutorials"+os.sep+"build" in root: continue 916 if os.path.basename(root.rstrip("/")) == 'output': continue 917 if self.verbose: print(root) 918 self.genPetscTests(root,dirs,files,dataDict) 919 920 # If checking output, report results 921 if self.check_output: 922 if self.missing_files: 923 for file in set(self.missing_files): # set uniqifies 924 print(file) 925 sys.exit(1) 926 927 # Now summarize this dictionary 928 if self.verbose: self.genPetscTests_summarize(dataDict) 929 return dataDict 930 931 def gen_gnumake(self, fd): 932 """ 933 Overwrite of the method in the base PETSc class 934 """ 935 def write(stem, srcs): 936 for lang in LANGS: 937 if srcs[lang]['srcs']: 938 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 939 for pkg in self.pkg_pkgs: 940 srcs = self.gen_pkg(pkg) 941 write('testsrcs-' + pkg, srcs) 942 # Handle dependencies 943 for lang in LANGS: 944 for exfile in srcs[lang]['srcs']: 945 if exfile in srcs[lang]: 946 ex='$(TESTDIR)/'+getlangsplit(exfile) 947 exfo=ex+'.o' 948 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 949 if deps: 950 # The executable literally depends on the object file because it is linked 951 fd.write(ex +": " + " ".join(deps) +'\n') 952 # The object file containing 'main' does not normally depend on other object 953 # files, but it does when it includes their modules. This dependency is 954 # overly blunt and could be reduced to only depend on object files for 955 # modules that are used, like "*f90aux.o". 956 fd.write(exfo +": " + " ".join(deps) +'\n') 957 958 return self.gendeps 959 960 def gen_pkg(self, pkg): 961 """ 962 Overwrite of the method in the base PETSc class 963 """ 964 return self.sources[pkg] 965 966 def write_gnumake(self, dataDict, output=None): 967 """ 968 Write out something similar to files from gmakegen.py 969 970 Test depends on script which also depends on source 971 file, but since I don't have a good way generating 972 acting on a single file (oops) just depend on 973 executable which in turn will depend on src file 974 """ 975 # Different options for how to set up the targets 976 compileExecsFirst=False 977 978 # Open file 979 with open(output, 'w') as fd: 980 # Write out the sources 981 gendeps = self.gen_gnumake(fd) 982 983 # Write out the tests and execname targets 984 fd.write("\n#Tests and executables\n") # Delimiter 985 986 for pkg in self.pkg_pkgs: 987 # These grab the ones that are built 988 for lang in LANGS: 989 testdeps=[] 990 for ftest in self.tests[pkg][lang]: 991 test=os.path.basename(ftest) 992 basedir=os.path.dirname(ftest) 993 testdeps.append(nameSpace(test,basedir)) 994 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 995 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 996 997 # test targets 998 for ftest in self.tests[pkg][lang]: 999 test=os.path.basename(ftest) 1000 basedir=os.path.dirname(ftest) 1001 testdir="${TESTDIR}/"+basedir+"/" 1002 nmtest=nameSpace(test,basedir) 1003 rundir=os.path.join(testdir,test) 1004 script=test+".sh" 1005 1006 # Deps 1007 exfile=self.tests[pkg][lang][ftest]['exfile'] 1008 fullex=os.path.join(self.srcdir,exfile) 1009 localexec=self.tests[pkg][lang][ftest]['exec'] 1010 execname=os.path.join(testdir,localexec) 1011 fullscript=os.path.join(testdir,script) 1012 tmpfile=os.path.join(testdir,test,test+".tmp") 1013 1014 # *.counts depends on the script and either executable (will 1015 # be run) or the example source file (SKIP or TODO) 1016 fd.write('%s.counts : %s %s' 1017 % (os.path.join('$(TESTDIR)/counts', nmtest), 1018 fullscript, 1019 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1020 ) 1021 if exfile in self.sources[pkg][lang]: 1022 for dep in self.sources[pkg][lang][exfile]: 1023 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1024 fd.write('\n') 1025 1026 # Now write the args: 1027 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1028 1029 return 1030 1031 def write_db(self, dataDict, testdir): 1032 """ 1033 Write out the dataDict into a pickle file 1034 """ 1035 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1036 pickle.dump(dataDict,fd) 1037 return 1038 1039def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1040 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1041 srcdir=None, testdir=None, check=False): 1042 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1043 testdir=os.path.normpath(testdir) 1044 if petsc_arch: 1045 petsc_arch=petsc_arch.rstrip(os.path.sep) 1046 if len(petsc_arch.split(os.path.sep))>1: 1047 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1048 output = os.path.join(testdir, 'testfiles') 1049 1050 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1051 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1052 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1053 testdir=testdir,check=check) 1054 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1055 if not pEx.check_output: 1056 pEx.write_gnumake(dataDict, output) 1057 pEx.write_db(dataDict, testdir) 1058 1059if __name__ == '__main__': 1060 import optparse 1061 parser = optparse.OptionParser() 1062 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1063 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1064 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1065 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1066 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1067 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1068 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1069 help='Check whether output files are in output director') 1070 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1071 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1072 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1073 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1074 1075 opts, extra_args = parser.parse_args() 1076 if extra_args: 1077 import sys 1078 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1079 exit(1) 1080 if opts.testdir is None: 1081 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1082 1083 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1084 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1085 verbose=opts.verbose, 1086 single_ex=opts.single_executable, srcdir=opts.srcdir, 1087 testdir=opts.testdir, check=opts.check_output) 1088