1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose) 85 86 self.single_ex=single_ex 87 self.srcdir=srcdir 88 89 # Set locations to handle movement 90 self.inInstallDir=self.getInInstallDir(thisscriptdir) 91 92 if self.inInstallDir: 93 # Case 2 discussed above 94 # set PETSC_ARCH to install directory to allow script to work in both 95 dirlist=thisscriptdir.split(os.path.sep) 96 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 97 self.arch_dir=installdir 98 if self.srcdir is None: 99 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 100 else: 101 if petsc_arch == '': 102 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 103 # Case 1 discussed above 104 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 105 if self.srcdir is None: 106 self.srcdir=os.path.join(self.petsc_dir,'src') 107 108 self.testroot_dir=os.path.abspath(testdir) 109 110 self.ptNaming=True 111 self.verbose=verbose 112 # Whether to write out a useful debugging 113 self.summarize=True if verbose else False 114 115 # For help in setting the requirements 116 self.precision_types="single double __float128 int32".split() 117 self.integer_types="int32 int64".split() 118 self.languages="fortran cuda cxx".split() # Always requires C so do not list 119 120 # Things that are not test 121 self.buildkeys=testparse.buildkeys 122 123 # Adding a dictionary for storing sources, objects, and tests 124 # to make building the dependency tree easier 125 self.sources={} 126 self.objects={} 127 self.tests={} 128 for pkg in PKGS: 129 self.sources[pkg]={} 130 self.objects[pkg]=[] 131 self.tests[pkg]={} 132 for lang in LANGS: 133 self.sources[pkg][lang]={} 134 self.sources[pkg][lang]['srcs']=[] 135 self.tests[pkg][lang]={} 136 137 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 138 139 self.indent=" " 140 if self.verbose: print('Finishing the constructor') 141 return 142 143 def srcrelpath(self,rdir): 144 """ 145 Get relative path to source directory 146 """ 147 return os.path.relpath(rdir,self.srcdir) 148 149 def getInInstallDir(self,thisscriptdir): 150 """ 151 When petsc is installed then this file in installed in: 152 <PREFIX>/share/petsc/examples/config/gmakegentest.py 153 otherwise the path is: 154 <PETSC_DIR>/config/gmakegentest.py 155 We use this difference to determine if we are in installdir 156 """ 157 dirlist=thisscriptdir.split(os.path.sep) 158 if len(dirlist)>4: 159 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 160 if lastfour==os.path.join('share','petsc','examples','config'): 161 return True 162 else: 163 return False 164 else: 165 return False 166 167 def nameSpace(self,srcfile,srcdir): 168 """ 169 Because the scripts have a non-unique naming, the pretty-printing 170 needs to convey the srcdir and srcfile. There are two ways of doing this. 171 """ 172 if self.ptNaming: 173 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 174 cdir=srcdir 175 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 176 nameString=prefix+srcfile 177 else: 178 #nameString=srcdir+": "+srcfile 179 nameString=srcfile 180 return nameString 181 182 def getLanguage(self,srcfile): 183 """ 184 Based on the source, determine associated language as found in gmakegen.LANGS 185 Can we just return srcext[1:\] now? 186 """ 187 langReq=None 188 srcext=os.path.splitext(srcfile)[-1] 189 if srcext in ".F90".split(): langReq="F90" 190 if srcext in ".F".split(): langReq="F" 191 if srcext in ".cxx".split(): langReq="cxx" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getLoopVars(self,inDict,testname, isSubtest=False): 198 """ 199 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 200 Return: 201 inDict['args']: -ksp_monitor 202 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 203 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 204 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 205 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 206 subst should be passed in instead of inDict 207 """ 208 loopVars={}; newargs=[] 209 lsuffix='_' 210 argregex = re.compile(' (?=-[a-zA-Z])') 211 from testparse import parseLoopArgs 212 for key in inDict: 213 if key in ('SKIP', 'regexes'): 214 continue 215 akey=('subargs' if key=='args' else key) # what to assign 216 if akey not in inDict: inDict[akey]='' 217 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 218 # Always generate a loop over nsize, even if there is only one value 219 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 220 keystr = str(inDict[key]) 221 varlist = [] 222 for varset in argregex.split(keystr): 223 if not varset.strip(): continue 224 if '{{' in varset: 225 keyvar,lvars,ftype=parseLoopArgs(varset) 226 if akey not in loopVars: loopVars[akey]={} 227 varlist.append(keyvar) 228 loopVars[akey][keyvar]=[keyvar,lvars] 229 if akey=='nsize': 230 if len(lvars.split()) > 1: 231 lsuffix += akey +'-${' + keyvar + '}' 232 else: 233 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 234 lsuffix+=keyvar+'-${' + keyvar + '}_' 235 else: 236 if key=='args': 237 newargs.append(varset.strip()) 238 if varlist: 239 loopVars[akey]['varlist']=varlist 240 241 # For subtests, args are always substituted in (not top level) 242 if isSubtest: 243 inDict['subargs'] += " ".join(newargs) 244 inDict['args']='' 245 if 'label_suffix' in inDict: 246 inDict['label_suffix']+=lsuffix.rstrip('_') 247 else: 248 inDict['label_suffix']=lsuffix.rstrip('_') 249 else: 250 if loopVars: 251 inDict['args'] = ' '.join(newargs) 252 inDict['label_suffix']=lsuffix.rstrip('_') 253 return loopVars 254 255 def getArgLabel(self,testDict): 256 """ 257 In all of the arguments in the test dictionary, create a simple 258 string for searching within the makefile system. For simplicity in 259 search, remove "-", for strings, etc. 260 Also, concatenate the arg commands 261 For now, ignore nsize -- seems hard to search for anyway 262 """ 263 # Collect all of the args associated with a test 264 argStr=("" if 'args' not in testDict else testDict['args']) 265 if 'subtests' in testDict: 266 for stest in testDict["subtests"]: 267 sd=testDict[stest] 268 argStr=argStr+("" if 'args' not in sd else sd['args']) 269 270 # Now go through and cleanup 271 argStr=re.sub('{{(.*?)}}',"",argStr) 272 argStr=re.sub('-'," ",argStr) 273 for digit in string.digits: argStr=re.sub(digit," ",argStr) 274 argStr=re.sub("\.","",argStr) 275 argStr=re.sub(",","",argStr) 276 argStr=re.sub('\+',' ',argStr) 277 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 278 return argStr.strip() 279 280 def addToSources(self,exfile,rpath,srcDict): 281 """ 282 Put into data structure that allows easy generation of makefile 283 """ 284 pkg=rpath.split(os.path.sep)[0] 285 relpfile=os.path.join(rpath,exfile) 286 lang=self.getLanguage(exfile) 287 if not lang: return 288 self.sources[pkg][lang]['srcs'].append(relpfile) 289 self.sources[pkg][lang][relpfile] = [] 290 if 'depends' in srcDict: 291 depSrcList=srcDict['depends'].split() 292 for depSrc in depSrcList: 293 depObj=os.path.splitext(depSrc)[0]+".o" 294 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 295 296 # In gmakefile, ${TESTDIR} var specifies the object compilation 297 testsdir=rpath+"/" 298 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 299 self.objects[pkg].append(objfile) 300 return 301 302 def addToTests(self,test,rpath,exfile,execname,testDict): 303 """ 304 Put into data structure that allows easy generation of makefile 305 Organized by languages to allow testing of languages 306 """ 307 pkg=rpath.split("/")[0] 308 nmtest=os.path.join(rpath,test) 309 lang=self.getLanguage(exfile) 310 if not lang: return 311 self.tests[pkg][lang][nmtest]={} 312 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 313 self.tests[pkg][lang][nmtest]['exec']=execname 314 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 315 return 316 317 def getExecname(self,exfile,rpath): 318 """ 319 Generate bash script using template found next to this file. 320 This file is read in at constructor time to avoid file I/O 321 """ 322 if self.single_ex: 323 execname=rpath.split("/")[1]+"-ex" 324 else: 325 execname=os.path.splitext(exfile)[0] 326 return execname 327 328 def getSubstVars(self,testDict,rpath,testname): 329 """ 330 Create a dictionary with all of the variables that get substituted 331 into the template commands found in example_template.py 332 """ 333 subst={} 334 335 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 336 if 'nsize' not in testDict: testDict['nsize'] = '1' 337 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 338 for ak in testparse.acceptedkeys: 339 if ak=='test': continue 340 subst[ak]=(testDict[ak] if ak in testDict else '') 341 342 # Now do other variables 343 subst['execname']=testDict['execname'] 344 if 'filter' in testDict: 345 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 346 347 # Others 348 subst['subargs']='' # Default. For variables override 349 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 350 subst['label_suffix']='' 351 subst['comments']="\n#".join(subst['comments'].split("\n")) 352 if subst['comments']: subst['comments']="#"+subst['comments'] 353 subst['exec']="../"+subst['execname'] 354 subst['testroot']=self.testroot_dir 355 subst['testname']=testname 356 dp = self.conf.get('DATAFILESPATH','') 357 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 358 359 # This is used to label some matrices 360 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 361 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 362 363 #Conf vars 364 if self.petsc_arch.find('valgrind')>=0: 365 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 366 else: 367 subst['mpiexec']=self.conf['MPIEXEC'] 368 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 369 subst['petsc_arch']=self.petsc_arch 370 if self.inInstallDir: 371 # Case 2 372 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 373 else: 374 # Case 1 375 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 376 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 377 subst['diff']=self.conf['DIFF'] 378 subst['rm']=self.conf['RM'] 379 subst['grep']=self.conf['GREP'] 380 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 381 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 382 383 # Output file is special because of subtests override 384 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 385 subst['defroot']=defroot 386 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 387 subst['redirect_file']=defroot+".tmp" 388 if 'output_file' not in testDict: 389 subst['output_file']="output/"+defroot+".out" 390 # Add in the full path here. 391 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 392 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 393 if not subst['TODO']: 394 print("Warning: "+subst['output_file']+" not found.") 395 # Worry about alt files here -- see 396 # src/snes/examples/tutorials/output/ex22*.out 397 altlist=[subst['output_file']] 398 basefile,ext = os.path.splitext(subst['output_file']) 399 for i in range(1,9): 400 altroot=basefile+"_alt" 401 if i > 1: altroot=altroot+"_"+str(i) 402 af=altroot+".out" 403 srcaf=os.path.join(subst['srcdir'],af) 404 fullaf=os.path.join(self.petsc_dir,srcaf) 405 if os.path.isfile(fullaf): altlist.append(srcaf) 406 if len(altlist)>1: subst['altfiles']=altlist 407 #if len(altlist)>1: print("Found alt files: ",altlist) 408 409 subst['regexes']={} 410 for subkey in subst: 411 if subkey=='regexes': continue 412 if not isinstance(subst[subkey],str): continue 413 patt="@"+subkey.upper()+"@" 414 subst['regexes'][subkey]=re.compile(patt) 415 416 return subst 417 418 def _substVars(self,subst,origStr): 419 """ 420 Substitute variables 421 """ 422 Str=origStr 423 for subkey in subst: 424 if subkey=='regexes': continue 425 if not isinstance(subst[subkey],str): continue 426 if subkey.upper() not in Str: continue 427 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 428 return Str 429 430 def getCmds(self,subst,i): 431 """ 432 Generate bash script using template found next to this file. 433 This file is read in at constructor time to avoid file I/O 434 """ 435 nindnt=i # the start and has to be consistent with below 436 cmdindnt=self.indent*nindnt 437 cmdLines="" 438 439 # MPI is the default -- but we have a few odd commands 440 if not subst['command']: 441 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 442 else: 443 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 444 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 445 446 cmdLines+=cmdindnt+'if test $res = 0; then\n' 447 diffindnt=self.indent*(nindnt+1) 448 if not subst['filter_output']: 449 if 'altfiles' not in subst: 450 cmd=diffindnt+self._substVars(subst,example_template.difftest) 451 else: 452 # Have to do it by hand a bit because of variable number of alt files 453 rf=subst['redirect_file'] 454 cmd=diffindnt+example_template.difftest.split('@')[0] 455 for i in range(len(subst['altfiles'])): 456 af=subst['altfiles'][i] 457 cmd+=af+' '+rf 458 if i!=len(subst['altfiles'])-1: 459 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 460 cmd+=' || ${diff_exe} ' 461 else: 462 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 463 cmd+=subst['label_suffix']+' ""' # Quotes are painful 464 else: 465 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 466 cmdLines+=cmd+"\n" 467 cmdLines+=cmdindnt+'else\n' 468 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 469 cmdLines+=cmdindnt+'fi\n' 470 return cmdLines 471 472 def _writeTodoSkip(self,fh,tors,reasons,footer): 473 """ 474 Write out the TODO and SKIP lines in the file 475 The TODO or SKIP variable, tors, should be lower case 476 """ 477 TORS=tors.upper() 478 template=eval("example_template."+tors+"line") 479 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 480 tab = '' 481 if reasons: 482 fh.write('if ! $force; then\n') 483 tab = tab + ' ' 484 if reasons == ["Requires DATAFILESPATH"]: 485 # The only reason not to run is DATAFILESPATH, which we check at run-time 486 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 487 tab = tab + ' ' 488 if reasons: 489 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 490 fh.write(tab+footer+"\n") 491 fh.write(tab+"exit\n") 492 if reasons == ["Requires DATAFILESPATH"]: 493 fh.write(' fi\n') 494 if reasons: 495 fh.write('fi\n') 496 fh.write('\n\n') 497 return 498 499 def getLoopVarsHead(self,loopVars,i,usedVars={}): 500 """ 501 Generate a nicely indented string with the format loops 502 Here is what the data structure looks like 503 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 504 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 505 loopVars['subargs']['pc_type']=["j","cholesky sor"] 506 """ 507 outstr=''; indnt=self.indent 508 509 for key in loopVars: 510 if key in usedVars: continue # Do not duplicate setting vars 511 for var in loopVars[key]['varlist']: 512 varval=loopVars[key][var] 513 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 514 outstr += "\n\n" 515 516 for key in loopVars: 517 for var in loopVars[key]['varlist']: 518 varval=loopVars[key][var] 519 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 520 i = i + 1 521 return (outstr,i) 522 523 def getLoopVarsFoot(self,loopVars,i): 524 outstr=''; indnt=self.indent 525 for key in loopVars: 526 for var in loopVars[key]['varlist']: 527 i = i - 1 528 outstr += indnt * i + "done\n" 529 return (outstr,i) 530 531 def genRunScript(self,testname,root,isRun,srcDict): 532 """ 533 Generate bash script using template found next to this file. 534 This file is read in at constructor time to avoid file I/O 535 """ 536 # runscript_dir directory has to be consistent with gmakefile 537 testDict=srcDict[testname] 538 rpath=self.srcrelpath(root) 539 runscript_dir=os.path.join(self.testroot_dir,rpath) 540 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 541 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 542 543 # Get variables to go into shell scripts. last time testDict used 544 subst=self.getSubstVars(testDict,rpath,testname) 545 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 546 if 'subtests' in testDict: 547 # The subtests inherit inDict, so we don't need top-level loops. 548 loopVars = {} 549 550 #Handle runfiles 551 for lfile in subst.get('localrunfiles','').split(): 552 install_files(os.path.join(root, lfile), 553 os.path.join(runscript_dir, os.path.dirname(lfile))) 554 # Check subtests for local runfiles 555 for stest in subst.get("subtests",[]): 556 for lfile in testDict[stest].get('localrunfiles','').split(): 557 install_files(os.path.join(root, lfile), 558 os.path.join(runscript_dir, os.path.dirname(lfile))) 559 560 # Now substitute the key variables into the header and footer 561 header=self._substVars(subst,example_template.header) 562 # The header is done twice to enable @...@ in header 563 header=self._substVars(subst,header) 564 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 565 566 # Start writing the file 567 fh.write(header+"\n") 568 569 # If there is a TODO or a SKIP then we do it before writing out the 570 # rest of the command (which is useful for working on the test) 571 # SKIP and TODO can be for the source file or for the runs 572 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 573 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 574 575 j=0 # for indentation 576 577 if loopVars: 578 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 579 if (loopHead): fh.write(loopHead+"\n") 580 581 # Subtests are special 582 allLoopVars=list(loopVars.keys()) 583 if 'subtests' in testDict: 584 substP=subst # Subtests can inherit args but be careful 585 k=0 # for label suffixes 586 for stest in testDict["subtests"]: 587 subst=substP.copy() 588 subst.update(testDict[stest]) 589 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 590 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 591 if sLoopVars: 592 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 593 allLoopVars+=list(sLoopVars.keys()) 594 fh.write(sLoopHead+"\n") 595 fh.write(self.getCmds(subst,j)+"\n") 596 if sLoopVars: 597 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 598 fh.write(sLoopFoot+"\n") 599 else: 600 fh.write(self.getCmds(subst,j)+"\n") 601 602 if loopVars: 603 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 604 fh.write(loopFoot+"\n") 605 606 fh.write(footer+"\n") 607 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 608 #if '10_9' in testname: sys.exit() 609 return 610 611 def genScriptsAndInfo(self,exfile,root,srcDict): 612 """ 613 Generate scripts from the source file, determine if built, etc. 614 For every test in the exfile with info in the srcDict: 615 1. Determine if it needs to be run for this arch 616 2. Generate the script 617 3. Generate the data needed to write out the makefile in a 618 convenient way 619 All tests are *always* run, but some may be SKIP'd per the TAP standard 620 """ 621 debug=False 622 rpath=self.srcrelpath(root) 623 execname=self.getExecname(exfile,rpath) 624 isBuilt=self._isBuilt(exfile,srcDict) 625 for test in srcDict: 626 if test in self.buildkeys: continue 627 if debug: print(self.nameSpace(exfile,root), test) 628 srcDict[test]['execname']=execname # Convenience in generating scripts 629 isRun=self._isRun(srcDict[test]) 630 self.genRunScript(test,root,isRun,srcDict) 631 srcDict[test]['isrun']=isRun 632 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 633 634 # This adds to datastructure for building deps 635 if isBuilt: self.addToSources(exfile,rpath,srcDict) 636 return 637 638 def _isBuilt(self,exfile,srcDict): 639 """ 640 Determine if this file should be built. 641 """ 642 # Get the language based on file extension 643 srcDict['SKIP'] = [] 644 lang=self.getLanguage(exfile) 645 if (lang=="F" or lang=="F90"): 646 if not self.have_fortran: 647 srcDict["SKIP"].append("Fortran required for this test") 648 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 649 srcDict["SKIP"].append("Fortran f90freeform required for this test") 650 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 651 srcDict["SKIP"].append("CUDA required for this test") 652 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 653 srcDict["SKIP"].append("C++ required for this test") 654 655 # Deprecated source files 656 if srcDict.get("TODO"): 657 return False 658 659 # isRun can work with srcDict to handle the requires 660 if "requires" in srcDict: 661 if srcDict["requires"]: 662 return self._isRun(srcDict) 663 664 return srcDict['SKIP'] == [] 665 666 667 def _isRun(self,testDict, debug=False): 668 """ 669 Based on the requirements listed in the src file and the petscconf.h 670 info, determine whether this test should be run or not. 671 """ 672 indent=" " 673 674 if 'SKIP' not in testDict: 675 testDict['SKIP'] = [] 676 # MPI requirements 677 if 'MPI_IS_MPIUNI' in self.conf: 678 if testDict.get('nsize', '1') != '1': 679 testDict['SKIP'].append("Parallel test with serial build") 680 681 # The requirements for the test are the sum of all the run subtests 682 if 'subtests' in testDict: 683 if 'requires' not in testDict: testDict['requires']="" 684 for stest in testDict['subtests']: 685 if 'requires' in testDict[stest]: 686 testDict['requires']+=" "+testDict[stest]['requires'] 687 if testDict.get('nsize', '1') != '1': 688 testDict['SKIP'].append("Parallel test with serial build") 689 break 690 691 # Now go through all requirements 692 if 'requires' in testDict: 693 for requirement in testDict['requires'].split(): 694 requirement=requirement.strip() 695 if not requirement: continue 696 if debug: print(indent+"Requirement: ", requirement) 697 isNull=False 698 if requirement.startswith("!"): 699 requirement=requirement[1:]; isNull=True 700 # Precision requirement for reals 701 if requirement in self.precision_types: 702 if self.conf['PETSC_PRECISION']==requirement: 703 if isNull: 704 testDict['SKIP'].append("not "+requirement+" required") 705 continue 706 continue # Success 707 elif not isNull: 708 testDict['SKIP'].append(requirement+" required") 709 continue 710 # Precision requirement for ints 711 if requirement in self.integer_types: 712 if requirement=="int32": 713 if self.conf['PETSC_SIZEOF_INT']==4: 714 if isNull: 715 testDict['SKIP'].append("not int32 required") 716 continue 717 continue # Success 718 elif not isNull: 719 testDict['SKIP'].append("int32 required") 720 continue 721 if requirement=="int64": 722 if self.conf['PETSC_SIZEOF_INT']==8: 723 if isNull: 724 testDict['SKIP'].append("NOT int64 required") 725 continue 726 continue # Success 727 elif not isNull: 728 testDict['SKIP'].append("int64 required") 729 continue 730 # Datafilespath 731 if requirement=="datafilespath" and not isNull: 732 testDict['SKIP'].append("Requires DATAFILESPATH") 733 continue 734 # Defines -- not sure I have comments matching 735 if "define(" in requirement.lower(): 736 reqdef=requirement.split("(")[1].split(")")[0] 737 if reqdef in self.conf: 738 if isNull: 739 testDict['SKIP'].append("Null requirement not met: "+requirement) 740 continue 741 continue # Success 742 elif not isNull: 743 testDict['SKIP'].append("Required: "+requirement) 744 continue 745 746 # Rest should be packages that we can just get from conf 747 if requirement == "complex": 748 petscconfvar="PETSC_USE_COMPLEX" 749 else: 750 petscconfvar="PETSC_HAVE_"+requirement.upper() 751 if self.conf.get(petscconfvar): 752 if isNull: 753 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 754 continue 755 continue # Success 756 elif not isNull: 757 if debug: print("requirement not found: ", requirement) 758 testDict['SKIP'].append(petscconfvar+" requirement not met") 759 continue 760 761 return testDict['SKIP'] == [] 762 763 def genPetscTests_summarize(self,dataDict): 764 """ 765 Required method to state what happened 766 """ 767 if not self.summarize: return 768 indent=" " 769 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 770 fh=open(fhname,"w") 771 for root in dataDict: 772 relroot=self.srcrelpath(root) 773 pkg=relroot.split("/")[1] 774 fh.write(relroot+"\n") 775 allSrcs=[] 776 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 777 for exfile in dataDict[root]: 778 # Basic information 779 rfile=os.path.join(relroot,exfile) 780 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 781 fh.write(indent+exfile+indent*4+builtStatus+"\n") 782 783 for test in dataDict[root][exfile]: 784 if test in self.buildkeys: continue 785 line=indent*2+test 786 fh.write(line+"\n") 787 # Looks nice to have the keys in order 788 #for key in dataDict[root][exfile][test]: 789 for key in "isrun abstracted nsize args requires script".split(): 790 if key not in dataDict[root][exfile][test]: continue 791 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 792 fh.write(line+"\n") 793 fh.write("\n") 794 fh.write("\n") 795 fh.write("\n") 796 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 797 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 798 fh.close() 799 return 800 801 def genPetscTests(self,root,dirs,files,dataDict): 802 """ 803 Go through and parse the source files in the directory to generate 804 the examples based on the metadata contained in the source files 805 """ 806 debug=False 807 # Use examplesAnalyze to get what the makefles think are sources 808 #self.examplesAnalyze(root,dirs,files,anlzDict) 809 810 dataDict[root]={} 811 812 for exfile in files: 813 #TST: Until we replace files, still leaving the orginals as is 814 #if not exfile.startswith("new_"+"ex"): continue 815 #if not exfile.startswith("ex"): continue 816 817 # Ignore emacs and other temporary files 818 if exfile.startswith("."): continue 819 if exfile.startswith("#"): continue 820 if exfile.endswith("~"): continue 821 # Only parse source files 822 ext=os.path.splitext(exfile)[-1].lstrip('.') 823 if ext not in LANGS: continue 824 825 # Convenience 826 fullex=os.path.join(root,exfile) 827 if self.verbose: print(' --> '+fullex) 828 dataDict[root].update(testparse.parseTestFile(fullex,0)) 829 if exfile in dataDict[root]: 830 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 831 832 return 833 834 def walktree(self,top): 835 """ 836 Walk a directory tree, starting from 'top' 837 """ 838 # Goal of action is to fill this dictionary 839 dataDict={} 840 for root, dirs, files in os.walk(top, topdown=True): 841 dirs.sort() 842 files.sort() 843 if not "examples" in root: continue 844 if "dSYM" in root: continue 845 if os.path.basename(root.rstrip("/")) == 'output': continue 846 if self.verbose: print(root) 847 self.genPetscTests(root,dirs,files,dataDict) 848 # Now summarize this dictionary 849 if self.verbose: self.genPetscTests_summarize(dataDict) 850 return dataDict 851 852 def gen_gnumake(self, fd): 853 """ 854 Overwrite of the method in the base PETSc class 855 """ 856 def write(stem, srcs): 857 for lang in LANGS: 858 if srcs[lang]['srcs']: 859 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 860 for pkg in PKGS: 861 srcs = self.gen_pkg(pkg) 862 write('testsrcs-' + pkg, srcs) 863 # Handle dependencies 864 for lang in LANGS: 865 for exfile in srcs[lang]['srcs']: 866 if exfile in srcs[lang]: 867 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 868 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 869 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 870 if deps: 871 # The executable literally depends on the object file because it is linked 872 fd.write(ex +": " + " ".join(deps) +'\n') 873 # The object file containing 'main' does not normally depend on other object 874 # files, but it does when it includes their modules. This dependency is 875 # overly blunt and could be reduced to only depend on object files for 876 # modules that are used, like "*f90aux.o". 877 fd.write(exfo +": " + " ".join(deps) +'\n') 878 879 return self.gendeps 880 881 def gen_pkg(self, pkg): 882 """ 883 Overwrite of the method in the base PETSc class 884 """ 885 return self.sources[pkg] 886 887 def write_gnumake(self, dataDict, output=None): 888 """ 889 Write out something similar to files from gmakegen.py 890 891 Test depends on script which also depends on source 892 file, but since I don't have a good way generating 893 acting on a single file (oops) just depend on 894 executable which in turn will depend on src file 895 """ 896 # Different options for how to set up the targets 897 compileExecsFirst=False 898 899 # Open file 900 fd = open(output, 'w') 901 902 # Write out the sources 903 gendeps = self.gen_gnumake(fd) 904 905 # Write out the tests and execname targets 906 fd.write("\n#Tests and executables\n") # Delimiter 907 908 for pkg in PKGS: 909 # These grab the ones that are built 910 for lang in LANGS: 911 testdeps=[] 912 for ftest in self.tests[pkg][lang]: 913 test=os.path.basename(ftest) 914 basedir=os.path.dirname(ftest) 915 testdeps.append(self.nameSpace(test,basedir)) 916 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 917 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 918 919 # test targets 920 for ftest in self.tests[pkg][lang]: 921 test=os.path.basename(ftest) 922 basedir=os.path.dirname(ftest) 923 testdir="${TESTDIR}/"+basedir+"/" 924 nmtest=self.nameSpace(test,basedir) 925 rundir=os.path.join(testdir,test) 926 script=test+".sh" 927 928 # Deps 929 exfile=self.tests[pkg][lang][ftest]['exfile'] 930 fullex=os.path.join(self.srcdir,exfile) 931 localexec=self.tests[pkg][lang][ftest]['exec'] 932 execname=os.path.join(testdir,localexec) 933 fullscript=os.path.join(testdir,script) 934 tmpfile=os.path.join(testdir,test,test+".tmp") 935 936 # *.counts depends on the script and either executable (will 937 # be run) or the example source file (SKIP or TODO) 938 fd.write('%s.counts : %s %s' 939 % (os.path.join('$(TESTDIR)/counts', nmtest), 940 fullscript, 941 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 942 ) 943 if exfile in self.sources[pkg][lang]: 944 for dep in self.sources[pkg][lang][exfile]: 945 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 946 fd.write('\n') 947 948 # Now write the args: 949 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 950 951 fd.close() 952 return 953 954def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 955 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 956 testdir=os.path.normpath(testdir) 957 if petsc_arch: 958 petsc_arch=petsc_arch.rstrip(os.path.sep) 959 if len(petsc_arch.split(os.path.sep))>1: 960 petsc_dir,petsc_arch=os.path.split(petsc_arch) 961 output = os.path.join(testdir, 'testfiles') 962 963 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 964 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 965 testdir=testdir) 966 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 967 pEx.write_gnumake(dataDict, output) 968 969if __name__ == '__main__': 970 import optparse 971 parser = optparse.OptionParser() 972 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 973 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 974 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 975 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 976 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 977 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 978 979 opts, extra_args = parser.parse_args() 980 if extra_args: 981 import sys 982 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 983 exit(1) 984 if opts.testdir is None: 985 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 986 987 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 988 verbose=opts.verbose, 989 single_ex=opts.single_executable, srcdir=opts.srcdir, 990 testdir=opts.testdir) 991