1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,testDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 testDict['args']: -ksp_monitor 97 testDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of testDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=testDict.keys() 105 for key in lkeys: 106 if type(testDict[key])!=types.StringType: continue 107 keystr = str(testDict[key]) 108 akey=('subargs' if key=='args' else key) # what to assign 109 if not testDict.has_key(akey): testDict[akey]='' 110 varlist=[] 111 for varset in re.split('-(?=[a-zA-Z])',keystr): 112 if not varset.strip(): continue 113 if len(re.findall('{{(.*?)}}',varset))>0: 114 if not loopVars.has_key(akey): loopVars[akey]={} 115 # Assuming only one for loop per var specification 116 keyvar=varset.split("{{")[0].strip() 117 if not keyvar.strip(): 118 if key=='nsize': 119 keyvar='nsize' 120 else: 121 print testname+": Problem in finding loop variables", keyvar, varset 122 varlist.append(keyvar) 123 loopVars[akey][keyvar]=[keyvar,re.findall('{{(.*?)}}',varset)[0]] 124 if akey=='nsize': 125 testDict[akey] = '${' + keyvar + '}' 126 else: 127 testDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 128 else: 129 if key=='args': newargs+=" -"+varset.strip() 130 if len(varlist)>0: loopVars[akey]['varlist']=varlist 131 132 133 # For subtests, args are always substituted in (not top level) 134 if isSubtest: 135 testDict['subargs']+=" "+newargs.strip() 136 testDict['args']='' 137 else: 138 if len(loopVars.keys())>0: testDict['args']=newargs.strip() 139 if len(loopVars.keys())>0: 140 return loopVars 141 else: 142 return None 143 144 def getArgLabel(self,testDict): 145 """ 146 In all of the arguments in the test dictionary, create a simple 147 string for searching within the makefile system. For simplicity in 148 search, remove "-", for strings, etc. 149 Also, concatenate the arg commands 150 For now, ignore nsize -- seems hard to search for anyway 151 """ 152 # Collect all of the args associated with a test 153 argStr=("" if not testDict.has_key('args') else testDict['args']) 154 if testDict.has_key('subtests'): 155 for stest in testDict["subtests"]: 156 sd=testDict[stest] 157 argStr=argStr+("" if not sd.has_key('args') else sd['args']) 158 159 # Now go through and cleanup 160 argStr=re.sub('{{(.*?)}}',"",argStr) 161 argStr=re.sub('-'," ",argStr) 162 for digit in string.digits: argStr=re.sub(digit," ",argStr) 163 argStr=re.sub("\.","",argStr) 164 argStr=re.sub(",","",argStr) 165 argStr=re.sub('\+',' ',argStr) 166 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 167 return argStr.strip() 168 169 def addToSources(self,exfile,root,srcDict): 170 """ 171 Put into data structure that allows easy generation of makefile 172 """ 173 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 174 fullfile=os.path.join(root,exfile) 175 relpfile=self.relpath(self.petsc_dir,fullfile) 176 lang=self.getLanguage(exfile) 177 if not lang: return 178 self.sources[pkg][lang]['srcs'].append(relpfile) 179 if srcDict.has_key('depends'): 180 depSrc=srcDict['depends'] 181 depObj=os.path.splitext(depSrc)[0]+".o" 182 self.sources[pkg][lang][exfile]=depObj 183 184 # In gmakefile, ${TESTDIR} var specifies the object compilation 185 testsdir=self.relpath(self.petsc_dir,root)+"/" 186 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 187 self.objects[pkg].append(objfile) 188 return 189 190 def addToTests(self,test,root,exfile,execname,testDict): 191 """ 192 Put into data structure that allows easy generation of makefile 193 Organized by languages to allow testing of languages 194 """ 195 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 196 #nmtest=self.nameSpace(test,root) 197 rpath=self.relpath(self.petsc_dir,root) 198 nmtest=os.path.join(rpath,test) 199 lang=self.getLanguage(exfile) 200 if not lang: return 201 self.tests[pkg][lang][nmtest]={} 202 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 203 self.tests[pkg][lang][nmtest]['exec']=execname 204 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 205 return 206 207 def getExecname(self,exfile,root): 208 """ 209 Generate bash script using template found next to this file. 210 This file is read in at constructor time to avoid file I/O 211 """ 212 rpath=self.relpath(self.petsc_dir,root) 213 if self.single_ex: 214 execname=rpath.split("/")[1]+"-ex" 215 else: 216 execname=os.path.splitext(exfile)[0] 217 return execname 218 219 def getSubstVars(self,testDict,rpath,testname): 220 """ 221 Create a dictionary with all of the variables that get substituted 222 into the template commands found in example_template.py 223 TODO: Cleanup 224 """ 225 subst={} 226 # Handle defaults 227 if not testDict.has_key('nsize'): testDict['nsize']=1 228 if not testDict.has_key('filter'): testDict['filter']="" 229 if not testDict.has_key('filter_output'): testDict['filter_output']="" 230 if not testDict.has_key('localrunfiles'): testDict['localrunfiles']="" 231 if not testDict.has_key('args'): testDict['args']="" 232 if not testDict.has_key('subargs'): testDict['subargs']="" 233 if not testDict.has_key('comments'): testDict['comments']="" 234 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 235 if not "_" in defroot: defroot=defroot+"_1" 236 if not testDict.has_key('redirect_file'): testDict['redirect_file']=defroot+".tmp" 237 238 # Output file is special because of subtests override 239 if testDict.has_key('output_file'): 240 subst['output_file']=testDict['output_file'] 241 subst['output_file_inSrc']=True 242 else: 243 subst['output_file']="output/"+defroot+".out" 244 subst['output_file_inSrc']=False 245 246 # Setup the variables in template_string that need to be substituted 247 subst['defroot']=defroot 248 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 249 subst['label']=self.nameSpace(defroot,subst['srcdir']) 250 subst['label_suffix']='' 251 subst['redirect_file']=testDict['redirect_file'] 252 subst['comments']="\n#".join(testDict['comments'].split("\n")) 253 if subst['comments']: subst['comments']="#"+subst['comments'] 254 subst['exec']="../"+testDict['execname'] 255 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky 256 subst['filter_output']=testDict['filter_output'] 257 subst['localrunfiles']=testDict['localrunfiles'] 258 subst['testroot']=self.testroot_dir 259 subst['testname']=testname 260 if self.conf['PETSC_HAVE_DATAFILESPATH']: 261 dp=self.conf['DATAFILESPATH'] 262 dpl='DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 263 else: 264 dpl='' 265 subst['datafilespath_line']=dpl 266 267 # Be careful with this 268 if testDict.has_key('command'): subst['command']=testDict['command'] 269 270 # These can have for loops and are treated separately later 271 if testDict.has_key('nsize'): subst['nsize']=str(testDict['nsize']) 272 if testDict.has_key('args'): subst['args']=testDict['args'] 273 274 #Conf vars 275 if self.petsc_arch.find('valgrind')>=0: 276 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 277 else: 278 subst['mpiexec']=self.conf['MPIEXEC'] 279 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 280 subst['diff']=self.conf['DIFF'] 281 subst['rm']=self.conf['RM'] 282 subst['grep']=self.conf['GREP'] 283 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 284 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 285 286 return subst 287 288 def getCmds(self,subst,i): 289 """ 290 Generate bash script using template found next to this file. 291 This file is read in at constructor time to avoid file I/O 292 """ 293 indnt=self.indent 294 nindnt=i # the start and has to be consistent with below 295 cmdLines="" 296 297 # Add in the full path here. 298 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 299 300 # MPI is the default -- but we have a few odd commands 301 if not subst.has_key('command'): 302 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 303 else: 304 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 305 cmdLines+=cmd+"\n\n" 306 307 if not subst['filter_output']: 308 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 309 else: 310 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 311 cmdLines+=cmd+"\n" 312 return cmdLines 313 314 def _substVars(self,subst,origStr): 315 """ 316 Substitute variables 317 """ 318 Str=origStr 319 for subkey in subst: 320 if type(subst[subkey])!=types.StringType: continue 321 patt="@"+subkey.upper()+"@" 322 Str=re.sub(patt,subst[subkey],Str) 323 return Str 324 325 def _writeTodoSkip(self,fh,tors,inDict,footer): 326 """ 327 Write out the TODO and SKIP lines in the file 328 The TODO or SKIP variable, tors, should be lower case 329 """ 330 TORS=tors.upper() 331 template=eval("example_template."+tors+"line") 332 tsStr=re.sub("@"+TORS+"COMMENT@",inDict[TORS],template) 333 fh.write(tsStr+"\ntotal=1; "+tors+"=1\n") 334 fh.write(footer+"\n") 335 fh.write("exit\n\n\n") 336 return 337 338 def getLoopVarsHead(self,loopVars,i): 339 """ 340 Generate a nicely indented string with the format loops 341 Here is what the data structure looks like 342 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 343 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 344 loopVars['subargs']['pc_type']=["j","cholesky sor"] 345 """ 346 outstr=''; indnt=self.indent 347 for key in loopVars: 348 for var in loopVars[key]['varlist']: 349 varval=loopVars[key][var] 350 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 351 i = i + 1 352 return (outstr,i) 353 354 def getLoopVarsFoot(self,loopVars,i): 355 outstr=''; indnt=self.indent 356 for key in loopVars: 357 for var in loopVars[key]['varlist']: 358 i = i - 1 359 outstr += indnt * i + "done\n" 360 return (outstr,i) 361 362 def genRunScript(self,testname,root,isRun,srcDict): 363 """ 364 Generate bash script using template found next to this file. 365 This file is read in at constructor time to avoid file I/O 366 """ 367 # runscript_dir directory has to be consistent with gmakefile 368 testDict=srcDict[testname] 369 rpath=self.relpath(self.petsc_dir,root) 370 runscript_dir=os.path.join(self.testroot_dir,rpath) 371 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 372 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 373 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 374 375 # This gets a datastructure for handling loops as well as alters testDict 376 subst=self.getSubstVars(testDict,rpath,testname) 377 loopVars = self._getLoopVars(subst,testname) 378 379 #Handle runfiles 380 if subst['localrunfiles']: 381 for lfile in subst['localrunfiles'].split(): 382 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 383 shutil.copy(fullfile,runscript_dir) 384 # Check subtests for local runfiles 385 if testDict.has_key("subtests"): 386 for stest in testDict["subtests"]: 387 if testDict[stest].has_key('localrunfiles'): 388 for lfile in testDict[stest]['localrunfiles'].split(): 389 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 390 shutil.copy(fullfile,self.runscript_dir) 391 392 # Now substitute the key variables into the header and footer 393 header=self._substVars(subst,example_template.header) 394 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 395 396 # Start writing the file 397 fh.write(header+"\n") 398 399 # If there is a TODO or a SKIP then we do it before writing out the 400 # rest of the command (which is useful for working on the test) 401 # SKIP and TODO can be for the source file or for the runs 402 for tors in "todo skip".split(): 403 for d in [srcDict,testDict]: 404 if d.has_key(tors.upper()): self._writeTodoSkip(fh,tors,d,footer) 405 406 j=0 # for indentation 407 408 if loopVars: 409 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 410 if (loopHead): fh.write(loopHead+"\n") 411 412 # Subtests are special 413 if testDict.has_key("subtests"): 414 substP=subst # Subtests can inherit args but be careful 415 for stest in testDict["subtests"]: 416 subst=substP.copy() 417 subst.update(testDict[stest]) 418 subst['nsize']=str(subst['nsize']) 419 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 420 #if '10_9' in testname: print sLoopVars 421 if sLoopVars: 422 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 423 fh.write(sLoopHead+"\n") 424 fh.write(self.getCmds(subst,j)+"\n") 425 if sLoopVars: 426 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 427 fh.write(sLoopFoot+"\n") 428 else: 429 fh.write(self.getCmds(subst,j)+"\n") 430 431 if loopVars: 432 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 433 fh.write(loopFoot+"\n") 434 435 fh.write(footer+"\n") 436 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 437 #if '10_9' in testname: sys.exit() 438 return 439 440 def genScriptsAndInfo(self,exfile,root,srcDict): 441 """ 442 Generate scripts from the source file, determine if built, etc. 443 For every test in the exfile with info in the srcDict: 444 1. Determine if it needs to be run for this arch 445 2. Generate the script 446 3. Generate the data needed to write out the makefile in a 447 convenient way 448 All tests are *always* run, but some may be SKIP'd per the TAP standard 449 """ 450 debug=False 451 fileIsTested=False 452 execname=self.getExecname(exfile,root) 453 isBuilt=self._isBuilt(exfile,srcDict) 454 for test in srcDict: 455 if test in self.buildkeys: continue 456 if debug: print self.nameSpace(exfile,root), test 457 srcDict[test]['execname']=execname # Convenience in generating scripts 458 isRun=self._isRun(srcDict[test]) 459 self.genRunScript(test,root,isRun,srcDict) 460 srcDict[test]['isrun']=isRun 461 if isRun: fileIsTested=True 462 self.addToTests(test,root,exfile,execname,srcDict[test]) 463 464 # This adds to datastructure for building deps 465 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 466 #print self.nameSpace(exfile,root), fileIsTested 467 return 468 469 def _isBuilt(self,exfile,srcDict): 470 """ 471 Determine if this file should be built. 472 """ 473 # Get the language based on file extension 474 lang=self.getLanguage(exfile) 475 if (lang=="F" or lang=="F90") and not self.have_fortran: 476 srcDict["SKIP"]="Fortran required for this test" 477 return False 478 if lang=="cu" and not self.conf.has_key('PETSC_HAVE_CUDA'): 479 srcDict["SKIP"]="CUDA required for this test" 480 return False 481 if lang=="cxx" and not self.conf.has_key('PETSC_HAVE_CXX'): 482 srcDict["SKIP"]="C++ required for this test" 483 return False 484 485 # Deprecated source files 486 if srcDict.has_key("TODO"): return False 487 488 # isRun can work with srcDict to handle the requires 489 if srcDict.has_key("requires"): 490 if len(srcDict["requires"])>0: 491 return self._isRun(srcDict) 492 493 return True 494 495 496 def _isRun(self,testDict): 497 """ 498 Based on the requirements listed in the src file and the petscconf.h 499 info, determine whether this test should be run or not. 500 """ 501 indent=" " 502 debug=False 503 504 # MPI requirements 505 if testDict.has_key('nsize'): 506 if testDict['nsize']>1 and self.conf.has_key('MPI_IS_MPIUNI'): 507 if debug: print indent+"Cannot run parallel tests" 508 testDict['SKIP']="Parallel test with serial build" 509 return False 510 511 # The requirements for the test are the sum of all the run subtests 512 if testDict.has_key('subtests'): 513 if not testDict.has_key('requires'): testDict['requires']="" 514 for stest in testDict['subtests']: 515 if testDict[stest].has_key('requires'): 516 testDict['requires']=testDict['requires']+" "+testDict[stest]['requires'] 517 518 519 # Now go through all requirements 520 if testDict.has_key('requires'): 521 for requirement in testDict['requires'].split(): 522 requirement=requirement.strip() 523 if not requirement: continue 524 if debug: print indent+"Requirement: ", requirement 525 isNull=False 526 if requirement.startswith("!"): 527 requirement=requirement[1:]; isNull=True 528 # Precision requirement for reals 529 if requirement in self.precision_types: 530 if self.conf['PETSC_PRECISION']==requirement: 531 testDict['SKIP']="not "+requirement+" required" 532 if isNull: return False 533 else: 534 testDict['SKIP']=requirement+" required" 535 return False 536 # Precision requirement for ints 537 if requirement in self.integer_types: 538 if requirement=="int32": 539 if self.conf['PETSC_SIZEOF_INT']==4: 540 testDict['SKIP']="not int32 required" 541 if isNull: return False 542 else: 543 testDict['SKIP']="int32 required" 544 return False 545 if requirement=="int64": 546 if self.conf['PETSC_SIZEOF_INT']==8: 547 testDict['SKIP']="NOT int64 required" 548 if isNull: return False 549 else: 550 testDict['SKIP']="int64 required" 551 return False 552 # Datafilespath 553 if requirement=="datafilespath": 554 if not self.conf['PETSC_HAVE_DATAFILESPATH']: 555 testDict['SKIP']="Requires DATAFILESPATH" 556 return False 557 # Defines -- not sure I have comments matching 558 if "define(" in requirement.lower(): 559 reqdef=requirement.split("(")[1].split(")")[0] 560 val=(reqdef.split()[1] if " " in reqdef else "") 561 if self.conf.has_key(reqdef): 562 if val: 563 if self.conf[reqdef]==val: 564 if isNull: 565 testDict['SKIP']="Null requirement not met: "+requirement 566 return False 567 else: 568 testDict['SKIP']="Required: "+requirement 569 return False 570 else: 571 if isNull: 572 testDict['SKIP']="Null requirement not met: "+requirement 573 return False 574 else: 575 return True 576 else: 577 testDict['SKIP']="Requirement not met: "+requirement 578 return False 579 580 # Rest should be packages that we can just get from conf 581 if requirement == "complex": petscconfvar="PETSC_USE_COMPLEX" 582 else: petscconfvar="PETSC_HAVE_"+requirement.upper() 583 if self.conf.get(petscconfvar): 584 if isNull: 585 testDict['SKIP']="Not "+petscconfvar+" requirement not met" 586 return False 587 elif not isNull: 588 if debug: print "requirement not found: ", requirement 589 testDict['SKIP']=petscconfvar+" requirement not met" 590 return False 591 592 return True 593 594 def genPetscTests_summarize(self,dataDict): 595 """ 596 Required method to state what happened 597 """ 598 if not self.summarize: return 599 indent=" " 600 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 601 fh=open(fhname,"w") 602 #print "See ", fhname 603 for root in dataDict: 604 relroot=self.relpath(self.petsc_dir,root) 605 pkg=relroot.split("/")[1] 606 fh.write(relroot+"\n") 607 allSrcs=[] 608 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 609 for exfile in dataDict[root]: 610 # Basic information 611 fullfile=os.path.join(root,exfile) 612 rfile=self.relpath(self.petsc_dir,fullfile) 613 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 614 fh.write(indent+exfile+indent*4+builtStatus+"\n") 615 616 for test in dataDict[root][exfile]: 617 if test in self.buildkeys: continue 618 line=indent*2+test 619 fh.write(line+"\n") 620 # Looks nice to have the keys in order 621 #for key in dataDict[root][exfile][test]: 622 for key in "isrun abstracted nsize args requires script".split(): 623 if not dataDict[root][exfile][test].has_key(key): continue 624 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 625 fh.write(line+"\n") 626 fh.write("\n") 627 fh.write("\n") 628 fh.write("\n") 629 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 630 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 631 fh.close() 632 return 633 634 def genPetscTests(self,root,dirs,files,dataDict): 635 """ 636 Go through and parse the source files in the directory to generate 637 the examples based on the metadata contained in the source files 638 """ 639 debug=False 640 # Use examplesAnalyze to get what the makefles think are sources 641 #self.examplesAnalyze(root,dirs,files,anlzDict) 642 643 dataDict[root]={} 644 645 for exfile in files: 646 #TST: Until we replace files, still leaving the orginals as is 647 #if not exfile.startswith("new_"+"ex"): continue 648 if not exfile.startswith("ex"): continue 649 650 # Convenience 651 fullex=os.path.join(root,exfile) 652 relpfile=self.relpath(self.petsc_dir,fullex) 653 if debug: print relpfile 654 dataDict[root].update(testparse.parseTestFile(fullex)) 655 # Need to check and make sure tests are in the file 656 # if verbosity>=1: print relpfile 657 if dataDict[root].has_key(exfile): 658 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 659 660 return 661 662 def walktree(self,top,action="printFiles"): 663 """ 664 Walk a directory tree, starting from 'top' 665 """ 666 #print "action", action 667 # Goal of action is to fill this dictionary 668 dataDict={} 669 for root, dirs, files in os.walk(top, topdown=False): 670 if not "examples" in root: continue 671 if not os.path.isfile(os.path.join(root,"makefile")): continue 672 bname=os.path.basename(root.rstrip("/")) 673 if bname=="tests" or bname=="tutorials": 674 eval("self."+action+"(root,dirs,files,dataDict)") 675 if type(top) != types.StringType: 676 raise TypeError("top must be a string") 677 # Now summarize this dictionary 678 eval("self."+action+"_summarize(dataDict)") 679 return dataDict 680 681 def gen_gnumake(self, fd): 682 """ 683 Overwrite of the method in the base PETSc class 684 """ 685 def write(stem, srcs): 686 for lang in LANGS: 687 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 688 for pkg in PKGS: 689 srcs = self.gen_pkg(pkg) 690 write('testsrcs-' + pkg, srcs) 691 return self.gendeps 692 693 def gen_pkg(self, pkg): 694 """ 695 Overwrite of the method in the base PETSc class 696 """ 697 return self.sources[pkg] 698 699 def write_gnumake(self,dataDict): 700 """ 701 Write out something similar to files from gmakegen.py 702 703 There is not a lot of has_key type checking because 704 should just work and need to know if there are bugs 705 706 Test depends on script which also depends on source 707 file, but since I don't have a good way generating 708 acting on a single file (oops) just depend on 709 executable which in turn will depend on src file 710 """ 711 # Different options for how to set up the targets 712 compileExecsFirst=False 713 714 # Open file 715 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 716 fd = open(arch_files, 'w') 717 718 # Write out the sources 719 gendeps = self.gen_gnumake(fd) 720 721 # Write out the tests and execname targets 722 fd.write("\n#Tests and executables\n") # Delimiter 723 724 for pkg in PKGS: 725 # These grab the ones that are built 726 for lang in LANGS: 727 testdeps=[] 728 for ftest in self.tests[pkg][lang]: 729 test=os.path.basename(ftest) 730 basedir=os.path.dirname(ftest) 731 testdeps.append(self.nameSpace(test,basedir)) 732 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 733 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 734 735 # test targets 736 for ftest in self.tests[pkg][lang]: 737 test=os.path.basename(ftest) 738 basedir=os.path.dirname(ftest) 739 testdir="${TESTDIR}/"+basedir+"/" 740 nmtest=self.nameSpace(test,basedir) 741 rundir=os.path.join(testdir,test) 742 #print test, nmtest 743 script=test+".sh" 744 745 # Deps 746 exfile=self.tests[pkg][lang][ftest]['exfile'] 747 fullex=os.path.join(self.petsc_dir,exfile) 748 localexec=self.tests[pkg][lang][ftest]['exec'] 749 execname=os.path.join(testdir,localexec) 750 fullscript=os.path.join(testdir,script) 751 tmpfile=os.path.join(testdir,test,test+".tmp") 752 753 # *.counts depends on the script and either executable (will 754 # be run) or the example source file (SKIP or TODO) 755 fd.write('%s.counts : %s %s\n' 756 % (os.path.join('$(TESTDIR)/counts', nmtest), 757 fullscript, 758 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 759 # Now write the args: 760 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 761 762 fd.close() 763 return 764 765 def writeHarness(self,output,dataDict): 766 """ 767 This is set up to write out multiple harness even if only gnumake 768 is supported now 769 """ 770 eval("self.write_"+output+"(dataDict)") 771 return 772 773def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 774 if output is None: 775 output = 'gnumake' 776 777 778 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 779 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 780 pEx.writeHarness(output,dataDict) 781 782if __name__ == '__main__': 783 import optparse 784 parser = optparse.OptionParser() 785 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 786 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 787 parser.add_option('--output', help='Location to write output file', default=None) 788 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 789 opts, extra_args = parser.parse_args() 790 if extra_args: 791 import sys 792 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 793 exit(1) 794 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 795