1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,testDict,i,isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 testDict['args']: -ksp_monitor 97 testDict['subargs']: -bs ${i} -pc_type ${j} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["i"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["j"],["cholesky sor"]] 101 subst should be passed in instead of testDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=testDict.keys() 105 for key in lkeys: 106 if type(testDict[key])!=types.StringType: continue 107 keystr = str(testDict[key]) 108 akey=('subargs' if key=='args' else key) # what to assign 109 if not testDict.has_key(akey): testDict[akey]='' 110 varlist=[] 111 for varset in re.split('-(?=[a-zA-Z])',keystr): 112 if not varset.strip(): continue 113 if len(re.findall('{{(.*?)}}',varset))>0: 114 if not loopVars.has_key(akey): loopVars[akey]={} 115 # Assuming only one for loop per var specification 116 keyvar=varset.split("{{")[0].strip() 117 varlist.append(keyvar) 118 lvar=string.ascii_lowercase[i]; i+=1 119 loopVars[akey][keyvar]=[lvar,re.findall('{{(.*?)}}',varset)[0]] 120 testDict[akey] += '-'+keyvar+' ${' + lvar + '} ' 121 else: 122 if key=='args': newargs+="-"+varset+" " 123 if len(varlist)>0: loopVars[akey]['varlist']=varlist 124 125 126 # For subtests, args are always substituted in (not top level) 127 if isSubtest: 128 testDict['subargs']+=newargs 129 testDict['args']='' 130 else: 131 if len(loopVars.keys())>0: 132 testDict['args']=newargs 133 if len(loopVars.keys())>0: 134 return loopVars 135 else: 136 return None 137 138 def getArgLabel(self,testDict): 139 """ 140 In all of the arguments in the test dictionary, create a simple 141 string for searching within the makefile system. For simplicity in 142 search, remove "-", for strings, etc. 143 Also, concatenate the arg commands 144 For now, ignore nsize -- seems hard to search for anyway 145 """ 146 # Collect all of the args associated with a test 147 argStr=("" if not testDict.has_key('args') else testDict['args']) 148 if testDict.has_key('subtests'): 149 for stest in testDict["subtests"]: 150 sd=testDict[stest] 151 argStr=argStr+("" if not sd.has_key('args') else sd['args']) 152 153 # Now go through and cleanup 154 argStr=re.sub('{{(.*?)}}',"",argStr) 155 argStr=re.sub('-'," ",argStr) 156 for digit in string.digits: argStr=re.sub(digit," ",argStr) 157 argStr=re.sub("\.","",argStr) 158 argStr=re.sub(",","",argStr) 159 argStr=re.sub('\+',' ',argStr) 160 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 161 return argStr.strip() 162 163 def addToSources(self,exfile,root,srcDict): 164 """ 165 Put into data structure that allows easy generation of makefile 166 """ 167 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 168 fullfile=os.path.join(root,exfile) 169 relpfile=self.relpath(self.petsc_dir,fullfile) 170 lang=self.getLanguage(exfile) 171 if not lang: return 172 self.sources[pkg][lang]['srcs'].append(relpfile) 173 if srcDict.has_key('depends'): 174 depSrc=srcDict['depends'] 175 depObj=os.path.splitext(depSrc)[0]+".o" 176 self.sources[pkg][lang][exfile]=depObj 177 178 # In gmakefile, ${TESTDIR} var specifies the object compilation 179 testsdir=self.relpath(self.petsc_dir,root)+"/" 180 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 181 self.objects[pkg].append(objfile) 182 return 183 184 def addToTests(self,test,root,exfile,execname,testDict): 185 """ 186 Put into data structure that allows easy generation of makefile 187 Organized by languages to allow testing of languages 188 """ 189 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 190 #nmtest=self.nameSpace(test,root) 191 rpath=self.relpath(self.petsc_dir,root) 192 nmtest=os.path.join(rpath,test) 193 lang=self.getLanguage(exfile) 194 if not lang: return 195 self.tests[pkg][lang][nmtest]={} 196 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 197 self.tests[pkg][lang][nmtest]['exec']=execname 198 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 199 return 200 201 def getExecname(self,exfile,root): 202 """ 203 Generate bash script using template found next to this file. 204 This file is read in at constructor time to avoid file I/O 205 """ 206 rpath=self.relpath(self.petsc_dir,root) 207 if self.single_ex: 208 execname=rpath.split("/")[1]+"-ex" 209 else: 210 execname=os.path.splitext(exfile)[0] 211 return execname 212 213 def getSubstVars(self,testDict,rpath,testname): 214 """ 215 Create a dictionary with all of the variables that get substituted 216 into the template commands found in example_template.py 217 TODO: Cleanup 218 """ 219 subst={} 220 # Handle defaults 221 if not testDict.has_key('nsize'): testDict['nsize']=1 222 if not testDict.has_key('filter'): testDict['filter']="" 223 if not testDict.has_key('filter_output'): testDict['filter_output']="" 224 if not testDict.has_key('localrunfiles'): testDict['localrunfiles']="" 225 if not testDict.has_key('args'): testDict['args']="" 226 if not testDict.has_key('subargs'): testDict['subargs']="" 227 if not testDict.has_key('comments'): testDict['comments']="" 228 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 229 if not "_" in defroot: defroot=defroot+"_1" 230 if not testDict.has_key('redirect_file'): testDict['redirect_file']=defroot+".tmp" 231 232 # Output file is special because of subtests override 233 if testDict.has_key('output_file'): 234 subst['output_file']=testDict['output_file'] 235 subst['output_file_inSrc']=True 236 else: 237 subst['output_file']="output/"+defroot+".out" 238 subst['output_file_inSrc']=False 239 240 # Setup the variables in template_string that need to be substituted 241 subst['defroot']=defroot 242 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 243 subst['label']=self.nameSpace(defroot,subst['srcdir']) 244 subst['label_suffix']='' 245 subst['redirect_file']=testDict['redirect_file'] 246 subst['comments']="\n#".join(testDict['comments'].split("\n")) 247 if subst['comments']: subst['comments']="#"+subst['comments'] 248 subst['exec']="../"+testDict['execname'] 249 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky 250 subst['filter_output']=testDict['filter_output'] 251 subst['localrunfiles']=testDict['localrunfiles'] 252 subst['testroot']=self.testroot_dir 253 subst['testname']=testname 254 if self.conf['PETSC_HAVE_DATAFILESPATH']: 255 dp=self.conf['DATAFILESPATH'] 256 dpl='DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 257 else: 258 dpl='' 259 subst['datafilespath_line']=dpl 260 261 # Be careful with this 262 if testDict.has_key('command'): subst['command']=testDict['command'] 263 264 # These can have for loops and are treated separately later 265 if testDict.has_key('nsize'): subst['nsize']=str(testDict['nsize']) 266 if testDict.has_key('args'): subst['args']=testDict['args'] 267 268 #Conf vars 269 if self.petsc_arch.find('valgrind')>=0: 270 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 271 else: 272 subst['mpiexec']=self.conf['MPIEXEC'] 273 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 274 subst['diff']=self.conf['DIFF'] 275 subst['rm']=self.conf['RM'] 276 subst['grep']=self.conf['GREP'] 277 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 278 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 279 280 for key in subst: 281 if type(subst[key])!=types.StringType: continue 282 subst[key] = self._substVars(testDict,subst[key]) 283 284 return subst 285 286 def getCmds(self,subst,i): 287 """ 288 Generate bash script using template found next to this file. 289 This file is read in at constructor time to avoid file I/O 290 """ 291 indnt=self.indent 292 nindnt=i # the start and has to be consistent with below 293 cmdLines="" 294 295 # Add in the full path here. 296 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 297 298 # MPI is the default -- but we have a few odd commands 299 if not subst.has_key('command'): 300 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 301 else: 302 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 303 cmdLines+=cmd+"\n\n" 304 305 if not subst['filter_output']: 306 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 307 else: 308 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 309 cmdLines+=cmd+"\n" 310 return cmdLines 311 312 def _substVars(self,subst,origStr): 313 """ 314 Substitute varial 315 """ 316 Str=origStr 317 for subkey in subst: 318 if type(subst[subkey])!=types.StringType: continue 319 patt="@"+subkey.upper()+"@" 320 Str=re.sub(patt,subst[subkey],Str) 321 return Str 322 323 def _writeTodoSkip(self,fh,tors,inDict,footer): 324 """ 325 Write out the TODO and SKIP lines in the file 326 The TODO or SKIP variable, tors, should be lower case 327 """ 328 TORS=tors.upper() 329 template=eval("example_template."+tors+"line") 330 tsStr=re.sub("@"+TORS+"COMMENT@",inDict[TORS],template) 331 fh.write(tsStr+"\ntotal=1; "+tors+"=1\n") 332 fh.write(footer+"\n") 333 fh.write("exit\n\n\n") 334 return 335 336 def getLoopVarsHead(self,loopVars,i): 337 """ 338 Generate a nicely indented string with the format loops 339 Here is what the data structure looks like 340 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 341 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 342 loopVars['subargs']['pc_type']=["j","cholesky sor"] 343 """ 344 outstr=''; indnt=self.indent 345 for key in loopVars: 346 for var in loopVars[key]['varlist']: 347 varval=loopVars[key][var] 348 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 349 i = i + 1 350 return (outstr,i) 351 352 def getLoopVarsFoot(self,loopVars,i): 353 outstr=''; indnt=self.indent 354 for key in loopVars: 355 for var in loopVars[key]['varlist']: 356 i = i - 1 357 outstr += indnt * i + "done\n" 358 return (outstr,i) 359 360 def genRunScript(self,testname,root,isRun,srcDict): 361 """ 362 Generate bash script using template found next to this file. 363 This file is read in at constructor time to avoid file I/O 364 """ 365 print testname 366 # runscript_dir directory has to be consistent with gmakefile 367 testDict=srcDict[testname] 368 rpath=self.relpath(self.petsc_dir,root) 369 runscript_dir=os.path.join(self.testroot_dir,rpath) 370 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 371 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 372 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 373 374 i=8 # for loop counters 375 # This gets a datastructure for handling loops as well as alters testDict 376 subst=self.getSubstVars(testDict,rpath,testname) 377 loopVars = self._getLoopVars(subst,i) 378 379 #Handle runfiles 380 if subst['localrunfiles']: 381 for lfile in subst['localrunfiles'].split(): 382 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 383 shutil.copy(fullfile,runscript_dir) 384 # Check subtests for local runfiles 385 if testDict.has_key("subtests"): 386 for stest in testDict["subtests"]: 387 if testDict[stest].has_key('localrunfiles'): 388 for lfile in testDict[stest]['localrunfiles'].split(): 389 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 390 shutil.copy(fullfile,self.runscript_dir) 391 392 # Now substitute the key variables into the header and footer 393 header=self._substVars(subst,example_template.header) 394 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 395 396 # Start writing the file 397 fh.write(header+"\n") 398 399 # If there is a TODO or a SKIP then we do it before writing out the 400 # rest of the command (which is useful for working on the test) 401 # SKIP and TODO can be for the source file or for the runs 402 for tors in "todo skip".split(): 403 for d in [srcDict,testDict]: 404 if d.has_key(tors.upper()): self._writeTodoSkip(fh,tors,d,footer) 405 406 j=0 # for indentation 407 408 if loopVars: 409 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 410 if (loopHead): fh.write(loopHead+"\n") 411 412 # Subtests are special 413 if testDict.has_key("subtests"): 414 substP=subst # Subtests can inherit args but be careful 415 for stest in testDict["subtests"]: 416 subst=substP.copy() 417 subst.update(testDict[stest]) 418 subst['nsize']=str(subst['nsize']) 419 sLoopVars = self._getLoopVars(subst,i,isSubtest=True) 420 if sLoopVars: 421 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 422 fh.write(sLoopHead+"\n") 423 fh.write(self.getCmds(subst,j)+"\n") 424 if sLoopVars: 425 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 426 fh.write(sLoopFoot+"\n") 427 else: 428 fh.write(self.getCmds(subst,j)+"\n") 429 430 if loopVars: 431 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 432 fh.write(loopFoot+"\n") 433 434 fh.write(footer+"\n") 435 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 436 return 437 438 def genScriptsAndInfo(self,exfile,root,srcDict): 439 """ 440 Generate scripts from the source file, determine if built, etc. 441 For every test in the exfile with info in the srcDict: 442 1. Determine if it needs to be run for this arch 443 2. Generate the script 444 3. Generate the data needed to write out the makefile in a 445 convenient way 446 All tests are *always* run, but some may be SKIP'd per the TAP standard 447 """ 448 debug=False 449 fileIsTested=False 450 execname=self.getExecname(exfile,root) 451 isBuilt=self._isBuilt(exfile,srcDict) 452 for test in srcDict: 453 if test in self.buildkeys: continue 454 if debug: print self.nameSpace(exfile,root), test 455 srcDict[test]['execname']=execname # Convenience in generating scripts 456 isRun=self._isRun(srcDict[test]) 457 self.genRunScript(test,root,isRun,srcDict) 458 srcDict[test]['isrun']=isRun 459 if isRun: fileIsTested=True 460 self.addToTests(test,root,exfile,execname,srcDict[test]) 461 462 # This adds to datastructure for building deps 463 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 464 #print self.nameSpace(exfile,root), fileIsTested 465 return 466 467 def _isBuilt(self,exfile,srcDict): 468 """ 469 Determine if this file should be built. 470 """ 471 # Get the language based on file extension 472 lang=self.getLanguage(exfile) 473 if (lang=="F" or lang=="F90") and not self.have_fortran: 474 srcDict["SKIP"]="Fortran required for this test" 475 return False 476 if lang=="cu" and not self.conf.has_key('PETSC_HAVE_CUDA'): 477 srcDict["SKIP"]="CUDA required for this test" 478 return False 479 if lang=="cxx" and not self.conf.has_key('PETSC_HAVE_CXX'): 480 srcDict["SKIP"]="C++ required for this test" 481 return False 482 483 # Deprecated source files 484 if srcDict.has_key("TODO"): return False 485 486 # isRun can work with srcDict to handle the requires 487 if srcDict.has_key("requires"): 488 if len(srcDict["requires"])>0: 489 return self._isRun(srcDict) 490 491 return True 492 493 494 def _isRun(self,testDict): 495 """ 496 Based on the requirements listed in the src file and the petscconf.h 497 info, determine whether this test should be run or not. 498 """ 499 indent=" " 500 debug=False 501 502 # MPI requirements 503 if testDict.has_key('nsize'): 504 if testDict['nsize']>1 and self.conf.has_key('MPI_IS_MPIUNI'): 505 if debug: print indent+"Cannot run parallel tests" 506 testDict['SKIP']="Parallel test with serial build" 507 return False 508 509 # The requirements for the test are the sum of all the run subtests 510 if testDict.has_key('subtests'): 511 if not testDict.has_key('requires'): testDict['requires']="" 512 for stest in testDict['subtests']: 513 if testDict[stest].has_key('requires'): 514 testDict['requires']=testDict['requires']+" "+testDict[stest]['requires'] 515 516 517 # Now go through all requirements 518 if testDict.has_key('requires'): 519 for requirement in testDict['requires'].split(): 520 requirement=requirement.strip() 521 if not requirement: continue 522 if debug: print indent+"Requirement: ", requirement 523 isNull=False 524 if requirement.startswith("!"): 525 requirement=requirement[1:]; isNull=True 526 # Precision requirement for reals 527 if requirement in self.precision_types: 528 if self.conf['PETSC_PRECISION']==requirement: 529 testDict['SKIP']="not "+requirement+" required" 530 if isNull: return False 531 else: 532 testDict['SKIP']=requirement+" required" 533 return False 534 # Precision requirement for ints 535 if requirement in self.integer_types: 536 if requirement=="int32": 537 if self.conf['PETSC_SIZEOF_INT']==4: 538 testDict['SKIP']="not int32 required" 539 if isNull: return False 540 else: 541 testDict['SKIP']="int32 required" 542 return False 543 if requirement=="int64": 544 if self.conf['PETSC_SIZEOF_INT']==8: 545 testDict['SKIP']="NOT int64 required" 546 if isNull: return False 547 else: 548 testDict['SKIP']="int64 required" 549 return False 550 # Datafilespath 551 if requirement=="datafilespath": 552 if not self.conf['PETSC_HAVE_DATAFILESPATH']: 553 testDict['SKIP']="Requires DATAFILESPATH" 554 return False 555 # Defines -- not sure I have comments matching 556 if "define(" in requirement.lower(): 557 reqdef=requirement.split("(")[1].split(")")[0] 558 val=(reqdef.split()[1] if " " in reqdef else "") 559 if self.conf.has_key(reqdef): 560 if val: 561 if self.conf[reqdef]==val: 562 if isNull: 563 testDict['SKIP']="Null requirement not met: "+requirement 564 return False 565 else: 566 testDict['SKIP']="Required: "+requirement 567 return False 568 else: 569 if isNull: 570 testDict['SKIP']="Null requirement not met: "+requirement 571 return False 572 else: 573 return True 574 else: 575 testDict['SKIP']="Requirement not met: "+requirement 576 return False 577 578 # Rest should be packages that we can just get from conf 579 if requirement == "complex": petscconfvar="PETSC_USE_COMPLEX" 580 else: petscconfvar="PETSC_HAVE_"+requirement.upper() 581 if self.conf.get(petscconfvar): 582 if isNull: 583 testDict['SKIP']="Not "+petscconfvar+" requirement not met" 584 return False 585 elif not isNull: 586 if debug: print "requirement not found: ", requirement 587 testDict['SKIP']=petscconfvar+" requirement not met" 588 return False 589 590 return True 591 592 def genPetscTests_summarize(self,dataDict): 593 """ 594 Required method to state what happened 595 """ 596 if not self.summarize: return 597 indent=" " 598 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 599 fh=open(fhname,"w") 600 #print "See ", fhname 601 for root in dataDict: 602 relroot=self.relpath(self.petsc_dir,root) 603 pkg=relroot.split("/")[1] 604 fh.write(relroot+"\n") 605 allSrcs=[] 606 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 607 for exfile in dataDict[root]: 608 # Basic information 609 fullfile=os.path.join(root,exfile) 610 rfile=self.relpath(self.petsc_dir,fullfile) 611 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 612 fh.write(indent+exfile+indent*4+builtStatus+"\n") 613 614 for test in dataDict[root][exfile]: 615 if test in self.buildkeys: continue 616 line=indent*2+test 617 fh.write(line+"\n") 618 # Looks nice to have the keys in order 619 #for key in dataDict[root][exfile][test]: 620 for key in "isrun abstracted nsize args requires script".split(): 621 if not dataDict[root][exfile][test].has_key(key): continue 622 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 623 fh.write(line+"\n") 624 fh.write("\n") 625 fh.write("\n") 626 fh.write("\n") 627 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 628 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 629 fh.close() 630 return 631 632 def genPetscTests(self,root,dirs,files,dataDict): 633 """ 634 Go through and parse the source files in the directory to generate 635 the examples based on the metadata contained in the source files 636 """ 637 debug=False 638 # Use examplesAnalyze to get what the makefles think are sources 639 #self.examplesAnalyze(root,dirs,files,anlzDict) 640 641 dataDict[root]={} 642 643 for exfile in files: 644 #TST: Until we replace files, still leaving the orginals as is 645 #if not exfile.startswith("new_"+"ex"): continue 646 if not exfile.startswith("ex"): continue 647 648 # Convenience 649 fullex=os.path.join(root,exfile) 650 relpfile=self.relpath(self.petsc_dir,fullex) 651 if debug: print relpfile 652 dataDict[root].update(testparse.parseTestFile(fullex)) 653 # Need to check and make sure tests are in the file 654 # if verbosity>=1: print relpfile 655 if dataDict[root].has_key(exfile): 656 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 657 658 return 659 660 def walktree(self,top,action="printFiles"): 661 """ 662 Walk a directory tree, starting from 'top' 663 """ 664 #print "action", action 665 # Goal of action is to fill this dictionary 666 dataDict={} 667 for root, dirs, files in os.walk(top, topdown=False): 668 if not "examples" in root: continue 669 if not os.path.isfile(os.path.join(root,"makefile")): continue 670 bname=os.path.basename(root.rstrip("/")) 671 if bname=="tests" or bname=="tutorials": 672 eval("self."+action+"(root,dirs,files,dataDict)") 673 if type(top) != types.StringType: 674 raise TypeError("top must be a string") 675 # Now summarize this dictionary 676 eval("self."+action+"_summarize(dataDict)") 677 return dataDict 678 679 def gen_gnumake(self, fd): 680 """ 681 Overwrite of the method in the base PETSc class 682 """ 683 def write(stem, srcs): 684 for lang in LANGS: 685 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 686 for pkg in PKGS: 687 srcs = self.gen_pkg(pkg) 688 write('testsrcs-' + pkg, srcs) 689 return self.gendeps 690 691 def gen_pkg(self, pkg): 692 """ 693 Overwrite of the method in the base PETSc class 694 """ 695 return self.sources[pkg] 696 697 def write_gnumake(self,dataDict): 698 """ 699 Write out something similar to files from gmakegen.py 700 701 There is not a lot of has_key type checking because 702 should just work and need to know if there are bugs 703 704 Test depends on script which also depends on source 705 file, but since I don't have a good way generating 706 acting on a single file (oops) just depend on 707 executable which in turn will depend on src file 708 """ 709 # Different options for how to set up the targets 710 compileExecsFirst=False 711 712 # Open file 713 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 714 fd = open(arch_files, 'w') 715 716 # Write out the sources 717 gendeps = self.gen_gnumake(fd) 718 719 # Write out the tests and execname targets 720 fd.write("\n#Tests and executables\n") # Delimiter 721 722 for pkg in PKGS: 723 # These grab the ones that are built 724 for lang in LANGS: 725 testdeps=[] 726 for ftest in self.tests[pkg][lang]: 727 test=os.path.basename(ftest) 728 basedir=os.path.dirname(ftest) 729 testdeps.append(self.nameSpace(test,basedir)) 730 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 731 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 732 733 # test targets 734 for ftest in self.tests[pkg][lang]: 735 test=os.path.basename(ftest) 736 basedir=os.path.dirname(ftest) 737 testdir="${TESTDIR}/"+basedir+"/" 738 nmtest=self.nameSpace(test,basedir) 739 rundir=os.path.join(testdir,test) 740 #print test, nmtest 741 script=test+".sh" 742 743 # Deps 744 exfile=self.tests[pkg][lang][ftest]['exfile'] 745 fullex=os.path.join(self.petsc_dir,exfile) 746 localexec=self.tests[pkg][lang][ftest]['exec'] 747 execname=os.path.join(testdir,localexec) 748 fullscript=os.path.join(testdir,script) 749 tmpfile=os.path.join(testdir,test,test+".tmp") 750 751 # *.counts depends on the script and either executable (will 752 # be run) or the example source file (SKIP or TODO) 753 fd.write('%s.counts : %s %s\n' 754 % (os.path.join('$(TESTDIR)/counts', nmtest), 755 fullscript, 756 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 757 # Now write the args: 758 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 759 760 fd.close() 761 return 762 763 def writeHarness(self,output,dataDict): 764 """ 765 This is set up to write out multiple harness even if only gnumake 766 is supported now 767 """ 768 eval("self.write_"+output+"(dataDict)") 769 return 770 771def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 772 if output is None: 773 output = 'gnumake' 774 775 776 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 777 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 778 pEx.writeHarness(output,dataDict) 779 780if __name__ == '__main__': 781 import optparse 782 parser = optparse.OptionParser() 783 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 784 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 785 parser.add_option('--output', help='Location to write output file', default=None) 786 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 787 opts, extra_args = parser.parse_args() 788 if extra_args: 789 import sys 790 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 791 exit(1) 792 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 793