1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,testDict,i,isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 testDict['args']: -ksp_monitor 97 testDict['subargs']: -bs ${i} -pc_type ${j} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["i"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["j"],["cholesky sor"]] 101 subst should be passed in instead of testDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=testDict.keys() 105 for key in lkeys: 106 if type(testDict[key])!=types.StringType: continue 107 keystr = str(testDict[key]) 108 akey=('subargs' if key=='args' else key) # what to assign 109 if not testDict.has_key(akey): testDict[akey]='' 110 varlist=[] 111 for varset in re.split('-(?=[a-zA-Z])',keystr): 112 if not varset.strip(): continue 113 if len(re.findall('{{(.*?)}}',varset))>0: 114 if not loopVars.has_key(akey): loopVars[akey]={} 115 # Assuming only one for loop per var specification 116 keyvar=varset.split("{{")[0].strip() 117 varlist.append(keyvar) 118 lvar=string.ascii_lowercase[i]; i+=1 119 loopVars[akey][keyvar]=[lvar,re.findall('{{(.*?)}}',varset)[0]] 120 testDict[akey] += '-'+keyvar+' ${' + lvar + '} ' 121 else: 122 if key=='args': 123 newargs+="-"+varset+" " 124 # Also move the datafilespath variables up to localrunfiles 125 keyvar=varset.split(" ")[0].strip() 126 if keyvar=='f0' or keyvar=='f1': 127 if testDict.has_key('localrunfiles'): 128 keyvar= 129 if len(varlist)>0: loopVars[akey]['varlist']=varlist 130 131 132 # For subtests, args are always substituted in (not top level) 133 if isSubtest: 134 testDict['subargs']+=newargs 135 testDict['args']='' 136 else: 137 if len(loopVars.keys())>0: 138 testDict['args']=newargs 139 if len(loopVars.keys())>0: 140 return loopVars 141 else: 142 return None 143 144 def getArgLabel(self,testDict): 145 """ 146 In all of the arguments in the test dictionary, create a simple 147 string for searching within the makefile system. For simplicity in 148 search, remove "-", for strings, etc. 149 Also, concatenate the arg commands 150 For now, ignore nsize -- seems hard to search for anyway 151 """ 152 # Collect all of the args associated with a test 153 argStr=("" if not testDict.has_key('args') else testDict['args']) 154 if testDict.has_key('subtests'): 155 for stest in testDict["subtests"]: 156 sd=testDict[stest] 157 argStr=argStr+("" if not sd.has_key('args') else sd['args']) 158 159 # Now go through and cleanup 160 argStr=re.sub('{{(.*?)}}',"",argStr) 161 argStr=re.sub('-'," ",argStr) 162 for digit in string.digits: argStr=re.sub(digit," ",argStr) 163 argStr=re.sub("\.","",argStr) 164 argStr=re.sub(",","",argStr) 165 argStr=re.sub('\+',' ',argStr) 166 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 167 return argStr.strip() 168 169 def addToSources(self,exfile,root,srcDict): 170 """ 171 Put into data structure that allows easy generation of makefile 172 """ 173 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 174 fullfile=os.path.join(root,exfile) 175 relpfile=self.relpath(self.petsc_dir,fullfile) 176 lang=self.getLanguage(exfile) 177 if not lang: return 178 self.sources[pkg][lang]['srcs'].append(relpfile) 179 if srcDict.has_key('depends'): 180 depSrc=srcDict['depends'] 181 depObj=os.path.splitext(depSrc)[0]+".o" 182 self.sources[pkg][lang][exfile]=depObj 183 184 # In gmakefile, ${TESTDIR} var specifies the object compilation 185 testsdir=self.relpath(self.petsc_dir,root)+"/" 186 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 187 self.objects[pkg].append(objfile) 188 return 189 190 def addToTests(self,test,root,exfile,execname,testDict): 191 """ 192 Put into data structure that allows easy generation of makefile 193 Organized by languages to allow testing of languages 194 """ 195 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 196 #nmtest=self.nameSpace(test,root) 197 rpath=self.relpath(self.petsc_dir,root) 198 nmtest=os.path.join(rpath,test) 199 lang=self.getLanguage(exfile) 200 if not lang: return 201 self.tests[pkg][lang][nmtest]={} 202 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 203 self.tests[pkg][lang][nmtest]['exec']=execname 204 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 205 return 206 207 def getExecname(self,exfile,root): 208 """ 209 Generate bash script using template found next to this file. 210 This file is read in at constructor time to avoid file I/O 211 """ 212 rpath=self.relpath(self.petsc_dir,root) 213 if self.single_ex: 214 execname=rpath.split("/")[1]+"-ex" 215 else: 216 execname=os.path.splitext(exfile)[0] 217 return execname 218 219 def getSubstVars(self,testDict,rpath,testname): 220 """ 221 Create a dictionary with all of the variables that get substituted 222 into the template commands found in example_template.py 223 TODO: Cleanup 224 """ 225 subst={} 226 # Handle defaults 227 if not testDict.has_key('nsize'): testDict['nsize']=1 228 if not testDict.has_key('filter'): testDict['filter']="" 229 if not testDict.has_key('filter_output'): testDict['filter_output']="" 230 if not testDict.has_key('localrunfiles'): testDict['localrunfiles']="" 231 if not testDict.has_key('args'): testDict['args']="" 232 if not testDict.has_key('subargs'): testDict['subargs']="" 233 if not testDict.has_key('comments'): testDict['comments']="" 234 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 235 if not "_" in defroot: defroot=defroot+"_1" 236 if not testDict.has_key('redirect_file'): testDict['redirect_file']=defroot+".tmp" 237 238 # Output file is special because of subtests override 239 if testDict.has_key('output_file'): 240 subst['output_file']=testDict['output_file'] 241 subst['output_file_inSrc']=True 242 else: 243 subst['output_file']="output/"+defroot+".out" 244 subst['output_file_inSrc']=False 245 246 # Setup the variables in template_string that need to be substituted 247 subst['defroot']=defroot 248 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 249 subst['label']=self.nameSpace(defroot,subst['srcdir']) 250 subst['label_suffix']='' 251 subst['redirect_file']=testDict['redirect_file'] 252 subst['comments']="\n#".join(testDict['comments'].split("\n")) 253 if subst['comments']: subst['comments']="#"+subst['comments'] 254 subst['exec']="../"+testDict['execname'] 255 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky 256 subst['filter_output']=testDict['filter_output'] 257 subst['localrunfiles']=testDict['localrunfiles'] 258 subst['testroot']=self.testroot_dir 259 subst['testname']=testname 260 if self.conf['PETSC_HAVE_DATAFILESPATH']: 261 dp=self.conf['DATAFILESPATH'] 262 dpl='DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 263 else: 264 dpl='' 265 subst['datafilespath_line']=dpl 266 267 # Be careful with this 268 if testDict.has_key('command'): subst['command']=testDict['command'] 269 270 # These can have for loops and are treated separately later 271 if testDict.has_key('nsize'): subst['nsize']=str(testDict['nsize']) 272 if testDict.has_key('args'): subst['args']=testDict['args'] 273 274 #Conf vars 275 if self.petsc_arch.find('valgrind')>=0: 276 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 277 else: 278 subst['mpiexec']=self.conf['MPIEXEC'] 279 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 280 subst['diff']=self.conf['DIFF'] 281 subst['rm']=self.conf['RM'] 282 subst['grep']=self.conf['GREP'] 283 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 284 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 285 286 for key in subst: 287 if type(subst[key])!=types.StringType: continue 288 subst[key] = self._substVars(testDict,subst[key]) 289 290 return subst 291 292 def getCmds(self,subst,i): 293 """ 294 Generate bash script using template found next to this file. 295 This file is read in at constructor time to avoid file I/O 296 """ 297 indnt=self.indent 298 nindnt=i # the start and has to be consistent with below 299 cmdLines="" 300 301 # Add in the full path here. 302 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 303 304 # MPI is the default -- but we have a few odd commands 305 if not subst.has_key('command'): 306 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 307 else: 308 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 309 cmdLines+=cmd+"\n\n" 310 311 if not subst['filter_output']: 312 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 313 else: 314 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 315 cmdLines+=cmd+"\n" 316 return cmdLines 317 318 def _substVars(self,subst,origStr): 319 """ 320 Substitute varial 321 """ 322 Str=origStr 323 for subkey in subst: 324 if type(subst[subkey])!=types.StringType: continue 325 patt="@"+subkey.upper()+"@" 326 Str=re.sub(patt,subst[subkey],Str) 327 return Str 328 329 def _writeTodoSkip(self,fh,tors,inDict,footer): 330 """ 331 Write out the TODO and SKIP lines in the file 332 The TODO or SKIP variable, tors, should be lower case 333 """ 334 TORS=tors.upper() 335 template=eval("example_template."+tors+"line") 336 tsStr=re.sub("@"+TORS+"COMMENT@",inDict[TORS],template) 337 fh.write(tsStr+"\ntotal=1; "+tors+"=1\n") 338 fh.write(footer+"\n") 339 fh.write("exit\n\n\n") 340 return 341 342 def getLoopVarsHead(self,loopVars,i): 343 """ 344 Generate a nicely indented string with the format loops 345 Here is what the data structure looks like 346 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 347 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 348 loopVars['subargs']['pc_type']=["j","cholesky sor"] 349 """ 350 outstr=''; indnt=self.indent 351 for key in loopVars: 352 for var in loopVars[key]['varlist']: 353 varval=loopVars[key][var] 354 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 355 i = i + 1 356 return (outstr,i) 357 358 def getLoopVarsFoot(self,loopVars,i): 359 outstr=''; indnt=self.indent 360 for key in loopVars: 361 for var in loopVars[key]['varlist']: 362 i = i - 1 363 outstr += indnt * i + "done\n" 364 return (outstr,i) 365 366 def genRunScript(self,testname,root,isRun,srcDict): 367 """ 368 Generate bash script using template found next to this file. 369 This file is read in at constructor time to avoid file I/O 370 """ 371 # runscript_dir directory has to be consistent with gmakefile 372 testDict=srcDict[testname] 373 rpath=self.relpath(self.petsc_dir,root) 374 runscript_dir=os.path.join(self.testroot_dir,rpath) 375 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 376 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 377 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 378 379 i=8 # for loop counters 380 # This gets a datastructure for handling loops as well as alters testDict 381 subst=self.getSubstVars(testDict,rpath,testname) 382 loopVars = self._getLoopVars(subst,i) 383 384 #Handle runfiles 385 if subst['localrunfiles']: 386 for lfile in subst['localrunfiles'].split(): 387 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 388 shutil.copy(fullfile,runscript_dir) 389 # Check subtests for local runfiles 390 if testDict.has_key("subtests"): 391 if 'mumps_cholesky' in testname: print testDict["subtests"] 392 for stest in testDict["subtests"]: 393 if testDict[stest].has_key('localrunfiles'): 394 for lfile in testDict[stest]['localrunfiles'].split(): 395 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 396 shutil.copy(fullfile,self.runscript_dir) 397 398 # Now substitute the key variables into the header and footer 399 header=self._substVars(subst,example_template.header) 400 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 401 402 # Start writing the file 403 fh.write(header+"\n") 404 405 # If there is a TODO or a SKIP then we do it before writing out the 406 # rest of the command (which is useful for working on the test) 407 # SKIP and TODO can be for the source file or for the runs 408 for tors in "todo skip".split(): 409 for d in [srcDict,testDict]: 410 if d.has_key(tors.upper()): self._writeTodoSkip(fh,tors,d,footer) 411 412 j=0 # for indentation 413 414 if loopVars: 415 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 416 if (loopHead): fh.write(loopHead+"\n") 417 418 # Subtests are special 419 if testDict.has_key("subtests"): 420 substP=subst # Subtests can inherit args but be careful 421 for stest in testDict["subtests"]: 422 subst=substP.copy() 423 subst.update(testDict[stest]) 424 subst['nsize']=str(subst['nsize']) 425 sLoopVars = self._getLoopVars(subst,i,isSubtest=True) 426 if sLoopVars: 427 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 428 fh.write(sLoopHead+"\n") 429 fh.write(self.getCmds(subst,j)+"\n") 430 if sLoopVars: 431 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 432 fh.write(sLoopFoot+"\n") 433 else: 434 fh.write(self.getCmds(subst,j)+"\n") 435 436 if loopVars: 437 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 438 fh.write(loopFoot+"\n") 439 440 fh.write(footer+"\n") 441 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 442 return 443 444 def genScriptsAndInfo(self,exfile,root,srcDict): 445 """ 446 Generate scripts from the source file, determine if built, etc. 447 For every test in the exfile with info in the srcDict: 448 1. Determine if it needs to be run for this arch 449 2. Generate the script 450 3. Generate the data needed to write out the makefile in a 451 convenient way 452 All tests are *always* run, but some may be SKIP'd per the TAP standard 453 """ 454 debug=False 455 fileIsTested=False 456 execname=self.getExecname(exfile,root) 457 isBuilt=self._isBuilt(exfile,srcDict) 458 for test in srcDict: 459 if test in self.buildkeys: continue 460 if debug: print self.nameSpace(exfile,root), test 461 srcDict[test]['execname']=execname # Convenience in generating scripts 462 isRun=self._isRun(srcDict[test]) 463 self.genRunScript(test,root,isRun,srcDict) 464 srcDict[test]['isrun']=isRun 465 if isRun: fileIsTested=True 466 self.addToTests(test,root,exfile,execname,srcDict[test]) 467 468 # This adds to datastructure for building deps 469 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 470 #print self.nameSpace(exfile,root), fileIsTested 471 return 472 473 def _isBuilt(self,exfile,srcDict): 474 """ 475 Determine if this file should be built. 476 """ 477 # Get the language based on file extension 478 lang=self.getLanguage(exfile) 479 if (lang=="F" or lang=="F90") and not self.have_fortran: 480 srcDict["SKIP"]="Fortran required for this test" 481 return False 482 if lang=="cu" and not self.conf.has_key('PETSC_HAVE_CUDA'): 483 srcDict["SKIP"]="CUDA required for this test" 484 return False 485 if lang=="cxx" and not self.conf.has_key('PETSC_HAVE_CXX'): 486 srcDict["SKIP"]="C++ required for this test" 487 return False 488 489 # Deprecated source files 490 if srcDict.has_key("TODO"): return False 491 492 # isRun can work with srcDict to handle the requires 493 if srcDict.has_key("requires"): 494 if len(srcDict["requires"])>0: 495 return self._isRun(srcDict) 496 497 return True 498 499 500 def _isRun(self,testDict): 501 """ 502 Based on the requirements listed in the src file and the petscconf.h 503 info, determine whether this test should be run or not. 504 """ 505 indent=" " 506 debug=False 507 508 # MPI requirements 509 if testDict.has_key('nsize'): 510 if testDict['nsize']>1 and self.conf.has_key('MPI_IS_MPIUNI'): 511 if debug: print indent+"Cannot run parallel tests" 512 testDict['SKIP']="Parallel test with serial build" 513 return False 514 515 # The requirements for the test are the sum of all the run subtests 516 if testDict.has_key('subtests'): 517 if not testDict.has_key('requires'): testDict['requires']="" 518 for stest in testDict['subtests']: 519 if testDict[stest].has_key('requires'): 520 testDict['requires']=testDict['requires']+" "+testDict[stest]['requires'] 521 522 523 # Now go through all requirements 524 if testDict.has_key('requires'): 525 for requirement in testDict['requires'].split(): 526 requirement=requirement.strip() 527 if not requirement: continue 528 if debug: print indent+"Requirement: ", requirement 529 isNull=False 530 if requirement.startswith("!"): 531 requirement=requirement[1:]; isNull=True 532 # Precision requirement for reals 533 if requirement in self.precision_types: 534 if self.conf['PETSC_PRECISION']==requirement: 535 testDict['SKIP']="not "+requirement+" required" 536 if isNull: return False 537 else: 538 testDict['SKIP']=requirement+" required" 539 return False 540 # Precision requirement for ints 541 if requirement in self.integer_types: 542 if requirement=="int32": 543 if self.conf['PETSC_SIZEOF_INT']==4: 544 testDict['SKIP']="not int32 required" 545 if isNull: return False 546 else: 547 testDict['SKIP']="int32 required" 548 return False 549 if requirement=="int64": 550 if self.conf['PETSC_SIZEOF_INT']==8: 551 testDict['SKIP']="NOT int64 required" 552 if isNull: return False 553 else: 554 testDict['SKIP']="int64 required" 555 return False 556 # Datafilespath 557 if requirement=="datafilespath": 558 if not self.conf['PETSC_HAVE_DATAFILESPATH']: 559 testDict['SKIP']="Requires DATAFILESPATH" 560 return False 561 # Defines -- not sure I have comments matching 562 if "define(" in requirement.lower(): 563 reqdef=requirement.split("(")[1].split(")")[0] 564 val=(reqdef.split()[1] if " " in reqdef else "") 565 if self.conf.has_key(reqdef): 566 if val: 567 if self.conf[reqdef]==val: 568 if isNull: 569 testDict['SKIP']="Null requirement not met: "+requirement 570 return False 571 else: 572 testDict['SKIP']="Required: "+requirement 573 return False 574 else: 575 if isNull: 576 testDict['SKIP']="Null requirement not met: "+requirement 577 return False 578 else: 579 return True 580 else: 581 testDict['SKIP']="Requirement not met: "+requirement 582 return False 583 584 # Rest should be packages that we can just get from conf 585 if requirement == "complex": petscconfvar="PETSC_USE_COMPLEX" 586 else: petscconfvar="PETSC_HAVE_"+requirement.upper() 587 if self.conf.get(petscconfvar): 588 if isNull: 589 testDict['SKIP']="Not "+petscconfvar+" requirement not met" 590 return False 591 elif not isNull: 592 if debug: print "requirement not found: ", requirement 593 testDict['SKIP']=petscconfvar+" requirement not met" 594 return False 595 596 return True 597 598 def genPetscTests_summarize(self,dataDict): 599 """ 600 Required method to state what happened 601 """ 602 if not self.summarize: return 603 indent=" " 604 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 605 fh=open(fhname,"w") 606 #print "See ", fhname 607 for root in dataDict: 608 relroot=self.relpath(self.petsc_dir,root) 609 pkg=relroot.split("/")[1] 610 fh.write(relroot+"\n") 611 allSrcs=[] 612 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 613 for exfile in dataDict[root]: 614 # Basic information 615 fullfile=os.path.join(root,exfile) 616 rfile=self.relpath(self.petsc_dir,fullfile) 617 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 618 fh.write(indent+exfile+indent*4+builtStatus+"\n") 619 620 for test in dataDict[root][exfile]: 621 if test in self.buildkeys: continue 622 line=indent*2+test 623 fh.write(line+"\n") 624 # Looks nice to have the keys in order 625 #for key in dataDict[root][exfile][test]: 626 for key in "isrun abstracted nsize args requires script".split(): 627 if not dataDict[root][exfile][test].has_key(key): continue 628 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 629 fh.write(line+"\n") 630 fh.write("\n") 631 fh.write("\n") 632 fh.write("\n") 633 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 634 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 635 fh.close() 636 return 637 638 def genPetscTests(self,root,dirs,files,dataDict): 639 """ 640 Go through and parse the source files in the directory to generate 641 the examples based on the metadata contained in the source files 642 """ 643 debug=False 644 # Use examplesAnalyze to get what the makefles think are sources 645 #self.examplesAnalyze(root,dirs,files,anlzDict) 646 647 dataDict[root]={} 648 649 for exfile in files: 650 #TST: Until we replace files, still leaving the orginals as is 651 #if not exfile.startswith("new_"+"ex"): continue 652 if not exfile.startswith("ex"): continue 653 654 # Convenience 655 fullex=os.path.join(root,exfile) 656 relpfile=self.relpath(self.petsc_dir,fullex) 657 if debug: print relpfile 658 dataDict[root].update(testparse.parseTestFile(fullex)) 659 # Need to check and make sure tests are in the file 660 # if verbosity>=1: print relpfile 661 if dataDict[root].has_key(exfile): 662 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 663 664 return 665 666 def walktree(self,top,action="printFiles"): 667 """ 668 Walk a directory tree, starting from 'top' 669 """ 670 #print "action", action 671 # Goal of action is to fill this dictionary 672 dataDict={} 673 for root, dirs, files in os.walk(top, topdown=False): 674 if not "examples" in root: continue 675 if not os.path.isfile(os.path.join(root,"makefile")): continue 676 bname=os.path.basename(root.rstrip("/")) 677 if bname=="tests" or bname=="tutorials": 678 eval("self."+action+"(root,dirs,files,dataDict)") 679 if type(top) != types.StringType: 680 raise TypeError("top must be a string") 681 # Now summarize this dictionary 682 eval("self."+action+"_summarize(dataDict)") 683 return dataDict 684 685 def gen_gnumake(self, fd): 686 """ 687 Overwrite of the method in the base PETSc class 688 """ 689 def write(stem, srcs): 690 for lang in LANGS: 691 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 692 for pkg in PKGS: 693 srcs = self.gen_pkg(pkg) 694 write('testsrcs-' + pkg, srcs) 695 return self.gendeps 696 697 def gen_pkg(self, pkg): 698 """ 699 Overwrite of the method in the base PETSc class 700 """ 701 return self.sources[pkg] 702 703 def write_gnumake(self,dataDict): 704 """ 705 Write out something similar to files from gmakegen.py 706 707 There is not a lot of has_key type checking because 708 should just work and need to know if there are bugs 709 710 Test depends on script which also depends on source 711 file, but since I don't have a good way generating 712 acting on a single file (oops) just depend on 713 executable which in turn will depend on src file 714 """ 715 # Different options for how to set up the targets 716 compileExecsFirst=False 717 718 # Open file 719 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 720 fd = open(arch_files, 'w') 721 722 # Write out the sources 723 gendeps = self.gen_gnumake(fd) 724 725 # Write out the tests and execname targets 726 fd.write("\n#Tests and executables\n") # Delimiter 727 728 for pkg in PKGS: 729 # These grab the ones that are built 730 for lang in LANGS: 731 testdeps=[] 732 for ftest in self.tests[pkg][lang]: 733 test=os.path.basename(ftest) 734 basedir=os.path.dirname(ftest) 735 testdeps.append(self.nameSpace(test,basedir)) 736 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 737 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 738 739 # test targets 740 for ftest in self.tests[pkg][lang]: 741 test=os.path.basename(ftest) 742 basedir=os.path.dirname(ftest) 743 testdir="${TESTDIR}/"+basedir+"/" 744 nmtest=self.nameSpace(test,basedir) 745 rundir=os.path.join(testdir,test) 746 #print test, nmtest 747 script=test+".sh" 748 749 # Deps 750 exfile=self.tests[pkg][lang][ftest]['exfile'] 751 fullex=os.path.join(self.petsc_dir,exfile) 752 localexec=self.tests[pkg][lang][ftest]['exec'] 753 execname=os.path.join(testdir,localexec) 754 fullscript=os.path.join(testdir,script) 755 tmpfile=os.path.join(testdir,test,test+".tmp") 756 757 # *.counts depends on the script and either executable (will 758 # be run) or the example source file (SKIP or TODO) 759 fd.write('%s.counts : %s %s\n' 760 % (os.path.join('$(TESTDIR)/counts', nmtest), 761 fullscript, 762 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 763 # Now write the args: 764 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 765 766 fd.close() 767 return 768 769 def writeHarness(self,output,dataDict): 770 """ 771 This is set up to write out multiple harness even if only gnumake 772 is supported now 773 """ 774 eval("self.write_"+output+"(dataDict)") 775 return 776 777def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 778 if output is None: 779 output = 'gnumake' 780 781 782 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 783 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 784 pEx.writeHarness(output,dataDict) 785 786if __name__ == '__main__': 787 import optparse 788 parser = optparse.OptionParser() 789 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 790 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 791 parser.add_option('--output', help='Location to write output file', default=None) 792 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 793 opts, extra_args = parser.parse_args() 794 if extra_args: 795 import sys 796 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 797 exit(1) 798 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 799