1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 for key in lkeys: 107 if type(inDict[key])!=types.StringType: continue 108 keystr = str(inDict[key]) 109 akey=('subargs' if key=='args' else key) # what to assign 110 if not inDict.has_key(akey): inDict[akey]='' 111 varlist=[] 112 for varset in re.split('-(?=[a-zA-Z])',keystr): 113 if not varset.strip(): continue 114 if len(re.findall('{{(.*?)}}',varset))>0: 115 if not loopVars.has_key(akey): loopVars[akey]={} 116 # Assuming only one for loop per var specification 117 keyvar=varset.split("{{")[0].strip() 118 if not keyvar.strip(): 119 if key=='nsize': 120 keyvar='nsize' 121 else: 122 print testname+": Problem in finding loop variables", keyvar, varset 123 varlist.append(keyvar) 124 loopVars[akey][keyvar]=[keyvar,re.findall('{{(.*?)}}',varset)[0]] 125 if akey=='nsize': 126 inDict[akey] = '${' + keyvar + '}' 127 lsuffix+=akey+'-'+inDict[akey]+'_' 128 else: 129 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 130 lsuffix+=keyvar+'-${' + keyvar + '}_' 131 else: 132 if key=='args': newargs+=" -"+varset.strip() 133 if len(varlist)>0: loopVars[akey]['varlist']=varlist 134 135 136 # For subtests, args are always substituted in (not top level) 137 if isSubtest: 138 inDict['subargs']+=" "+newargs.strip() 139 inDict['args']='' 140 if inDict.has_key('label_suffix'): 141 inDict['label_suffix']+=lsuffix.rstrip('_') 142 else: 143 inDict['label_suffix']=lsuffix.rstrip('_') 144 else: 145 if len(loopVars.keys())>0: 146 inDict['args']=newargs.strip() 147 inDict['label_suffix']=lsuffix.rstrip('_') 148 if len(loopVars.keys())>0: 149 return loopVars 150 else: 151 return None 152 153 def getArgLabel(self,testDict): 154 """ 155 In all of the arguments in the test dictionary, create a simple 156 string for searching within the makefile system. For simplicity in 157 search, remove "-", for strings, etc. 158 Also, concatenate the arg commands 159 For now, ignore nsize -- seems hard to search for anyway 160 """ 161 # Collect all of the args associated with a test 162 argStr=("" if not testDict.has_key('args') else testDict['args']) 163 if testDict.has_key('subtests'): 164 for stest in testDict["subtests"]: 165 sd=testDict[stest] 166 argStr=argStr+("" if not sd.has_key('args') else sd['args']) 167 168 # Now go through and cleanup 169 argStr=re.sub('{{(.*?)}}',"",argStr) 170 argStr=re.sub('-'," ",argStr) 171 for digit in string.digits: argStr=re.sub(digit," ",argStr) 172 argStr=re.sub("\.","",argStr) 173 argStr=re.sub(",","",argStr) 174 argStr=re.sub('\+',' ',argStr) 175 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 176 return argStr.strip() 177 178 def addToSources(self,exfile,root,srcDict): 179 """ 180 Put into data structure that allows easy generation of makefile 181 """ 182 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 183 fullfile=os.path.join(root,exfile) 184 relpfile=self.relpath(self.petsc_dir,fullfile) 185 lang=self.getLanguage(exfile) 186 if not lang: return 187 self.sources[pkg][lang]['srcs'].append(relpfile) 188 if srcDict.has_key('depends'): 189 depSrc=srcDict['depends'] 190 depObj=os.path.splitext(depSrc)[0]+".o" 191 self.sources[pkg][lang][exfile]=depObj 192 193 # In gmakefile, ${TESTDIR} var specifies the object compilation 194 testsdir=self.relpath(self.petsc_dir,root)+"/" 195 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 196 self.objects[pkg].append(objfile) 197 return 198 199 def addToTests(self,test,root,exfile,execname,testDict): 200 """ 201 Put into data structure that allows easy generation of makefile 202 Organized by languages to allow testing of languages 203 """ 204 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 205 #nmtest=self.nameSpace(test,root) 206 rpath=self.relpath(self.petsc_dir,root) 207 nmtest=os.path.join(rpath,test) 208 lang=self.getLanguage(exfile) 209 if not lang: return 210 self.tests[pkg][lang][nmtest]={} 211 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 212 self.tests[pkg][lang][nmtest]['exec']=execname 213 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 214 return 215 216 def getExecname(self,exfile,root): 217 """ 218 Generate bash script using template found next to this file. 219 This file is read in at constructor time to avoid file I/O 220 """ 221 rpath=self.relpath(self.petsc_dir,root) 222 if self.single_ex: 223 execname=rpath.split("/")[1]+"-ex" 224 else: 225 execname=os.path.splitext(exfile)[0] 226 return execname 227 228 def getSubstVars(self,testDict,rpath,testname): 229 """ 230 Create a dictionary with all of the variables that get substituted 231 into the template commands found in example_template.py 232 """ 233 subst={} 234 235 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 236 if not testDict.has_key('nsize'): testDict['nsize']=1 237 for ak in testparse.acceptedkeys: 238 if ak=='test': continue 239 subst[ak]=(testDict[ak] if testDict.has_key(ak) else '') 240 241 # Now do other variables 242 subst['execname']=testDict['execname'] 243 if testDict.has_key('filter'): 244 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 245 246 # Output file is special because of subtests override 247 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 248 subst['defroot']=defroot 249 if not "_" in defroot: defroot=defroot+"_1" 250 if not testDict.has_key('output_file'): 251 subst['output_file']="output/"+defroot+".out" 252 subst['redirect_file']=defroot+".tmp" 253 254 # Others 255 subst['subargs']='' # Default. For variables override 256 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 257 subst['label']=self.nameSpace(defroot,subst['srcdir']) 258 subst['label_suffix']='' 259 subst['comments']="\n#".join(subst['comments'].split("\n")) 260 if subst['comments']: subst['comments']="#"+subst['comments'] 261 subst['exec']="../"+subst['execname'] 262 subst['testroot']=self.testroot_dir 263 subst['testname']=testname 264 if self.conf['PETSC_HAVE_DATAFILESPATH']: 265 dp=self.conf['DATAFILESPATH'] 266 dpl='DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 267 else: 268 dpl='' 269 subst['datafilespath_line']=dpl 270 271 # These can have for loops and are treated separately later 272 subst['nsize']=str(subst['nsize']) 273 274 #Conf vars 275 if self.petsc_arch.find('valgrind')>=0: 276 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 277 else: 278 subst['mpiexec']=self.conf['MPIEXEC'] 279 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 280 subst['diff']=self.conf['DIFF'] 281 subst['rm']=self.conf['RM'] 282 subst['grep']=self.conf['GREP'] 283 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 284 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 285 286 return subst 287 288 def getCmds(self,subst,i): 289 """ 290 Generate bash script using template found next to this file. 291 This file is read in at constructor time to avoid file I/O 292 """ 293 indnt=self.indent 294 nindnt=i # the start and has to be consistent with below 295 cmdLines="" 296 297 # Add in the full path here. 298 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 299 300 # MPI is the default -- but we have a few odd commands 301 if not subst['command']: 302 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 303 else: 304 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 305 cmdLines+=cmd+"\n\n" 306 307 if not subst['filter_output']: 308 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 309 else: 310 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 311 cmdLines+=cmd+"\n" 312 return cmdLines 313 314 def _substVars(self,subst,origStr): 315 """ 316 Substitute variables 317 """ 318 Str=origStr 319 for subkey in subst: 320 if type(subst[subkey])!=types.StringType: continue 321 patt="@"+subkey.upper()+"@" 322 Str=re.sub(patt,subst[subkey],Str) 323 return Str 324 325 def _writeTodoSkip(self,fh,tors,inDict,footer): 326 """ 327 Write out the TODO and SKIP lines in the file 328 The TODO or SKIP variable, tors, should be lower case 329 """ 330 TORS=tors.upper() 331 template=eval("example_template."+tors+"line") 332 tsStr=re.sub("@"+TORS+"COMMENT@",inDict[TORS],template) 333 fh.write(tsStr+"\ntotal=1; "+tors+"=1\n") 334 fh.write(footer+"\n") 335 fh.write("exit\n\n\n") 336 return 337 338 def getLoopVarsHead(self,loopVars,i): 339 """ 340 Generate a nicely indented string with the format loops 341 Here is what the data structure looks like 342 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 343 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 344 loopVars['subargs']['pc_type']=["j","cholesky sor"] 345 """ 346 outstr=''; indnt=self.indent 347 for key in loopVars: 348 for var in loopVars[key]['varlist']: 349 varval=loopVars[key][var] 350 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 351 i = i + 1 352 return (outstr,i) 353 354 def getLoopVarsFoot(self,loopVars,i): 355 outstr=''; indnt=self.indent 356 for key in loopVars: 357 for var in loopVars[key]['varlist']: 358 i = i - 1 359 outstr += indnt * i + "done\n" 360 return (outstr,i) 361 362 def genRunScript(self,testname,root,isRun,srcDict): 363 """ 364 Generate bash script using template found next to this file. 365 This file is read in at constructor time to avoid file I/O 366 """ 367 # runscript_dir directory has to be consistent with gmakefile 368 testDict=srcDict[testname] 369 rpath=self.relpath(self.petsc_dir,root) 370 runscript_dir=os.path.join(self.testroot_dir,rpath) 371 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 372 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 373 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 374 375 # Get variables to go into shell scripts. last time testDict used 376 subst=self.getSubstVars(testDict,rpath,testname) 377 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 378 #if '33_' in testname: print subst['subargs'] 379 380 #Handle runfiles 381 if subst.has_key('localrunfiles'): 382 for lfile in subst['localrunfiles'].split(): 383 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 384 shutil.copy(fullfile,runscript_dir) 385 # Check subtests for local runfiles 386 if subst.has_key("subtests"): 387 for stest in subst["subtests"]: 388 if subst[stest].has_key('localrunfiles'): 389 for lfile in testDict[stest]['localrunfiles'].split(): 390 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 391 shutil.copy(fullfile,self.runscript_dir) 392 393 # Now substitute the key variables into the header and footer 394 header=self._substVars(subst,example_template.header) 395 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 396 397 # Start writing the file 398 fh.write(header+"\n") 399 400 # If there is a TODO or a SKIP then we do it before writing out the 401 # rest of the command (which is useful for working on the test) 402 # SKIP and TODO can be for the source file or for the runs 403 for tors in "todo skip".split(): 404 for d in [srcDict,testDict]: 405 if d.has_key(tors.upper()): self._writeTodoSkip(fh,tors,d,footer) 406 407 j=0 # for indentation 408 409 if loopVars: 410 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 411 if (loopHead): fh.write(loopHead+"\n") 412 413 # Subtests are special 414 if testDict.has_key("subtests"): 415 substP=subst # Subtests can inherit args but be careful 416 for stest in testDict["subtests"]: 417 subst=substP.copy() 418 subst.update(testDict[stest]) 419 subst['nsize']=str(subst['nsize']) 420 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 421 #if '10_9' in testname: print sLoopVars 422 if sLoopVars: 423 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 424 fh.write(sLoopHead+"\n") 425 fh.write(self.getCmds(subst,j)+"\n") 426 if sLoopVars: 427 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 428 fh.write(sLoopFoot+"\n") 429 else: 430 fh.write(self.getCmds(subst,j)+"\n") 431 432 if loopVars: 433 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 434 fh.write(loopFoot+"\n") 435 436 fh.write(footer+"\n") 437 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 438 #if '10_9' in testname: sys.exit() 439 return 440 441 def genScriptsAndInfo(self,exfile,root,srcDict): 442 """ 443 Generate scripts from the source file, determine if built, etc. 444 For every test in the exfile with info in the srcDict: 445 1. Determine if it needs to be run for this arch 446 2. Generate the script 447 3. Generate the data needed to write out the makefile in a 448 convenient way 449 All tests are *always* run, but some may be SKIP'd per the TAP standard 450 """ 451 debug=False 452 fileIsTested=False 453 execname=self.getExecname(exfile,root) 454 isBuilt=self._isBuilt(exfile,srcDict) 455 for test in srcDict: 456 if test in self.buildkeys: continue 457 if debug: print self.nameSpace(exfile,root), test 458 srcDict[test]['execname']=execname # Convenience in generating scripts 459 isRun=self._isRun(srcDict[test]) 460 self.genRunScript(test,root,isRun,srcDict) 461 srcDict[test]['isrun']=isRun 462 if isRun: fileIsTested=True 463 self.addToTests(test,root,exfile,execname,srcDict[test]) 464 465 # This adds to datastructure for building deps 466 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 467 #print self.nameSpace(exfile,root), fileIsTested 468 return 469 470 def _isBuilt(self,exfile,srcDict): 471 """ 472 Determine if this file should be built. 473 """ 474 # Get the language based on file extension 475 lang=self.getLanguage(exfile) 476 if (lang=="F" or lang=="F90") and not self.have_fortran: 477 srcDict["SKIP"]="Fortran required for this test" 478 return False 479 if lang=="cu" and not self.conf.has_key('PETSC_HAVE_CUDA'): 480 srcDict["SKIP"]="CUDA required for this test" 481 return False 482 if lang=="cxx" and not self.conf.has_key('PETSC_HAVE_CXX'): 483 srcDict["SKIP"]="C++ required for this test" 484 return False 485 486 # Deprecated source files 487 if srcDict.has_key("TODO"): return False 488 489 # isRun can work with srcDict to handle the requires 490 if srcDict.has_key("requires"): 491 if len(srcDict["requires"])>0: 492 return self._isRun(srcDict) 493 494 return True 495 496 497 def _isRun(self,testDict): 498 """ 499 Based on the requirements listed in the src file and the petscconf.h 500 info, determine whether this test should be run or not. 501 """ 502 indent=" " 503 debug=False 504 505 # MPI requirements 506 if testDict.has_key('nsize'): 507 if testDict['nsize']>1 and self.conf.has_key('MPI_IS_MPIUNI'): 508 if debug: print indent+"Cannot run parallel tests" 509 testDict['SKIP']="Parallel test with serial build" 510 return False 511 512 # The requirements for the test are the sum of all the run subtests 513 if testDict.has_key('subtests'): 514 if not testDict.has_key('requires'): testDict['requires']="" 515 for stest in testDict['subtests']: 516 if testDict[stest].has_key('requires'): 517 testDict['requires']=testDict['requires']+" "+testDict[stest]['requires'] 518 519 520 # Now go through all requirements 521 if testDict.has_key('requires'): 522 for requirement in testDict['requires'].split(): 523 requirement=requirement.strip() 524 if not requirement: continue 525 if debug: print indent+"Requirement: ", requirement 526 isNull=False 527 if requirement.startswith("!"): 528 requirement=requirement[1:]; isNull=True 529 # Precision requirement for reals 530 if requirement in self.precision_types: 531 if self.conf['PETSC_PRECISION']==requirement: 532 testDict['SKIP']="not "+requirement+" required" 533 if isNull: return False 534 else: 535 testDict['SKIP']=requirement+" required" 536 return False 537 # Precision requirement for ints 538 if requirement in self.integer_types: 539 if requirement=="int32": 540 if self.conf['PETSC_SIZEOF_INT']==4: 541 testDict['SKIP']="not int32 required" 542 if isNull: return False 543 else: 544 testDict['SKIP']="int32 required" 545 return False 546 if requirement=="int64": 547 if self.conf['PETSC_SIZEOF_INT']==8: 548 testDict['SKIP']="NOT int64 required" 549 if isNull: return False 550 else: 551 testDict['SKIP']="int64 required" 552 return False 553 # Datafilespath 554 if requirement=="datafilespath": 555 if not self.conf['PETSC_HAVE_DATAFILESPATH']: 556 testDict['SKIP']="Requires DATAFILESPATH" 557 return False 558 # Defines -- not sure I have comments matching 559 if "define(" in requirement.lower(): 560 reqdef=requirement.split("(")[1].split(")")[0] 561 val=(reqdef.split()[1] if " " in reqdef else "") 562 if self.conf.has_key(reqdef): 563 if val: 564 if self.conf[reqdef]==val: 565 if isNull: 566 testDict['SKIP']="Null requirement not met: "+requirement 567 return False 568 else: 569 testDict['SKIP']="Required: "+requirement 570 return False 571 else: 572 if isNull: 573 testDict['SKIP']="Null requirement not met: "+requirement 574 return False 575 else: 576 return True 577 else: 578 testDict['SKIP']="Requirement not met: "+requirement 579 return False 580 581 # Rest should be packages that we can just get from conf 582 if requirement == "complex": petscconfvar="PETSC_USE_COMPLEX" 583 else: petscconfvar="PETSC_HAVE_"+requirement.upper() 584 if self.conf.get(petscconfvar): 585 if isNull: 586 testDict['SKIP']="Not "+petscconfvar+" requirement not met" 587 return False 588 elif not isNull: 589 if debug: print "requirement not found: ", requirement 590 testDict['SKIP']=petscconfvar+" requirement not met" 591 return False 592 593 return True 594 595 def genPetscTests_summarize(self,dataDict): 596 """ 597 Required method to state what happened 598 """ 599 if not self.summarize: return 600 indent=" " 601 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 602 fh=open(fhname,"w") 603 #print "See ", fhname 604 for root in dataDict: 605 relroot=self.relpath(self.petsc_dir,root) 606 pkg=relroot.split("/")[1] 607 fh.write(relroot+"\n") 608 allSrcs=[] 609 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 610 for exfile in dataDict[root]: 611 # Basic information 612 fullfile=os.path.join(root,exfile) 613 rfile=self.relpath(self.petsc_dir,fullfile) 614 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 615 fh.write(indent+exfile+indent*4+builtStatus+"\n") 616 617 for test in dataDict[root][exfile]: 618 if test in self.buildkeys: continue 619 line=indent*2+test 620 fh.write(line+"\n") 621 # Looks nice to have the keys in order 622 #for key in dataDict[root][exfile][test]: 623 for key in "isrun abstracted nsize args requires script".split(): 624 if not dataDict[root][exfile][test].has_key(key): continue 625 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 626 fh.write(line+"\n") 627 fh.write("\n") 628 fh.write("\n") 629 fh.write("\n") 630 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 631 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 632 fh.close() 633 return 634 635 def genPetscTests(self,root,dirs,files,dataDict): 636 """ 637 Go through and parse the source files in the directory to generate 638 the examples based on the metadata contained in the source files 639 """ 640 debug=False 641 # Use examplesAnalyze to get what the makefles think are sources 642 #self.examplesAnalyze(root,dirs,files,anlzDict) 643 644 dataDict[root]={} 645 646 for exfile in files: 647 #TST: Until we replace files, still leaving the orginals as is 648 #if not exfile.startswith("new_"+"ex"): continue 649 if not exfile.startswith("ex"): continue 650 651 # Convenience 652 fullex=os.path.join(root,exfile) 653 relpfile=self.relpath(self.petsc_dir,fullex) 654 if debug: print relpfile 655 dataDict[root].update(testparse.parseTestFile(fullex,0)) 656 # Need to check and make sure tests are in the file 657 # if verbosity>=1: print relpfile 658 if dataDict[root].has_key(exfile): 659 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 660 661 return 662 663 def walktree(self,top,action="printFiles"): 664 """ 665 Walk a directory tree, starting from 'top' 666 """ 667 #print "action", action 668 # Goal of action is to fill this dictionary 669 dataDict={} 670 for root, dirs, files in os.walk(top, topdown=False): 671 if not "examples" in root: continue 672 if not os.path.isfile(os.path.join(root,"makefile")): continue 673 bname=os.path.basename(root.rstrip("/")) 674 if bname=="tests" or bname=="tutorials": 675 eval("self."+action+"(root,dirs,files,dataDict)") 676 if type(top) != types.StringType: 677 raise TypeError("top must be a string") 678 # Now summarize this dictionary 679 eval("self."+action+"_summarize(dataDict)") 680 return dataDict 681 682 def gen_gnumake(self, fd): 683 """ 684 Overwrite of the method in the base PETSc class 685 """ 686 def write(stem, srcs): 687 for lang in LANGS: 688 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 689 for pkg in PKGS: 690 srcs = self.gen_pkg(pkg) 691 write('testsrcs-' + pkg, srcs) 692 return self.gendeps 693 694 def gen_pkg(self, pkg): 695 """ 696 Overwrite of the method in the base PETSc class 697 """ 698 return self.sources[pkg] 699 700 def write_gnumake(self,dataDict): 701 """ 702 Write out something similar to files from gmakegen.py 703 704 There is not a lot of has_key type checking because 705 should just work and need to know if there are bugs 706 707 Test depends on script which also depends on source 708 file, but since I don't have a good way generating 709 acting on a single file (oops) just depend on 710 executable which in turn will depend on src file 711 """ 712 # Different options for how to set up the targets 713 compileExecsFirst=False 714 715 # Open file 716 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 717 fd = open(arch_files, 'w') 718 719 # Write out the sources 720 gendeps = self.gen_gnumake(fd) 721 722 # Write out the tests and execname targets 723 fd.write("\n#Tests and executables\n") # Delimiter 724 725 for pkg in PKGS: 726 # These grab the ones that are built 727 for lang in LANGS: 728 testdeps=[] 729 for ftest in self.tests[pkg][lang]: 730 test=os.path.basename(ftest) 731 basedir=os.path.dirname(ftest) 732 testdeps.append(self.nameSpace(test,basedir)) 733 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 734 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 735 736 # test targets 737 for ftest in self.tests[pkg][lang]: 738 test=os.path.basename(ftest) 739 basedir=os.path.dirname(ftest) 740 testdir="${TESTDIR}/"+basedir+"/" 741 nmtest=self.nameSpace(test,basedir) 742 rundir=os.path.join(testdir,test) 743 #print test, nmtest 744 script=test+".sh" 745 746 # Deps 747 exfile=self.tests[pkg][lang][ftest]['exfile'] 748 fullex=os.path.join(self.petsc_dir,exfile) 749 localexec=self.tests[pkg][lang][ftest]['exec'] 750 execname=os.path.join(testdir,localexec) 751 fullscript=os.path.join(testdir,script) 752 tmpfile=os.path.join(testdir,test,test+".tmp") 753 754 # *.counts depends on the script and either executable (will 755 # be run) or the example source file (SKIP or TODO) 756 fd.write('%s.counts : %s %s\n' 757 % (os.path.join('$(TESTDIR)/counts', nmtest), 758 fullscript, 759 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 760 # Now write the args: 761 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 762 763 fd.close() 764 return 765 766 def writeHarness(self,output,dataDict): 767 """ 768 This is set up to write out multiple harness even if only gnumake 769 is supported now 770 """ 771 eval("self.write_"+output+"(dataDict)") 772 return 773 774def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 775 if output is None: 776 output = 'gnumake' 777 778 779 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 780 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 781 pEx.writeHarness(output,dataDict) 782 783if __name__ == '__main__': 784 import optparse 785 parser = optparse.OptionParser() 786 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 787 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 788 parser.add_option('--output', help='Location to write output file', default=None) 789 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 790 opts, extra_args = parser.parse_args() 791 if extra_args: 792 import sys 793 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 794 exit(1) 795 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 796