1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 for key in lkeys: 107 if type(inDict[key])!=types.StringType: continue 108 keystr = str(inDict[key]) 109 akey=('subargs' if key=='args' else key) # what to assign 110 if not inDict.has_key(akey): inDict[akey]='' 111 varlist=[] 112 for varset in re.split('-(?=[a-zA-Z])',keystr): 113 if not varset.strip(): continue 114 if len(re.findall('{{(.*?)}(?:[\w\s]*)?}',varset))>0: 115 if not loopVars.has_key(akey): loopVars[akey]={} 116 # Assuming only one for loop per var specification 117 keyvar=varset.split("{{")[0].strip() 118 if not keyvar.strip(): 119 if key=='nsize': 120 keyvar='nsize' 121 else: 122 print testname+": Problem in finding loop variables", keyvar, varset 123 varlist.append(keyvar) 124 loopVars[akey][keyvar]=[keyvar,re.findall('{{(.*?)}(?:[\w\s]*)?}',varset)[0]] 125 if akey=='nsize': 126 inDict[akey] = '${' + keyvar + '}' 127 lsuffix+=akey+'-'+inDict[akey]+'_' 128 else: 129 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 130 lsuffix+=keyvar+'-${' + keyvar + '}_' 131 else: 132 if key=='args': newargs+=" -"+varset.strip() 133 if len(varlist)>0: loopVars[akey]['varlist']=varlist 134 135 136 # For subtests, args are always substituted in (not top level) 137 if isSubtest: 138 inDict['subargs']+=" "+newargs.strip() 139 inDict['args']='' 140 if inDict.has_key('label_suffix'): 141 inDict['label_suffix']+=lsuffix.rstrip('_') 142 else: 143 inDict['label_suffix']=lsuffix.rstrip('_') 144 else: 145 if len(loopVars.keys())>0: 146 inDict['args']=newargs.strip() 147 inDict['label_suffix']=lsuffix.rstrip('_') 148 if len(loopVars.keys())>0: 149 return loopVars 150 else: 151 return None 152 153 def getArgLabel(self,testDict): 154 """ 155 In all of the arguments in the test dictionary, create a simple 156 string for searching within the makefile system. For simplicity in 157 search, remove "-", for strings, etc. 158 Also, concatenate the arg commands 159 For now, ignore nsize -- seems hard to search for anyway 160 """ 161 # Collect all of the args associated with a test 162 argStr=("" if not testDict.has_key('args') else testDict['args']) 163 if testDict.has_key('subtests'): 164 for stest in testDict["subtests"]: 165 sd=testDict[stest] 166 argStr=argStr+("" if not sd.has_key('args') else sd['args']) 167 168 # Now go through and cleanup 169 argStr=re.sub('{{(.*?)}}',"",argStr) 170 argStr=re.sub('-'," ",argStr) 171 for digit in string.digits: argStr=re.sub(digit," ",argStr) 172 argStr=re.sub("\.","",argStr) 173 argStr=re.sub(",","",argStr) 174 argStr=re.sub('\+',' ',argStr) 175 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 176 return argStr.strip() 177 178 def addToSources(self,exfile,root,srcDict): 179 """ 180 Put into data structure that allows easy generation of makefile 181 """ 182 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 183 fullfile=os.path.join(root,exfile) 184 relpfile=self.relpath(self.petsc_dir,fullfile) 185 lang=self.getLanguage(exfile) 186 if not lang: return 187 self.sources[pkg][lang]['srcs'].append(relpfile) 188 if srcDict.has_key('depends'): 189 depSrc=srcDict['depends'] 190 depObj=os.path.splitext(depSrc)[0]+".o" 191 self.sources[pkg][lang][exfile]=depObj 192 193 # In gmakefile, ${TESTDIR} var specifies the object compilation 194 testsdir=self.relpath(self.petsc_dir,root)+"/" 195 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 196 self.objects[pkg].append(objfile) 197 return 198 199 def addToTests(self,test,root,exfile,execname,testDict): 200 """ 201 Put into data structure that allows easy generation of makefile 202 Organized by languages to allow testing of languages 203 """ 204 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 205 #nmtest=self.nameSpace(test,root) 206 rpath=self.relpath(self.petsc_dir,root) 207 nmtest=os.path.join(rpath,test) 208 lang=self.getLanguage(exfile) 209 if not lang: return 210 self.tests[pkg][lang][nmtest]={} 211 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 212 self.tests[pkg][lang][nmtest]['exec']=execname 213 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 214 return 215 216 def getExecname(self,exfile,root): 217 """ 218 Generate bash script using template found next to this file. 219 This file is read in at constructor time to avoid file I/O 220 """ 221 rpath=self.relpath(self.petsc_dir,root) 222 if self.single_ex: 223 execname=rpath.split("/")[1]+"-ex" 224 else: 225 execname=os.path.splitext(exfile)[0] 226 return execname 227 228 def getSubstVars(self,testDict,rpath,testname): 229 """ 230 Create a dictionary with all of the variables that get substituted 231 into the template commands found in example_template.py 232 """ 233 subst={} 234 235 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 236 if not testDict.has_key('nsize'): testDict['nsize']=1 237 for ak in testparse.acceptedkeys: 238 if ak=='test': continue 239 subst[ak]=(testDict[ak] if testDict.has_key(ak) else '') 240 241 # Now do other variables 242 subst['execname']=testDict['execname'] 243 if testDict.has_key('filter'): 244 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 245 246 # Output file is special because of subtests override 247 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 248 subst['defroot']=defroot 249 if not "_" in defroot: defroot=defroot+"_1" 250 if not testDict.has_key('output_file'): 251 subst['output_file']="output/"+defroot+".out" 252 subst['redirect_file']=defroot+".tmp" 253 254 # Others 255 subst['subargs']='' # Default. For variables override 256 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 257 subst['label']=self.nameSpace(defroot,subst['srcdir']) 258 subst['label_suffix']='' 259 subst['comments']="\n#".join(subst['comments'].split("\n")) 260 if subst['comments']: subst['comments']="#"+subst['comments'] 261 subst['exec']="../"+subst['execname'] 262 subst['testroot']=self.testroot_dir 263 subst['testname']=testname 264 if self.conf['PETSC_HAVE_DATAFILESPATH']: 265 dp=self.conf['DATAFILESPATH'] 266 dpl='DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 267 else: 268 dpl='' 269 subst['datafilespath_line']=dpl 270 271 # This is used to label some matrices 272 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 273 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 274 275 # These can have for loops and are treated separately later 276 subst['nsize']=str(subst['nsize']) 277 278 #Conf vars 279 if self.petsc_arch.find('valgrind')>=0: 280 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 281 else: 282 subst['mpiexec']=self.conf['MPIEXEC'] 283 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 284 subst['diff']=self.conf['DIFF'] 285 subst['rm']=self.conf['RM'] 286 subst['grep']=self.conf['GREP'] 287 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 288 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 289 290 return subst 291 292 def getCmds(self,subst,i): 293 """ 294 Generate bash script using template found next to this file. 295 This file is read in at constructor time to avoid file I/O 296 """ 297 indnt=self.indent 298 nindnt=i # the start and has to be consistent with below 299 cmdLines="" 300 301 # Add in the full path here. 302 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 303 304 # MPI is the default -- but we have a few odd commands 305 if not subst['command']: 306 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 307 else: 308 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 309 cmdLines+=cmd+"\n\n" 310 311 if not subst['filter_output']: 312 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 313 else: 314 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 315 cmdLines+=cmd+"\n" 316 return cmdLines 317 318 def _substVars(self,subst,origStr): 319 """ 320 Substitute variables 321 """ 322 Str=origStr 323 for subkey in subst: 324 if type(subst[subkey])!=types.StringType: continue 325 patt="@"+subkey.upper()+"@" 326 Str=re.sub(patt,subst[subkey],Str) 327 return Str 328 329 def _writeTodoSkip(self,fh,tors,inDict,footer): 330 """ 331 Write out the TODO and SKIP lines in the file 332 The TODO or SKIP variable, tors, should be lower case 333 """ 334 TORS=tors.upper() 335 template=eval("example_template."+tors+"line") 336 tsStr=re.sub("@"+TORS+"COMMENT@",inDict[TORS],template) 337 fh.write(tsStr+"\ntotal=1; "+tors+"=1\n") 338 fh.write(footer+"\n") 339 fh.write("exit\n\n\n") 340 return 341 342 def getLoopVarsHead(self,loopVars,i): 343 """ 344 Generate a nicely indented string with the format loops 345 Here is what the data structure looks like 346 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 347 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 348 loopVars['subargs']['pc_type']=["j","cholesky sor"] 349 """ 350 outstr=''; indnt=self.indent 351 for key in loopVars: 352 for var in loopVars[key]['varlist']: 353 varval=loopVars[key][var] 354 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 355 i = i + 1 356 return (outstr,i) 357 358 def getLoopVarsFoot(self,loopVars,i): 359 outstr=''; indnt=self.indent 360 for key in loopVars: 361 for var in loopVars[key]['varlist']: 362 i = i - 1 363 outstr += indnt * i + "done\n" 364 return (outstr,i) 365 366 def genRunScript(self,testname,root,isRun,srcDict): 367 """ 368 Generate bash script using template found next to this file. 369 This file is read in at constructor time to avoid file I/O 370 """ 371 # runscript_dir directory has to be consistent with gmakefile 372 testDict=srcDict[testname] 373 rpath=self.relpath(self.petsc_dir,root) 374 runscript_dir=os.path.join(self.testroot_dir,rpath) 375 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 376 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 377 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 378 379 # Get variables to go into shell scripts. last time testDict used 380 subst=self.getSubstVars(testDict,rpath,testname) 381 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 382 #if '33_' in testname: print subst['subargs'] 383 384 #Handle runfiles 385 if subst.has_key('localrunfiles'): 386 for lfile in subst['localrunfiles'].split(): 387 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 388 shutil.copy(fullfile,runscript_dir) 389 # Check subtests for local runfiles 390 if subst.has_key("subtests"): 391 for stest in subst["subtests"]: 392 if subst[stest].has_key('localrunfiles'): 393 for lfile in testDict[stest]['localrunfiles'].split(): 394 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 395 shutil.copy(fullfile,self.runscript_dir) 396 397 # Now substitute the key variables into the header and footer 398 header=self._substVars(subst,example_template.header) 399 # The header is done twice to enable @...@ in header 400 header=self._substVars(subst,header) 401 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 402 403 # Start writing the file 404 fh.write(header+"\n") 405 406 # If there is a TODO or a SKIP then we do it before writing out the 407 # rest of the command (which is useful for working on the test) 408 # SKIP and TODO can be for the source file or for the runs 409 for tors in "todo skip".split(): 410 for d in [srcDict,testDict]: 411 if d.has_key(tors.upper()): self._writeTodoSkip(fh,tors,d,footer) 412 413 j=0 # for indentation 414 415 if loopVars: 416 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 417 if (loopHead): fh.write(loopHead+"\n") 418 419 # Subtests are special 420 if testDict.has_key("subtests"): 421 substP=subst # Subtests can inherit args but be careful 422 for stest in testDict["subtests"]: 423 subst=substP.copy() 424 subst.update(testDict[stest]) 425 subst['nsize']=str(subst['nsize']) 426 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 427 #if '10_9' in testname: print sLoopVars 428 if sLoopVars: 429 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 430 fh.write(sLoopHead+"\n") 431 fh.write(self.getCmds(subst,j)+"\n") 432 if sLoopVars: 433 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 434 fh.write(sLoopFoot+"\n") 435 else: 436 fh.write(self.getCmds(subst,j)+"\n") 437 438 if loopVars: 439 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 440 fh.write(loopFoot+"\n") 441 442 fh.write(footer+"\n") 443 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 444 #if '10_9' in testname: sys.exit() 445 return 446 447 def genScriptsAndInfo(self,exfile,root,srcDict): 448 """ 449 Generate scripts from the source file, determine if built, etc. 450 For every test in the exfile with info in the srcDict: 451 1. Determine if it needs to be run for this arch 452 2. Generate the script 453 3. Generate the data needed to write out the makefile in a 454 convenient way 455 All tests are *always* run, but some may be SKIP'd per the TAP standard 456 """ 457 debug=False 458 fileIsTested=False 459 execname=self.getExecname(exfile,root) 460 isBuilt=self._isBuilt(exfile,srcDict) 461 for test in srcDict: 462 if test in self.buildkeys: continue 463 if debug: print self.nameSpace(exfile,root), test 464 srcDict[test]['execname']=execname # Convenience in generating scripts 465 isRun=self._isRun(srcDict[test]) 466 self.genRunScript(test,root,isRun,srcDict) 467 srcDict[test]['isrun']=isRun 468 if isRun: fileIsTested=True 469 self.addToTests(test,root,exfile,execname,srcDict[test]) 470 471 # This adds to datastructure for building deps 472 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 473 #print self.nameSpace(exfile,root), fileIsTested 474 return 475 476 def _isBuilt(self,exfile,srcDict): 477 """ 478 Determine if this file should be built. 479 """ 480 # Get the language based on file extension 481 lang=self.getLanguage(exfile) 482 if (lang=="F" or lang=="F90") and not self.have_fortran: 483 srcDict["SKIP"]="Fortran required for this test" 484 return False 485 if lang=="cu" and not self.conf.has_key('PETSC_HAVE_CUDA'): 486 srcDict["SKIP"]="CUDA required for this test" 487 return False 488 if lang=="cxx" and not self.conf.has_key('PETSC_HAVE_CXX'): 489 srcDict["SKIP"]="C++ required for this test" 490 return False 491 492 # Deprecated source files 493 if srcDict.has_key("TODO"): return False 494 495 # isRun can work with srcDict to handle the requires 496 if srcDict.has_key("requires"): 497 if len(srcDict["requires"])>0: 498 return self._isRun(srcDict) 499 500 return True 501 502 503 def _isRun(self,testDict): 504 """ 505 Based on the requirements listed in the src file and the petscconf.h 506 info, determine whether this test should be run or not. 507 """ 508 indent=" " 509 debug=False 510 511 # MPI requirements 512 if testDict.has_key('nsize'): 513 if testDict['nsize']>1 and self.conf.has_key('MPI_IS_MPIUNI'): 514 if debug: print indent+"Cannot run parallel tests" 515 testDict['SKIP']="Parallel test with serial build" 516 return False 517 518 # The requirements for the test are the sum of all the run subtests 519 if testDict.has_key('subtests'): 520 if not testDict.has_key('requires'): testDict['requires']="" 521 for stest in testDict['subtests']: 522 if testDict[stest].has_key('requires'): 523 testDict['requires']=testDict['requires']+" "+testDict[stest]['requires'] 524 525 526 # Now go through all requirements 527 if testDict.has_key('requires'): 528 for requirement in testDict['requires'].split(): 529 requirement=requirement.strip() 530 if not requirement: continue 531 if debug: print indent+"Requirement: ", requirement 532 isNull=False 533 if requirement.startswith("!"): 534 requirement=requirement[1:]; isNull=True 535 # Precision requirement for reals 536 if requirement in self.precision_types: 537 if self.conf['PETSC_PRECISION']==requirement: 538 testDict['SKIP']="not "+requirement+" required" 539 if isNull: return False 540 else: 541 testDict['SKIP']=requirement+" required" 542 return False 543 # Precision requirement for ints 544 if requirement in self.integer_types: 545 if requirement=="int32": 546 if self.conf['PETSC_SIZEOF_INT']==4: 547 testDict['SKIP']="not int32 required" 548 if isNull: return False 549 else: 550 testDict['SKIP']="int32 required" 551 return False 552 if requirement=="int64": 553 if self.conf['PETSC_SIZEOF_INT']==8: 554 testDict['SKIP']="NOT int64 required" 555 if isNull: return False 556 else: 557 testDict['SKIP']="int64 required" 558 return False 559 # Datafilespath 560 if requirement=="datafilespath": 561 if not self.conf['PETSC_HAVE_DATAFILESPATH']: 562 testDict['SKIP']="Requires DATAFILESPATH" 563 return False 564 # Defines -- not sure I have comments matching 565 if "define(" in requirement.lower(): 566 reqdef=requirement.split("(")[1].split(")")[0] 567 val=(reqdef.split()[1] if " " in reqdef else "") 568 if self.conf.has_key(reqdef): 569 if val: 570 if self.conf[reqdef]==val: 571 if isNull: 572 testDict['SKIP']="Null requirement not met: "+requirement 573 return False 574 else: 575 testDict['SKIP']="Required: "+requirement 576 return False 577 else: 578 if isNull: 579 testDict['SKIP']="Null requirement not met: "+requirement 580 return False 581 else: 582 return True 583 else: 584 testDict['SKIP']="Requirement not met: "+requirement 585 return False 586 587 # Rest should be packages that we can just get from conf 588 if requirement == "complex": petscconfvar="PETSC_USE_COMPLEX" 589 else: petscconfvar="PETSC_HAVE_"+requirement.upper() 590 if self.conf.get(petscconfvar): 591 if isNull: 592 testDict['SKIP']="Not "+petscconfvar+" requirement not met" 593 return False 594 elif not isNull: 595 if debug: print "requirement not found: ", requirement 596 testDict['SKIP']=petscconfvar+" requirement not met" 597 return False 598 599 return True 600 601 def genPetscTests_summarize(self,dataDict): 602 """ 603 Required method to state what happened 604 """ 605 if not self.summarize: return 606 indent=" " 607 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 608 fh=open(fhname,"w") 609 #print "See ", fhname 610 for root in dataDict: 611 relroot=self.relpath(self.petsc_dir,root) 612 pkg=relroot.split("/")[1] 613 fh.write(relroot+"\n") 614 allSrcs=[] 615 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 616 for exfile in dataDict[root]: 617 # Basic information 618 fullfile=os.path.join(root,exfile) 619 rfile=self.relpath(self.petsc_dir,fullfile) 620 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 621 fh.write(indent+exfile+indent*4+builtStatus+"\n") 622 623 for test in dataDict[root][exfile]: 624 if test in self.buildkeys: continue 625 line=indent*2+test 626 fh.write(line+"\n") 627 # Looks nice to have the keys in order 628 #for key in dataDict[root][exfile][test]: 629 for key in "isrun abstracted nsize args requires script".split(): 630 if not dataDict[root][exfile][test].has_key(key): continue 631 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 632 fh.write(line+"\n") 633 fh.write("\n") 634 fh.write("\n") 635 fh.write("\n") 636 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 637 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 638 fh.close() 639 return 640 641 def genPetscTests(self,root,dirs,files,dataDict): 642 """ 643 Go through and parse the source files in the directory to generate 644 the examples based on the metadata contained in the source files 645 """ 646 debug=False 647 # Use examplesAnalyze to get what the makefles think are sources 648 #self.examplesAnalyze(root,dirs,files,anlzDict) 649 650 dataDict[root]={} 651 652 for exfile in files: 653 #TST: Until we replace files, still leaving the orginals as is 654 #if not exfile.startswith("new_"+"ex"): continue 655 if not exfile.startswith("ex"): continue 656 657 # Convenience 658 fullex=os.path.join(root,exfile) 659 relpfile=self.relpath(self.petsc_dir,fullex) 660 if debug: print relpfile 661 dataDict[root].update(testparse.parseTestFile(fullex,0)) 662 # Need to check and make sure tests are in the file 663 # if verbosity>=1: print relpfile 664 if dataDict[root].has_key(exfile): 665 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 666 667 return 668 669 def walktree(self,top,action="printFiles"): 670 """ 671 Walk a directory tree, starting from 'top' 672 """ 673 #print "action", action 674 # Goal of action is to fill this dictionary 675 dataDict={} 676 for root, dirs, files in os.walk(top, topdown=False): 677 if not "examples" in root: continue 678 if not os.path.isfile(os.path.join(root,"makefile")): continue 679 bname=os.path.basename(root.rstrip("/")) 680 if bname=="tests" or bname=="tutorials": 681 eval("self."+action+"(root,dirs,files,dataDict)") 682 if type(top) != types.StringType: 683 raise TypeError("top must be a string") 684 # Now summarize this dictionary 685 eval("self."+action+"_summarize(dataDict)") 686 return dataDict 687 688 def gen_gnumake(self, fd): 689 """ 690 Overwrite of the method in the base PETSc class 691 """ 692 def write(stem, srcs): 693 for lang in LANGS: 694 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 695 for pkg in PKGS: 696 srcs = self.gen_pkg(pkg) 697 write('testsrcs-' + pkg, srcs) 698 return self.gendeps 699 700 def gen_pkg(self, pkg): 701 """ 702 Overwrite of the method in the base PETSc class 703 """ 704 return self.sources[pkg] 705 706 def write_gnumake(self,dataDict): 707 """ 708 Write out something similar to files from gmakegen.py 709 710 There is not a lot of has_key type checking because 711 should just work and need to know if there are bugs 712 713 Test depends on script which also depends on source 714 file, but since I don't have a good way generating 715 acting on a single file (oops) just depend on 716 executable which in turn will depend on src file 717 """ 718 # Different options for how to set up the targets 719 compileExecsFirst=False 720 721 # Open file 722 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 723 fd = open(arch_files, 'w') 724 725 # Write out the sources 726 gendeps = self.gen_gnumake(fd) 727 728 # Write out the tests and execname targets 729 fd.write("\n#Tests and executables\n") # Delimiter 730 731 for pkg in PKGS: 732 # These grab the ones that are built 733 for lang in LANGS: 734 testdeps=[] 735 for ftest in self.tests[pkg][lang]: 736 test=os.path.basename(ftest) 737 basedir=os.path.dirname(ftest) 738 testdeps.append(self.nameSpace(test,basedir)) 739 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 740 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 741 742 # test targets 743 for ftest in self.tests[pkg][lang]: 744 test=os.path.basename(ftest) 745 basedir=os.path.dirname(ftest) 746 testdir="${TESTDIR}/"+basedir+"/" 747 nmtest=self.nameSpace(test,basedir) 748 rundir=os.path.join(testdir,test) 749 #print test, nmtest 750 script=test+".sh" 751 752 # Deps 753 exfile=self.tests[pkg][lang][ftest]['exfile'] 754 fullex=os.path.join(self.petsc_dir,exfile) 755 localexec=self.tests[pkg][lang][ftest]['exec'] 756 execname=os.path.join(testdir,localexec) 757 fullscript=os.path.join(testdir,script) 758 tmpfile=os.path.join(testdir,test,test+".tmp") 759 760 # *.counts depends on the script and either executable (will 761 # be run) or the example source file (SKIP or TODO) 762 fd.write('%s.counts : %s %s\n' 763 % (os.path.join('$(TESTDIR)/counts', nmtest), 764 fullscript, 765 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 766 # Now write the args: 767 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 768 769 fd.close() 770 return 771 772 def writeHarness(self,output,dataDict): 773 """ 774 This is set up to write out multiple harness even if only gnumake 775 is supported now 776 """ 777 eval("self.write_"+output+"(dataDict)") 778 return 779 780def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 781 if output is None: 782 output = 'gnumake' 783 784 785 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 786 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 787 pEx.writeHarness(output,dataDict) 788 789if __name__ == '__main__': 790 import optparse 791 parser = optparse.OptionParser() 792 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 793 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 794 parser.add_option('--output', help='Location to write output file', default=None) 795 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 796 opts, extra_args = parser.parse_args() 797 if extra_args: 798 import sys 799 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 800 exit(1) 801 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 802