1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 return 60 61 def nameSpace(self,srcfile,srcdir): 62 """ 63 Because the scripts have a non-unique naming, the pretty-printing 64 needs to convey the srcdir and srcfile. There are two ways of doing this. 65 """ 66 if self.ptNaming: 67 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 68 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 69 nameString=prefix+srcfile 70 else: 71 #nameString=srcdir+": "+srcfile 72 nameString=srcfile 73 return nameString 74 75 def getLanguage(self,srcfile): 76 """ 77 Based on the source, determine associated language as found in gmakegen.LANGS 78 Can we just return srcext[1:\] now? 79 """ 80 langReq=None 81 srcext=os.path.splitext(srcfile)[-1] 82 if srcext in ".F90".split(): langReq="F90" 83 if srcext in ".F".split(): langReq="F" 84 if srcext in ".cxx".split(): langReq="cxx" 85 if srcext == ".cu": langReq="cu" 86 if srcext == ".c": langReq="c" 87 #if not langReq: print "ERROR: ", srcext, srcfile 88 return langReq 89 90 def _getLoopVars(self,testDict,i): 91 """ 92 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 93 Return: 94 args: -ksp_monitor 95 loopVars[bs]['val']="1 2 3 4 5" 96 loopVars[pc_type]['val']="cholesky sor" 97 """ 98 loopVars={} 99 newargs="" 100 for key in testDict: 101 if type(testDict[key])!=types.StringType: continue 102 keystr = str(testDict[key]) 103 for varset in re.split('-(?=[a-zA-Z])',keystr): 104 if not varset.strip(): continue 105 if len(re.findall('{{(.*?)}}',varset))>0: 106 # Assuming only one for loop per var specification 107 forvar=varset.split("{{")[0].strip() 108 lvar=string.ascii_lowercase[i]; i+=1 109 loopVars[key]={} 110 #print key, testDict 111 sys.exit() 112 loopVars[key][forvar]={} 113 loopVars[key][forvar]=re.findall('{{(.*?)}}',varset)[0] 114 akey=('subargs' if key=='args' else key) 115 testDict[akey] = '${' + lvar + '}' 116 else: 117 if key=='args': newargs+="-"+varset+" " 118 119 testDict['args']=newargs 120 121 return loopVars 122 123 def getArgLabel(self,testDict): 124 """ 125 In all of the arguments in the test dictionary, create a simple 126 string for searching within the makefile system. For simplicity in 127 search, remove "-", for strings, etc. 128 Also, concatenate the arg commands 129 For now, ignore nsize -- seems hard to search for anyway 130 """ 131 # Collect all of the args associated with a test 132 argStr=("" if not testDict.has_key('args') else testDict['args']) 133 if testDict.has_key('subtests'): 134 for stest in testDict["subtests"]: 135 sd=testDict[stest] 136 argStr=argStr+("" if not sd.has_key('args') else sd['args']) 137 138 # Now go through and cleanup 139 argStr=re.sub('{{(.*?)}}',"",argStr) 140 argStr=re.sub('-'," ",argStr) 141 for digit in string.digits: argStr=re.sub(digit," ",argStr) 142 argStr=re.sub("\.","",argStr) 143 argStr=re.sub(",","",argStr) 144 argStr=re.sub('\+',' ',argStr) 145 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 146 return argStr.strip() 147 148 def addToSources(self,exfile,root,srcDict): 149 """ 150 Put into data structure that allows easy generation of makefile 151 """ 152 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 153 fullfile=os.path.join(root,exfile) 154 relpfile=self.relpath(self.petsc_dir,fullfile) 155 lang=self.getLanguage(exfile) 156 if not lang: return 157 self.sources[pkg][lang]['srcs'].append(relpfile) 158 if srcDict.has_key('depends'): 159 depSrc=srcDict['depends'] 160 depObj=os.path.splitext(depSrc)[0]+".o" 161 self.sources[pkg][lang][exfile]=depObj 162 163 # In gmakefile, ${TESTDIR} var specifies the object compilation 164 testsdir=self.relpath(self.petsc_dir,root)+"/" 165 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 166 self.objects[pkg].append(objfile) 167 return 168 169 def addToTests(self,test,root,exfile,execname,testDict): 170 """ 171 Put into data structure that allows easy generation of makefile 172 Organized by languages to allow testing of languages 173 """ 174 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 175 #nmtest=self.nameSpace(test,root) 176 rpath=self.relpath(self.petsc_dir,root) 177 nmtest=os.path.join(rpath,test) 178 lang=self.getLanguage(exfile) 179 if not lang: return 180 self.tests[pkg][lang][nmtest]={} 181 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 182 self.tests[pkg][lang][nmtest]['exec']=execname 183 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 184 return 185 186 def getLines(self,forvar,fordict,i,j): 187 """ 188 Get the for and done lines 189 """ 190 indent=" " 191 lindex=string.ascii_lowercase[i] 192 forStr=fordict[forvar]['val'] 193 fordict[forvar]['loopvar']=lindex 194 forline=indent*j+"for "+lindex+" in "+forStr+"; do" 195 #subline=re.sub("{{"+loopvals+"}}","${"+lindex+"}",forStr) 196 subline="${"+lindex+"}" 197 donline=indent*j+"done" 198 return forline,subline,donline 199 200 def getFor(self,subst,i,j,parentD,subtest=False): 201 """ 202 Get the for and done lines 203 """ 204 forlines="" 205 donlines="" 206 nsizeStr=subst['nsize'] 207 ofname="" 208 if "{{" in subst['nsize']: 209 nd={'nsize':{'val':subst['nsize']}} 210 fline,sline,dline=self.getLines('nsize',nd,i,j) 211 forlines=fline+"\n"; donlines=dline+"\n" 212 i=i+1; j=j+1 213 subst['nsize']=sline 214 215 if not "{{" in subst['args']: return forlines,donlines,i,j,{} 216 217 # Now work on args 218 forlist,fordict,newargs=testparse.extractForVars(subst['args']) 219 if not subtest: subst['args']=newargs.strip() # promote to control arguments 220 forargs="" 221 for farg in forlist: 222 if farg in parentD.keys(): 223 subline="${"+parentD[farg]['loopvar']+"}" 224 ofname+=farg+"-"+subline+"-" 225 else: 226 forline,subline,donline=self.getLines(farg,fordict,i,j) 227 forlines+=forline+"\n"; donlines+=donline+"\n" 228 forargs=forargs+" -"+farg+" "+subline 229 ofname=ofname+farg+"-"+subline+"-" 230 i=i+1; j=j+1 231 subst['forargs']=forargs.strip() 232 if subtest: subst['forargs']+=" "+newargs 233 # Do not overwrite the output file name if it is in source 234 if not subst['output_file_inSrc']: 235 subst['output_file']="output/"+subst['defroot']+"_"+ofname.strip('-')+".out" 236 subst['redirect_file']=subst['defroot']+"_"+ofname.strip('-')+".tmp" 237 subst['label_suffix']="_"+ofname.strip('-') 238 239 # The do lines have reverse order with respect to indentation 240 dl=donlines.rstrip("\n").split("\n") 241 dl.reverse() 242 donlines="\n".join(dl)+"\n" 243 244 return forlines,donlines,i,j,fordict 245 246 247 def getExecname(self,exfile,root): 248 """ 249 Generate bash script using template found next to this file. 250 This file is read in at constructor time to avoid file I/O 251 """ 252 rpath=self.relpath(self.petsc_dir,root) 253 if self.single_ex: 254 execname=rpath.split("/")[1]+"-ex" 255 else: 256 execname=os.path.splitext(exfile)[0] 257 return execname 258 259 def getSubstVars(self,testDict,rpath,testname): 260 """ 261 Create a dictionary with all of the variables that get substituted 262 into the template commands found in example_template.py 263 TODO: Cleanup 264 """ 265 subst={} 266 # Handle defaults 267 if not testDict.has_key('nsize'): testDict['nsize']=1 268 if not testDict.has_key('filter'): testDict['filter']="" 269 if not testDict.has_key('filter_output'): testDict['filter_output']="" 270 if not testDict.has_key('localrunfiles'): testDict['localrunfiles']="" 271 if not testDict.has_key('args'): testDict['args']="" 272 if not testDict.has_key('forargs'): testDict['forargs']="" 273 if not testDict.has_key('comments'): testDict['comments']="" 274 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 275 if not "_" in defroot: defroot=defroot+"_1" 276 if not testDict.has_key('redirect_file'): testDict['redirect_file']=defroot+".tmp" 277 278 # Output file is special because of subtests override 279 if testDict.has_key('output_file'): 280 subst['output_file']=testDict['output_file'] 281 subst['output_file_inSrc']=True 282 else: 283 subst['output_file']="output/"+defroot+".out" 284 subst['output_file_inSrc']=False 285 286 # Setup the variables in template_string that need to be substituted 287 subst['defroot']=defroot 288 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 289 subst['label']=self.nameSpace(defroot,subst['srcdir']) 290 subst['label_suffix']='' 291 subst['redirect_file']=testDict['redirect_file'] 292 subst['comments']="\n#".join(testDict['comments'].split("\n")) 293 if subst['comments']: subst['comments']="#"+subst['comments'] 294 subst['exec']="../"+testDict['execname'] 295 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky 296 subst['filter_output']=testDict['filter_output'] 297 subst['localrunfiles']=testDict['localrunfiles'] 298 subst['testroot']=self.testroot_dir 299 subst['testname']=testname 300 if self.conf['PETSC_HAVE_DATAFILESPATH']: 301 dp=self.conf['DATAFILESPATH'] 302 dpl='DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 303 else: 304 dpl='' 305 subst['datafilespath_line']=dpl 306 307 # Be careful with this 308 if testDict.has_key('command'): subst['command']=testDict['command'] 309 310 # These can have for loops and are treated separately later 311 if testDict.has_key('nsize'): subst['nsize']=str(testDict['nsize']) 312 if testDict.has_key('args'): subst['args']=testDict['args'] 313 314 #Conf vars 315 if self.petsc_arch.find('valgrind')>=0: 316 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 317 else: 318 subst['mpiexec']=self.conf['MPIEXEC'] 319 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 320 subst['diff']=self.conf['DIFF'] 321 subst['rm']=self.conf['RM'] 322 subst['grep']=self.conf['GREP'] 323 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 324 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 325 326 for key in subst: 327 if type(subst[key])!=types.StringType: continue 328 subst[key] = self._substVars(testDict,subst[key]) 329 330 return subst 331 332 def getCmds(self,subst,i,subtest=False): 333 """ 334 Generate bash script using template found next to this file. 335 This file is read in at constructor time to avoid file I/O 336 """ 337 indent=" " 338 nindent=i # the start and has to be consistent with below 339 cmdLines="" 340 341 # Add in the full path here. 342 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 343 344 # MPI is the default -- but we have a few odd commands 345 if not subst.has_key('command'): 346 cmd=indent*nindent+self._substVars(subst,example_template.mpitest) 347 else: 348 cmd=indent*nindent+self._substVars(subst,example_template.commandtest) 349 cmdLines+=cmd+"\n\n" 350 351 if not subst['filter_output']: 352 cmd=indent*nindent+self._substVars(subst,example_template.difftest) 353 else: 354 cmd=indent*nindent+self._substVars(subst,example_template.filterdifftest) 355 cmdLines+=cmd+"\n" 356 return cmdLines 357 358 def _substVars(self,subst,origStr): 359 """ 360 Substitute varial 361 """ 362 Str=origStr 363 for subkey in subst: 364 if type(subst[subkey])!=types.StringType: continue 365 patt="@"+subkey.upper()+"@" 366 Str=re.sub(patt,subst[subkey],Str) 367 return Str 368 369 def _writeTodoSkip(self,fh,tors,inDict,footer): 370 """ 371 Write out the TODO and SKIP lines in the file 372 The TODO or SKIP variable, tors, should be lower case 373 """ 374 TORS=tors.upper() 375 template=eval("example_template."+tors+"line") 376 tsStr=re.sub("@"+TORS+"COMMENT@",inDict[TORS],template) 377 fh.write(tsStr+"\ntotal=1; "+tors+"=1\n") 378 fh.write(footer+"\n") 379 fh.write("exit\n\n\n") 380 return 381 382 def getLoopVarsHead(self,loopVars,i): 383 outstr='' 384 indent=" " 385 for key in loopVars: 386 newstr = indent * i + "for {0} in {1}; do\n".format(key,' '.join(['"'+val+'"' for val in loopVars[key]])) 387 outstr = outstr + newstr 388 i = i + 1 389 return (outstr,i) 390 391 def getLoopVarsFoot(self,loopVars,i): 392 outstr='' 393 indent=" " 394 for key in loopVars: 395 i = i - 1 396 newstr = indent * i + "done\n" 397 outstr = outstr + newstr 398 return (outstr,i) 399 400 def genRunScript(self,testname,root,isRun,srcDict): 401 """ 402 Generate bash script using template found next to this file. 403 This file is read in at constructor time to avoid file I/O 404 """ 405 print testname 406 # runscript_dir directory has to be consistent with gmakefile 407 testDict=srcDict[testname] 408 rpath=self.relpath(self.petsc_dir,root) 409 runscript_dir=os.path.join(self.testroot_dir,rpath) 410 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 411 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 412 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 413 414 subst=self.getSubstVars(testDict,rpath,testname) 415 416 #Handle runfiles 417 if subst['localrunfiles']: 418 for lfile in subst['localrunfiles'].split(): 419 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 420 shutil.copy(fullfile,runscript_dir) 421 # Check subtests for local runfiles 422 if testDict.has_key("subtests"): 423 for stest in testDict["subtests"]: 424 if testDict[stest].has_key('localrunfiles'): 425 for lfile in testDict[stest]['localrunfiles'].split(): 426 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 427 shutil.copy(fullfile,self.runscript_dir) 428 429 # Now substitute the key variables into the header and footer 430 header=self._substVars(subst,example_template.header) 431 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 432 433 # Start writing the file 434 fh.write(header+"\n") 435 436 # If there is a TODO or a SKIP then we do it before writing out the 437 # rest of the command (which is useful for working on the test) 438 # SKIP and TODO can be for the source file or for the runs 439 for tors in "todo skip".split(): 440 for d in [srcDict,testDict]: 441 if d.has_key(tors.upper()): self._writeTodoSkip(fh,tors,d,footer) 442 443 # Need to handle loops 444 i=8 # for loop counters 445 j=0 # for indentation 446 indent=" " 447 448 loopVars = self._getLoopVars(testDict,i) 449 if len(loopVars.keys())>0: print testname, loopVars 450 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 451 if (loopHead): fh.write(loopHead+"\n") 452 453 # Subtests are special 454 if testDict.has_key("subtests"): 455 substP=subst # Subtests can inherit args but be careful 456 for stest in testDict["subtests"]: 457 sdict = testDict[stest] 458 if not sdict.has_key('args'): sdict['args'] = "" 459 sLoopVars = self._getLoopVars(sdict,i) 460 subst=substP.copy() 461 subst.update(sdict) 462 if testDict[stest].has_key('output_file'): 463 subst['output_file']=os.path.join(subst['srcdir'],testDict[stest]['output_file']) 464 for key in subst: 465 if type(subst[key])!=types.StringType: continue 466 subst[key] = self._substVars(sdict,subst[key]) 467 subst[key] = self._substVars(testDict,subst[key]) 468 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 469 if (sLoopHead): fh.write(sLoopHead+"\n") 470 subst['subargs']=subst['args'] # Need to check here 471 fh.write(self.getCmds(subst,j,subtest=True)+"\n") 472 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 473 if (sLoopFoot): fh.write(sLoopFoot+"\n") 474 else: 475 fh.write(self.getCmds(subst,j)+"\n") 476 477 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 478 if (loopFoot): fh.write(loopFoot+"\n") 479 480 fh.write(footer+"\n") 481 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 482 return 483 484 def genScriptsAndInfo(self,exfile,root,srcDict): 485 """ 486 Generate scripts from the source file, determine if built, etc. 487 For every test in the exfile with info in the srcDict: 488 1. Determine if it needs to be run for this arch 489 2. Generate the script 490 3. Generate the data needed to write out the makefile in a 491 convenient way 492 All tests are *always* run, but some may be SKIP'd per the TAP standard 493 """ 494 debug=False 495 fileIsTested=False 496 execname=self.getExecname(exfile,root) 497 isBuilt=self._isBuilt(exfile,srcDict) 498 for test in srcDict: 499 if test in self.buildkeys: continue 500 if debug: print self.nameSpace(exfile,root), test 501 srcDict[test]['execname']=execname # Convenience in generating scripts 502 isRun=self._isRun(srcDict[test]) 503 self.genRunScript(test,root,isRun,srcDict) 504 srcDict[test]['isrun']=isRun 505 if isRun: fileIsTested=True 506 self.addToTests(test,root,exfile,execname,srcDict[test]) 507 508 # This adds to datastructure for building deps 509 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 510 #print self.nameSpace(exfile,root), fileIsTested 511 return 512 513 def _isBuilt(self,exfile,srcDict): 514 """ 515 Determine if this file should be built. 516 """ 517 # Get the language based on file extension 518 lang=self.getLanguage(exfile) 519 if (lang=="F" or lang=="F90") and not self.have_fortran: 520 srcDict["SKIP"]="Fortran required for this test" 521 return False 522 if lang=="cu" and not self.conf.has_key('PETSC_HAVE_CUDA'): 523 srcDict["SKIP"]="CUDA required for this test" 524 return False 525 if lang=="cxx" and not self.conf.has_key('PETSC_HAVE_CXX'): 526 srcDict["SKIP"]="C++ required for this test" 527 return False 528 529 # Deprecated source files 530 if srcDict.has_key("TODO"): return False 531 532 # isRun can work with srcDict to handle the requires 533 if srcDict.has_key("requires"): 534 if len(srcDict["requires"])>0: 535 return self._isRun(srcDict) 536 537 return True 538 539 540 def _isRun(self,testDict): 541 """ 542 Based on the requirements listed in the src file and the petscconf.h 543 info, determine whether this test should be run or not. 544 """ 545 indent=" " 546 debug=False 547 548 # MPI requirements 549 if testDict.has_key('nsize'): 550 if testDict['nsize']>1 and self.conf.has_key('MPI_IS_MPIUNI'): 551 if debug: print indent+"Cannot run parallel tests" 552 testDict['SKIP']="Parallel test with serial build" 553 return False 554 555 # The requirements for the test are the sum of all the run subtests 556 if testDict.has_key('subtests'): 557 if not testDict.has_key('requires'): testDict['requires']="" 558 for stest in testDict['subtests']: 559 if testDict[stest].has_key('requires'): 560 testDict['requires']=testDict['requires']+" "+testDict[stest]['requires'] 561 562 563 # Now go through all requirements 564 if testDict.has_key('requires'): 565 for requirement in testDict['requires'].split(): 566 requirement=requirement.strip() 567 if not requirement: continue 568 if debug: print indent+"Requirement: ", requirement 569 isNull=False 570 if requirement.startswith("!"): 571 requirement=requirement[1:]; isNull=True 572 # Precision requirement for reals 573 if requirement in self.precision_types: 574 if self.conf['PETSC_PRECISION']==requirement: 575 testDict['SKIP']="not "+requirement+" required" 576 if isNull: return False 577 else: 578 testDict['SKIP']=requirement+" required" 579 return False 580 # Precision requirement for ints 581 if requirement in self.integer_types: 582 if requirement=="int32": 583 if self.conf['PETSC_SIZEOF_INT']==4: 584 testDict['SKIP']="not int32 required" 585 if isNull: return False 586 else: 587 testDict['SKIP']="int32 required" 588 return False 589 if requirement=="int64": 590 if self.conf['PETSC_SIZEOF_INT']==8: 591 testDict['SKIP']="NOT int64 required" 592 if isNull: return False 593 else: 594 testDict['SKIP']="int64 required" 595 return False 596 # Datafilespath 597 if requirement=="datafilespath": 598 if not self.conf['PETSC_HAVE_DATAFILESPATH']: 599 testDict['SKIP']="Requires DATAFILESPATH" 600 return False 601 # Defines -- not sure I have comments matching 602 if "define(" in requirement.lower(): 603 reqdef=requirement.split("(")[1].split(")")[0] 604 val=(reqdef.split()[1] if " " in reqdef else "") 605 if self.conf.has_key(reqdef): 606 if val: 607 if self.conf[reqdef]==val: 608 if isNull: 609 testDict['SKIP']="Null requirement not met: "+requirement 610 return False 611 else: 612 testDict['SKIP']="Required: "+requirement 613 return False 614 else: 615 if isNull: 616 testDict['SKIP']="Null requirement not met: "+requirement 617 return False 618 else: 619 return True 620 else: 621 testDict['SKIP']="Requirement not met: "+requirement 622 return False 623 624 # Rest should be packages that we can just get from conf 625 if requirement == "complex": petscconfvar="PETSC_USE_COMPLEX" 626 else: petscconfvar="PETSC_HAVE_"+requirement.upper() 627 if self.conf.get(petscconfvar): 628 if isNull: 629 testDict['SKIP']="Not "+petscconfvar+" requirement not met" 630 return False 631 elif not isNull: 632 if debug: print "requirement not found: ", requirement 633 testDict['SKIP']=petscconfvar+" requirement not met" 634 return False 635 636 return True 637 638 def genPetscTests_summarize(self,dataDict): 639 """ 640 Required method to state what happened 641 """ 642 if not self.summarize: return 643 indent=" " 644 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 645 fh=open(fhname,"w") 646 #print "See ", fhname 647 for root in dataDict: 648 relroot=self.relpath(self.petsc_dir,root) 649 pkg=relroot.split("/")[1] 650 fh.write(relroot+"\n") 651 allSrcs=[] 652 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 653 for exfile in dataDict[root]: 654 # Basic information 655 fullfile=os.path.join(root,exfile) 656 rfile=self.relpath(self.petsc_dir,fullfile) 657 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 658 fh.write(indent+exfile+indent*4+builtStatus+"\n") 659 660 for test in dataDict[root][exfile]: 661 if test in self.buildkeys: continue 662 line=indent*2+test 663 fh.write(line+"\n") 664 # Looks nice to have the keys in order 665 #for key in dataDict[root][exfile][test]: 666 for key in "isrun abstracted nsize args requires script".split(): 667 if not dataDict[root][exfile][test].has_key(key): continue 668 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 669 fh.write(line+"\n") 670 fh.write("\n") 671 fh.write("\n") 672 fh.write("\n") 673 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 674 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 675 fh.close() 676 return 677 678 def genPetscTests(self,root,dirs,files,dataDict): 679 """ 680 Go through and parse the source files in the directory to generate 681 the examples based on the metadata contained in the source files 682 """ 683 debug=False 684 # Use examplesAnalyze to get what the makefles think are sources 685 #self.examplesAnalyze(root,dirs,files,anlzDict) 686 687 dataDict[root]={} 688 689 for exfile in files: 690 #TST: Until we replace files, still leaving the orginals as is 691 #if not exfile.startswith("new_"+"ex"): continue 692 if not exfile.startswith("ex"): continue 693 694 # Convenience 695 fullex=os.path.join(root,exfile) 696 relpfile=self.relpath(self.petsc_dir,fullex) 697 if debug: print relpfile 698 dataDict[root].update(testparse.parseTestFile(fullex)) 699 # Need to check and make sure tests are in the file 700 # if verbosity>=1: print relpfile 701 if dataDict[root].has_key(exfile): 702 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 703 704 return 705 706 def walktree(self,top,action="printFiles"): 707 """ 708 Walk a directory tree, starting from 'top' 709 """ 710 #print "action", action 711 # Goal of action is to fill this dictionary 712 dataDict={} 713 for root, dirs, files in os.walk(top, topdown=False): 714 if not "examples" in root: continue 715 if not os.path.isfile(os.path.join(root,"makefile")): continue 716 bname=os.path.basename(root.rstrip("/")) 717 if bname=="tests" or bname=="tutorials": 718 eval("self."+action+"(root,dirs,files,dataDict)") 719 if type(top) != types.StringType: 720 raise TypeError("top must be a string") 721 # Now summarize this dictionary 722 eval("self."+action+"_summarize(dataDict)") 723 return dataDict 724 725 def gen_gnumake(self, fd): 726 """ 727 Overwrite of the method in the base PETSc class 728 """ 729 def write(stem, srcs): 730 for lang in LANGS: 731 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 732 for pkg in PKGS: 733 srcs = self.gen_pkg(pkg) 734 write('testsrcs-' + pkg, srcs) 735 return self.gendeps 736 737 def gen_pkg(self, pkg): 738 """ 739 Overwrite of the method in the base PETSc class 740 """ 741 return self.sources[pkg] 742 743 def write_gnumake(self,dataDict): 744 """ 745 Write out something similar to files from gmakegen.py 746 747 There is not a lot of has_key type checking because 748 should just work and need to know if there are bugs 749 750 Test depends on script which also depends on source 751 file, but since I don't have a good way generating 752 acting on a single file (oops) just depend on 753 executable which in turn will depend on src file 754 """ 755 # Different options for how to set up the targets 756 compileExecsFirst=False 757 758 # Open file 759 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 760 fd = open(arch_files, 'w') 761 762 # Write out the sources 763 gendeps = self.gen_gnumake(fd) 764 765 # Write out the tests and execname targets 766 fd.write("\n#Tests and executables\n") # Delimiter 767 768 for pkg in PKGS: 769 # These grab the ones that are built 770 for lang in LANGS: 771 testdeps=[] 772 for ftest in self.tests[pkg][lang]: 773 test=os.path.basename(ftest) 774 basedir=os.path.dirname(ftest) 775 testdeps.append(self.nameSpace(test,basedir)) 776 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 777 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 778 779 # test targets 780 for ftest in self.tests[pkg][lang]: 781 test=os.path.basename(ftest) 782 basedir=os.path.dirname(ftest) 783 testdir="${TESTDIR}/"+basedir+"/" 784 nmtest=self.nameSpace(test,basedir) 785 rundir=os.path.join(testdir,test) 786 #print test, nmtest 787 script=test+".sh" 788 789 # Deps 790 exfile=self.tests[pkg][lang][ftest]['exfile'] 791 fullex=os.path.join(self.petsc_dir,exfile) 792 localexec=self.tests[pkg][lang][ftest]['exec'] 793 execname=os.path.join(testdir,localexec) 794 fullscript=os.path.join(testdir,script) 795 tmpfile=os.path.join(testdir,test,test+".tmp") 796 797 # *.counts depends on the script and either executable (will 798 # be run) or the example source file (SKIP or TODO) 799 fd.write('%s.counts : %s %s\n' 800 % (os.path.join('$(TESTDIR)/counts', nmtest), 801 fullscript, 802 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 803 # Now write the args: 804 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 805 806 fd.close() 807 return 808 809 def writeHarness(self,output,dataDict): 810 """ 811 This is set up to write out multiple harness even if only gnumake 812 is supported now 813 """ 814 eval("self.write_"+output+"(dataDict)") 815 return 816 817def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 818 if output is None: 819 output = 'gnumake' 820 821 822 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 823 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 824 pEx.writeHarness(output,dataDict) 825 826if __name__ == '__main__': 827 import optparse 828 parser = optparse.OptionParser() 829 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 830 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 831 parser.add_option('--output', help='Location to write output file', default=None) 832 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 833 opts, extra_args = parser.parse_args() 834 if extra_args: 835 import sys 836 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 837 exit(1) 838 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 839