1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,testDict,i,isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 testDict['args']: -ksp_monitor 97 testDict['subargs']: -bs ${i} -pc_type ${j} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["i"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["j"],["cholesky sor"]] 101 subst should be passed in instead of testDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=testDict.keys() 105 for key in lkeys: 106 if type(testDict[key])!=types.StringType: continue 107 keystr = str(testDict[key]) 108 akey=('subargs' if key=='args' else key) # what to assign 109 if not testDict.has_key(akey): testDict[akey]='' 110 varlist=[] 111 for varset in re.split('-(?=[a-zA-Z])',keystr): 112 if not varset.strip(): continue 113 if len(re.findall('{{(.*?)}}',varset))>0: 114 if not loopVars.has_key(akey): loopVars[akey]={} 115 # Assuming only one for loop per var specification 116 keyvar=varset.split("{{")[0].strip() 117 varlist.append(keyvar) 118 lvar=string.ascii_lowercase[i]; i+=1 119 loopVars[akey][keyvar]=[lvar,re.findall('{{(.*?)}}',varset)[0]] 120 testDict[akey] += '-'+keyvar+' ${' + lvar + '} ' 121 else: 122 if key=='args': newargs+="-"+varset+" " 123 if len(varlist)>0: loopVars[akey]['varlist']=varlist 124 125 126 # For subtests, args are always substituted in (not top level) 127 if isSubtest: 128 testDict['subargs']+=newargs 129 testDict['args']='' 130 else: 131 if len(loopVars.keys())>0: 132 testDict['args']=newargs 133 if len(loopVars.keys())>0: 134 return loopVars 135 else: 136 return None 137 138 def getArgLabel(self,testDict): 139 """ 140 In all of the arguments in the test dictionary, create a simple 141 string for searching within the makefile system. For simplicity in 142 search, remove "-", for strings, etc. 143 Also, concatenate the arg commands 144 For now, ignore nsize -- seems hard to search for anyway 145 """ 146 # Collect all of the args associated with a test 147 argStr=("" if not testDict.has_key('args') else testDict['args']) 148 if testDict.has_key('subtests'): 149 for stest in testDict["subtests"]: 150 sd=testDict[stest] 151 argStr=argStr+("" if not sd.has_key('args') else sd['args']) 152 153 # Now go through and cleanup 154 argStr=re.sub('{{(.*?)}}',"",argStr) 155 argStr=re.sub('-'," ",argStr) 156 for digit in string.digits: argStr=re.sub(digit," ",argStr) 157 argStr=re.sub("\.","",argStr) 158 argStr=re.sub(",","",argStr) 159 argStr=re.sub('\+',' ',argStr) 160 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 161 return argStr.strip() 162 163 def addToSources(self,exfile,root,srcDict): 164 """ 165 Put into data structure that allows easy generation of makefile 166 """ 167 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 168 fullfile=os.path.join(root,exfile) 169 relpfile=self.relpath(self.petsc_dir,fullfile) 170 lang=self.getLanguage(exfile) 171 if not lang: return 172 self.sources[pkg][lang]['srcs'].append(relpfile) 173 if srcDict.has_key('depends'): 174 depSrc=srcDict['depends'] 175 depObj=os.path.splitext(depSrc)[0]+".o" 176 self.sources[pkg][lang][exfile]=depObj 177 178 # In gmakefile, ${TESTDIR} var specifies the object compilation 179 testsdir=self.relpath(self.petsc_dir,root)+"/" 180 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 181 self.objects[pkg].append(objfile) 182 return 183 184 def addToTests(self,test,root,exfile,execname,testDict): 185 """ 186 Put into data structure that allows easy generation of makefile 187 Organized by languages to allow testing of languages 188 """ 189 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 190 #nmtest=self.nameSpace(test,root) 191 rpath=self.relpath(self.petsc_dir,root) 192 nmtest=os.path.join(rpath,test) 193 lang=self.getLanguage(exfile) 194 if not lang: return 195 self.tests[pkg][lang][nmtest]={} 196 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 197 self.tests[pkg][lang][nmtest]['exec']=execname 198 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 199 return 200 201 def getLines(self,forvar,fordict,i,j): 202 """ 203 Get the for and done lines 204 """ 205 indent=self.indent 206 lindex=string.ascii_lowercase[i] 207 forStr=fordict[forvar]['val'] 208 fordict[forvar]['loopvar']=lindex 209 forline=indent*j+"for "+lindex+" in "+forStr+"; do" 210 #subline=re.sub("{{"+loopvals+"}}","${"+lindex+"}",forStr) 211 subline="${"+lindex+"}" 212 donline=indent*j+"done" 213 return forline,subline,donline 214 215 def getFor(self,subst,i,j,parentD,subtest=False): 216 """ 217 Get the for and done lines 218 """ 219 forlines="" 220 donlines="" 221 nsizeStr=subst['nsize'] 222 ofname="" 223 if "{{" in subst['nsize']: 224 nd={'nsize':{'val':subst['nsize']}} 225 fline,sline,dline=self.getLines('nsize',nd,i,j) 226 forlines=fline+"\n"; donlines=dline+"\n" 227 i=i+1; j=j+1 228 subst['nsize']=sline 229 230 if not "{{" in subst['args']: return forlines,donlines,i,j,{} 231 232 # Now work on args 233 forlist,fordict,newargs=testparse.extractForVars(subst['args']) 234 if not subtest: subst['args']=newargs.strip() # promote to control arguments 235 subargs="" 236 for farg in forlist: 237 if farg in parentD.keys(): 238 subline="${"+parentD[farg]['loopvar']+"}" 239 ofname+=farg+"-"+subline+"-" 240 else: 241 forline,subline,donline=self.getLines(farg,fordict,i,j) 242 forlines+=forline+"\n"; donlines+=donline+"\n" 243 subargs=subargs+" -"+farg+" "+subline 244 ofname=ofname+farg+"-"+subline+"-" 245 i=i+1; j=j+1 246 subst['subargs']=subargs.strip() 247 if subtest: subst['subargs']+=" "+newargs 248 # Do not overwrite the output file name if it is in source 249 if not subst['output_file_inSrc']: 250 subst['output_file']="output/"+subst['defroot']+"_"+ofname.strip('-')+".out" 251 subst['redirect_file']=subst['defroot']+"_"+ofname.strip('-')+".tmp" 252 subst['label_suffix']="_"+ofname.strip('-') 253 254 # The do lines have reverse order with respect to indentation 255 dl=donlines.rstrip("\n").split("\n") 256 dl.reverse() 257 donlines="\n".join(dl)+"\n" 258 259 return forlines,donlines,i,j,fordict 260 261 262 def getExecname(self,exfile,root): 263 """ 264 Generate bash script using template found next to this file. 265 This file is read in at constructor time to avoid file I/O 266 """ 267 rpath=self.relpath(self.petsc_dir,root) 268 if self.single_ex: 269 execname=rpath.split("/")[1]+"-ex" 270 else: 271 execname=os.path.splitext(exfile)[0] 272 return execname 273 274 def getSubstVars(self,testDict,rpath,testname): 275 """ 276 Create a dictionary with all of the variables that get substituted 277 into the template commands found in example_template.py 278 TODO: Cleanup 279 """ 280 subst={} 281 # Handle defaults 282 if not testDict.has_key('nsize'): testDict['nsize']=1 283 if not testDict.has_key('filter'): testDict['filter']="" 284 if not testDict.has_key('filter_output'): testDict['filter_output']="" 285 if not testDict.has_key('localrunfiles'): testDict['localrunfiles']="" 286 if not testDict.has_key('args'): testDict['args']="" 287 if not testDict.has_key('subargs'): testDict['subargs']="" 288 if not testDict.has_key('comments'): testDict['comments']="" 289 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 290 if not "_" in defroot: defroot=defroot+"_1" 291 if not testDict.has_key('redirect_file'): testDict['redirect_file']=defroot+".tmp" 292 293 # Output file is special because of subtests override 294 if testDict.has_key('output_file'): 295 subst['output_file']=testDict['output_file'] 296 subst['output_file_inSrc']=True 297 else: 298 subst['output_file']="output/"+defroot+".out" 299 subst['output_file_inSrc']=False 300 301 # Setup the variables in template_string that need to be substituted 302 subst['defroot']=defroot 303 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 304 subst['label']=self.nameSpace(defroot,subst['srcdir']) 305 subst['label_suffix']='' 306 subst['redirect_file']=testDict['redirect_file'] 307 subst['comments']="\n#".join(testDict['comments'].split("\n")) 308 if subst['comments']: subst['comments']="#"+subst['comments'] 309 subst['exec']="../"+testDict['execname'] 310 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky 311 subst['filter_output']=testDict['filter_output'] 312 subst['localrunfiles']=testDict['localrunfiles'] 313 subst['testroot']=self.testroot_dir 314 subst['testname']=testname 315 if self.conf['PETSC_HAVE_DATAFILESPATH']: 316 dp=self.conf['DATAFILESPATH'] 317 dpl='DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 318 else: 319 dpl='' 320 subst['datafilespath_line']=dpl 321 322 # Be careful with this 323 if testDict.has_key('command'): subst['command']=testDict['command'] 324 325 # These can have for loops and are treated separately later 326 if testDict.has_key('nsize'): subst['nsize']=str(testDict['nsize']) 327 if testDict.has_key('args'): subst['args']=testDict['args'] 328 329 #Conf vars 330 if self.petsc_arch.find('valgrind')>=0: 331 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 332 else: 333 subst['mpiexec']=self.conf['MPIEXEC'] 334 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 335 subst['diff']=self.conf['DIFF'] 336 subst['rm']=self.conf['RM'] 337 subst['grep']=self.conf['GREP'] 338 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 339 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 340 341 for key in subst: 342 if type(subst[key])!=types.StringType: continue 343 subst[key] = self._substVars(testDict,subst[key]) 344 345 return subst 346 347 def getCmds(self,subst,i): 348 """ 349 Generate bash script using template found next to this file. 350 This file is read in at constructor time to avoid file I/O 351 """ 352 indnt=self.indent 353 nindnt=i # the start and has to be consistent with below 354 cmdLines="" 355 356 # Add in the full path here. 357 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 358 359 # MPI is the default -- but we have a few odd commands 360 if not subst.has_key('command'): 361 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 362 else: 363 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 364 cmdLines+=cmd+"\n\n" 365 366 if not subst['filter_output']: 367 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 368 else: 369 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 370 cmdLines+=cmd+"\n" 371 return cmdLines 372 373 def _substVars(self,subst,origStr): 374 """ 375 Substitute varial 376 """ 377 Str=origStr 378 for subkey in subst: 379 if type(subst[subkey])!=types.StringType: continue 380 patt="@"+subkey.upper()+"@" 381 Str=re.sub(patt,subst[subkey],Str) 382 return Str 383 384 def _writeTodoSkip(self,fh,tors,inDict,footer): 385 """ 386 Write out the TODO and SKIP lines in the file 387 The TODO or SKIP variable, tors, should be lower case 388 """ 389 TORS=tors.upper() 390 template=eval("example_template."+tors+"line") 391 tsStr=re.sub("@"+TORS+"COMMENT@",inDict[TORS],template) 392 fh.write(tsStr+"\ntotal=1; "+tors+"=1\n") 393 fh.write(footer+"\n") 394 fh.write("exit\n\n\n") 395 return 396 397 def getLoopVarsHead(self,loopVars,i): 398 """ 399 Generate a nicely indented string with the format loops 400 Here is what the data structure looks like 401 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 402 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 403 loopVars['subargs']['pc_type']=["j","cholesky sor"] 404 """ 405 outstr=''; indnt=self.indent 406 for key in loopVars: 407 for var in loopVars[key]['varlist']: 408 varval=loopVars[key][var] 409 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 410 i = i + 1 411 return (outstr,i) 412 413 def getLoopVarsFoot(self,loopVars,i): 414 outstr=''; indnt=self.indent 415 for key in loopVars: 416 for var in loopVars[key]['varlist']: 417 i = i - 1 418 outstr += indnt * i + "done\n" 419 return (outstr,i) 420 421 def genRunScript(self,testname,root,isRun,srcDict): 422 """ 423 Generate bash script using template found next to this file. 424 This file is read in at constructor time to avoid file I/O 425 """ 426 # runscript_dir directory has to be consistent with gmakefile 427 testDict=srcDict[testname] 428 rpath=self.relpath(self.petsc_dir,root) 429 runscript_dir=os.path.join(self.testroot_dir,rpath) 430 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 431 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 432 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 433 434 i=8 # for loop counters 435 # This gets a datastructure for handling loops as well as alters testDict 436 subst=self.getSubstVars(testDict,rpath,testname) 437 loopVars = self._getLoopVars(subst,i) 438 439 #Handle runfiles 440 if subst['localrunfiles']: 441 for lfile in subst['localrunfiles'].split(): 442 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 443 shutil.copy(fullfile,runscript_dir) 444 # Check subtests for local runfiles 445 if testDict.has_key("subtests"): 446 for stest in testDict["subtests"]: 447 if testDict[stest].has_key('localrunfiles'): 448 for lfile in testDict[stest]['localrunfiles'].split(): 449 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 450 shutil.copy(fullfile,self.runscript_dir) 451 452 # Now substitute the key variables into the header and footer 453 header=self._substVars(subst,example_template.header) 454 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 455 456 # Start writing the file 457 fh.write(header+"\n") 458 459 # If there is a TODO or a SKIP then we do it before writing out the 460 # rest of the command (which is useful for working on the test) 461 # SKIP and TODO can be for the source file or for the runs 462 for tors in "todo skip".split(): 463 for d in [srcDict,testDict]: 464 if d.has_key(tors.upper()): self._writeTodoSkip(fh,tors,d,footer) 465 466 j=0 # for indentation 467 468 if loopVars: 469 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 470 if (loopHead): fh.write(loopHead+"\n") 471 472 # Subtests are special 473 if testDict.has_key("subtests"): 474 substP=subst # Subtests can inherit args but be careful 475 for stest in testDict["subtests"]: 476 subst=substP.copy() 477 subst.update(testDict[stest]) 478 subst['nsize']=str(subst['nsize']) 479 sLoopVars = self._getLoopVars(subst,i,isSubtest=True) 480 if sLoopVars: 481 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 482 fh.write(sLoopHead+"\n") 483 fh.write(self.getCmds(subst,j)+"\n") 484 if sLoopVars: 485 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 486 fh.write(sLoopFoot+"\n") 487 else: 488 fh.write(self.getCmds(subst,j)+"\n") 489 490 if loopVars: 491 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 492 fh.write(loopFoot+"\n") 493 494 fh.write(footer+"\n") 495 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 496 return 497 498 def genScriptsAndInfo(self,exfile,root,srcDict): 499 """ 500 Generate scripts from the source file, determine if built, etc. 501 For every test in the exfile with info in the srcDict: 502 1. Determine if it needs to be run for this arch 503 2. Generate the script 504 3. Generate the data needed to write out the makefile in a 505 convenient way 506 All tests are *always* run, but some may be SKIP'd per the TAP standard 507 """ 508 debug=False 509 fileIsTested=False 510 execname=self.getExecname(exfile,root) 511 isBuilt=self._isBuilt(exfile,srcDict) 512 for test in srcDict: 513 if test in self.buildkeys: continue 514 if debug: print self.nameSpace(exfile,root), test 515 srcDict[test]['execname']=execname # Convenience in generating scripts 516 isRun=self._isRun(srcDict[test]) 517 self.genRunScript(test,root,isRun,srcDict) 518 srcDict[test]['isrun']=isRun 519 if isRun: fileIsTested=True 520 self.addToTests(test,root,exfile,execname,srcDict[test]) 521 522 # This adds to datastructure for building deps 523 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 524 #print self.nameSpace(exfile,root), fileIsTested 525 return 526 527 def _isBuilt(self,exfile,srcDict): 528 """ 529 Determine if this file should be built. 530 """ 531 # Get the language based on file extension 532 lang=self.getLanguage(exfile) 533 if (lang=="F" or lang=="F90") and not self.have_fortran: 534 srcDict["SKIP"]="Fortran required for this test" 535 return False 536 if lang=="cu" and not self.conf.has_key('PETSC_HAVE_CUDA'): 537 srcDict["SKIP"]="CUDA required for this test" 538 return False 539 if lang=="cxx" and not self.conf.has_key('PETSC_HAVE_CXX'): 540 srcDict["SKIP"]="C++ required for this test" 541 return False 542 543 # Deprecated source files 544 if srcDict.has_key("TODO"): return False 545 546 # isRun can work with srcDict to handle the requires 547 if srcDict.has_key("requires"): 548 if len(srcDict["requires"])>0: 549 return self._isRun(srcDict) 550 551 return True 552 553 554 def _isRun(self,testDict): 555 """ 556 Based on the requirements listed in the src file and the petscconf.h 557 info, determine whether this test should be run or not. 558 """ 559 indent=" " 560 debug=False 561 562 # MPI requirements 563 if testDict.has_key('nsize'): 564 if testDict['nsize']>1 and self.conf.has_key('MPI_IS_MPIUNI'): 565 if debug: print indent+"Cannot run parallel tests" 566 testDict['SKIP']="Parallel test with serial build" 567 return False 568 569 # The requirements for the test are the sum of all the run subtests 570 if testDict.has_key('subtests'): 571 if not testDict.has_key('requires'): testDict['requires']="" 572 for stest in testDict['subtests']: 573 if testDict[stest].has_key('requires'): 574 testDict['requires']=testDict['requires']+" "+testDict[stest]['requires'] 575 576 577 # Now go through all requirements 578 if testDict.has_key('requires'): 579 for requirement in testDict['requires'].split(): 580 requirement=requirement.strip() 581 if not requirement: continue 582 if debug: print indent+"Requirement: ", requirement 583 isNull=False 584 if requirement.startswith("!"): 585 requirement=requirement[1:]; isNull=True 586 # Precision requirement for reals 587 if requirement in self.precision_types: 588 if self.conf['PETSC_PRECISION']==requirement: 589 testDict['SKIP']="not "+requirement+" required" 590 if isNull: return False 591 else: 592 testDict['SKIP']=requirement+" required" 593 return False 594 # Precision requirement for ints 595 if requirement in self.integer_types: 596 if requirement=="int32": 597 if self.conf['PETSC_SIZEOF_INT']==4: 598 testDict['SKIP']="not int32 required" 599 if isNull: return False 600 else: 601 testDict['SKIP']="int32 required" 602 return False 603 if requirement=="int64": 604 if self.conf['PETSC_SIZEOF_INT']==8: 605 testDict['SKIP']="NOT int64 required" 606 if isNull: return False 607 else: 608 testDict['SKIP']="int64 required" 609 return False 610 # Datafilespath 611 if requirement=="datafilespath": 612 if not self.conf['PETSC_HAVE_DATAFILESPATH']: 613 testDict['SKIP']="Requires DATAFILESPATH" 614 return False 615 # Defines -- not sure I have comments matching 616 if "define(" in requirement.lower(): 617 reqdef=requirement.split("(")[1].split(")")[0] 618 val=(reqdef.split()[1] if " " in reqdef else "") 619 if self.conf.has_key(reqdef): 620 if val: 621 if self.conf[reqdef]==val: 622 if isNull: 623 testDict['SKIP']="Null requirement not met: "+requirement 624 return False 625 else: 626 testDict['SKIP']="Required: "+requirement 627 return False 628 else: 629 if isNull: 630 testDict['SKIP']="Null requirement not met: "+requirement 631 return False 632 else: 633 return True 634 else: 635 testDict['SKIP']="Requirement not met: "+requirement 636 return False 637 638 # Rest should be packages that we can just get from conf 639 if requirement == "complex": petscconfvar="PETSC_USE_COMPLEX" 640 else: petscconfvar="PETSC_HAVE_"+requirement.upper() 641 if self.conf.get(petscconfvar): 642 if isNull: 643 testDict['SKIP']="Not "+petscconfvar+" requirement not met" 644 return False 645 elif not isNull: 646 if debug: print "requirement not found: ", requirement 647 testDict['SKIP']=petscconfvar+" requirement not met" 648 return False 649 650 return True 651 652 def genPetscTests_summarize(self,dataDict): 653 """ 654 Required method to state what happened 655 """ 656 if not self.summarize: return 657 indent=" " 658 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 659 fh=open(fhname,"w") 660 #print "See ", fhname 661 for root in dataDict: 662 relroot=self.relpath(self.petsc_dir,root) 663 pkg=relroot.split("/")[1] 664 fh.write(relroot+"\n") 665 allSrcs=[] 666 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 667 for exfile in dataDict[root]: 668 # Basic information 669 fullfile=os.path.join(root,exfile) 670 rfile=self.relpath(self.petsc_dir,fullfile) 671 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 672 fh.write(indent+exfile+indent*4+builtStatus+"\n") 673 674 for test in dataDict[root][exfile]: 675 if test in self.buildkeys: continue 676 line=indent*2+test 677 fh.write(line+"\n") 678 # Looks nice to have the keys in order 679 #for key in dataDict[root][exfile][test]: 680 for key in "isrun abstracted nsize args requires script".split(): 681 if not dataDict[root][exfile][test].has_key(key): continue 682 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 683 fh.write(line+"\n") 684 fh.write("\n") 685 fh.write("\n") 686 fh.write("\n") 687 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 688 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 689 fh.close() 690 return 691 692 def genPetscTests(self,root,dirs,files,dataDict): 693 """ 694 Go through and parse the source files in the directory to generate 695 the examples based on the metadata contained in the source files 696 """ 697 debug=False 698 # Use examplesAnalyze to get what the makefles think are sources 699 #self.examplesAnalyze(root,dirs,files,anlzDict) 700 701 dataDict[root]={} 702 703 for exfile in files: 704 #TST: Until we replace files, still leaving the orginals as is 705 #if not exfile.startswith("new_"+"ex"): continue 706 if not exfile.startswith("ex"): continue 707 708 # Convenience 709 fullex=os.path.join(root,exfile) 710 relpfile=self.relpath(self.petsc_dir,fullex) 711 if debug: print relpfile 712 dataDict[root].update(testparse.parseTestFile(fullex)) 713 # Need to check and make sure tests are in the file 714 # if verbosity>=1: print relpfile 715 if dataDict[root].has_key(exfile): 716 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 717 718 return 719 720 def walktree(self,top,action="printFiles"): 721 """ 722 Walk a directory tree, starting from 'top' 723 """ 724 #print "action", action 725 # Goal of action is to fill this dictionary 726 dataDict={} 727 for root, dirs, files in os.walk(top, topdown=False): 728 if not "examples" in root: continue 729 if not os.path.isfile(os.path.join(root,"makefile")): continue 730 bname=os.path.basename(root.rstrip("/")) 731 if bname=="tests" or bname=="tutorials": 732 eval("self."+action+"(root,dirs,files,dataDict)") 733 if type(top) != types.StringType: 734 raise TypeError("top must be a string") 735 # Now summarize this dictionary 736 eval("self."+action+"_summarize(dataDict)") 737 return dataDict 738 739 def gen_gnumake(self, fd): 740 """ 741 Overwrite of the method in the base PETSc class 742 """ 743 def write(stem, srcs): 744 for lang in LANGS: 745 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 746 for pkg in PKGS: 747 srcs = self.gen_pkg(pkg) 748 write('testsrcs-' + pkg, srcs) 749 return self.gendeps 750 751 def gen_pkg(self, pkg): 752 """ 753 Overwrite of the method in the base PETSc class 754 """ 755 return self.sources[pkg] 756 757 def write_gnumake(self,dataDict): 758 """ 759 Write out something similar to files from gmakegen.py 760 761 There is not a lot of has_key type checking because 762 should just work and need to know if there are bugs 763 764 Test depends on script which also depends on source 765 file, but since I don't have a good way generating 766 acting on a single file (oops) just depend on 767 executable which in turn will depend on src file 768 """ 769 # Different options for how to set up the targets 770 compileExecsFirst=False 771 772 # Open file 773 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 774 fd = open(arch_files, 'w') 775 776 # Write out the sources 777 gendeps = self.gen_gnumake(fd) 778 779 # Write out the tests and execname targets 780 fd.write("\n#Tests and executables\n") # Delimiter 781 782 for pkg in PKGS: 783 # These grab the ones that are built 784 for lang in LANGS: 785 testdeps=[] 786 for ftest in self.tests[pkg][lang]: 787 test=os.path.basename(ftest) 788 basedir=os.path.dirname(ftest) 789 testdeps.append(self.nameSpace(test,basedir)) 790 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 791 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 792 793 # test targets 794 for ftest in self.tests[pkg][lang]: 795 test=os.path.basename(ftest) 796 basedir=os.path.dirname(ftest) 797 testdir="${TESTDIR}/"+basedir+"/" 798 nmtest=self.nameSpace(test,basedir) 799 rundir=os.path.join(testdir,test) 800 #print test, nmtest 801 script=test+".sh" 802 803 # Deps 804 exfile=self.tests[pkg][lang][ftest]['exfile'] 805 fullex=os.path.join(self.petsc_dir,exfile) 806 localexec=self.tests[pkg][lang][ftest]['exec'] 807 execname=os.path.join(testdir,localexec) 808 fullscript=os.path.join(testdir,script) 809 tmpfile=os.path.join(testdir,test,test+".tmp") 810 811 # *.counts depends on the script and either executable (will 812 # be run) or the example source file (SKIP or TODO) 813 fd.write('%s.counts : %s %s\n' 814 % (os.path.join('$(TESTDIR)/counts', nmtest), 815 fullscript, 816 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 817 # Now write the args: 818 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 819 820 fd.close() 821 return 822 823 def writeHarness(self,output,dataDict): 824 """ 825 This is set up to write out multiple harness even if only gnumake 826 is supported now 827 """ 828 eval("self.write_"+output+"(dataDict)") 829 return 830 831def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 832 if output is None: 833 output = 'gnumake' 834 835 836 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 837 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 838 pEx.writeHarness(output,dataDict) 839 840if __name__ == '__main__': 841 import optparse 842 parser = optparse.OptionParser() 843 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 844 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 845 parser.add_option('--output', help='Location to write output file', default=None) 846 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 847 opts, extra_args = parser.parse_args() 848 if extra_args: 849 import sys 850 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 851 exit(1) 852 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 853