1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double __float128 int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 from testparse import parseLoopArgs 107 for key in lkeys: 108 if type(inDict[key])!=types.StringType: continue 109 keystr = str(inDict[key]) 110 akey=('subargs' if key=='args' else key) # what to assign 111 if akey not in inDict: inDict[akey]='' 112 varlist=[] 113 for varset in re.split('-(?=[a-zA-Z])',keystr): 114 if not varset.strip(): continue 115 if '{{' in varset: 116 keyvar,lvars,ftype=parseLoopArgs(varset) 117 if akey not in loopVars: loopVars[akey]={} 118 varlist.append(keyvar) 119 loopVars[akey][keyvar]=[keyvar,lvars] 120 if akey=='nsize': 121 inDict[akey] = '${' + keyvar + '}' 122 lsuffix+=akey+'-'+inDict[akey]+'_' 123 else: 124 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 125 lsuffix+=keyvar+'-${' + keyvar + '}_' 126 else: 127 if key=='args': newargs+=" -"+varset.strip() 128 if len(varlist)>0: loopVars[akey]['varlist']=varlist 129 130 131 # For subtests, args are always substituted in (not top level) 132 if isSubtest: 133 inDict['subargs']+=" "+newargs.strip() 134 inDict['args']='' 135 if 'label_suffix' in inDict: 136 inDict['label_suffix']+=lsuffix.rstrip('_') 137 else: 138 inDict['label_suffix']=lsuffix.rstrip('_') 139 else: 140 if len(loopVars.keys())>0: 141 inDict['args']=newargs.strip() 142 inDict['label_suffix']=lsuffix.rstrip('_') 143 if len(loopVars.keys())>0: 144 return loopVars 145 else: 146 return None 147 148 def getArgLabel(self,testDict): 149 """ 150 In all of the arguments in the test dictionary, create a simple 151 string for searching within the makefile system. For simplicity in 152 search, remove "-", for strings, etc. 153 Also, concatenate the arg commands 154 For now, ignore nsize -- seems hard to search for anyway 155 """ 156 # Collect all of the args associated with a test 157 argStr=("" if 'args' not in testDict else testDict['args']) 158 if 'subtests' in testDict: 159 for stest in testDict["subtests"]: 160 sd=testDict[stest] 161 argStr=argStr+("" if 'args' not in sd else sd['args']) 162 163 # Now go through and cleanup 164 argStr=re.sub('{{(.*?)}}',"",argStr) 165 argStr=re.sub('-'," ",argStr) 166 for digit in string.digits: argStr=re.sub(digit," ",argStr) 167 argStr=re.sub("\.","",argStr) 168 argStr=re.sub(",","",argStr) 169 argStr=re.sub('\+',' ',argStr) 170 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 171 return argStr.strip() 172 173 def addToSources(self,exfile,root,srcDict): 174 """ 175 Put into data structure that allows easy generation of makefile 176 """ 177 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 178 fullfile=os.path.join(root,exfile) 179 relpfile=self.relpath(self.petsc_dir,fullfile) 180 lang=self.getLanguage(exfile) 181 if not lang: return 182 self.sources[pkg][lang]['srcs'].append(relpfile) 183 if 'depends' in srcDict: 184 depSrc=srcDict['depends'] 185 depObj=os.path.splitext(depSrc)[0]+".o" 186 self.sources[pkg][lang][exfile]=depObj 187 188 # In gmakefile, ${TESTDIR} var specifies the object compilation 189 testsdir=self.relpath(self.petsc_dir,root)+"/" 190 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 191 self.objects[pkg].append(objfile) 192 return 193 194 def addToTests(self,test,root,exfile,execname,testDict): 195 """ 196 Put into data structure that allows easy generation of makefile 197 Organized by languages to allow testing of languages 198 """ 199 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 200 #nmtest=self.nameSpace(test,root) 201 rpath=self.relpath(self.petsc_dir,root) 202 nmtest=os.path.join(rpath,test) 203 lang=self.getLanguage(exfile) 204 if not lang: return 205 self.tests[pkg][lang][nmtest]={} 206 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 207 self.tests[pkg][lang][nmtest]['exec']=execname 208 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 209 return 210 211 def getExecname(self,exfile,root): 212 """ 213 Generate bash script using template found next to this file. 214 This file is read in at constructor time to avoid file I/O 215 """ 216 rpath=self.relpath(self.petsc_dir,root) 217 if self.single_ex: 218 execname=rpath.split("/")[1]+"-ex" 219 else: 220 execname=os.path.splitext(exfile)[0] 221 return execname 222 223 def getSubstVars(self,testDict,rpath,testname): 224 """ 225 Create a dictionary with all of the variables that get substituted 226 into the template commands found in example_template.py 227 """ 228 subst={} 229 230 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 231 if 'nsize' not in testDict: testDict['nsize']=1 232 for ak in testparse.acceptedkeys: 233 if ak=='test': continue 234 subst[ak]=(testDict[ak] if ak in testDict else '') 235 236 # Now do other variables 237 subst['execname']=testDict['execname'] 238 if 'filter' in testDict: 239 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 240 241 # Others 242 subst['subargs']='' # Default. For variables override 243 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 244 subst['label_suffix']='' 245 subst['comments']="\n#".join(subst['comments'].split("\n")) 246 if subst['comments']: subst['comments']="#"+subst['comments'] 247 subst['exec']="../"+subst['execname'] 248 subst['testroot']=self.testroot_dir 249 subst['testname']=testname 250 dp = self.conf.get('DATAFILESPATH','') 251 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 252 253 # This is used to label some matrices 254 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 255 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 256 257 # These can have for loops and are treated separately later 258 subst['nsize']=str(subst['nsize']) 259 260 #Conf vars 261 if self.petsc_arch.find('valgrind')>=0: 262 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 263 else: 264 subst['mpiexec']=self.conf['MPIEXEC'] 265 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 266 subst['diff']=self.conf['DIFF'] 267 subst['rm']=self.conf['RM'] 268 subst['grep']=self.conf['GREP'] 269 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 270 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 271 272 # Output file is special because of subtests override 273 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 274 if not "_" in defroot: defroot=defroot+"_1" 275 subst['defroot']=defroot 276 subst['label']=self.nameSpace(defroot,subst['srcdir']) 277 subst['redirect_file']=defroot+".tmp" 278 if 'output_file' not in testDict: 279 subst['output_file']="output/"+defroot+".out" 280 # Add in the full path here. 281 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 282 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 283 if not subst['TODO']: 284 print "Warning: "+subst['output_file']+" not found." 285 # Worry about alt files here -- see 286 # src/snes/examples/tutorials/output/ex22*.out 287 altlist=[subst['output_file']] 288 for i in range(1,3): 289 altroot=defroot+"_alt" 290 if i==2: altroot=altroot+"_2" 291 af="output/"+altroot+".out" 292 srcaf=os.path.join(subst['srcdir'],af) 293 fullaf=os.path.join(self.petsc_dir,srcaf) 294 if os.path.isfile(fullaf): altlist.append(srcaf) 295 if len(altlist)>1: subst['altfiles']=altlist 296 #if len(altlist)>1: print "Found alt files: ",altlist 297 298 return subst 299 300 def getCmds(self,subst,i): 301 """ 302 Generate bash script using template found next to this file. 303 This file is read in at constructor time to avoid file I/O 304 """ 305 nindnt=i # the start and has to be consistent with below 306 cmdindnt=self.indent*nindnt 307 cmdLines="" 308 309 # MPI is the default -- but we have a few odd commands 310 if not subst['command']: 311 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 312 else: 313 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 314 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 315 316 cmdLines+=cmdindnt+'if test $res = 0; then\n' 317 diffindnt=self.indent*(nindnt+1) 318 if not subst['filter_output']: 319 if 'altfiles' not in subst: 320 cmd=diffindnt+self._substVars(subst,example_template.difftest) 321 else: 322 # Have to do it by hand a bit because of variable number of alt files 323 rf=subst['redirect_file'] 324 cmd=diffindnt+example_template.difftest.split('@')[0] 325 for i in range(len(subst['altfiles'])): 326 af=subst['altfiles'][i] 327 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 328 if i!=len(subst['altfiles'])-1: 329 cmd+=' || ${diff_exe} ' 330 else: 331 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 332 cmd+=subst['label_suffix']+' ""' # Quotes are painful 333 else: 334 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 335 cmdLines+=cmd+"\n" 336 cmdLines+=cmdindnt+'else\n' 337 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 338 cmdLines+=cmdindnt+'fi\n' 339 return cmdLines 340 341 def _substVars(self,subst,origStr): 342 """ 343 Substitute variables 344 """ 345 Str=origStr 346 for subkey in subst: 347 if type(subst[subkey])!=types.StringType: continue 348 patt="@"+subkey.upper()+"@" 349 Str=re.sub(patt,subst[subkey],Str) 350 return Str 351 352 def _writeTodoSkip(self,fh,tors,reasons,footer): 353 """ 354 Write out the TODO and SKIP lines in the file 355 The TODO or SKIP variable, tors, should be lower case 356 """ 357 TORS=tors.upper() 358 template=eval("example_template."+tors+"line") 359 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 360 tab = '' 361 if reasons: 362 fh.write('if ! $force; then\n') 363 tab = tab + ' ' 364 if reasons == ["Requires DATAFILESPATH"]: 365 # The only reason not to run is DATAFILESPATH, which we check at run-time 366 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 367 tab = tab + ' ' 368 if reasons: 369 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 370 fh.write(tab+footer+"\n") 371 fh.write(tab+"exit\n") 372 if reasons == ["Requires DATAFILESPATH"]: 373 fh.write(' fi\n') 374 if reasons: 375 fh.write('fi\n') 376 fh.write('\n\n') 377 return 378 379 def getLoopVarsHead(self,loopVars,i): 380 """ 381 Generate a nicely indented string with the format loops 382 Here is what the data structure looks like 383 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 384 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 385 loopVars['subargs']['pc_type']=["j","cholesky sor"] 386 """ 387 outstr=''; indnt=self.indent 388 for key in loopVars: 389 for var in loopVars[key]['varlist']: 390 varval=loopVars[key][var] 391 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 392 i = i + 1 393 return (outstr,i) 394 395 def getLoopVarsFoot(self,loopVars,i): 396 outstr=''; indnt=self.indent 397 for key in loopVars: 398 for var in loopVars[key]['varlist']: 399 i = i - 1 400 outstr += indnt * i + "done\n" 401 return (outstr,i) 402 403 def genRunScript(self,testname,root,isRun,srcDict): 404 """ 405 Generate bash script using template found next to this file. 406 This file is read in at constructor time to avoid file I/O 407 """ 408 # runscript_dir directory has to be consistent with gmakefile 409 testDict=srcDict[testname] 410 rpath=self.relpath(self.petsc_dir,root) 411 runscript_dir=os.path.join(self.testroot_dir,rpath) 412 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 413 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 414 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 415 416 # Get variables to go into shell scripts. last time testDict used 417 subst=self.getSubstVars(testDict,rpath,testname) 418 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 419 #if '33_' in testname: print subst['subargs'] 420 421 #Handle runfiles 422 for lfile in subst.get('localrunfiles','').split(): 423 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 424 shutil.copy(fullfile,runscript_dir) 425 # Check subtests for local runfiles 426 for stest in subst.get("subtests",[]): 427 for lfile in testDict[stest].get('localrunfiles','').split(): 428 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 429 shutil.copy(fullfile,self.runscript_dir) 430 431 # Now substitute the key variables into the header and footer 432 header=self._substVars(subst,example_template.header) 433 # The header is done twice to enable @...@ in header 434 header=self._substVars(subst,header) 435 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 436 437 # Start writing the file 438 fh.write(header+"\n") 439 440 # If there is a TODO or a SKIP then we do it before writing out the 441 # rest of the command (which is useful for working on the test) 442 # SKIP and TODO can be for the source file or for the runs 443 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 444 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 445 446 j=0 # for indentation 447 448 if loopVars: 449 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 450 if (loopHead): fh.write(loopHead+"\n") 451 452 # Subtests are special 453 if 'subtests' in testDict: 454 substP=subst # Subtests can inherit args but be careful 455 for stest in testDict["subtests"]: 456 subst=substP.copy() 457 subst.update(testDict[stest]) 458 subst['nsize']=str(subst['nsize']) 459 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 460 #if '10_9' in testname: print sLoopVars 461 if sLoopVars: 462 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 463 fh.write(sLoopHead+"\n") 464 fh.write(self.getCmds(subst,j)+"\n") 465 if sLoopVars: 466 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 467 fh.write(sLoopFoot+"\n") 468 else: 469 fh.write(self.getCmds(subst,j)+"\n") 470 471 if loopVars: 472 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 473 fh.write(loopFoot+"\n") 474 475 fh.write(footer+"\n") 476 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 477 #if '10_9' in testname: sys.exit() 478 return 479 480 def genScriptsAndInfo(self,exfile,root,srcDict): 481 """ 482 Generate scripts from the source file, determine if built, etc. 483 For every test in the exfile with info in the srcDict: 484 1. Determine if it needs to be run for this arch 485 2. Generate the script 486 3. Generate the data needed to write out the makefile in a 487 convenient way 488 All tests are *always* run, but some may be SKIP'd per the TAP standard 489 """ 490 debug=False 491 execname=self.getExecname(exfile,root) 492 isBuilt=self._isBuilt(exfile,srcDict) 493 for test in srcDict: 494 if test in self.buildkeys: continue 495 if debug: print self.nameSpace(exfile,root), test 496 srcDict[test]['execname']=execname # Convenience in generating scripts 497 isRun=self._isRun(srcDict[test]) 498 self.genRunScript(test,root,isRun,srcDict) 499 srcDict[test]['isrun']=isRun 500 self.addToTests(test,root,exfile,execname,srcDict[test]) 501 502 # This adds to datastructure for building deps 503 if isBuilt: self.addToSources(exfile,root,srcDict) 504 return 505 506 def _isBuilt(self,exfile,srcDict): 507 """ 508 Determine if this file should be built. 509 """ 510 # Get the language based on file extension 511 srcDict['SKIP'] = [] 512 lang=self.getLanguage(exfile) 513 if (lang=="F" or lang=="F90") and not self.have_fortran: 514 srcDict["SKIP"].append("Fortran required for this test") 515 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 516 srcDict["SKIP"].append("CUDA required for this test") 517 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 518 srcDict["SKIP"].append("C++ required for this test") 519 520 # Deprecated source files 521 if srcDict.get("TODO"): 522 return False 523 524 # isRun can work with srcDict to handle the requires 525 if "requires" in srcDict: 526 if len(srcDict["requires"])>0: 527 return self._isRun(srcDict) 528 529 return srcDict['SKIP'] == [] 530 531 532 def _isRun(self,testDict): 533 """ 534 Based on the requirements listed in the src file and the petscconf.h 535 info, determine whether this test should be run or not. 536 """ 537 indent=" " 538 debug=False 539 540 if 'SKIP' not in testDict: 541 testDict['SKIP'] = [] 542 # MPI requirements 543 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 544 if debug: print indent+"Cannot run parallel tests" 545 testDict['SKIP'].append("Parallel test with serial build") 546 547 # The requirements for the test are the sum of all the run subtests 548 if 'subtests' in testDict: 549 if 'requires' not in testDict: testDict['requires']="" 550 for stest in testDict['subtests']: 551 if 'requires' in testDict[stest]: 552 testDict['requires']+=" "+testDict[stest]['requires'] 553 554 555 # Now go through all requirements 556 if 'requires' in testDict: 557 for requirement in testDict['requires'].split(): 558 requirement=requirement.strip() 559 if not requirement: continue 560 if debug: print indent+"Requirement: ", requirement 561 isNull=False 562 if requirement.startswith("!"): 563 requirement=requirement[1:]; isNull=True 564 # Precision requirement for reals 565 if requirement in self.precision_types: 566 if self.conf['PETSC_PRECISION']==requirement: 567 if isNull: 568 testDict['SKIP'].append("not "+requirement+" required") 569 continue 570 continue # Success 571 elif not isNull: 572 testDict['SKIP'].append(requirement+" required") 573 continue 574 # Precision requirement for ints 575 if requirement in self.integer_types: 576 if requirement=="int32": 577 if self.conf['PETSC_SIZEOF_INT']==4: 578 if isNull: 579 testDict['SKIP'].append("not int32 required") 580 continue 581 continue # Success 582 elif not isNull: 583 testDict['SKIP'].append("int32 required") 584 continue 585 if requirement=="int64": 586 if self.conf['PETSC_SIZEOF_INT']==8: 587 if isNull: 588 testDict['SKIP'].append("NOT int64 required") 589 continue 590 continue # Success 591 elif not isNull: 592 testDict['SKIP'].append("int64 required") 593 continue 594 # Datafilespath 595 if requirement=="datafilespath" and not isNull: 596 testDict['SKIP'].append("Requires DATAFILESPATH") 597 continue 598 # Defines -- not sure I have comments matching 599 if "define(" in requirement.lower(): 600 reqdef=requirement.split("(")[1].split(")")[0] 601 if reqdef in self.conf: 602 if isNull: 603 testDict['SKIP'].append("Null requirement not met: "+requirement) 604 continue 605 continue # Success 606 elif not isNull: 607 testDict['SKIP'].append("Required: "+requirement) 608 continue 609 610 # Rest should be packages that we can just get from conf 611 if requirement == "complex": 612 petscconfvar="PETSC_USE_COMPLEX" 613 else: 614 petscconfvar="PETSC_HAVE_"+requirement.upper() 615 if self.conf.get(petscconfvar): 616 if isNull: 617 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 618 continue 619 continue # Success 620 elif not isNull: 621 if debug: print "requirement not found: ", requirement 622 testDict['SKIP'].append(petscconfvar+" requirement not met") 623 continue 624 625 return testDict['SKIP'] == [] 626 627 def genPetscTests_summarize(self,dataDict): 628 """ 629 Required method to state what happened 630 """ 631 if not self.summarize: return 632 indent=" " 633 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 634 fh=open(fhname,"w") 635 #print "See ", fhname 636 for root in dataDict: 637 relroot=self.relpath(self.petsc_dir,root) 638 pkg=relroot.split("/")[1] 639 fh.write(relroot+"\n") 640 allSrcs=[] 641 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 642 for exfile in dataDict[root]: 643 # Basic information 644 fullfile=os.path.join(root,exfile) 645 rfile=self.relpath(self.petsc_dir,fullfile) 646 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 647 fh.write(indent+exfile+indent*4+builtStatus+"\n") 648 649 for test in dataDict[root][exfile]: 650 if test in self.buildkeys: continue 651 line=indent*2+test 652 fh.write(line+"\n") 653 # Looks nice to have the keys in order 654 #for key in dataDict[root][exfile][test]: 655 for key in "isrun abstracted nsize args requires script".split(): 656 if key not in dataDict[root][exfile][test]: continue 657 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 658 fh.write(line+"\n") 659 fh.write("\n") 660 fh.write("\n") 661 fh.write("\n") 662 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 663 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 664 fh.close() 665 return 666 667 def genPetscTests(self,root,dirs,files,dataDict): 668 """ 669 Go through and parse the source files in the directory to generate 670 the examples based on the metadata contained in the source files 671 """ 672 debug=False 673 # Use examplesAnalyze to get what the makefles think are sources 674 #self.examplesAnalyze(root,dirs,files,anlzDict) 675 676 dataDict[root]={} 677 678 for exfile in files: 679 #TST: Until we replace files, still leaving the orginals as is 680 #if not exfile.startswith("new_"+"ex"): continue 681 if not exfile.startswith("ex"): continue 682 683 # Convenience 684 fullex=os.path.join(root,exfile) 685 relpfile=self.relpath(self.petsc_dir,fullex) 686 if debug: print relpfile 687 dataDict[root].update(testparse.parseTestFile(fullex,0)) 688 # Need to check and make sure tests are in the file 689 # if verbosity>=1: print relpfile 690 if exfile in dataDict[root]: 691 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 692 693 return 694 695 def walktree(self,top,action="printFiles"): 696 """ 697 Walk a directory tree, starting from 'top' 698 """ 699 #print "action", action 700 # Goal of action is to fill this dictionary 701 dataDict={} 702 for root, dirs, files in os.walk(top, topdown=False): 703 if not "examples" in root: continue 704 if not os.path.isfile(os.path.join(root,"makefile")): continue 705 bname=os.path.basename(root.rstrip("/")) 706 if bname=="tests" or bname=="tutorials": 707 eval("self."+action+"(root,dirs,files,dataDict)") 708 if type(top) != types.StringType: 709 raise TypeError("top must be a string") 710 # Now summarize this dictionary 711 eval("self."+action+"_summarize(dataDict)") 712 return dataDict 713 714 def gen_gnumake(self, fd): 715 """ 716 Overwrite of the method in the base PETSc class 717 """ 718 def write(stem, srcs): 719 for lang in LANGS: 720 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 721 for pkg in PKGS: 722 srcs = self.gen_pkg(pkg) 723 write('testsrcs-' + pkg, srcs) 724 return self.gendeps 725 726 def gen_pkg(self, pkg): 727 """ 728 Overwrite of the method in the base PETSc class 729 """ 730 return self.sources[pkg] 731 732 def write_gnumake(self,dataDict): 733 """ 734 Write out something similar to files from gmakegen.py 735 736 Test depends on script which also depends on source 737 file, but since I don't have a good way generating 738 acting on a single file (oops) just depend on 739 executable which in turn will depend on src file 740 """ 741 # Different options for how to set up the targets 742 compileExecsFirst=False 743 744 # Open file 745 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 746 fd = open(arch_files, 'w') 747 748 # Write out the sources 749 gendeps = self.gen_gnumake(fd) 750 751 # Write out the tests and execname targets 752 fd.write("\n#Tests and executables\n") # Delimiter 753 754 for pkg in PKGS: 755 # These grab the ones that are built 756 for lang in LANGS: 757 testdeps=[] 758 for ftest in self.tests[pkg][lang]: 759 test=os.path.basename(ftest) 760 basedir=os.path.dirname(ftest) 761 testdeps.append(self.nameSpace(test,basedir)) 762 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 763 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 764 765 # test targets 766 for ftest in self.tests[pkg][lang]: 767 test=os.path.basename(ftest) 768 basedir=os.path.dirname(ftest) 769 testdir="${TESTDIR}/"+basedir+"/" 770 nmtest=self.nameSpace(test,basedir) 771 rundir=os.path.join(testdir,test) 772 #print test, nmtest 773 script=test+".sh" 774 775 # Deps 776 exfile=self.tests[pkg][lang][ftest]['exfile'] 777 fullex=os.path.join(self.petsc_dir,exfile) 778 localexec=self.tests[pkg][lang][ftest]['exec'] 779 execname=os.path.join(testdir,localexec) 780 fullscript=os.path.join(testdir,script) 781 tmpfile=os.path.join(testdir,test,test+".tmp") 782 783 # *.counts depends on the script and either executable (will 784 # be run) or the example source file (SKIP or TODO) 785 fd.write('%s.counts : %s %s\n' 786 % (os.path.join('$(TESTDIR)/counts', nmtest), 787 fullscript, 788 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 789 # Now write the args: 790 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 791 792 fd.close() 793 return 794 795 def writeHarness(self,output,dataDict): 796 """ 797 This is set up to write out multiple harness even if only gnumake 798 is supported now 799 """ 800 eval("self.write_"+output+"(dataDict)") 801 return 802 803def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 804 if output is None: 805 output = 'gnumake' 806 807 808 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 809 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 810 pEx.writeHarness(output,dataDict) 811 812if __name__ == '__main__': 813 import optparse 814 parser = optparse.OptionParser() 815 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 816 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 817 parser.add_option('--output', help='Location to write output file', default=None) 818 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 819 opts, extra_args = parser.parse_args() 820 if extra_args: 821 import sys 822 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 823 exit(1) 824 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 825