1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double __float128 int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 from testparse import parseLoopArgs 107 for key in lkeys: 108 if type(inDict[key])!=types.StringType: continue 109 keystr = str(inDict[key]) 110 akey=('subargs' if key=='args' else key) # what to assign 111 if akey not in inDict: inDict[akey]='' 112 varlist=[] 113 for varset in re.split('-(?=[a-zA-Z])',keystr): 114 if not varset.strip(): continue 115 if '{{' in varset: 116 keyvar,lvars,ftype=parseLoopArgs(varset) 117 if akey not in loopVars: loopVars[akey]={} 118 varlist.append(keyvar) 119 loopVars[akey][keyvar]=[keyvar,lvars] 120 if akey=='nsize': 121 inDict[akey] = '${' + keyvar + '}' 122 lsuffix+=akey+'-'+inDict[akey]+'_' 123 else: 124 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 125 lsuffix+=keyvar+'-${' + keyvar + '}_' 126 else: 127 if key=='args': newargs+=" -"+varset.strip() 128 if len(varlist)>0: loopVars[akey]['varlist']=varlist 129 130 131 # For subtests, args are always substituted in (not top level) 132 if isSubtest: 133 inDict['subargs']+=" "+newargs.strip() 134 inDict['args']='' 135 if 'label_suffix' in inDict: 136 inDict['label_suffix']+=lsuffix.rstrip('_') 137 else: 138 inDict['label_suffix']=lsuffix.rstrip('_') 139 else: 140 if len(loopVars.keys())>0: 141 inDict['args']=newargs.strip() 142 inDict['label_suffix']=lsuffix.rstrip('_') 143 if len(loopVars.keys())>0: 144 return loopVars 145 else: 146 return None 147 148 def getArgLabel(self,testDict): 149 """ 150 In all of the arguments in the test dictionary, create a simple 151 string for searching within the makefile system. For simplicity in 152 search, remove "-", for strings, etc. 153 Also, concatenate the arg commands 154 For now, ignore nsize -- seems hard to search for anyway 155 """ 156 # Collect all of the args associated with a test 157 argStr=("" if 'args' not in testDict else testDict['args']) 158 if 'subtests' in testDict: 159 for stest in testDict["subtests"]: 160 sd=testDict[stest] 161 argStr=argStr+("" if 'args' not in sd else sd['args']) 162 163 # Now go through and cleanup 164 argStr=re.sub('{{(.*?)}}',"",argStr) 165 argStr=re.sub('-'," ",argStr) 166 for digit in string.digits: argStr=re.sub(digit," ",argStr) 167 argStr=re.sub("\.","",argStr) 168 argStr=re.sub(",","",argStr) 169 argStr=re.sub('\+',' ',argStr) 170 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 171 return argStr.strip() 172 173 def addToSources(self,exfile,root,srcDict): 174 """ 175 Put into data structure that allows easy generation of makefile 176 """ 177 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 178 fullfile=os.path.join(root,exfile) 179 relpfile=self.relpath(self.petsc_dir,fullfile) 180 lang=self.getLanguage(exfile) 181 if not lang: return 182 self.sources[pkg][lang]['srcs'].append(relpfile) 183 if 'depends' in srcDict: 184 depSrc=srcDict['depends'] 185 depObj=os.path.splitext(depSrc)[0]+".o" 186 self.sources[pkg][lang][exfile]=depObj 187 188 # In gmakefile, ${TESTDIR} var specifies the object compilation 189 testsdir=self.relpath(self.petsc_dir,root)+"/" 190 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 191 self.objects[pkg].append(objfile) 192 return 193 194 def addToTests(self,test,root,exfile,execname,testDict): 195 """ 196 Put into data structure that allows easy generation of makefile 197 Organized by languages to allow testing of languages 198 """ 199 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 200 #nmtest=self.nameSpace(test,root) 201 rpath=self.relpath(self.petsc_dir,root) 202 nmtest=os.path.join(rpath,test) 203 lang=self.getLanguage(exfile) 204 if not lang: return 205 self.tests[pkg][lang][nmtest]={} 206 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 207 self.tests[pkg][lang][nmtest]['exec']=execname 208 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 209 return 210 211 def getExecname(self,exfile,root): 212 """ 213 Generate bash script using template found next to this file. 214 This file is read in at constructor time to avoid file I/O 215 """ 216 rpath=self.relpath(self.petsc_dir,root) 217 if self.single_ex: 218 execname=rpath.split("/")[1]+"-ex" 219 else: 220 execname=os.path.splitext(exfile)[0] 221 return execname 222 223 def getSubstVars(self,testDict,rpath,testname): 224 """ 225 Create a dictionary with all of the variables that get substituted 226 into the template commands found in example_template.py 227 """ 228 subst={} 229 230 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 231 if 'nsize' not in testDict: testDict['nsize']=1 232 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 233 for ak in testparse.acceptedkeys: 234 if ak=='test': continue 235 subst[ak]=(testDict[ak] if ak in testDict else '') 236 237 # Now do other variables 238 subst['execname']=testDict['execname'] 239 if 'filter' in testDict: 240 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 241 242 # Others 243 subst['subargs']='' # Default. For variables override 244 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 245 subst['label_suffix']='' 246 subst['comments']="\n#".join(subst['comments'].split("\n")) 247 if subst['comments']: subst['comments']="#"+subst['comments'] 248 subst['exec']="../"+subst['execname'] 249 subst['testroot']=self.testroot_dir 250 subst['testname']=testname 251 dp = self.conf.get('DATAFILESPATH','') 252 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 253 254 # This is used to label some matrices 255 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 256 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 257 258 # These can have for loops and are treated separately later 259 subst['nsize']=str(subst['nsize']) 260 261 #Conf vars 262 if self.petsc_arch.find('valgrind')>=0: 263 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 264 else: 265 subst['mpiexec']=self.conf['MPIEXEC'] 266 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 267 subst['petsc_arch']=self.petsc_arch 268 subst['diff']=self.conf['DIFF'] 269 subst['rm']=self.conf['RM'] 270 subst['grep']=self.conf['GREP'] 271 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 272 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 273 274 # Output file is special because of subtests override 275 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 276 if not "_" in defroot: defroot=defroot+"_1" 277 subst['defroot']=defroot 278 subst['label']=self.nameSpace(defroot,subst['srcdir']) 279 subst['redirect_file']=defroot+".tmp" 280 if 'output_file' not in testDict: 281 subst['output_file']="output/"+defroot+".out" 282 # Add in the full path here. 283 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 284 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 285 if not subst['TODO']: 286 print "Warning: "+subst['output_file']+" not found." 287 # Worry about alt files here -- see 288 # src/snes/examples/tutorials/output/ex22*.out 289 altlist=[subst['output_file']] 290 for i in range(1,3): 291 altroot=defroot+"_alt" 292 if i==2: altroot=altroot+"_2" 293 af="output/"+altroot+".out" 294 srcaf=os.path.join(subst['srcdir'],af) 295 fullaf=os.path.join(self.petsc_dir,srcaf) 296 if os.path.isfile(fullaf): altlist.append(srcaf) 297 if len(altlist)>1: subst['altfiles']=altlist 298 #if len(altlist)>1: print "Found alt files: ",altlist 299 300 return subst 301 302 def getCmds(self,subst,i): 303 """ 304 Generate bash script using template found next to this file. 305 This file is read in at constructor time to avoid file I/O 306 """ 307 nindnt=i # the start and has to be consistent with below 308 cmdindnt=self.indent*nindnt 309 cmdLines="" 310 311 # MPI is the default -- but we have a few odd commands 312 if not subst['command']: 313 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 314 else: 315 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 316 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 317 318 cmdLines+=cmdindnt+'if test $res = 0; then\n' 319 diffindnt=self.indent*(nindnt+1) 320 if not subst['filter_output']: 321 if 'altfiles' not in subst: 322 cmd=diffindnt+self._substVars(subst,example_template.difftest) 323 else: 324 # Have to do it by hand a bit because of variable number of alt files 325 rf=subst['redirect_file'] 326 cmd=diffindnt+example_template.difftest.split('@')[0] 327 for i in range(len(subst['altfiles'])): 328 af=subst['altfiles'][i] 329 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 330 if i!=len(subst['altfiles'])-1: 331 cmd+=' || ${diff_exe} ' 332 else: 333 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 334 cmd+=subst['label_suffix']+' ""' # Quotes are painful 335 else: 336 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 337 cmdLines+=cmd+"\n" 338 cmdLines+=cmdindnt+'else\n' 339 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 340 cmdLines+=cmdindnt+'fi\n' 341 return cmdLines 342 343 def _substVars(self,subst,origStr): 344 """ 345 Substitute variables 346 """ 347 Str=origStr 348 for subkey in subst: 349 if type(subst[subkey])!=types.StringType: continue 350 patt="@"+subkey.upper()+"@" 351 Str=re.sub(patt,subst[subkey],Str) 352 return Str 353 354 def _writeTodoSkip(self,fh,tors,reasons,footer): 355 """ 356 Write out the TODO and SKIP lines in the file 357 The TODO or SKIP variable, tors, should be lower case 358 """ 359 TORS=tors.upper() 360 template=eval("example_template."+tors+"line") 361 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 362 tab = '' 363 if reasons: 364 fh.write('if ! $force; then\n') 365 tab = tab + ' ' 366 if reasons == ["Requires DATAFILESPATH"]: 367 # The only reason not to run is DATAFILESPATH, which we check at run-time 368 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 369 tab = tab + ' ' 370 if reasons: 371 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 372 fh.write(tab+footer+"\n") 373 fh.write(tab+"exit\n") 374 if reasons == ["Requires DATAFILESPATH"]: 375 fh.write(' fi\n') 376 if reasons: 377 fh.write('fi\n') 378 fh.write('\n\n') 379 return 380 381 def getLoopVarsHead(self,loopVars,i): 382 """ 383 Generate a nicely indented string with the format loops 384 Here is what the data structure looks like 385 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 386 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 387 loopVars['subargs']['pc_type']=["j","cholesky sor"] 388 """ 389 outstr=''; indnt=self.indent 390 for key in loopVars: 391 for var in loopVars[key]['varlist']: 392 varval=loopVars[key][var] 393 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 394 i = i + 1 395 return (outstr,i) 396 397 def getLoopVarsFoot(self,loopVars,i): 398 outstr=''; indnt=self.indent 399 for key in loopVars: 400 for var in loopVars[key]['varlist']: 401 i = i - 1 402 outstr += indnt * i + "done\n" 403 return (outstr,i) 404 405 def genRunScript(self,testname,root,isRun,srcDict): 406 """ 407 Generate bash script using template found next to this file. 408 This file is read in at constructor time to avoid file I/O 409 """ 410 # runscript_dir directory has to be consistent with gmakefile 411 testDict=srcDict[testname] 412 rpath=self.relpath(self.petsc_dir,root) 413 runscript_dir=os.path.join(self.testroot_dir,rpath) 414 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 415 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 416 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 417 418 # Get variables to go into shell scripts. last time testDict used 419 subst=self.getSubstVars(testDict,rpath,testname) 420 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 421 #if '33_' in testname: print subst['subargs'] 422 423 #Handle runfiles 424 for lfile in subst.get('localrunfiles','').split(): 425 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 426 shutil.copy(fullfile,runscript_dir) 427 # Check subtests for local runfiles 428 for stest in subst.get("subtests",[]): 429 for lfile in testDict[stest].get('localrunfiles','').split(): 430 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 431 shutil.copy(fullfile,self.runscript_dir) 432 433 # Now substitute the key variables into the header and footer 434 header=self._substVars(subst,example_template.header) 435 # The header is done twice to enable @...@ in header 436 header=self._substVars(subst,header) 437 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 438 439 # Start writing the file 440 fh.write(header+"\n") 441 442 # If there is a TODO or a SKIP then we do it before writing out the 443 # rest of the command (which is useful for working on the test) 444 # SKIP and TODO can be for the source file or for the runs 445 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 446 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 447 448 j=0 # for indentation 449 450 if loopVars: 451 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 452 if (loopHead): fh.write(loopHead+"\n") 453 454 # Subtests are special 455 if 'subtests' in testDict: 456 substP=subst # Subtests can inherit args but be careful 457 for stest in testDict["subtests"]: 458 subst=substP.copy() 459 subst.update(testDict[stest]) 460 subst['nsize']=str(subst['nsize']) 461 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 462 #if '10_9' in testname: print sLoopVars 463 if sLoopVars: 464 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 465 fh.write(sLoopHead+"\n") 466 fh.write(self.getCmds(subst,j)+"\n") 467 if sLoopVars: 468 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 469 fh.write(sLoopFoot+"\n") 470 else: 471 fh.write(self.getCmds(subst,j)+"\n") 472 473 if loopVars: 474 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 475 fh.write(loopFoot+"\n") 476 477 fh.write(footer+"\n") 478 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 479 #if '10_9' in testname: sys.exit() 480 return 481 482 def genScriptsAndInfo(self,exfile,root,srcDict): 483 """ 484 Generate scripts from the source file, determine if built, etc. 485 For every test in the exfile with info in the srcDict: 486 1. Determine if it needs to be run for this arch 487 2. Generate the script 488 3. Generate the data needed to write out the makefile in a 489 convenient way 490 All tests are *always* run, but some may be SKIP'd per the TAP standard 491 """ 492 debug=False 493 execname=self.getExecname(exfile,root) 494 isBuilt=self._isBuilt(exfile,srcDict) 495 for test in srcDict: 496 if test in self.buildkeys: continue 497 if debug: print self.nameSpace(exfile,root), test 498 srcDict[test]['execname']=execname # Convenience in generating scripts 499 isRun=self._isRun(srcDict[test]) 500 self.genRunScript(test,root,isRun,srcDict) 501 srcDict[test]['isrun']=isRun 502 self.addToTests(test,root,exfile,execname,srcDict[test]) 503 504 # This adds to datastructure for building deps 505 if isBuilt: self.addToSources(exfile,root,srcDict) 506 return 507 508 def _isBuilt(self,exfile,srcDict): 509 """ 510 Determine if this file should be built. 511 """ 512 # Get the language based on file extension 513 srcDict['SKIP'] = [] 514 lang=self.getLanguage(exfile) 515 if (lang=="F" or lang=="F90") and not self.have_fortran: 516 srcDict["SKIP"].append("Fortran required for this test") 517 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 518 srcDict["SKIP"].append("CUDA required for this test") 519 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 520 srcDict["SKIP"].append("C++ required for this test") 521 522 # Deprecated source files 523 if srcDict.get("TODO"): 524 return False 525 526 # isRun can work with srcDict to handle the requires 527 if "requires" in srcDict: 528 if len(srcDict["requires"])>0: 529 return self._isRun(srcDict) 530 531 return srcDict['SKIP'] == [] 532 533 534 def _isRun(self,testDict): 535 """ 536 Based on the requirements listed in the src file and the petscconf.h 537 info, determine whether this test should be run or not. 538 """ 539 indent=" " 540 debug=False 541 542 if 'SKIP' not in testDict: 543 testDict['SKIP'] = [] 544 # MPI requirements 545 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 546 if debug: print indent+"Cannot run parallel tests" 547 testDict['SKIP'].append("Parallel test with serial build") 548 549 # The requirements for the test are the sum of all the run subtests 550 if 'subtests' in testDict: 551 if 'requires' not in testDict: testDict['requires']="" 552 for stest in testDict['subtests']: 553 if 'requires' in testDict[stest]: 554 testDict['requires']+=" "+testDict[stest]['requires'] 555 556 557 # Now go through all requirements 558 if 'requires' in testDict: 559 for requirement in testDict['requires'].split(): 560 requirement=requirement.strip() 561 if not requirement: continue 562 if debug: print indent+"Requirement: ", requirement 563 isNull=False 564 if requirement.startswith("!"): 565 requirement=requirement[1:]; isNull=True 566 # Precision requirement for reals 567 if requirement in self.precision_types: 568 if self.conf['PETSC_PRECISION']==requirement: 569 if isNull: 570 testDict['SKIP'].append("not "+requirement+" required") 571 continue 572 continue # Success 573 elif not isNull: 574 testDict['SKIP'].append(requirement+" required") 575 continue 576 # Precision requirement for ints 577 if requirement in self.integer_types: 578 if requirement=="int32": 579 if self.conf['PETSC_SIZEOF_INT']==4: 580 if isNull: 581 testDict['SKIP'].append("not int32 required") 582 continue 583 continue # Success 584 elif not isNull: 585 testDict['SKIP'].append("int32 required") 586 continue 587 if requirement=="int64": 588 if self.conf['PETSC_SIZEOF_INT']==8: 589 if isNull: 590 testDict['SKIP'].append("NOT int64 required") 591 continue 592 continue # Success 593 elif not isNull: 594 testDict['SKIP'].append("int64 required") 595 continue 596 # Datafilespath 597 if requirement=="datafilespath" and not isNull: 598 testDict['SKIP'].append("Requires DATAFILESPATH") 599 continue 600 # Defines -- not sure I have comments matching 601 if "define(" in requirement.lower(): 602 reqdef=requirement.split("(")[1].split(")")[0] 603 if reqdef in self.conf: 604 if isNull: 605 testDict['SKIP'].append("Null requirement not met: "+requirement) 606 continue 607 continue # Success 608 elif not isNull: 609 testDict['SKIP'].append("Required: "+requirement) 610 continue 611 612 # Rest should be packages that we can just get from conf 613 if requirement == "complex": 614 petscconfvar="PETSC_USE_COMPLEX" 615 else: 616 petscconfvar="PETSC_HAVE_"+requirement.upper() 617 if self.conf.get(petscconfvar): 618 if isNull: 619 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 620 continue 621 continue # Success 622 elif not isNull: 623 if debug: print "requirement not found: ", requirement 624 testDict['SKIP'].append(petscconfvar+" requirement not met") 625 continue 626 627 return testDict['SKIP'] == [] 628 629 def genPetscTests_summarize(self,dataDict): 630 """ 631 Required method to state what happened 632 """ 633 if not self.summarize: return 634 indent=" " 635 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 636 fh=open(fhname,"w") 637 #print "See ", fhname 638 for root in dataDict: 639 relroot=self.relpath(self.petsc_dir,root) 640 pkg=relroot.split("/")[1] 641 fh.write(relroot+"\n") 642 allSrcs=[] 643 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 644 for exfile in dataDict[root]: 645 # Basic information 646 fullfile=os.path.join(root,exfile) 647 rfile=self.relpath(self.petsc_dir,fullfile) 648 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 649 fh.write(indent+exfile+indent*4+builtStatus+"\n") 650 651 for test in dataDict[root][exfile]: 652 if test in self.buildkeys: continue 653 line=indent*2+test 654 fh.write(line+"\n") 655 # Looks nice to have the keys in order 656 #for key in dataDict[root][exfile][test]: 657 for key in "isrun abstracted nsize args requires script".split(): 658 if key not in dataDict[root][exfile][test]: continue 659 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 660 fh.write(line+"\n") 661 fh.write("\n") 662 fh.write("\n") 663 fh.write("\n") 664 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 665 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 666 fh.close() 667 return 668 669 def genPetscTests(self,root,dirs,files,dataDict): 670 """ 671 Go through and parse the source files in the directory to generate 672 the examples based on the metadata contained in the source files 673 """ 674 debug=False 675 # Use examplesAnalyze to get what the makefles think are sources 676 #self.examplesAnalyze(root,dirs,files,anlzDict) 677 678 dataDict[root]={} 679 680 for exfile in files: 681 #TST: Until we replace files, still leaving the orginals as is 682 #if not exfile.startswith("new_"+"ex"): continue 683 if not exfile.startswith("ex"): continue 684 685 # Convenience 686 fullex=os.path.join(root,exfile) 687 relpfile=self.relpath(self.petsc_dir,fullex) 688 if debug: print relpfile 689 dataDict[root].update(testparse.parseTestFile(fullex,0)) 690 # Need to check and make sure tests are in the file 691 # if verbosity>=1: print relpfile 692 if exfile in dataDict[root]: 693 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 694 695 return 696 697 def walktree(self,top,action="printFiles"): 698 """ 699 Walk a directory tree, starting from 'top' 700 """ 701 #print "action", action 702 # Goal of action is to fill this dictionary 703 dataDict={} 704 for root, dirs, files in os.walk(top, topdown=False): 705 if not "examples" in root: continue 706 if not os.path.isfile(os.path.join(root,"makefile")): continue 707 bname=os.path.basename(root.rstrip("/")) 708 if bname=="tests" or bname=="tutorials": 709 eval("self."+action+"(root,dirs,files,dataDict)") 710 if type(top) != types.StringType: 711 raise TypeError("top must be a string") 712 # Now summarize this dictionary 713 eval("self."+action+"_summarize(dataDict)") 714 return dataDict 715 716 def gen_gnumake(self, fd): 717 """ 718 Overwrite of the method in the base PETSc class 719 """ 720 def write(stem, srcs): 721 for lang in LANGS: 722 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 723 for pkg in PKGS: 724 srcs = self.gen_pkg(pkg) 725 write('testsrcs-' + pkg, srcs) 726 return self.gendeps 727 728 def gen_pkg(self, pkg): 729 """ 730 Overwrite of the method in the base PETSc class 731 """ 732 return self.sources[pkg] 733 734 def write_gnumake(self,dataDict): 735 """ 736 Write out something similar to files from gmakegen.py 737 738 Test depends on script which also depends on source 739 file, but since I don't have a good way generating 740 acting on a single file (oops) just depend on 741 executable which in turn will depend on src file 742 """ 743 # Different options for how to set up the targets 744 compileExecsFirst=False 745 746 # Open file 747 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 748 fd = open(arch_files, 'w') 749 750 # Write out the sources 751 gendeps = self.gen_gnumake(fd) 752 753 # Write out the tests and execname targets 754 fd.write("\n#Tests and executables\n") # Delimiter 755 756 for pkg in PKGS: 757 # These grab the ones that are built 758 for lang in LANGS: 759 testdeps=[] 760 for ftest in self.tests[pkg][lang]: 761 test=os.path.basename(ftest) 762 basedir=os.path.dirname(ftest) 763 testdeps.append(self.nameSpace(test,basedir)) 764 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 765 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 766 767 # test targets 768 for ftest in self.tests[pkg][lang]: 769 test=os.path.basename(ftest) 770 basedir=os.path.dirname(ftest) 771 testdir="${TESTDIR}/"+basedir+"/" 772 nmtest=self.nameSpace(test,basedir) 773 rundir=os.path.join(testdir,test) 774 #print test, nmtest 775 script=test+".sh" 776 777 # Deps 778 exfile=self.tests[pkg][lang][ftest]['exfile'] 779 fullex=os.path.join(self.petsc_dir,exfile) 780 localexec=self.tests[pkg][lang][ftest]['exec'] 781 execname=os.path.join(testdir,localexec) 782 fullscript=os.path.join(testdir,script) 783 tmpfile=os.path.join(testdir,test,test+".tmp") 784 785 # *.counts depends on the script and either executable (will 786 # be run) or the example source file (SKIP or TODO) 787 fd.write('%s.counts : %s %s\n' 788 % (os.path.join('$(TESTDIR)/counts', nmtest), 789 fullscript, 790 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 791 # Now write the args: 792 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 793 794 fd.close() 795 return 796 797 def writeHarness(self,output,dataDict): 798 """ 799 This is set up to write out multiple harness even if only gnumake 800 is supported now 801 """ 802 eval("self.write_"+output+"(dataDict)") 803 return 804 805def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 806 if output is None: 807 output = 'gnumake' 808 809 810 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 811 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 812 pEx.writeHarness(output,dataDict) 813 814if __name__ == '__main__': 815 import optparse 816 parser = optparse.OptionParser() 817 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 818 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 819 parser.add_option('--output', help='Location to write output file', default=None) 820 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 821 opts, extra_args = parser.parse_args() 822 if extra_args: 823 import sys 824 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 825 exit(1) 826 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 827