1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 for key in lkeys: 107 if type(inDict[key])!=types.StringType: continue 108 keystr = str(inDict[key]) 109 akey=('subargs' if key=='args' else key) # what to assign 110 if not inDict.has_key(akey): inDict[akey]='' 111 varlist=[] 112 for varset in re.split('-(?=[a-zA-Z])',keystr): 113 if not varset.strip(): continue 114 if len(re.findall('{{(.*?)}(?:[\w\s]*)?}',varset))>0: 115 if not loopVars.has_key(akey): loopVars[akey]={} 116 # Assuming only one for loop per var specification 117 keyvar=varset.split("{{")[0].strip() 118 if not keyvar.strip(): 119 if key=='nsize': 120 keyvar='nsize' 121 else: 122 print testname+": Problem in finding loop variables", keyvar, varset 123 varlist.append(keyvar) 124 loopVars[akey][keyvar]=[keyvar,re.findall('{{(.*?)}(?:[\w\s]*)?}',varset)[0]] 125 if akey=='nsize': 126 inDict[akey] = '${' + keyvar + '}' 127 lsuffix+=akey+'-'+inDict[akey]+'_' 128 else: 129 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 130 lsuffix+=keyvar+'-${' + keyvar + '}_' 131 else: 132 if key=='args': newargs+=" -"+varset.strip() 133 if len(varlist)>0: loopVars[akey]['varlist']=varlist 134 135 136 # For subtests, args are always substituted in (not top level) 137 if isSubtest: 138 inDict['subargs']+=" "+newargs.strip() 139 inDict['args']='' 140 if inDict.has_key('label_suffix'): 141 inDict['label_suffix']+=lsuffix.rstrip('_') 142 else: 143 inDict['label_suffix']=lsuffix.rstrip('_') 144 else: 145 if len(loopVars.keys())>0: 146 inDict['args']=newargs.strip() 147 inDict['label_suffix']=lsuffix.rstrip('_') 148 if len(loopVars.keys())>0: 149 return loopVars 150 else: 151 return None 152 153 def getArgLabel(self,testDict): 154 """ 155 In all of the arguments in the test dictionary, create a simple 156 string for searching within the makefile system. For simplicity in 157 search, remove "-", for strings, etc. 158 Also, concatenate the arg commands 159 For now, ignore nsize -- seems hard to search for anyway 160 """ 161 # Collect all of the args associated with a test 162 argStr=("" if not testDict.has_key('args') else testDict['args']) 163 if testDict.has_key('subtests'): 164 for stest in testDict["subtests"]: 165 sd=testDict[stest] 166 argStr=argStr+("" if not sd.has_key('args') else sd['args']) 167 168 # Now go through and cleanup 169 argStr=re.sub('{{(.*?)}}',"",argStr) 170 argStr=re.sub('-'," ",argStr) 171 for digit in string.digits: argStr=re.sub(digit," ",argStr) 172 argStr=re.sub("\.","",argStr) 173 argStr=re.sub(",","",argStr) 174 argStr=re.sub('\+',' ',argStr) 175 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 176 return argStr.strip() 177 178 def addToSources(self,exfile,root,srcDict): 179 """ 180 Put into data structure that allows easy generation of makefile 181 """ 182 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 183 fullfile=os.path.join(root,exfile) 184 relpfile=self.relpath(self.petsc_dir,fullfile) 185 lang=self.getLanguage(exfile) 186 if not lang: return 187 self.sources[pkg][lang]['srcs'].append(relpfile) 188 if srcDict.has_key('depends'): 189 depSrc=srcDict['depends'] 190 depObj=os.path.splitext(depSrc)[0]+".o" 191 self.sources[pkg][lang][exfile]=depObj 192 193 # In gmakefile, ${TESTDIR} var specifies the object compilation 194 testsdir=self.relpath(self.petsc_dir,root)+"/" 195 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 196 self.objects[pkg].append(objfile) 197 return 198 199 def addToTests(self,test,root,exfile,execname,testDict): 200 """ 201 Put into data structure that allows easy generation of makefile 202 Organized by languages to allow testing of languages 203 """ 204 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 205 #nmtest=self.nameSpace(test,root) 206 rpath=self.relpath(self.petsc_dir,root) 207 nmtest=os.path.join(rpath,test) 208 lang=self.getLanguage(exfile) 209 if not lang: return 210 self.tests[pkg][lang][nmtest]={} 211 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 212 self.tests[pkg][lang][nmtest]['exec']=execname 213 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 214 return 215 216 def getExecname(self,exfile,root): 217 """ 218 Generate bash script using template found next to this file. 219 This file is read in at constructor time to avoid file I/O 220 """ 221 rpath=self.relpath(self.petsc_dir,root) 222 if self.single_ex: 223 execname=rpath.split("/")[1]+"-ex" 224 else: 225 execname=os.path.splitext(exfile)[0] 226 return execname 227 228 def getSubstVars(self,testDict,rpath,testname): 229 """ 230 Create a dictionary with all of the variables that get substituted 231 into the template commands found in example_template.py 232 """ 233 subst={} 234 235 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 236 if not testDict.has_key('nsize'): testDict['nsize']=1 237 for ak in testparse.acceptedkeys: 238 if ak=='test': continue 239 subst[ak]=(testDict[ak] if testDict.has_key(ak) else '') 240 241 # Now do other variables 242 subst['execname']=testDict['execname'] 243 if testDict.has_key('filter'): 244 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 245 246 # Others 247 subst['subargs']='' # Default. For variables override 248 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 249 subst['label_suffix']='' 250 subst['comments']="\n#".join(subst['comments'].split("\n")) 251 if subst['comments']: subst['comments']="#"+subst['comments'] 252 subst['exec']="../"+subst['execname'] 253 subst['testroot']=self.testroot_dir 254 subst['testname']=testname 255 dp = self.conf.get('DATAFILESPATH','') 256 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 257 258 # This is used to label some matrices 259 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 260 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 261 262 # These can have for loops and are treated separately later 263 subst['nsize']=str(subst['nsize']) 264 265 #Conf vars 266 if self.petsc_arch.find('valgrind')>=0: 267 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 268 else: 269 subst['mpiexec']=self.conf['MPIEXEC'] 270 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 271 subst['diff']=self.conf['DIFF'] 272 subst['rm']=self.conf['RM'] 273 subst['grep']=self.conf['GREP'] 274 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 275 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 276 277 # Output file is special because of subtests override 278 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 279 if not "_" in defroot: defroot=defroot+"_1" 280 subst['defroot']=defroot 281 subst['label']=self.nameSpace(defroot,subst['srcdir']) 282 subst['redirect_file']=defroot+".tmp" 283 if not testDict.has_key('output_file'): 284 subst['output_file']="output/"+defroot+".out" 285 # Add in the full path here. 286 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 287 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 288 if not subst['TODO']: 289 print "Warning: "+subst['output_file']+" not found." 290 # Worry about alt files here -- see 291 # src/snes/examples/tutorials/output/ex22*.out 292 altlist=[subst['output_file']] 293 for i in range(1,3): 294 altroot=defroot+"_alt" 295 if i==2: altroot=altroot+"_2" 296 af="output/"+altroot+".out" 297 srcaf=os.path.join(subst['srcdir'],af) 298 fullaf=os.path.join(self.petsc_dir,srcaf) 299 if os.path.isfile(fullaf): altlist.append(srcaf) 300 if len(altlist)>1: subst['altfiles']=altlist 301 #if len(altlist)>1: print "Found alt files: ",altlist 302 303 return subst 304 305 def getCmds(self,subst,i): 306 """ 307 Generate bash script using template found next to this file. 308 This file is read in at constructor time to avoid file I/O 309 """ 310 indnt=self.indent 311 nindnt=i # the start and has to be consistent with below 312 cmdLines="" 313 314 # MPI is the default -- but we have a few odd commands 315 if not subst['command']: 316 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 317 else: 318 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 319 cmdLines+=cmd+"\n\n" 320 321 if not subst['filter_output']: 322 if not subst.has_key('altfiles'): 323 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 324 else: 325 # Have to do it by hand a bit because of variable number of alt files 326 rf=subst['redirect_file'] 327 cmd=indnt*nindnt+example_template.difftest.split('@')[0] 328 for i in range(len(subst['altfiles'])): 329 af=subst['altfiles'][i] 330 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 331 if i!=len(subst['altfiles'])-1: 332 cmd+=' || ${diff_exe} ' 333 else: 334 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 335 cmd+=subst['label_suffix']+' ""' # Quotes are painful 336 else: 337 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 338 cmdLines+=cmd+"\n" 339 return cmdLines 340 341 def _substVars(self,subst,origStr): 342 """ 343 Substitute variables 344 """ 345 Str=origStr 346 for subkey in subst: 347 if type(subst[subkey])!=types.StringType: continue 348 patt="@"+subkey.upper()+"@" 349 Str=re.sub(patt,subst[subkey],Str) 350 return Str 351 352 def _writeTodoSkip(self,fh,tors,reasons,footer): 353 """ 354 Write out the TODO and SKIP lines in the file 355 The TODO or SKIP variable, tors, should be lower case 356 """ 357 TORS=tors.upper() 358 template=eval("example_template."+tors+"line") 359 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 360 tab = '' 361 if reasons == ["Requires DATAFILESPATH"]: 362 # The only reason not to run is DATAFILESPATH, which we check at run-time 363 tab = ' ' 364 fh.write('if test -z "${DATAFILESPATH}"; then\n') 365 if reasons: 366 fh.write(tab+tsStr+"\ntotal=1; "+tors+"=1\n") 367 fh.write(tab+footer+"\n") 368 fh.write(tab+"exit\n") 369 if reasons == ["Requires DATAFILESPATH"]: 370 fh.write('fi\n') 371 fh.write('\n\n') 372 return 373 374 def getLoopVarsHead(self,loopVars,i): 375 """ 376 Generate a nicely indented string with the format loops 377 Here is what the data structure looks like 378 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 379 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 380 loopVars['subargs']['pc_type']=["j","cholesky sor"] 381 """ 382 outstr=''; indnt=self.indent 383 for key in loopVars: 384 for var in loopVars[key]['varlist']: 385 varval=loopVars[key][var] 386 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 387 i = i + 1 388 return (outstr,i) 389 390 def getLoopVarsFoot(self,loopVars,i): 391 outstr=''; indnt=self.indent 392 for key in loopVars: 393 for var in loopVars[key]['varlist']: 394 i = i - 1 395 outstr += indnt * i + "done\n" 396 return (outstr,i) 397 398 def genRunScript(self,testname,root,isRun,srcDict): 399 """ 400 Generate bash script using template found next to this file. 401 This file is read in at constructor time to avoid file I/O 402 """ 403 # runscript_dir directory has to be consistent with gmakefile 404 testDict=srcDict[testname] 405 rpath=self.relpath(self.petsc_dir,root) 406 runscript_dir=os.path.join(self.testroot_dir,rpath) 407 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 408 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 409 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 410 411 # Get variables to go into shell scripts. last time testDict used 412 subst=self.getSubstVars(testDict,rpath,testname) 413 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 414 #if '33_' in testname: print subst['subargs'] 415 416 #Handle runfiles 417 if subst.has_key('localrunfiles'): 418 for lfile in subst['localrunfiles'].split(): 419 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 420 shutil.copy(fullfile,runscript_dir) 421 # Check subtests for local runfiles 422 if subst.has_key("subtests"): 423 for stest in subst["subtests"]: 424 if subst[stest].has_key('localrunfiles'): 425 for lfile in testDict[stest]['localrunfiles'].split(): 426 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 427 shutil.copy(fullfile,self.runscript_dir) 428 429 # Now substitute the key variables into the header and footer 430 header=self._substVars(subst,example_template.header) 431 # The header is done twice to enable @...@ in header 432 header=self._substVars(subst,header) 433 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 434 435 # Start writing the file 436 fh.write(header+"\n") 437 438 # If there is a TODO or a SKIP then we do it before writing out the 439 # rest of the command (which is useful for working on the test) 440 # SKIP and TODO can be for the source file or for the runs 441 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 442 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 443 444 j=0 # for indentation 445 446 if loopVars: 447 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 448 if (loopHead): fh.write(loopHead+"\n") 449 450 # Subtests are special 451 if testDict.has_key("subtests"): 452 substP=subst # Subtests can inherit args but be careful 453 for stest in testDict["subtests"]: 454 subst=substP.copy() 455 subst.update(testDict[stest]) 456 subst['nsize']=str(subst['nsize']) 457 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 458 #if '10_9' in testname: print sLoopVars 459 if sLoopVars: 460 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 461 fh.write(sLoopHead+"\n") 462 fh.write(self.getCmds(subst,j)+"\n") 463 if sLoopVars: 464 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 465 fh.write(sLoopFoot+"\n") 466 else: 467 fh.write(self.getCmds(subst,j)+"\n") 468 469 if loopVars: 470 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 471 fh.write(loopFoot+"\n") 472 473 fh.write(footer+"\n") 474 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 475 #if '10_9' in testname: sys.exit() 476 return 477 478 def genScriptsAndInfo(self,exfile,root,srcDict): 479 """ 480 Generate scripts from the source file, determine if built, etc. 481 For every test in the exfile with info in the srcDict: 482 1. Determine if it needs to be run for this arch 483 2. Generate the script 484 3. Generate the data needed to write out the makefile in a 485 convenient way 486 All tests are *always* run, but some may be SKIP'd per the TAP standard 487 """ 488 debug=False 489 fileIsTested=False 490 execname=self.getExecname(exfile,root) 491 isBuilt=self._isBuilt(exfile,srcDict) 492 for test in srcDict: 493 if test in self.buildkeys: continue 494 if debug: print self.nameSpace(exfile,root), test 495 srcDict[test]['execname']=execname # Convenience in generating scripts 496 isRun=self._isRun(srcDict[test]) 497 self.genRunScript(test,root,isRun,srcDict) 498 srcDict[test]['isrun']=isRun 499 if isRun: fileIsTested=True 500 self.addToTests(test,root,exfile,execname,srcDict[test]) 501 502 # This adds to datastructure for building deps 503 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 504 #print self.nameSpace(exfile,root), fileIsTested 505 return 506 507 def _isBuilt(self,exfile,srcDict): 508 """ 509 Determine if this file should be built. 510 """ 511 # Get the language based on file extension 512 srcDict['SKIP'] = [] 513 lang=self.getLanguage(exfile) 514 if (lang=="F" or lang=="F90") and not self.have_fortran: 515 srcDict["SKIP"].append("Fortran required for this test") 516 if lang=="cu" and not self.conf.has_key('PETSC_HAVE_CUDA'): 517 srcDict["SKIP"].append("CUDA required for this test") 518 if lang=="cxx" and not self.conf.has_key('PETSC_HAVE_CXX'): 519 srcDict["SKIP"].append("C++ required for this test") 520 521 # Deprecated source files 522 if srcDict.get("TODO"): 523 return False 524 525 # isRun can work with srcDict to handle the requires 526 if srcDict.has_key("requires"): 527 if len(srcDict["requires"])>0: 528 return self._isRun(srcDict) 529 530 return srcDict['SKIP'] == [] 531 532 533 def _isRun(self,testDict): 534 """ 535 Based on the requirements listed in the src file and the petscconf.h 536 info, determine whether this test should be run or not. 537 """ 538 indent=" " 539 debug=False 540 541 if 'SKIP' not in testDict: 542 testDict['SKIP'] = [] 543 # MPI requirements 544 if testDict.has_key('nsize'): 545 if testDict['nsize']>1 and self.conf.has_key('MPI_IS_MPIUNI'): 546 if debug: print indent+"Cannot run parallel tests" 547 testDict['SKIP'].append("Parallel test with serial build") 548 549 # The requirements for the test are the sum of all the run subtests 550 if testDict.has_key('subtests'): 551 if not testDict.has_key('requires'): testDict['requires']="" 552 for stest in testDict['subtests']: 553 if testDict[stest].has_key('requires'): 554 testDict['requires']=testDict['requires']+" "+testDict[stest]['requires'] 555 556 557 # Now go through all requirements 558 if testDict.has_key('requires'): 559 for requirement in testDict['requires'].split(): 560 requirement=requirement.strip() 561 if not requirement: continue 562 if debug: print indent+"Requirement: ", requirement 563 isNull=False 564 if requirement.startswith("!"): 565 requirement=requirement[1:]; isNull=True 566 # Precision requirement for reals 567 if requirement in self.precision_types: 568 if self.conf['PETSC_PRECISION']==requirement: 569 if isNull: 570 testDict['SKIP'].append("not "+requirement+" required") 571 continue 572 elif not isNull: 573 testDict['SKIP'].append(requirement+" required") 574 continue 575 # Precision requirement for ints 576 if requirement in self.integer_types: 577 if requirement=="int32": 578 if self.conf['PETSC_SIZEOF_INT']==4: 579 if isNull: 580 testDict['SKIP'].append("not int32 required") 581 continue 582 elif not isNull: 583 testDict['SKIP'].append("int32 required") 584 continue 585 if requirement=="int64": 586 if self.conf['PETSC_SIZEOF_INT']==8: 587 if isNull: 588 testDict['SKIP'].append("NOT int64 required") 589 continue 590 elif not isNull: 591 testDict['SKIP'].append("int64 required") 592 continue 593 # Datafilespath 594 if requirement=="datafilespath" and not isNull: 595 testDict['SKIP'].append("Requires DATAFILESPATH") 596 continue 597 # Defines -- not sure I have comments matching 598 if "define(" in requirement.lower(): 599 reqdef=requirement.split("(")[1].split(")")[0] 600 if reqdef in self.conf: 601 if isNull: 602 testDict['SKIP'].append("Null requirement not met: "+requirement) 603 continue 604 elif isNull: 605 testDict['SKIP'].append("Required: "+requirement) 606 continue 607 608 # Rest should be packages that we can just get from conf 609 if requirement == "complex": 610 petscconfvar="PETSC_USE_COMPLEX" 611 else: 612 petscconfvar="PETSC_HAVE_"+requirement.upper() 613 if self.conf.get(petscconfvar): 614 if isNull: 615 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 616 continue 617 elif not isNull: 618 if debug: print "requirement not found: ", requirement 619 testDict['SKIP'].append(petscconfvar+" requirement not met") 620 continue 621 622 return testDict['SKIP'] == [] 623 624 def genPetscTests_summarize(self,dataDict): 625 """ 626 Required method to state what happened 627 """ 628 if not self.summarize: return 629 indent=" " 630 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 631 fh=open(fhname,"w") 632 #print "See ", fhname 633 for root in dataDict: 634 relroot=self.relpath(self.petsc_dir,root) 635 pkg=relroot.split("/")[1] 636 fh.write(relroot+"\n") 637 allSrcs=[] 638 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 639 for exfile in dataDict[root]: 640 # Basic information 641 fullfile=os.path.join(root,exfile) 642 rfile=self.relpath(self.petsc_dir,fullfile) 643 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 644 fh.write(indent+exfile+indent*4+builtStatus+"\n") 645 646 for test in dataDict[root][exfile]: 647 if test in self.buildkeys: continue 648 line=indent*2+test 649 fh.write(line+"\n") 650 # Looks nice to have the keys in order 651 #for key in dataDict[root][exfile][test]: 652 for key in "isrun abstracted nsize args requires script".split(): 653 if not dataDict[root][exfile][test].has_key(key): continue 654 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 655 fh.write(line+"\n") 656 fh.write("\n") 657 fh.write("\n") 658 fh.write("\n") 659 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 660 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 661 fh.close() 662 return 663 664 def genPetscTests(self,root,dirs,files,dataDict): 665 """ 666 Go through and parse the source files in the directory to generate 667 the examples based on the metadata contained in the source files 668 """ 669 debug=False 670 # Use examplesAnalyze to get what the makefles think are sources 671 #self.examplesAnalyze(root,dirs,files,anlzDict) 672 673 dataDict[root]={} 674 675 for exfile in files: 676 #TST: Until we replace files, still leaving the orginals as is 677 #if not exfile.startswith("new_"+"ex"): continue 678 if not exfile.startswith("ex"): continue 679 680 # Convenience 681 fullex=os.path.join(root,exfile) 682 relpfile=self.relpath(self.petsc_dir,fullex) 683 if debug: print relpfile 684 dataDict[root].update(testparse.parseTestFile(fullex,0)) 685 # Need to check and make sure tests are in the file 686 # if verbosity>=1: print relpfile 687 if dataDict[root].has_key(exfile): 688 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 689 690 return 691 692 def walktree(self,top,action="printFiles"): 693 """ 694 Walk a directory tree, starting from 'top' 695 """ 696 #print "action", action 697 # Goal of action is to fill this dictionary 698 dataDict={} 699 for root, dirs, files in os.walk(top, topdown=False): 700 if not "examples" in root: continue 701 if not os.path.isfile(os.path.join(root,"makefile")): continue 702 bname=os.path.basename(root.rstrip("/")) 703 if bname=="tests" or bname=="tutorials": 704 eval("self."+action+"(root,dirs,files,dataDict)") 705 if type(top) != types.StringType: 706 raise TypeError("top must be a string") 707 # Now summarize this dictionary 708 eval("self."+action+"_summarize(dataDict)") 709 return dataDict 710 711 def gen_gnumake(self, fd): 712 """ 713 Overwrite of the method in the base PETSc class 714 """ 715 def write(stem, srcs): 716 for lang in LANGS: 717 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 718 for pkg in PKGS: 719 srcs = self.gen_pkg(pkg) 720 write('testsrcs-' + pkg, srcs) 721 return self.gendeps 722 723 def gen_pkg(self, pkg): 724 """ 725 Overwrite of the method in the base PETSc class 726 """ 727 return self.sources[pkg] 728 729 def write_gnumake(self,dataDict): 730 """ 731 Write out something similar to files from gmakegen.py 732 733 There is not a lot of has_key type checking because 734 should just work and need to know if there are bugs 735 736 Test depends on script which also depends on source 737 file, but since I don't have a good way generating 738 acting on a single file (oops) just depend on 739 executable which in turn will depend on src file 740 """ 741 # Different options for how to set up the targets 742 compileExecsFirst=False 743 744 # Open file 745 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 746 fd = open(arch_files, 'w') 747 748 # Write out the sources 749 gendeps = self.gen_gnumake(fd) 750 751 # Write out the tests and execname targets 752 fd.write("\n#Tests and executables\n") # Delimiter 753 754 for pkg in PKGS: 755 # These grab the ones that are built 756 for lang in LANGS: 757 testdeps=[] 758 for ftest in self.tests[pkg][lang]: 759 test=os.path.basename(ftest) 760 basedir=os.path.dirname(ftest) 761 testdeps.append(self.nameSpace(test,basedir)) 762 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 763 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 764 765 # test targets 766 for ftest in self.tests[pkg][lang]: 767 test=os.path.basename(ftest) 768 basedir=os.path.dirname(ftest) 769 testdir="${TESTDIR}/"+basedir+"/" 770 nmtest=self.nameSpace(test,basedir) 771 rundir=os.path.join(testdir,test) 772 #print test, nmtest 773 script=test+".sh" 774 775 # Deps 776 exfile=self.tests[pkg][lang][ftest]['exfile'] 777 fullex=os.path.join(self.petsc_dir,exfile) 778 localexec=self.tests[pkg][lang][ftest]['exec'] 779 execname=os.path.join(testdir,localexec) 780 fullscript=os.path.join(testdir,script) 781 tmpfile=os.path.join(testdir,test,test+".tmp") 782 783 # *.counts depends on the script and either executable (will 784 # be run) or the example source file (SKIP or TODO) 785 fd.write('%s.counts : %s %s\n' 786 % (os.path.join('$(TESTDIR)/counts', nmtest), 787 fullscript, 788 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 789 # Now write the args: 790 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 791 792 fd.close() 793 return 794 795 def writeHarness(self,output,dataDict): 796 """ 797 This is set up to write out multiple harness even if only gnumake 798 is supported now 799 """ 800 eval("self.write_"+output+"(dataDict)") 801 return 802 803def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 804 if output is None: 805 output = 'gnumake' 806 807 808 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 809 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 810 pEx.writeHarness(output,dataDict) 811 812if __name__ == '__main__': 813 import optparse 814 parser = optparse.OptionParser() 815 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 816 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 817 parser.add_option('--output', help='Location to write output file', default=None) 818 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 819 opts, extra_args = parser.parse_args() 820 if extra_args: 821 import sys 822 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 823 exit(1) 824 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 825