1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double quad int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 from testparse import parseLoopArgs 107 for key in lkeys: 108 if type(inDict[key])!=types.StringType: continue 109 keystr = str(inDict[key]) 110 akey=('subargs' if key=='args' else key) # what to assign 111 if akey not in inDict: inDict[akey]='' 112 varlist=[] 113 for varset in re.split('-(?=[a-zA-Z])',keystr): 114 if not varset.strip(): continue 115 if '{{' in varset: 116 keyvar,lvars,ftype=parseLoopArgs(varset) 117 if akey not in loopVars: loopVars[akey]={} 118 varlist.append(keyvar) 119 loopVars[akey][keyvar]=[keyvar,lvars] 120 if akey=='nsize': 121 inDict[akey] = '${' + keyvar + '}' 122 lsuffix+=akey+'-'+inDict[akey]+'_' 123 else: 124 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 125 lsuffix+=keyvar+'-${' + keyvar + '}_' 126 else: 127 if key=='args': newargs+=" -"+varset.strip() 128 if len(varlist)>0: loopVars[akey]['varlist']=varlist 129 130 131 # For subtests, args are always substituted in (not top level) 132 if isSubtest: 133 inDict['subargs']+=" "+newargs.strip() 134 inDict['args']='' 135 if 'label_suffix' in inDict: 136 inDict['label_suffix']+=lsuffix.rstrip('_') 137 else: 138 inDict['label_suffix']=lsuffix.rstrip('_') 139 else: 140 if len(loopVars.keys())>0: 141 inDict['args']=newargs.strip() 142 inDict['label_suffix']=lsuffix.rstrip('_') 143 if len(loopVars.keys())>0: 144 return loopVars 145 else: 146 return None 147 148 def getArgLabel(self,testDict): 149 """ 150 In all of the arguments in the test dictionary, create a simple 151 string for searching within the makefile system. For simplicity in 152 search, remove "-", for strings, etc. 153 Also, concatenate the arg commands 154 For now, ignore nsize -- seems hard to search for anyway 155 """ 156 # Collect all of the args associated with a test 157 argStr=("" if 'args' not in testDict else testDict['args']) 158 if 'subtests' in testDict: 159 for stest in testDict["subtests"]: 160 sd=testDict[stest] 161 argStr=argStr+("" if 'args' not in sd else sd['args']) 162 163 # Now go through and cleanup 164 argStr=re.sub('{{(.*?)}}',"",argStr) 165 argStr=re.sub('-'," ",argStr) 166 for digit in string.digits: argStr=re.sub(digit," ",argStr) 167 argStr=re.sub("\.","",argStr) 168 argStr=re.sub(",","",argStr) 169 argStr=re.sub('\+',' ',argStr) 170 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 171 return argStr.strip() 172 173 def addToSources(self,exfile,root,srcDict): 174 """ 175 Put into data structure that allows easy generation of makefile 176 """ 177 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 178 fullfile=os.path.join(root,exfile) 179 relpfile=self.relpath(self.petsc_dir,fullfile) 180 lang=self.getLanguage(exfile) 181 if not lang: return 182 self.sources[pkg][lang]['srcs'].append(relpfile) 183 if 'depends' in srcDict: 184 depSrc=srcDict['depends'] 185 depObj=os.path.splitext(depSrc)[0]+".o" 186 self.sources[pkg][lang][exfile]=depObj 187 188 # In gmakefile, ${TESTDIR} var specifies the object compilation 189 testsdir=self.relpath(self.petsc_dir,root)+"/" 190 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 191 self.objects[pkg].append(objfile) 192 return 193 194 def addToTests(self,test,root,exfile,execname,testDict): 195 """ 196 Put into data structure that allows easy generation of makefile 197 Organized by languages to allow testing of languages 198 """ 199 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 200 #nmtest=self.nameSpace(test,root) 201 rpath=self.relpath(self.petsc_dir,root) 202 nmtest=os.path.join(rpath,test) 203 lang=self.getLanguage(exfile) 204 if not lang: return 205 self.tests[pkg][lang][nmtest]={} 206 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 207 self.tests[pkg][lang][nmtest]['exec']=execname 208 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 209 return 210 211 def getExecname(self,exfile,root): 212 """ 213 Generate bash script using template found next to this file. 214 This file is read in at constructor time to avoid file I/O 215 """ 216 rpath=self.relpath(self.petsc_dir,root) 217 if self.single_ex: 218 execname=rpath.split("/")[1]+"-ex" 219 else: 220 execname=os.path.splitext(exfile)[0] 221 return execname 222 223 def getSubstVars(self,testDict,rpath,testname): 224 """ 225 Create a dictionary with all of the variables that get substituted 226 into the template commands found in example_template.py 227 """ 228 subst={} 229 230 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 231 if 'nsize' not in testDict: testDict['nsize']=1 232 for ak in testparse.acceptedkeys: 233 if ak=='test': continue 234 subst[ak]=(testDict[ak] if ak in testDict else '') 235 236 # Now do other variables 237 subst['execname']=testDict['execname'] 238 if 'filter' in testDict: 239 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 240 241 # Others 242 subst['subargs']='' # Default. For variables override 243 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 244 subst['label_suffix']='' 245 subst['comments']="\n#".join(subst['comments'].split("\n")) 246 if subst['comments']: subst['comments']="#"+subst['comments'] 247 subst['exec']="../"+subst['execname'] 248 subst['testroot']=self.testroot_dir 249 subst['testname']=testname 250 dp = self.conf.get('DATAFILESPATH','') 251 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 252 253 # This is used to label some matrices 254 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 255 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 256 257 # These can have for loops and are treated separately later 258 subst['nsize']=str(subst['nsize']) 259 260 #Conf vars 261 if self.petsc_arch.find('valgrind')>=0: 262 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 263 else: 264 subst['mpiexec']=self.conf['MPIEXEC'] 265 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 266 subst['diff']=self.conf['DIFF'] 267 subst['rm']=self.conf['RM'] 268 subst['grep']=self.conf['GREP'] 269 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 270 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 271 272 # Output file is special because of subtests override 273 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 274 if not "_" in defroot: defroot=defroot+"_1" 275 subst['defroot']=defroot 276 subst['label']=self.nameSpace(defroot,subst['srcdir']) 277 subst['redirect_file']=defroot+".tmp" 278 if 'output_file' not in testDict: 279 subst['output_file']="output/"+defroot+".out" 280 # Add in the full path here. 281 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 282 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 283 if not subst['TODO']: 284 print "Warning: "+subst['output_file']+" not found." 285 # Worry about alt files here -- see 286 # src/snes/examples/tutorials/output/ex22*.out 287 altlist=[subst['output_file']] 288 for i in range(1,3): 289 altroot=defroot+"_alt" 290 if i==2: altroot=altroot+"_2" 291 af="output/"+altroot+".out" 292 srcaf=os.path.join(subst['srcdir'],af) 293 fullaf=os.path.join(self.petsc_dir,srcaf) 294 if os.path.isfile(fullaf): altlist.append(srcaf) 295 if len(altlist)>1: subst['altfiles']=altlist 296 #if len(altlist)>1: print "Found alt files: ",altlist 297 298 return subst 299 300 def getCmds(self,subst,i): 301 """ 302 Generate bash script using template found next to this file. 303 This file is read in at constructor time to avoid file I/O 304 """ 305 indnt=self.indent 306 nindnt=i # the start and has to be consistent with below 307 cmdLines="" 308 309 # MPI is the default -- but we have a few odd commands 310 if not subst['command']: 311 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 312 else: 313 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 314 cmdLines+=cmd+"\n\n" 315 316 if not subst['filter_output']: 317 if 'altfiles' not in subst: 318 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 319 else: 320 # Have to do it by hand a bit because of variable number of alt files 321 rf=subst['redirect_file'] 322 cmd=indnt*nindnt+example_template.difftest.split('@')[0] 323 for i in range(len(subst['altfiles'])): 324 af=subst['altfiles'][i] 325 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 326 if i!=len(subst['altfiles'])-1: 327 cmd+=' || ${diff_exe} ' 328 else: 329 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 330 cmd+=subst['label_suffix']+' ""' # Quotes are painful 331 else: 332 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 333 cmdLines+=cmd+"\n" 334 return cmdLines 335 336 def _substVars(self,subst,origStr): 337 """ 338 Substitute variables 339 """ 340 Str=origStr 341 for subkey in subst: 342 if type(subst[subkey])!=types.StringType: continue 343 patt="@"+subkey.upper()+"@" 344 Str=re.sub(patt,subst[subkey],Str) 345 return Str 346 347 def _writeTodoSkip(self,fh,tors,reasons,footer): 348 """ 349 Write out the TODO and SKIP lines in the file 350 The TODO or SKIP variable, tors, should be lower case 351 """ 352 TORS=tors.upper() 353 template=eval("example_template."+tors+"line") 354 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 355 tab = '' 356 if reasons == ["Requires DATAFILESPATH"]: 357 # The only reason not to run is DATAFILESPATH, which we check at run-time 358 tab = ' ' 359 fh.write('if test -z "${DATAFILESPATH}"; then\n') 360 if reasons: 361 fh.write(tab+tsStr+"\ntotal=1; "+tors+"=1\n") 362 fh.write(tab+footer+"\n") 363 fh.write(tab+"exit\n") 364 if reasons == ["Requires DATAFILESPATH"]: 365 fh.write('fi\n') 366 fh.write('\n\n') 367 return 368 369 def getLoopVarsHead(self,loopVars,i): 370 """ 371 Generate a nicely indented string with the format loops 372 Here is what the data structure looks like 373 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 374 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 375 loopVars['subargs']['pc_type']=["j","cholesky sor"] 376 """ 377 outstr=''; indnt=self.indent 378 for key in loopVars: 379 for var in loopVars[key]['varlist']: 380 varval=loopVars[key][var] 381 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 382 i = i + 1 383 return (outstr,i) 384 385 def getLoopVarsFoot(self,loopVars,i): 386 outstr=''; indnt=self.indent 387 for key in loopVars: 388 for var in loopVars[key]['varlist']: 389 i = i - 1 390 outstr += indnt * i + "done\n" 391 return (outstr,i) 392 393 def genRunScript(self,testname,root,isRun,srcDict): 394 """ 395 Generate bash script using template found next to this file. 396 This file is read in at constructor time to avoid file I/O 397 """ 398 # runscript_dir directory has to be consistent with gmakefile 399 testDict=srcDict[testname] 400 rpath=self.relpath(self.petsc_dir,root) 401 runscript_dir=os.path.join(self.testroot_dir,rpath) 402 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 403 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 404 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 405 406 # Get variables to go into shell scripts. last time testDict used 407 subst=self.getSubstVars(testDict,rpath,testname) 408 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 409 #if '33_' in testname: print subst['subargs'] 410 411 #Handle runfiles 412 for lfile in subst.get('localrunfiles','').split(): 413 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 414 shutil.copy(fullfile,runscript_dir) 415 # Check subtests for local runfiles 416 for stest in subst.get("subtests",[]): 417 for lfile in testDict[stest].get('localrunfiles','').split(): 418 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 419 shutil.copy(fullfile,self.runscript_dir) 420 421 # Now substitute the key variables into the header and footer 422 header=self._substVars(subst,example_template.header) 423 # The header is done twice to enable @...@ in header 424 header=self._substVars(subst,header) 425 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 426 427 # Start writing the file 428 fh.write(header+"\n") 429 430 # If there is a TODO or a SKIP then we do it before writing out the 431 # rest of the command (which is useful for working on the test) 432 # SKIP and TODO can be for the source file or for the runs 433 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 434 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 435 436 j=0 # for indentation 437 438 if loopVars: 439 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 440 if (loopHead): fh.write(loopHead+"\n") 441 442 # Subtests are special 443 if 'subtests' in testDict: 444 substP=subst # Subtests can inherit args but be careful 445 for stest in testDict["subtests"]: 446 subst=substP.copy() 447 subst.update(testDict[stest]) 448 subst['nsize']=str(subst['nsize']) 449 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 450 #if '10_9' in testname: print sLoopVars 451 if sLoopVars: 452 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 453 fh.write(sLoopHead+"\n") 454 fh.write(self.getCmds(subst,j)+"\n") 455 if sLoopVars: 456 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 457 fh.write(sLoopFoot+"\n") 458 else: 459 fh.write(self.getCmds(subst,j)+"\n") 460 461 if loopVars: 462 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 463 fh.write(loopFoot+"\n") 464 465 fh.write(footer+"\n") 466 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 467 #if '10_9' in testname: sys.exit() 468 return 469 470 def genScriptsAndInfo(self,exfile,root,srcDict): 471 """ 472 Generate scripts from the source file, determine if built, etc. 473 For every test in the exfile with info in the srcDict: 474 1. Determine if it needs to be run for this arch 475 2. Generate the script 476 3. Generate the data needed to write out the makefile in a 477 convenient way 478 All tests are *always* run, but some may be SKIP'd per the TAP standard 479 """ 480 debug=False 481 fileIsTested=False 482 execname=self.getExecname(exfile,root) 483 isBuilt=self._isBuilt(exfile,srcDict) 484 for test in srcDict: 485 if test in self.buildkeys: continue 486 if debug: print self.nameSpace(exfile,root), test 487 srcDict[test]['execname']=execname # Convenience in generating scripts 488 isRun=self._isRun(srcDict[test]) 489 self.genRunScript(test,root,isRun,srcDict) 490 srcDict[test]['isrun']=isRun 491 if isRun: fileIsTested=True 492 self.addToTests(test,root,exfile,execname,srcDict[test]) 493 494 # This adds to datastructure for building deps 495 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 496 #print self.nameSpace(exfile,root), fileIsTested 497 return 498 499 def _isBuilt(self,exfile,srcDict): 500 """ 501 Determine if this file should be built. 502 """ 503 # Get the language based on file extension 504 srcDict['SKIP'] = [] 505 lang=self.getLanguage(exfile) 506 if (lang=="F" or lang=="F90") and not self.have_fortran: 507 srcDict["SKIP"].append("Fortran required for this test") 508 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 509 srcDict["SKIP"].append("CUDA required for this test") 510 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 511 srcDict["SKIP"].append("C++ required for this test") 512 513 # Deprecated source files 514 if srcDict.get("TODO"): 515 return False 516 517 # isRun can work with srcDict to handle the requires 518 if "requires" in srcDict: 519 if len(srcDict["requires"])>0: 520 return self._isRun(srcDict) 521 522 return srcDict['SKIP'] == [] 523 524 525 def _isRun(self,testDict): 526 """ 527 Based on the requirements listed in the src file and the petscconf.h 528 info, determine whether this test should be run or not. 529 """ 530 indent=" " 531 debug=False 532 533 if 'SKIP' not in testDict: 534 testDict['SKIP'] = [] 535 # MPI requirements 536 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 537 if debug: print indent+"Cannot run parallel tests" 538 testDict['SKIP'].append("Parallel test with serial build") 539 540 # The requirements for the test are the sum of all the run subtests 541 if 'subtests' in testDict: 542 if 'requires' not in testDict: testDict['requires']="" 543 for stest in testDict['subtests']: 544 if 'requires' in testDict[stest]: 545 testDict['requires']+=" "+testDict[stest]['requires'] 546 547 548 # Now go through all requirements 549 if 'requires' in testDict: 550 for requirement in testDict['requires'].split(): 551 requirement=requirement.strip() 552 if not requirement: continue 553 if debug: print indent+"Requirement: ", requirement 554 isNull=False 555 if requirement.startswith("!"): 556 requirement=requirement[1:]; isNull=True 557 # Precision requirement for reals 558 if requirement in self.precision_types: 559 if self.conf['PETSC_PRECISION']==requirement: 560 if isNull: 561 testDict['SKIP'].append("not "+requirement+" required") 562 continue 563 elif not isNull: 564 testDict['SKIP'].append(requirement+" required") 565 continue 566 # Precision requirement for ints 567 if requirement in self.integer_types: 568 if requirement=="int32": 569 if self.conf['PETSC_SIZEOF_INT']==4: 570 if isNull: 571 testDict['SKIP'].append("not int32 required") 572 continue 573 elif not isNull: 574 testDict['SKIP'].append("int32 required") 575 continue 576 if requirement=="int64": 577 if self.conf['PETSC_SIZEOF_INT']==8: 578 if isNull: 579 testDict['SKIP'].append("NOT int64 required") 580 continue 581 elif not isNull: 582 testDict['SKIP'].append("int64 required") 583 continue 584 # Datafilespath 585 if requirement=="datafilespath" and not isNull: 586 testDict['SKIP'].append("Requires DATAFILESPATH") 587 continue 588 # Defines -- not sure I have comments matching 589 if "define(" in requirement.lower(): 590 reqdef=requirement.split("(")[1].split(")")[0] 591 if reqdef in self.conf: 592 if isNull: 593 testDict['SKIP'].append("Null requirement not met: "+requirement) 594 continue 595 elif isNull: 596 testDict['SKIP'].append("Required: "+requirement) 597 continue 598 599 # Rest should be packages that we can just get from conf 600 if requirement == "complex": 601 petscconfvar="PETSC_USE_COMPLEX" 602 else: 603 petscconfvar="PETSC_HAVE_"+requirement.upper() 604 if self.conf.get(petscconfvar): 605 if isNull: 606 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 607 continue 608 elif not isNull: 609 if debug: print "requirement not found: ", requirement 610 testDict['SKIP'].append(petscconfvar+" requirement not met") 611 continue 612 613 return testDict['SKIP'] == [] 614 615 def genPetscTests_summarize(self,dataDict): 616 """ 617 Required method to state what happened 618 """ 619 if not self.summarize: return 620 indent=" " 621 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 622 fh=open(fhname,"w") 623 #print "See ", fhname 624 for root in dataDict: 625 relroot=self.relpath(self.petsc_dir,root) 626 pkg=relroot.split("/")[1] 627 fh.write(relroot+"\n") 628 allSrcs=[] 629 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 630 for exfile in dataDict[root]: 631 # Basic information 632 fullfile=os.path.join(root,exfile) 633 rfile=self.relpath(self.petsc_dir,fullfile) 634 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 635 fh.write(indent+exfile+indent*4+builtStatus+"\n") 636 637 for test in dataDict[root][exfile]: 638 if test in self.buildkeys: continue 639 line=indent*2+test 640 fh.write(line+"\n") 641 # Looks nice to have the keys in order 642 #for key in dataDict[root][exfile][test]: 643 for key in "isrun abstracted nsize args requires script".split(): 644 if key not in dataDict[root][exfile][test]: continue 645 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 646 fh.write(line+"\n") 647 fh.write("\n") 648 fh.write("\n") 649 fh.write("\n") 650 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 651 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 652 fh.close() 653 return 654 655 def genPetscTests(self,root,dirs,files,dataDict): 656 """ 657 Go through and parse the source files in the directory to generate 658 the examples based on the metadata contained in the source files 659 """ 660 debug=False 661 # Use examplesAnalyze to get what the makefles think are sources 662 #self.examplesAnalyze(root,dirs,files,anlzDict) 663 664 dataDict[root]={} 665 666 for exfile in files: 667 #TST: Until we replace files, still leaving the orginals as is 668 #if not exfile.startswith("new_"+"ex"): continue 669 if not exfile.startswith("ex"): continue 670 671 # Convenience 672 fullex=os.path.join(root,exfile) 673 relpfile=self.relpath(self.petsc_dir,fullex) 674 if debug: print relpfile 675 dataDict[root].update(testparse.parseTestFile(fullex,0)) 676 # Need to check and make sure tests are in the file 677 # if verbosity>=1: print relpfile 678 if exfile in dataDict[root]: 679 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 680 681 return 682 683 def walktree(self,top,action="printFiles"): 684 """ 685 Walk a directory tree, starting from 'top' 686 """ 687 #print "action", action 688 # Goal of action is to fill this dictionary 689 dataDict={} 690 for root, dirs, files in os.walk(top, topdown=False): 691 if not "examples" in root: continue 692 if not os.path.isfile(os.path.join(root,"makefile")): continue 693 bname=os.path.basename(root.rstrip("/")) 694 if bname=="tests" or bname=="tutorials": 695 eval("self."+action+"(root,dirs,files,dataDict)") 696 if type(top) != types.StringType: 697 raise TypeError("top must be a string") 698 # Now summarize this dictionary 699 eval("self."+action+"_summarize(dataDict)") 700 return dataDict 701 702 def gen_gnumake(self, fd): 703 """ 704 Overwrite of the method in the base PETSc class 705 """ 706 def write(stem, srcs): 707 for lang in LANGS: 708 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 709 for pkg in PKGS: 710 srcs = self.gen_pkg(pkg) 711 write('testsrcs-' + pkg, srcs) 712 return self.gendeps 713 714 def gen_pkg(self, pkg): 715 """ 716 Overwrite of the method in the base PETSc class 717 """ 718 return self.sources[pkg] 719 720 def write_gnumake(self,dataDict): 721 """ 722 Write out something similar to files from gmakegen.py 723 724 Test depends on script which also depends on source 725 file, but since I don't have a good way generating 726 acting on a single file (oops) just depend on 727 executable which in turn will depend on src file 728 """ 729 # Different options for how to set up the targets 730 compileExecsFirst=False 731 732 # Open file 733 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 734 fd = open(arch_files, 'w') 735 736 # Write out the sources 737 gendeps = self.gen_gnumake(fd) 738 739 # Write out the tests and execname targets 740 fd.write("\n#Tests and executables\n") # Delimiter 741 742 for pkg in PKGS: 743 # These grab the ones that are built 744 for lang in LANGS: 745 testdeps=[] 746 for ftest in self.tests[pkg][lang]: 747 test=os.path.basename(ftest) 748 basedir=os.path.dirname(ftest) 749 testdeps.append(self.nameSpace(test,basedir)) 750 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 751 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 752 753 # test targets 754 for ftest in self.tests[pkg][lang]: 755 test=os.path.basename(ftest) 756 basedir=os.path.dirname(ftest) 757 testdir="${TESTDIR}/"+basedir+"/" 758 nmtest=self.nameSpace(test,basedir) 759 rundir=os.path.join(testdir,test) 760 #print test, nmtest 761 script=test+".sh" 762 763 # Deps 764 exfile=self.tests[pkg][lang][ftest]['exfile'] 765 fullex=os.path.join(self.petsc_dir,exfile) 766 localexec=self.tests[pkg][lang][ftest]['exec'] 767 execname=os.path.join(testdir,localexec) 768 fullscript=os.path.join(testdir,script) 769 tmpfile=os.path.join(testdir,test,test+".tmp") 770 771 # *.counts depends on the script and either executable (will 772 # be run) or the example source file (SKIP or TODO) 773 fd.write('%s.counts : %s %s\n' 774 % (os.path.join('$(TESTDIR)/counts', nmtest), 775 fullscript, 776 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 777 # Now write the args: 778 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 779 780 fd.close() 781 return 782 783 def writeHarness(self,output,dataDict): 784 """ 785 This is set up to write out multiple harness even if only gnumake 786 is supported now 787 """ 788 eval("self.write_"+output+"(dataDict)") 789 return 790 791def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 792 if output is None: 793 output = 'gnumake' 794 795 796 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 797 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 798 pEx.writeHarness(output,dataDict) 799 800if __name__ == '__main__': 801 import optparse 802 parser = optparse.OptionParser() 803 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 804 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 805 parser.add_option('--output', help='Location to write output file', default=None) 806 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 807 opts, extra_args = parser.parse_args() 808 if extra_args: 809 import sys 810 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 811 exit(1) 812 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 813