1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double __float128 int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 from testparse import parseLoopArgs 107 for key in lkeys: 108 if type(inDict[key])!=types.StringType: continue 109 keystr = str(inDict[key]) 110 akey=('subargs' if key=='args' else key) # what to assign 111 if akey not in inDict: inDict[akey]='' 112 varlist=[] 113 for varset in re.split('-(?=[a-zA-Z])',keystr): 114 if not varset.strip(): continue 115 if '{{' in varset: 116 keyvar,lvars,ftype=parseLoopArgs(varset) 117 if akey not in loopVars: loopVars[akey]={} 118 varlist.append(keyvar) 119 loopVars[akey][keyvar]=[keyvar,lvars] 120 if akey=='nsize': 121 inDict[akey] = '${' + keyvar + '}' 122 lsuffix+=akey+'-'+inDict[akey]+'_' 123 else: 124 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 125 lsuffix+=keyvar+'-${' + keyvar + '}_' 126 else: 127 if key=='args': newargs+=" -"+varset.strip() 128 if len(varlist)>0: loopVars[akey]['varlist']=varlist 129 130 131 # For subtests, args are always substituted in (not top level) 132 if isSubtest: 133 inDict['subargs']+=" "+newargs.strip() 134 inDict['args']='' 135 if 'label_suffix' in inDict: 136 inDict['label_suffix']+=lsuffix.rstrip('_') 137 else: 138 inDict['label_suffix']=lsuffix.rstrip('_') 139 else: 140 if len(loopVars.keys())>0: 141 inDict['args']=newargs.strip() 142 inDict['label_suffix']=lsuffix.rstrip('_') 143 if len(loopVars.keys())>0: 144 return loopVars 145 else: 146 return None 147 148 def getArgLabel(self,testDict): 149 """ 150 In all of the arguments in the test dictionary, create a simple 151 string for searching within the makefile system. For simplicity in 152 search, remove "-", for strings, etc. 153 Also, concatenate the arg commands 154 For now, ignore nsize -- seems hard to search for anyway 155 """ 156 # Collect all of the args associated with a test 157 argStr=("" if 'args' not in testDict else testDict['args']) 158 if 'subtests' in testDict: 159 for stest in testDict["subtests"]: 160 sd=testDict[stest] 161 argStr=argStr+("" if 'args' not in sd else sd['args']) 162 163 # Now go through and cleanup 164 argStr=re.sub('{{(.*?)}}',"",argStr) 165 argStr=re.sub('-'," ",argStr) 166 for digit in string.digits: argStr=re.sub(digit," ",argStr) 167 argStr=re.sub("\.","",argStr) 168 argStr=re.sub(",","",argStr) 169 argStr=re.sub('\+',' ',argStr) 170 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 171 return argStr.strip() 172 173 def addToSources(self,exfile,root,srcDict): 174 """ 175 Put into data structure that allows easy generation of makefile 176 """ 177 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 178 fullfile=os.path.join(root,exfile) 179 relpfile=self.relpath(self.petsc_dir,fullfile) 180 lang=self.getLanguage(exfile) 181 if not lang: return 182 self.sources[pkg][lang]['srcs'].append(relpfile) 183 if 'depends' in srcDict: 184 depSrc=srcDict['depends'] 185 depObj=os.path.splitext(depSrc)[0]+".o" 186 self.sources[pkg][lang][exfile]=depObj 187 188 # In gmakefile, ${TESTDIR} var specifies the object compilation 189 testsdir=self.relpath(self.petsc_dir,root)+"/" 190 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 191 self.objects[pkg].append(objfile) 192 return 193 194 def addToTests(self,test,root,exfile,execname,testDict): 195 """ 196 Put into data structure that allows easy generation of makefile 197 Organized by languages to allow testing of languages 198 """ 199 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 200 #nmtest=self.nameSpace(test,root) 201 rpath=self.relpath(self.petsc_dir,root) 202 nmtest=os.path.join(rpath,test) 203 lang=self.getLanguage(exfile) 204 if not lang: return 205 self.tests[pkg][lang][nmtest]={} 206 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 207 self.tests[pkg][lang][nmtest]['exec']=execname 208 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 209 return 210 211 def getExecname(self,exfile,root): 212 """ 213 Generate bash script using template found next to this file. 214 This file is read in at constructor time to avoid file I/O 215 """ 216 rpath=self.relpath(self.petsc_dir,root) 217 if self.single_ex: 218 execname=rpath.split("/")[1]+"-ex" 219 else: 220 execname=os.path.splitext(exfile)[0] 221 return execname 222 223 def getSubstVars(self,testDict,rpath,testname): 224 """ 225 Create a dictionary with all of the variables that get substituted 226 into the template commands found in example_template.py 227 """ 228 subst={} 229 230 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 231 if 'nsize' not in testDict: testDict['nsize']=1 232 for ak in testparse.acceptedkeys: 233 if ak=='test': continue 234 subst[ak]=(testDict[ak] if ak in testDict else '') 235 236 # Now do other variables 237 subst['execname']=testDict['execname'] 238 if 'filter' in testDict: 239 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 240 241 # Others 242 subst['subargs']='' # Default. For variables override 243 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 244 subst['label_suffix']='' 245 subst['comments']="\n#".join(subst['comments'].split("\n")) 246 if subst['comments']: subst['comments']="#"+subst['comments'] 247 subst['exec']="../"+subst['execname'] 248 subst['testroot']=self.testroot_dir 249 subst['testname']=testname 250 dp = self.conf.get('DATAFILESPATH','') 251 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 252 253 # This is used to label some matrices 254 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 255 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 256 257 # These can have for loops and are treated separately later 258 subst['nsize']=str(subst['nsize']) 259 260 #Conf vars 261 if self.petsc_arch.find('valgrind')>=0: 262 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 263 else: 264 subst['mpiexec']=self.conf['MPIEXEC'] 265 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 266 subst['diff']=self.conf['DIFF'] 267 subst['rm']=self.conf['RM'] 268 subst['grep']=self.conf['GREP'] 269 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 270 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 271 272 # Output file is special because of subtests override 273 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 274 if not "_" in defroot: defroot=defroot+"_1" 275 subst['defroot']=defroot 276 subst['label']=self.nameSpace(defroot,subst['srcdir']) 277 subst['redirect_file']=defroot+".tmp" 278 if 'output_file' not in testDict: 279 subst['output_file']="output/"+defroot+".out" 280 # Add in the full path here. 281 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 282 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 283 if not subst['TODO']: 284 print "Warning: "+subst['output_file']+" not found." 285 # Worry about alt files here -- see 286 # src/snes/examples/tutorials/output/ex22*.out 287 altlist=[subst['output_file']] 288 for i in range(1,3): 289 altroot=defroot+"_alt" 290 if i==2: altroot=altroot+"_2" 291 af="output/"+altroot+".out" 292 srcaf=os.path.join(subst['srcdir'],af) 293 fullaf=os.path.join(self.petsc_dir,srcaf) 294 if os.path.isfile(fullaf): altlist.append(srcaf) 295 if len(altlist)>1: subst['altfiles']=altlist 296 #if len(altlist)>1: print "Found alt files: ",altlist 297 298 return subst 299 300 def getCmds(self,subst,i): 301 """ 302 Generate bash script using template found next to this file. 303 This file is read in at constructor time to avoid file I/O 304 """ 305 indnt=self.indent 306 nindnt=i # the start and has to be consistent with below 307 cmdLines="" 308 309 # MPI is the default -- but we have a few odd commands 310 if not subst['command']: 311 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 312 else: 313 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 314 cmdLines+=cmd+"\n\n" 315 316 if not subst['filter_output']: 317 if 'altfiles' not in subst: 318 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 319 else: 320 # Have to do it by hand a bit because of variable number of alt files 321 rf=subst['redirect_file'] 322 cmd=indnt*nindnt+example_template.difftest.split('@')[0] 323 for i in range(len(subst['altfiles'])): 324 af=subst['altfiles'][i] 325 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 326 if i!=len(subst['altfiles'])-1: 327 cmd+=' || ${diff_exe} ' 328 else: 329 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 330 cmd+=subst['label_suffix']+' ""' # Quotes are painful 331 else: 332 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 333 cmdLines+=cmd+"\n" 334 return cmdLines 335 336 def _substVars(self,subst,origStr): 337 """ 338 Substitute variables 339 """ 340 Str=origStr 341 for subkey in subst: 342 if type(subst[subkey])!=types.StringType: continue 343 patt="@"+subkey.upper()+"@" 344 Str=re.sub(patt,subst[subkey],Str) 345 return Str 346 347 def _writeTodoSkip(self,fh,tors,reasons,footer): 348 """ 349 Write out the TODO and SKIP lines in the file 350 The TODO or SKIP variable, tors, should be lower case 351 """ 352 TORS=tors.upper() 353 template=eval("example_template."+tors+"line") 354 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 355 tab = '' 356 if reasons == ["Requires DATAFILESPATH"]: 357 # The only reason not to run is DATAFILESPATH, which we check at run-time 358 tab = ' ' 359 fh.write('if test -z "${DATAFILESPATH}"; then\n') 360 if reasons: 361 fh.write(tab+tsStr+"\ntotal=1; "+tors+"=1\n") 362 fh.write(tab+footer+"\n") 363 fh.write(tab+"exit\n") 364 if reasons == ["Requires DATAFILESPATH"]: 365 fh.write('fi\n') 366 fh.write('\n\n') 367 return 368 369 def getLoopVarsHead(self,loopVars,i): 370 """ 371 Generate a nicely indented string with the format loops 372 Here is what the data structure looks like 373 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 374 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 375 loopVars['subargs']['pc_type']=["j","cholesky sor"] 376 """ 377 outstr=''; indnt=self.indent 378 for key in loopVars: 379 for var in loopVars[key]['varlist']: 380 varval=loopVars[key][var] 381 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 382 i = i + 1 383 return (outstr,i) 384 385 def getLoopVarsFoot(self,loopVars,i): 386 outstr=''; indnt=self.indent 387 for key in loopVars: 388 for var in loopVars[key]['varlist']: 389 i = i - 1 390 outstr += indnt * i + "done\n" 391 return (outstr,i) 392 393 def genRunScript(self,testname,root,isRun,srcDict): 394 """ 395 Generate bash script using template found next to this file. 396 This file is read in at constructor time to avoid file I/O 397 """ 398 # runscript_dir directory has to be consistent with gmakefile 399 testDict=srcDict[testname] 400 rpath=self.relpath(self.petsc_dir,root) 401 runscript_dir=os.path.join(self.testroot_dir,rpath) 402 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 403 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 404 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 405 406 # Get variables to go into shell scripts. last time testDict used 407 subst=self.getSubstVars(testDict,rpath,testname) 408 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 409 #if '33_' in testname: print subst['subargs'] 410 411 #Handle runfiles 412 for lfile in subst.get('localrunfiles','').split(): 413 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 414 shutil.copy(fullfile,runscript_dir) 415 # Check subtests for local runfiles 416 for stest in subst.get("subtests",[]): 417 for lfile in testDict[stest].get('localrunfiles','').split(): 418 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 419 shutil.copy(fullfile,self.runscript_dir) 420 421 # Now substitute the key variables into the header and footer 422 header=self._substVars(subst,example_template.header) 423 # The header is done twice to enable @...@ in header 424 header=self._substVars(subst,header) 425 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 426 427 # Start writing the file 428 fh.write(header+"\n") 429 430 # If there is a TODO or a SKIP then we do it before writing out the 431 # rest of the command (which is useful for working on the test) 432 # SKIP and TODO can be for the source file or for the runs 433 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 434 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 435 436 j=0 # for indentation 437 438 if loopVars: 439 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 440 if (loopHead): fh.write(loopHead+"\n") 441 442 # Subtests are special 443 if 'subtests' in testDict: 444 substP=subst # Subtests can inherit args but be careful 445 for stest in testDict["subtests"]: 446 subst=substP.copy() 447 subst.update(testDict[stest]) 448 subst['nsize']=str(subst['nsize']) 449 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 450 #if '10_9' in testname: print sLoopVars 451 if sLoopVars: 452 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 453 fh.write(sLoopHead+"\n") 454 fh.write(self.getCmds(subst,j)+"\n") 455 if sLoopVars: 456 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 457 fh.write(sLoopFoot+"\n") 458 else: 459 fh.write(self.getCmds(subst,j)+"\n") 460 461 if loopVars: 462 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 463 fh.write(loopFoot+"\n") 464 465 fh.write(footer+"\n") 466 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 467 #if '10_9' in testname: sys.exit() 468 return 469 470 def genScriptsAndInfo(self,exfile,root,srcDict): 471 """ 472 Generate scripts from the source file, determine if built, etc. 473 For every test in the exfile with info in the srcDict: 474 1. Determine if it needs to be run for this arch 475 2. Generate the script 476 3. Generate the data needed to write out the makefile in a 477 convenient way 478 All tests are *always* run, but some may be SKIP'd per the TAP standard 479 """ 480 debug=False 481 execname=self.getExecname(exfile,root) 482 isBuilt=self._isBuilt(exfile,srcDict) 483 for test in srcDict: 484 if test in self.buildkeys: continue 485 if debug: print self.nameSpace(exfile,root), test 486 srcDict[test]['execname']=execname # Convenience in generating scripts 487 isRun=self._isRun(srcDict[test]) 488 self.genRunScript(test,root,isRun,srcDict) 489 srcDict[test]['isrun']=isRun 490 self.addToTests(test,root,exfile,execname,srcDict[test]) 491 492 # This adds to datastructure for building deps 493 if isBuilt: self.addToSources(exfile,root,srcDict) 494 return 495 496 def _isBuilt(self,exfile,srcDict): 497 """ 498 Determine if this file should be built. 499 """ 500 # Get the language based on file extension 501 srcDict['SKIP'] = [] 502 lang=self.getLanguage(exfile) 503 if (lang=="F" or lang=="F90") and not self.have_fortran: 504 srcDict["SKIP"].append("Fortran required for this test") 505 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 506 srcDict["SKIP"].append("CUDA required for this test") 507 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 508 srcDict["SKIP"].append("C++ required for this test") 509 510 # Deprecated source files 511 if srcDict.get("TODO"): 512 return False 513 514 # isRun can work with srcDict to handle the requires 515 if "requires" in srcDict: 516 if len(srcDict["requires"])>0: 517 return self._isRun(srcDict) 518 519 return srcDict['SKIP'] == [] 520 521 522 def _isRun(self,testDict): 523 """ 524 Based on the requirements listed in the src file and the petscconf.h 525 info, determine whether this test should be run or not. 526 """ 527 indent=" " 528 debug=False 529 530 if 'SKIP' not in testDict: 531 testDict['SKIP'] = [] 532 # MPI requirements 533 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 534 if debug: print indent+"Cannot run parallel tests" 535 testDict['SKIP'].append("Parallel test with serial build") 536 537 # The requirements for the test are the sum of all the run subtests 538 if 'subtests' in testDict: 539 if 'requires' not in testDict: testDict['requires']="" 540 for stest in testDict['subtests']: 541 if 'requires' in testDict[stest]: 542 testDict['requires']+=" "+testDict[stest]['requires'] 543 544 545 # Now go through all requirements 546 if 'requires' in testDict: 547 for requirement in testDict['requires'].split(): 548 requirement=requirement.strip() 549 if not requirement: continue 550 if debug: print indent+"Requirement: ", requirement 551 isNull=False 552 if requirement.startswith("!"): 553 requirement=requirement[1:]; isNull=True 554 # Precision requirement for reals 555 if requirement in self.precision_types: 556 if self.conf['PETSC_PRECISION']==requirement: 557 if isNull: 558 testDict['SKIP'].append("not "+requirement+" required") 559 continue 560 continue # Success 561 elif not isNull: 562 testDict['SKIP'].append(requirement+" required") 563 continue 564 # Precision requirement for ints 565 if requirement in self.integer_types: 566 if requirement=="int32": 567 if self.conf['PETSC_SIZEOF_INT']==4: 568 if isNull: 569 testDict['SKIP'].append("not int32 required") 570 continue 571 continue # Success 572 elif not isNull: 573 testDict['SKIP'].append("int32 required") 574 continue 575 if requirement=="int64": 576 if self.conf['PETSC_SIZEOF_INT']==8: 577 if isNull: 578 testDict['SKIP'].append("NOT int64 required") 579 continue 580 continue # Success 581 elif not isNull: 582 testDict['SKIP'].append("int64 required") 583 continue 584 # Datafilespath 585 if requirement=="datafilespath" and not isNull: 586 testDict['SKIP'].append("Requires DATAFILESPATH") 587 continue 588 # Defines -- not sure I have comments matching 589 if "define(" in requirement.lower(): 590 reqdef=requirement.split("(")[1].split(")")[0] 591 if reqdef in self.conf: 592 if isNull: 593 testDict['SKIP'].append("Null requirement not met: "+requirement) 594 continue 595 continue # Success 596 elif not isNull: 597 testDict['SKIP'].append("Required: "+requirement) 598 continue 599 600 # Rest should be packages that we can just get from conf 601 if requirement == "complex": 602 petscconfvar="PETSC_USE_COMPLEX" 603 else: 604 petscconfvar="PETSC_HAVE_"+requirement.upper() 605 if self.conf.get(petscconfvar): 606 if isNull: 607 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 608 continue 609 continue # Success 610 elif not isNull: 611 if debug: print "requirement not found: ", requirement 612 testDict['SKIP'].append(petscconfvar+" requirement not met") 613 continue 614 615 return testDict['SKIP'] == [] 616 617 def genPetscTests_summarize(self,dataDict): 618 """ 619 Required method to state what happened 620 """ 621 if not self.summarize: return 622 indent=" " 623 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 624 fh=open(fhname,"w") 625 #print "See ", fhname 626 for root in dataDict: 627 relroot=self.relpath(self.petsc_dir,root) 628 pkg=relroot.split("/")[1] 629 fh.write(relroot+"\n") 630 allSrcs=[] 631 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 632 for exfile in dataDict[root]: 633 # Basic information 634 fullfile=os.path.join(root,exfile) 635 rfile=self.relpath(self.petsc_dir,fullfile) 636 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 637 fh.write(indent+exfile+indent*4+builtStatus+"\n") 638 639 for test in dataDict[root][exfile]: 640 if test in self.buildkeys: continue 641 line=indent*2+test 642 fh.write(line+"\n") 643 # Looks nice to have the keys in order 644 #for key in dataDict[root][exfile][test]: 645 for key in "isrun abstracted nsize args requires script".split(): 646 if key not in dataDict[root][exfile][test]: continue 647 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 648 fh.write(line+"\n") 649 fh.write("\n") 650 fh.write("\n") 651 fh.write("\n") 652 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 653 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 654 fh.close() 655 return 656 657 def genPetscTests(self,root,dirs,files,dataDict): 658 """ 659 Go through and parse the source files in the directory to generate 660 the examples based on the metadata contained in the source files 661 """ 662 debug=False 663 # Use examplesAnalyze to get what the makefles think are sources 664 #self.examplesAnalyze(root,dirs,files,anlzDict) 665 666 dataDict[root]={} 667 668 for exfile in files: 669 #TST: Until we replace files, still leaving the orginals as is 670 #if not exfile.startswith("new_"+"ex"): continue 671 if not exfile.startswith("ex"): continue 672 673 # Convenience 674 fullex=os.path.join(root,exfile) 675 relpfile=self.relpath(self.petsc_dir,fullex) 676 if debug: print relpfile 677 dataDict[root].update(testparse.parseTestFile(fullex,0)) 678 # Need to check and make sure tests are in the file 679 # if verbosity>=1: print relpfile 680 if exfile in dataDict[root]: 681 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 682 683 return 684 685 def walktree(self,top,action="printFiles"): 686 """ 687 Walk a directory tree, starting from 'top' 688 """ 689 #print "action", action 690 # Goal of action is to fill this dictionary 691 dataDict={} 692 for root, dirs, files in os.walk(top, topdown=False): 693 if not "examples" in root: continue 694 if not os.path.isfile(os.path.join(root,"makefile")): continue 695 bname=os.path.basename(root.rstrip("/")) 696 if bname=="tests" or bname=="tutorials": 697 eval("self."+action+"(root,dirs,files,dataDict)") 698 if type(top) != types.StringType: 699 raise TypeError("top must be a string") 700 # Now summarize this dictionary 701 eval("self."+action+"_summarize(dataDict)") 702 return dataDict 703 704 def gen_gnumake(self, fd): 705 """ 706 Overwrite of the method in the base PETSc class 707 """ 708 def write(stem, srcs): 709 for lang in LANGS: 710 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 711 for pkg in PKGS: 712 srcs = self.gen_pkg(pkg) 713 write('testsrcs-' + pkg, srcs) 714 return self.gendeps 715 716 def gen_pkg(self, pkg): 717 """ 718 Overwrite of the method in the base PETSc class 719 """ 720 return self.sources[pkg] 721 722 def write_gnumake(self,dataDict): 723 """ 724 Write out something similar to files from gmakegen.py 725 726 Test depends on script which also depends on source 727 file, but since I don't have a good way generating 728 acting on a single file (oops) just depend on 729 executable which in turn will depend on src file 730 """ 731 # Different options for how to set up the targets 732 compileExecsFirst=False 733 734 # Open file 735 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 736 fd = open(arch_files, 'w') 737 738 # Write out the sources 739 gendeps = self.gen_gnumake(fd) 740 741 # Write out the tests and execname targets 742 fd.write("\n#Tests and executables\n") # Delimiter 743 744 for pkg in PKGS: 745 # These grab the ones that are built 746 for lang in LANGS: 747 testdeps=[] 748 for ftest in self.tests[pkg][lang]: 749 test=os.path.basename(ftest) 750 basedir=os.path.dirname(ftest) 751 testdeps.append(self.nameSpace(test,basedir)) 752 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 753 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 754 755 # test targets 756 for ftest in self.tests[pkg][lang]: 757 test=os.path.basename(ftest) 758 basedir=os.path.dirname(ftest) 759 testdir="${TESTDIR}/"+basedir+"/" 760 nmtest=self.nameSpace(test,basedir) 761 rundir=os.path.join(testdir,test) 762 #print test, nmtest 763 script=test+".sh" 764 765 # Deps 766 exfile=self.tests[pkg][lang][ftest]['exfile'] 767 fullex=os.path.join(self.petsc_dir,exfile) 768 localexec=self.tests[pkg][lang][ftest]['exec'] 769 execname=os.path.join(testdir,localexec) 770 fullscript=os.path.join(testdir,script) 771 tmpfile=os.path.join(testdir,test,test+".tmp") 772 773 # *.counts depends on the script and either executable (will 774 # be run) or the example source file (SKIP or TODO) 775 fd.write('%s.counts : %s %s\n' 776 % (os.path.join('$(TESTDIR)/counts', nmtest), 777 fullscript, 778 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 779 # Now write the args: 780 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 781 782 fd.close() 783 return 784 785 def writeHarness(self,output,dataDict): 786 """ 787 This is set up to write out multiple harness even if only gnumake 788 is supported now 789 """ 790 eval("self.write_"+output+"(dataDict)") 791 return 792 793def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 794 if output is None: 795 output = 'gnumake' 796 797 798 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 799 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 800 pEx.writeHarness(output,dataDict) 801 802if __name__ == '__main__': 803 import optparse 804 parser = optparse.OptionParser() 805 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 806 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 807 parser.add_option('--output', help='Location to write output file', default=None) 808 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 809 opts, extra_args = parser.parse_args() 810 if extra_args: 811 import sys 812 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 813 exit(1) 814 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 815