1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 # Whether to write out a useful debugging 53 #if verbose: self.summarize=True 54 self.summarize=True 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 return 82 83 def srcrelpath(self,rdir): 84 """ 85 Get relative path to source directory 86 """ 87 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 88 89 def getInInstallDir(self,thisscriptdir): 90 """ 91 When petsc is installed then this file in installed in: 92 <PREFIX>/share/petsc/examples/config/gmakegentest.py 93 otherwise the path is: 94 <PETSC_DIR>/config/gmakegentest.py 95 We use this difference to determine if we are in installdir 96 """ 97 dirlist=thisscriptdir.split(os.path.sep) 98 if len(dirlist)>4: 99 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 100 if lastfour==os.path.join('share','petsc','examples','config'): 101 return True 102 else: 103 return False 104 else: 105 return False 106 107 def nameSpace(self,srcfile,srcdir): 108 """ 109 Because the scripts have a non-unique naming, the pretty-printing 110 needs to convey the srcdir and srcfile. There are two ways of doing this. 111 """ 112 if self.ptNaming: 113 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 114 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 115 nameString=prefix+srcfile 116 else: 117 #nameString=srcdir+": "+srcfile 118 nameString=srcfile 119 return nameString 120 121 def getLanguage(self,srcfile): 122 """ 123 Based on the source, determine associated language as found in gmakegen.LANGS 124 Can we just return srcext[1:\] now? 125 """ 126 langReq=None 127 srcext=os.path.splitext(srcfile)[-1] 128 if srcext in ".F90".split(): langReq="F90" 129 if srcext in ".F".split(): langReq="F" 130 if srcext in ".cxx".split(): langReq="cxx" 131 if srcext == ".cu": langReq="cu" 132 if srcext == ".c": langReq="c" 133 #if not langReq: print "ERROR: ", srcext, srcfile 134 return langReq 135 136 def _getLoopVars(self,inDict,testname, isSubtest=False): 137 """ 138 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 139 Return: 140 inDict['args']: -ksp_monitor 141 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 142 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 143 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 144 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 145 subst should be passed in instead of inDict 146 """ 147 loopVars={}; newargs="" 148 lkeys=inDict.keys() 149 lsuffix='_' 150 from testparse import parseLoopArgs 151 for key in lkeys: 152 if type(inDict[key])!=types.StringType: continue 153 keystr = str(inDict[key]) 154 akey=('subargs' if key=='args' else key) # what to assign 155 if akey not in inDict: inDict[akey]='' 156 varlist=[] 157 for varset in re.split('-(?=[a-zA-Z])',keystr): 158 if not varset.strip(): continue 159 if '{{' in varset: 160 keyvar,lvars,ftype=parseLoopArgs(varset) 161 if akey not in loopVars: loopVars[akey]={} 162 varlist.append(keyvar) 163 loopVars[akey][keyvar]=[keyvar,lvars] 164 if akey=='nsize': 165 inDict[akey] = '${' + keyvar + '}' 166 lsuffix+=akey+'-'+inDict[akey]+'_' 167 else: 168 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 169 lsuffix+=keyvar+'-${' + keyvar + '}_' 170 else: 171 if key=='args': newargs+=" -"+varset.strip() 172 if len(varlist)>0: loopVars[akey]['varlist']=varlist 173 174 175 # For subtests, args are always substituted in (not top level) 176 if isSubtest: 177 inDict['subargs']+=" "+newargs.strip() 178 inDict['args']='' 179 if 'label_suffix' in inDict: 180 inDict['label_suffix']+=lsuffix.rstrip('_') 181 else: 182 inDict['label_suffix']=lsuffix.rstrip('_') 183 else: 184 if len(loopVars.keys())>0: 185 inDict['args']=newargs.strip() 186 inDict['label_suffix']=lsuffix.rstrip('_') 187 if len(loopVars.keys())>0: 188 return loopVars 189 else: 190 return None 191 192 def getArgLabel(self,testDict): 193 """ 194 In all of the arguments in the test dictionary, create a simple 195 string for searching within the makefile system. For simplicity in 196 search, remove "-", for strings, etc. 197 Also, concatenate the arg commands 198 For now, ignore nsize -- seems hard to search for anyway 199 """ 200 # Collect all of the args associated with a test 201 argStr=("" if 'args' not in testDict else testDict['args']) 202 if 'subtests' in testDict: 203 for stest in testDict["subtests"]: 204 sd=testDict[stest] 205 argStr=argStr+("" if 'args' not in sd else sd['args']) 206 207 # Now go through and cleanup 208 argStr=re.sub('{{(.*?)}}',"",argStr) 209 argStr=re.sub('-'," ",argStr) 210 for digit in string.digits: argStr=re.sub(digit," ",argStr) 211 argStr=re.sub("\.","",argStr) 212 argStr=re.sub(",","",argStr) 213 argStr=re.sub('\+',' ',argStr) 214 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 215 return argStr.strip() 216 217 def addToSources(self,exfile,root,srcDict): 218 """ 219 Put into data structure that allows easy generation of makefile 220 """ 221 rpath=self.srcrelpath(root) 222 pkg=rpath.split(os.path.sep)[1] 223 relpfile=os.path.join(rpath,exfile) 224 lang=self.getLanguage(exfile) 225 if not lang: return 226 self.sources[pkg][lang]['srcs'].append(relpfile) 227 if 'depends' in srcDict: 228 depSrc=srcDict['depends'] 229 depObj=os.path.splitext(depSrc)[0]+".o" 230 self.sources[pkg][lang][exfile]=depObj 231 232 # In gmakefile, ${TESTDIR} var specifies the object compilation 233 testsdir=self.srcrelpath(root)+"/" 234 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 235 self.objects[pkg].append(objfile) 236 return 237 238 def addToTests(self,test,root,exfile,execname,testDict): 239 """ 240 Put into data structure that allows easy generation of makefile 241 Organized by languages to allow testing of languages 242 """ 243 rpath=self.srcrelpath(root) 244 pkg=rpath.split("/")[1] 245 #nmtest=self.nameSpace(test,root) 246 nmtest=os.path.join(rpath,test) 247 lang=self.getLanguage(exfile) 248 if not lang: return 249 self.tests[pkg][lang][nmtest]={} 250 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 251 self.tests[pkg][lang][nmtest]['exec']=execname 252 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 253 return 254 255 def getExecname(self,exfile,root): 256 """ 257 Generate bash script using template found next to this file. 258 This file is read in at constructor time to avoid file I/O 259 """ 260 rpath=self.srcrelpath(root) 261 if self.single_ex: 262 execname=rpath.split("/")[1]+"-ex" 263 else: 264 execname=os.path.splitext(exfile)[0] 265 return execname 266 267 def getSubstVars(self,testDict,rpath,testname): 268 """ 269 Create a dictionary with all of the variables that get substituted 270 into the template commands found in example_template.py 271 """ 272 subst={} 273 274 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 275 if 'nsize' not in testDict: testDict['nsize']=1 276 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 277 for ak in testparse.acceptedkeys: 278 if ak=='test': continue 279 subst[ak]=(testDict[ak] if ak in testDict else '') 280 281 # Now do other variables 282 subst['execname']=testDict['execname'] 283 if 'filter' in testDict: 284 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 285 286 # Others 287 subst['subargs']='' # Default. For variables override 288 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 289 subst['label_suffix']='' 290 subst['comments']="\n#".join(subst['comments'].split("\n")) 291 if subst['comments']: subst['comments']="#"+subst['comments'] 292 subst['exec']="../"+subst['execname'] 293 subst['testroot']=self.testroot_dir 294 subst['testname']=testname 295 dp = self.conf.get('DATAFILESPATH','') 296 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 297 298 # This is used to label some matrices 299 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 300 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 301 302 # These can have for loops and are treated separately later 303 subst['nsize']=str(subst['nsize']) 304 305 #Conf vars 306 if self.petsc_arch.find('valgrind')>=0: 307 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 308 else: 309 subst['mpiexec']=self.conf['MPIEXEC'] 310 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 311 subst['petsc_arch']=self.petsc_arch 312 if not self.inInstallDir: 313 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 314 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 315 else: 316 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 317 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 318 subst['diff']=self.conf['DIFF'] 319 subst['rm']=self.conf['RM'] 320 subst['grep']=self.conf['GREP'] 321 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 322 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 323 324 # Output file is special because of subtests override 325 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 326 if not "_" in defroot: defroot=defroot+"_1" 327 subst['defroot']=defroot 328 subst['label']=self.nameSpace(defroot,subst['srcdir']) 329 subst['redirect_file']=defroot+".tmp" 330 if 'output_file' not in testDict: 331 subst['output_file']="output/"+defroot+".out" 332 # Add in the full path here. 333 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 334 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 335 if not subst['TODO']: 336 print "Warning: "+subst['output_file']+" not found." 337 # Worry about alt files here -- see 338 # src/snes/examples/tutorials/output/ex22*.out 339 altlist=[subst['output_file']] 340 for i in range(1,3): 341 altroot=defroot+"_alt" 342 if i==2: altroot=altroot+"_2" 343 af="output/"+altroot+".out" 344 srcaf=os.path.join(subst['srcdir'],af) 345 fullaf=os.path.join(self.petsc_dir,srcaf) 346 if os.path.isfile(fullaf): altlist.append(srcaf) 347 if len(altlist)>1: subst['altfiles']=altlist 348 #if len(altlist)>1: print "Found alt files: ",altlist 349 350 return subst 351 352 def getCmds(self,subst,i): 353 """ 354 Generate bash script using template found next to this file. 355 This file is read in at constructor time to avoid file I/O 356 """ 357 nindnt=i # the start and has to be consistent with below 358 cmdindnt=self.indent*nindnt 359 cmdLines="" 360 361 # MPI is the default -- but we have a few odd commands 362 if not subst['command']: 363 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 364 else: 365 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 366 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 367 368 cmdLines+=cmdindnt+'if test $res = 0; then\n' 369 diffindnt=self.indent*(nindnt+1) 370 if not subst['filter_output']: 371 if 'altfiles' not in subst: 372 cmd=diffindnt+self._substVars(subst,example_template.difftest) 373 else: 374 # Have to do it by hand a bit because of variable number of alt files 375 rf=subst['redirect_file'] 376 cmd=diffindnt+example_template.difftest.split('@')[0] 377 for i in range(len(subst['altfiles'])): 378 af=subst['altfiles'][i] 379 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 380 if i!=len(subst['altfiles'])-1: 381 cmd+=' || ${diff_exe} ' 382 else: 383 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 384 cmd+=subst['label_suffix']+' ""' # Quotes are painful 385 else: 386 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 387 cmdLines+=cmd+"\n" 388 cmdLines+=cmdindnt+'else\n' 389 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 390 cmdLines+=cmdindnt+'fi\n' 391 return cmdLines 392 393 def _substVars(self,subst,origStr): 394 """ 395 Substitute variables 396 """ 397 Str=origStr 398 for subkey in subst: 399 if type(subst[subkey])!=types.StringType: continue 400 patt="@"+subkey.upper()+"@" 401 Str=re.sub(patt,subst[subkey],Str) 402 return Str 403 404 def _writeTodoSkip(self,fh,tors,reasons,footer): 405 """ 406 Write out the TODO and SKIP lines in the file 407 The TODO or SKIP variable, tors, should be lower case 408 """ 409 TORS=tors.upper() 410 template=eval("example_template."+tors+"line") 411 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 412 tab = '' 413 if reasons: 414 fh.write('if ! $force; then\n') 415 tab = tab + ' ' 416 if reasons == ["Requires DATAFILESPATH"]: 417 # The only reason not to run is DATAFILESPATH, which we check at run-time 418 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 419 tab = tab + ' ' 420 if reasons: 421 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 422 fh.write(tab+footer+"\n") 423 fh.write(tab+"exit\n") 424 if reasons == ["Requires DATAFILESPATH"]: 425 fh.write(' fi\n') 426 if reasons: 427 fh.write('fi\n') 428 fh.write('\n\n') 429 return 430 431 def getLoopVarsHead(self,loopVars,i): 432 """ 433 Generate a nicely indented string with the format loops 434 Here is what the data structure looks like 435 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 436 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 437 loopVars['subargs']['pc_type']=["j","cholesky sor"] 438 """ 439 outstr=''; indnt=self.indent 440 for key in loopVars: 441 for var in loopVars[key]['varlist']: 442 varval=loopVars[key][var] 443 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 444 i = i + 1 445 return (outstr,i) 446 447 def getLoopVarsFoot(self,loopVars,i): 448 outstr=''; indnt=self.indent 449 for key in loopVars: 450 for var in loopVars[key]['varlist']: 451 i = i - 1 452 outstr += indnt * i + "done\n" 453 return (outstr,i) 454 455 def genRunScript(self,testname,root,isRun,srcDict): 456 """ 457 Generate bash script using template found next to this file. 458 This file is read in at constructor time to avoid file I/O 459 """ 460 # runscript_dir directory has to be consistent with gmakefile 461 testDict=srcDict[testname] 462 rpath=self.srcrelpath(root) 463 runscript_dir=os.path.join(self.testroot_dir,rpath) 464 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 465 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 466 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 467 468 # Get variables to go into shell scripts. last time testDict used 469 subst=self.getSubstVars(testDict,rpath,testname) 470 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 471 #if '33_' in testname: print subst['subargs'] 472 473 #Handle runfiles 474 for lfile in subst.get('localrunfiles','').split(): 475 fullfile=os.path.join(root,lfile) 476 shutil.copy(fullfile,runscript_dir) 477 # Check subtests for local runfiles 478 for stest in subst.get("subtests",[]): 479 for lfile in testDict[stest].get('localrunfiles','').split(): 480 fullfile=os.path.join(root,lfile) 481 shutil.copy(fullfile,self.runscript_dir) 482 483 # Now substitute the key variables into the header and footer 484 header=self._substVars(subst,example_template.header) 485 # The header is done twice to enable @...@ in header 486 header=self._substVars(subst,header) 487 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 488 489 # Start writing the file 490 fh.write(header+"\n") 491 492 # If there is a TODO or a SKIP then we do it before writing out the 493 # rest of the command (which is useful for working on the test) 494 # SKIP and TODO can be for the source file or for the runs 495 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 496 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 497 498 j=0 # for indentation 499 500 if loopVars: 501 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 502 if (loopHead): fh.write(loopHead+"\n") 503 504 # Subtests are special 505 if 'subtests' in testDict: 506 substP=subst # Subtests can inherit args but be careful 507 k=0 # for label suffixes 508 for stest in testDict["subtests"]: 509 subst=substP.copy() 510 subst.update(testDict[stest]) 511 # nsize is special because it is usually overwritten 512 if 'nsize' in testDict[stest]: 513 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 514 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 515 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 516 #if '10_9' in testname: print sLoopVars 517 if sLoopVars: 518 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 519 fh.write(sLoopHead+"\n") 520 fh.write(self.getCmds(subst,j)+"\n") 521 if sLoopVars: 522 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 523 fh.write(sLoopFoot+"\n") 524 else: 525 fh.write(self.getCmds(subst,j)+"\n") 526 527 if loopVars: 528 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 529 fh.write(loopFoot+"\n") 530 531 fh.write(footer+"\n") 532 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 533 #if '10_9' in testname: sys.exit() 534 return 535 536 def genScriptsAndInfo(self,exfile,root,srcDict): 537 """ 538 Generate scripts from the source file, determine if built, etc. 539 For every test in the exfile with info in the srcDict: 540 1. Determine if it needs to be run for this arch 541 2. Generate the script 542 3. Generate the data needed to write out the makefile in a 543 convenient way 544 All tests are *always* run, but some may be SKIP'd per the TAP standard 545 """ 546 debug=False 547 execname=self.getExecname(exfile,root) 548 isBuilt=self._isBuilt(exfile,srcDict) 549 for test in srcDict: 550 if test in self.buildkeys: continue 551 if debug: print self.nameSpace(exfile,root), test 552 srcDict[test]['execname']=execname # Convenience in generating scripts 553 isRun=self._isRun(srcDict[test]) 554 self.genRunScript(test,root,isRun,srcDict) 555 srcDict[test]['isrun']=isRun 556 self.addToTests(test,root,exfile,execname,srcDict[test]) 557 558 # This adds to datastructure for building deps 559 if isBuilt: self.addToSources(exfile,root,srcDict) 560 return 561 562 def _isBuilt(self,exfile,srcDict): 563 """ 564 Determine if this file should be built. 565 """ 566 # Get the language based on file extension 567 srcDict['SKIP'] = [] 568 lang=self.getLanguage(exfile) 569 if (lang=="F" or lang=="F90"): 570 if not self.have_fortran: 571 srcDict["SKIP"].append("Fortran required for this test") 572 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 573 srcDict["SKIP"].append("Fortran f90freeform required for this test") 574 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 575 srcDict["SKIP"].append("CUDA required for this test") 576 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 577 srcDict["SKIP"].append("C++ required for this test") 578 579 # Deprecated source files 580 if srcDict.get("TODO"): 581 return False 582 583 # isRun can work with srcDict to handle the requires 584 if "requires" in srcDict: 585 if len(srcDict["requires"])>0: 586 return self._isRun(srcDict) 587 588 return srcDict['SKIP'] == [] 589 590 591 def _isRun(self,testDict): 592 """ 593 Based on the requirements listed in the src file and the petscconf.h 594 info, determine whether this test should be run or not. 595 """ 596 indent=" " 597 debug=False 598 599 if 'SKIP' not in testDict: 600 testDict['SKIP'] = [] 601 # MPI requirements 602 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 603 if debug: print indent+"Cannot run parallel tests" 604 testDict['SKIP'].append("Parallel test with serial build") 605 606 # The requirements for the test are the sum of all the run subtests 607 if 'subtests' in testDict: 608 if 'requires' not in testDict: testDict['requires']="" 609 for stest in testDict['subtests']: 610 if 'requires' in testDict[stest]: 611 testDict['requires']+=" "+testDict[stest]['requires'] 612 613 614 # Now go through all requirements 615 if 'requires' in testDict: 616 for requirement in testDict['requires'].split(): 617 requirement=requirement.strip() 618 if not requirement: continue 619 if debug: print indent+"Requirement: ", requirement 620 isNull=False 621 if requirement.startswith("!"): 622 requirement=requirement[1:]; isNull=True 623 # Precision requirement for reals 624 if requirement in self.precision_types: 625 if self.conf['PETSC_PRECISION']==requirement: 626 if isNull: 627 testDict['SKIP'].append("not "+requirement+" required") 628 continue 629 continue # Success 630 elif not isNull: 631 testDict['SKIP'].append(requirement+" required") 632 continue 633 # Precision requirement for ints 634 if requirement in self.integer_types: 635 if requirement=="int32": 636 if self.conf['PETSC_SIZEOF_INT']==4: 637 if isNull: 638 testDict['SKIP'].append("not int32 required") 639 continue 640 continue # Success 641 elif not isNull: 642 testDict['SKIP'].append("int32 required") 643 continue 644 if requirement=="int64": 645 if self.conf['PETSC_SIZEOF_INT']==8: 646 if isNull: 647 testDict['SKIP'].append("NOT int64 required") 648 continue 649 continue # Success 650 elif not isNull: 651 testDict['SKIP'].append("int64 required") 652 continue 653 # Datafilespath 654 if requirement=="datafilespath" and not isNull: 655 testDict['SKIP'].append("Requires DATAFILESPATH") 656 continue 657 # Defines -- not sure I have comments matching 658 if "define(" in requirement.lower(): 659 reqdef=requirement.split("(")[1].split(")")[0] 660 if reqdef in self.conf: 661 if isNull: 662 testDict['SKIP'].append("Null requirement not met: "+requirement) 663 continue 664 continue # Success 665 elif not isNull: 666 testDict['SKIP'].append("Required: "+requirement) 667 continue 668 669 # Rest should be packages that we can just get from conf 670 if requirement == "complex": 671 petscconfvar="PETSC_USE_COMPLEX" 672 else: 673 petscconfvar="PETSC_HAVE_"+requirement.upper() 674 if self.conf.get(petscconfvar): 675 if isNull: 676 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 677 continue 678 continue # Success 679 elif not isNull: 680 if debug: print "requirement not found: ", requirement 681 testDict['SKIP'].append(petscconfvar+" requirement not met") 682 continue 683 684 return testDict['SKIP'] == [] 685 686 def genPetscTests_summarize(self,dataDict): 687 """ 688 Required method to state what happened 689 """ 690 if not self.summarize: return 691 indent=" " 692 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 693 fh=open(fhname,"w") 694 #print "See ", fhname 695 for root in dataDict: 696 relroot=self.srcrelpath(root) 697 pkg=relroot.split("/")[1] 698 fh.write(relroot+"\n") 699 allSrcs=[] 700 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 701 for exfile in dataDict[root]: 702 # Basic information 703 rfile=os.path.join(relroot,exfile) 704 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 705 fh.write(indent+exfile+indent*4+builtStatus+"\n") 706 707 for test in dataDict[root][exfile]: 708 if test in self.buildkeys: continue 709 line=indent*2+test 710 fh.write(line+"\n") 711 # Looks nice to have the keys in order 712 #for key in dataDict[root][exfile][test]: 713 for key in "isrun abstracted nsize args requires script".split(): 714 if key not in dataDict[root][exfile][test]: continue 715 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 716 fh.write(line+"\n") 717 fh.write("\n") 718 fh.write("\n") 719 fh.write("\n") 720 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 721 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 722 fh.close() 723 return 724 725 def genPetscTests(self,root,dirs,files,dataDict): 726 """ 727 Go through and parse the source files in the directory to generate 728 the examples based on the metadata contained in the source files 729 """ 730 debug=False 731 # Use examplesAnalyze to get what the makefles think are sources 732 #self.examplesAnalyze(root,dirs,files,anlzDict) 733 734 dataDict[root]={} 735 736 for exfile in files: 737 #TST: Until we replace files, still leaving the orginals as is 738 #if not exfile.startswith("new_"+"ex"): continue 739 #if not exfile.startswith("ex"): continue 740 741 # Ignore emacs files 742 if exfile.startswith("#"): continue 743 744 # Convenience 745 fullex=os.path.join(root,exfile) 746 relpfile=os.path.join(self.srcrelpath(root),exfile) 747 if debug: print relpfile 748 dataDict[root].update(testparse.parseTestFile(fullex,0)) 749 # Need to check and make sure tests are in the file 750 # if verbosity>=1: print relpfile 751 if exfile in dataDict[root]: 752 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 753 754 return 755 756 def walktree(self,top,action="printFiles"): 757 """ 758 Walk a directory tree, starting from 'top' 759 """ 760 #print "action", action 761 # Goal of action is to fill this dictionary 762 dataDict={} 763 for root, dirs, files in os.walk(top, topdown=False): 764 if not "examples" in root: continue 765 if not os.path.isfile(os.path.join(root,"makefile")): continue 766 bname=os.path.basename(root.rstrip("/")) 767 if bname=="tests" or bname=="tutorials": 768 eval("self."+action+"(root,dirs,files,dataDict)") 769 if type(top) != types.StringType: 770 raise TypeError("top must be a string") 771 # Now summarize this dictionary 772 eval("self."+action+"_summarize(dataDict)") 773 return dataDict 774 775 def gen_gnumake(self, fd): 776 """ 777 Overwrite of the method in the base PETSc class 778 """ 779 def write(stem, srcs): 780 for lang in LANGS: 781 if self.inInstallDir: 782 if len(srcs[lang]['srcs'])>0: 783 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 784 else: 785 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 786 else: 787 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 788 for pkg in PKGS: 789 srcs = self.gen_pkg(pkg) 790 write('testsrcs-' + pkg, srcs) 791 return self.gendeps 792 793 def gen_pkg(self, pkg): 794 """ 795 Overwrite of the method in the base PETSc class 796 """ 797 return self.sources[pkg] 798 799 def write_gnumake(self,dataDict): 800 """ 801 Write out something similar to files from gmakegen.py 802 803 Test depends on script which also depends on source 804 file, but since I don't have a good way generating 805 acting on a single file (oops) just depend on 806 executable which in turn will depend on src file 807 """ 808 # Different options for how to set up the targets 809 compileExecsFirst=False 810 811 # Open file 812 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 813 fd = open(arch_files, 'w') 814 815 # Write out the sources 816 gendeps = self.gen_gnumake(fd) 817 818 # Write out the tests and execname targets 819 fd.write("\n#Tests and executables\n") # Delimiter 820 821 for pkg in PKGS: 822 # These grab the ones that are built 823 for lang in LANGS: 824 testdeps=[] 825 for ftest in self.tests[pkg][lang]: 826 test=os.path.basename(ftest) 827 basedir=os.path.dirname(ftest) 828 testdeps.append(self.nameSpace(test,basedir)) 829 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 830 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 831 832 # test targets 833 for ftest in self.tests[pkg][lang]: 834 test=os.path.basename(ftest) 835 basedir=os.path.dirname(ftest) 836 testdir="${TESTDIR}/"+basedir+"/" 837 nmtest=self.nameSpace(test,basedir) 838 rundir=os.path.join(testdir,test) 839 #print test, nmtest 840 script=test+".sh" 841 842 # Deps 843 exfile=self.tests[pkg][lang][ftest]['exfile'] 844 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 845 localexec=self.tests[pkg][lang][ftest]['exec'] 846 execname=os.path.join(testdir,localexec) 847 fullscript=os.path.join(testdir,script) 848 tmpfile=os.path.join(testdir,test,test+".tmp") 849 850 # *.counts depends on the script and either executable (will 851 # be run) or the example source file (SKIP or TODO) 852 if exfile in self.sources[pkg][lang]: 853 fd.write('%s.counts : %s %s %s\n' 854 % (os.path.join('$(TESTDIR)/counts', nmtest), 855 fullscript, 856 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex, 857 os.path.join('$(TESTDIR)',self.sources[pkg][lang][exfile])) 858 ) 859 else: 860 fd.write('%s.counts : %s %s\n' 861 % (os.path.join('$(TESTDIR)/counts', nmtest), 862 fullscript, 863 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 864 ) 865 866 # Now write the args: 867 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 868 869 fd.close() 870 return 871 872 def writeHarness(self,output,dataDict): 873 """ 874 This is set up to write out multiple harness even if only gnumake 875 is supported now 876 """ 877 eval("self.write_"+output+"(dataDict)") 878 return 879 880def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 881 if output is None: 882 output = 'gnumake' 883 884 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 885 if len(petsc_arch.split(os.path.sep))>1: 886 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 887 888 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 889 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 890 testdir=testdir) 891 dataDict=pEx.walktree(os.path.join(pEx.srcdir),action="genPetscTests") 892 pEx.writeHarness(output,dataDict) 893 894if __name__ == '__main__': 895 import optparse 896 parser = optparse.OptionParser() 897 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 898 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 899 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 900 parser.add_option('--output', help='Location to write output file', default=None) 901 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 902 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 903 opts, extra_args = parser.parse_args() 904 opts, extra_args = parser.parse_args() 905 if extra_args: 906 import sys 907 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 908 exit(1) 909 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 910 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 911