1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 if not self.inInstallDir(thisscriptdir): 31 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 32 self.srcdir=os.path.join(self.petsc_dir,'src') 33 else: 34 # set PETSC_ARCH to install directory to allow script to work in both 35 dirlist=thisscriptdir.split(os.path.sep) 36 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 37 self.arch_dir=installdir 38 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 39 40 # Do some initialization 41 if testdir: 42 # If full path given, then use it, otherwise assume relative to arch_dir 43 if testdir.strip().startswith(os.path.sep): 44 self.testroot_dir=testdir.strip() 45 else: 46 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 47 else: 48 self.testroot_dir=os.path.join(self.arch_dir,"tests") 49 50 self.ptNaming=True 51 # Whether to write out a useful debugging 52 #if verbose: self.summarize=True 53 self.summarize=True 54 55 # For help in setting the requirements 56 self.precision_types="single double __float128 int32".split() 57 self.integer_types="int32 int64".split() 58 self.languages="fortran cuda cxx".split() # Always requires C so do not list 59 60 # Things that are not test 61 self.buildkeys=testparse.buildkeys 62 63 # Adding a dictionary for storing sources, objects, and tests 64 # to make building the dependency tree easier 65 self.sources={} 66 self.objects={} 67 self.tests={} 68 for pkg in PKGS: 69 self.sources[pkg]={} 70 self.objects[pkg]=[] 71 self.tests[pkg]={} 72 for lang in LANGS: 73 self.sources[pkg][lang]={} 74 self.sources[pkg][lang]['srcs']=[] 75 self.tests[pkg][lang]={} 76 77 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 78 79 self.indent=" " 80 return 81 82 def srcrelpath(self,rdir): 83 """ 84 Get relative path to source directory 85 """ 86 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 87 88 def inInstallDir(self,thisscriptdir): 89 """ 90 When petsc is installed then this file in installed in: 91 <PREFIX>/share/petsc/examples/config/gmakegentest.py 92 otherwise the path is: 93 <PETSC_DIR>/config/gmakegentest.py 94 We use this difference to determine if we are in installdir 95 """ 96 dirlist=thisscriptdir.split(os.path.sep) 97 if len(dirlist)>4: 98 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 99 if lastfour==os.path.join('share','petsc','examples','config'): 100 return True 101 else: 102 return False 103 else: 104 return False 105 106 def nameSpace(self,srcfile,srcdir): 107 """ 108 Because the scripts have a non-unique naming, the pretty-printing 109 needs to convey the srcdir and srcfile. There are two ways of doing this. 110 """ 111 if self.ptNaming: 112 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 113 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 114 nameString=prefix+srcfile 115 else: 116 #nameString=srcdir+": "+srcfile 117 nameString=srcfile 118 return nameString 119 120 def getLanguage(self,srcfile): 121 """ 122 Based on the source, determine associated language as found in gmakegen.LANGS 123 Can we just return srcext[1:\] now? 124 """ 125 langReq=None 126 srcext=os.path.splitext(srcfile)[-1] 127 if srcext in ".F90".split(): langReq="F90" 128 if srcext in ".F".split(): langReq="F" 129 if srcext in ".cxx".split(): langReq="cxx" 130 if srcext == ".cu": langReq="cu" 131 if srcext == ".c": langReq="c" 132 #if not langReq: print "ERROR: ", srcext, srcfile 133 return langReq 134 135 def _getLoopVars(self,inDict,testname, isSubtest=False): 136 """ 137 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 138 Return: 139 inDict['args']: -ksp_monitor 140 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 141 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 142 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 143 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 144 subst should be passed in instead of inDict 145 """ 146 loopVars={}; newargs="" 147 lkeys=inDict.keys() 148 lsuffix='_' 149 from testparse import parseLoopArgs 150 for key in lkeys: 151 if type(inDict[key])!=types.StringType: continue 152 keystr = str(inDict[key]) 153 akey=('subargs' if key=='args' else key) # what to assign 154 if akey not in inDict: inDict[akey]='' 155 varlist=[] 156 for varset in re.split('-(?=[a-zA-Z])',keystr): 157 if not varset.strip(): continue 158 if '{{' in varset: 159 keyvar,lvars,ftype=parseLoopArgs(varset) 160 if akey not in loopVars: loopVars[akey]={} 161 varlist.append(keyvar) 162 loopVars[akey][keyvar]=[keyvar,lvars] 163 if akey=='nsize': 164 inDict[akey] = '${' + keyvar + '}' 165 lsuffix+=akey+'-'+inDict[akey]+'_' 166 else: 167 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 168 lsuffix+=keyvar+'-${' + keyvar + '}_' 169 else: 170 if key=='args': newargs+=" -"+varset.strip() 171 if len(varlist)>0: loopVars[akey]['varlist']=varlist 172 173 174 # For subtests, args are always substituted in (not top level) 175 if isSubtest: 176 inDict['subargs']+=" "+newargs.strip() 177 inDict['args']='' 178 if 'label_suffix' in inDict: 179 inDict['label_suffix']+=lsuffix.rstrip('_') 180 else: 181 inDict['label_suffix']=lsuffix.rstrip('_') 182 else: 183 if len(loopVars.keys())>0: 184 inDict['args']=newargs.strip() 185 inDict['label_suffix']=lsuffix.rstrip('_') 186 if len(loopVars.keys())>0: 187 return loopVars 188 else: 189 return None 190 191 def getArgLabel(self,testDict): 192 """ 193 In all of the arguments in the test dictionary, create a simple 194 string for searching within the makefile system. For simplicity in 195 search, remove "-", for strings, etc. 196 Also, concatenate the arg commands 197 For now, ignore nsize -- seems hard to search for anyway 198 """ 199 # Collect all of the args associated with a test 200 argStr=("" if 'args' not in testDict else testDict['args']) 201 if 'subtests' in testDict: 202 for stest in testDict["subtests"]: 203 sd=testDict[stest] 204 argStr=argStr+("" if 'args' not in sd else sd['args']) 205 206 # Now go through and cleanup 207 argStr=re.sub('{{(.*?)}}',"",argStr) 208 argStr=re.sub('-'," ",argStr) 209 for digit in string.digits: argStr=re.sub(digit," ",argStr) 210 argStr=re.sub("\.","",argStr) 211 argStr=re.sub(",","",argStr) 212 argStr=re.sub('\+',' ',argStr) 213 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 214 return argStr.strip() 215 216 def addToSources(self,exfile,root,srcDict): 217 """ 218 Put into data structure that allows easy generation of makefile 219 """ 220 rpath=self.srcrelpath(root) 221 pkg=rpath.split(os.path.sep)[1] 222 relpfile=os.path.join(rpath,exfile) 223 lang=self.getLanguage(exfile) 224 if not lang: return 225 self.sources[pkg][lang]['srcs'].append(relpfile) 226 if 'depends' in srcDict: 227 depSrc=srcDict['depends'] 228 depObj=os.path.splitext(depSrc)[0]+".o" 229 self.sources[pkg][lang][exfile]=depObj 230 231 # In gmakefile, ${TESTDIR} var specifies the object compilation 232 testsdir=self.srcrelpath(root)+"/" 233 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 234 self.objects[pkg].append(objfile) 235 return 236 237 def addToTests(self,test,root,exfile,execname,testDict): 238 """ 239 Put into data structure that allows easy generation of makefile 240 Organized by languages to allow testing of languages 241 """ 242 rpath=self.srcrelpath(root) 243 pkg=rpath.split("/")[1] 244 #nmtest=self.nameSpace(test,root) 245 nmtest=os.path.join(rpath,test) 246 lang=self.getLanguage(exfile) 247 if not lang: return 248 self.tests[pkg][lang][nmtest]={} 249 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 250 self.tests[pkg][lang][nmtest]['exec']=execname 251 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 252 return 253 254 def getExecname(self,exfile,root): 255 """ 256 Generate bash script using template found next to this file. 257 This file is read in at constructor time to avoid file I/O 258 """ 259 rpath=self.srcrelpath(root) 260 if self.single_ex: 261 execname=rpath.split("/")[1]+"-ex" 262 else: 263 execname=os.path.splitext(exfile)[0] 264 return execname 265 266 def getSubstVars(self,testDict,rpath,testname): 267 """ 268 Create a dictionary with all of the variables that get substituted 269 into the template commands found in example_template.py 270 """ 271 subst={} 272 273 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 274 if 'nsize' not in testDict: testDict['nsize']=1 275 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 276 for ak in testparse.acceptedkeys: 277 if ak=='test': continue 278 subst[ak]=(testDict[ak] if ak in testDict else '') 279 280 # Now do other variables 281 subst['execname']=testDict['execname'] 282 if 'filter' in testDict: 283 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 284 285 # Others 286 subst['subargs']='' # Default. For variables override 287 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 288 subst['label_suffix']='' 289 subst['comments']="\n#".join(subst['comments'].split("\n")) 290 if subst['comments']: subst['comments']="#"+subst['comments'] 291 subst['exec']="../"+subst['execname'] 292 subst['testroot']=self.testroot_dir 293 subst['testname']=testname 294 dp = self.conf.get('DATAFILESPATH','') 295 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 296 297 # This is used to label some matrices 298 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 299 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 300 301 # These can have for loops and are treated separately later 302 subst['nsize']=str(subst['nsize']) 303 304 #Conf vars 305 if self.petsc_arch.find('valgrind')>=0: 306 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 307 else: 308 subst['mpiexec']=self.conf['MPIEXEC'] 309 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 310 subst['petsc_arch']=self.petsc_arch 311 if not self.inInstallDir: 312 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 313 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 314 else: 315 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 316 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 317 subst['diff']=self.conf['DIFF'] 318 subst['rm']=self.conf['RM'] 319 subst['grep']=self.conf['GREP'] 320 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 321 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 322 323 # Output file is special because of subtests override 324 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 325 if not "_" in defroot: defroot=defroot+"_1" 326 subst['defroot']=defroot 327 subst['label']=self.nameSpace(defroot,subst['srcdir']) 328 subst['redirect_file']=defroot+".tmp" 329 if 'output_file' not in testDict: 330 subst['output_file']="output/"+defroot+".out" 331 # Add in the full path here. 332 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 333 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 334 if not subst['TODO']: 335 print "Warning: "+subst['output_file']+" not found." 336 # Worry about alt files here -- see 337 # src/snes/examples/tutorials/output/ex22*.out 338 altlist=[subst['output_file']] 339 for i in range(1,3): 340 altroot=defroot+"_alt" 341 if i==2: altroot=altroot+"_2" 342 af="output/"+altroot+".out" 343 srcaf=os.path.join(subst['srcdir'],af) 344 fullaf=os.path.join(self.petsc_dir,srcaf) 345 if os.path.isfile(fullaf): altlist.append(srcaf) 346 if len(altlist)>1: subst['altfiles']=altlist 347 #if len(altlist)>1: print "Found alt files: ",altlist 348 349 return subst 350 351 def getCmds(self,subst,i): 352 """ 353 Generate bash script using template found next to this file. 354 This file is read in at constructor time to avoid file I/O 355 """ 356 nindnt=i # the start and has to be consistent with below 357 cmdindnt=self.indent*nindnt 358 cmdLines="" 359 360 # MPI is the default -- but we have a few odd commands 361 if not subst['command']: 362 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 363 else: 364 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 365 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 366 367 cmdLines+=cmdindnt+'if test $res = 0; then\n' 368 diffindnt=self.indent*(nindnt+1) 369 if not subst['filter_output']: 370 if 'altfiles' not in subst: 371 cmd=diffindnt+self._substVars(subst,example_template.difftest) 372 else: 373 # Have to do it by hand a bit because of variable number of alt files 374 rf=subst['redirect_file'] 375 cmd=diffindnt+example_template.difftest.split('@')[0] 376 for i in range(len(subst['altfiles'])): 377 af=subst['altfiles'][i] 378 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 379 if i!=len(subst['altfiles'])-1: 380 cmd+=' || ${diff_exe} ' 381 else: 382 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 383 cmd+=subst['label_suffix']+' ""' # Quotes are painful 384 else: 385 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 386 cmdLines+=cmd+"\n" 387 cmdLines+=cmdindnt+'else\n' 388 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 389 cmdLines+=cmdindnt+'fi\n' 390 return cmdLines 391 392 def _substVars(self,subst,origStr): 393 """ 394 Substitute variables 395 """ 396 Str=origStr 397 for subkey in subst: 398 if type(subst[subkey])!=types.StringType: continue 399 patt="@"+subkey.upper()+"@" 400 Str=re.sub(patt,subst[subkey],Str) 401 return Str 402 403 def _writeTodoSkip(self,fh,tors,reasons,footer): 404 """ 405 Write out the TODO and SKIP lines in the file 406 The TODO or SKIP variable, tors, should be lower case 407 """ 408 TORS=tors.upper() 409 template=eval("example_template."+tors+"line") 410 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 411 tab = '' 412 if reasons: 413 fh.write('if ! $force; then\n') 414 tab = tab + ' ' 415 if reasons == ["Requires DATAFILESPATH"]: 416 # The only reason not to run is DATAFILESPATH, which we check at run-time 417 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 418 tab = tab + ' ' 419 if reasons: 420 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 421 fh.write(tab+footer+"\n") 422 fh.write(tab+"exit\n") 423 if reasons == ["Requires DATAFILESPATH"]: 424 fh.write(' fi\n') 425 if reasons: 426 fh.write('fi\n') 427 fh.write('\n\n') 428 return 429 430 def getLoopVarsHead(self,loopVars,i): 431 """ 432 Generate a nicely indented string with the format loops 433 Here is what the data structure looks like 434 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 435 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 436 loopVars['subargs']['pc_type']=["j","cholesky sor"] 437 """ 438 outstr=''; indnt=self.indent 439 for key in loopVars: 440 for var in loopVars[key]['varlist']: 441 varval=loopVars[key][var] 442 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 443 i = i + 1 444 return (outstr,i) 445 446 def getLoopVarsFoot(self,loopVars,i): 447 outstr=''; indnt=self.indent 448 for key in loopVars: 449 for var in loopVars[key]['varlist']: 450 i = i - 1 451 outstr += indnt * i + "done\n" 452 return (outstr,i) 453 454 def genRunScript(self,testname,root,isRun,srcDict): 455 """ 456 Generate bash script using template found next to this file. 457 This file is read in at constructor time to avoid file I/O 458 """ 459 # runscript_dir directory has to be consistent with gmakefile 460 testDict=srcDict[testname] 461 rpath=self.srcrelpath(root) 462 runscript_dir=os.path.join(self.testroot_dir,rpath) 463 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 464 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 465 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 466 467 # Get variables to go into shell scripts. last time testDict used 468 subst=self.getSubstVars(testDict,rpath,testname) 469 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 470 #if '33_' in testname: print subst['subargs'] 471 472 #Handle runfiles 473 for lfile in subst.get('localrunfiles','').split(): 474 fullfile=os.path.join(root,lfile) 475 shutil.copy(fullfile,runscript_dir) 476 # Check subtests for local runfiles 477 for stest in subst.get("subtests",[]): 478 for lfile in testDict[stest].get('localrunfiles','').split(): 479 fullfile=os.path.join(root,lfile) 480 shutil.copy(fullfile,self.runscript_dir) 481 482 # Now substitute the key variables into the header and footer 483 header=self._substVars(subst,example_template.header) 484 # The header is done twice to enable @...@ in header 485 header=self._substVars(subst,header) 486 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 487 488 # Start writing the file 489 fh.write(header+"\n") 490 491 # If there is a TODO or a SKIP then we do it before writing out the 492 # rest of the command (which is useful for working on the test) 493 # SKIP and TODO can be for the source file or for the runs 494 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 495 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 496 497 j=0 # for indentation 498 499 if loopVars: 500 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 501 if (loopHead): fh.write(loopHead+"\n") 502 503 # Subtests are special 504 if 'subtests' in testDict: 505 substP=subst # Subtests can inherit args but be careful 506 for stest in testDict["subtests"]: 507 subst=substP.copy() 508 subst.update(testDict[stest]) 509 subst['nsize']=str(subst['nsize']) 510 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 511 #if '10_9' in testname: print sLoopVars 512 if sLoopVars: 513 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 514 fh.write(sLoopHead+"\n") 515 fh.write(self.getCmds(subst,j)+"\n") 516 if sLoopVars: 517 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 518 fh.write(sLoopFoot+"\n") 519 else: 520 fh.write(self.getCmds(subst,j)+"\n") 521 522 if loopVars: 523 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 524 fh.write(loopFoot+"\n") 525 526 fh.write(footer+"\n") 527 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 528 #if '10_9' in testname: sys.exit() 529 return 530 531 def genScriptsAndInfo(self,exfile,root,srcDict): 532 """ 533 Generate scripts from the source file, determine if built, etc. 534 For every test in the exfile with info in the srcDict: 535 1. Determine if it needs to be run for this arch 536 2. Generate the script 537 3. Generate the data needed to write out the makefile in a 538 convenient way 539 All tests are *always* run, but some may be SKIP'd per the TAP standard 540 """ 541 debug=False 542 execname=self.getExecname(exfile,root) 543 isBuilt=self._isBuilt(exfile,srcDict) 544 for test in srcDict: 545 if test in self.buildkeys: continue 546 if debug: print self.nameSpace(exfile,root), test 547 srcDict[test]['execname']=execname # Convenience in generating scripts 548 isRun=self._isRun(srcDict[test]) 549 self.genRunScript(test,root,isRun,srcDict) 550 srcDict[test]['isrun']=isRun 551 self.addToTests(test,root,exfile,execname,srcDict[test]) 552 553 # This adds to datastructure for building deps 554 if isBuilt: self.addToSources(exfile,root,srcDict) 555 return 556 557 def _isBuilt(self,exfile,srcDict): 558 """ 559 Determine if this file should be built. 560 """ 561 # Get the language based on file extension 562 srcDict['SKIP'] = [] 563 lang=self.getLanguage(exfile) 564 if (lang=="F" or lang=="F90"): 565 if not self.have_fortran: 566 srcDict["SKIP"].append("Fortran required for this test") 567 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 568 srcDict["SKIP"].append("Fortran f90freeform required for this test") 569 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 570 srcDict["SKIP"].append("CUDA required for this test") 571 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 572 srcDict["SKIP"].append("C++ required for this test") 573 574 # Deprecated source files 575 if srcDict.get("TODO"): 576 return False 577 578 # isRun can work with srcDict to handle the requires 579 if "requires" in srcDict: 580 if len(srcDict["requires"])>0: 581 return self._isRun(srcDict) 582 583 return srcDict['SKIP'] == [] 584 585 586 def _isRun(self,testDict): 587 """ 588 Based on the requirements listed in the src file and the petscconf.h 589 info, determine whether this test should be run or not. 590 """ 591 indent=" " 592 debug=False 593 594 if 'SKIP' not in testDict: 595 testDict['SKIP'] = [] 596 # MPI requirements 597 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 598 if debug: print indent+"Cannot run parallel tests" 599 testDict['SKIP'].append("Parallel test with serial build") 600 601 # The requirements for the test are the sum of all the run subtests 602 if 'subtests' in testDict: 603 if 'requires' not in testDict: testDict['requires']="" 604 for stest in testDict['subtests']: 605 if 'requires' in testDict[stest]: 606 testDict['requires']+=" "+testDict[stest]['requires'] 607 608 609 # Now go through all requirements 610 if 'requires' in testDict: 611 for requirement in testDict['requires'].split(): 612 requirement=requirement.strip() 613 if not requirement: continue 614 if debug: print indent+"Requirement: ", requirement 615 isNull=False 616 if requirement.startswith("!"): 617 requirement=requirement[1:]; isNull=True 618 # Precision requirement for reals 619 if requirement in self.precision_types: 620 if self.conf['PETSC_PRECISION']==requirement: 621 if isNull: 622 testDict['SKIP'].append("not "+requirement+" required") 623 continue 624 continue # Success 625 elif not isNull: 626 testDict['SKIP'].append(requirement+" required") 627 continue 628 # Precision requirement for ints 629 if requirement in self.integer_types: 630 if requirement=="int32": 631 if self.conf['PETSC_SIZEOF_INT']==4: 632 if isNull: 633 testDict['SKIP'].append("not int32 required") 634 continue 635 continue # Success 636 elif not isNull: 637 testDict['SKIP'].append("int32 required") 638 continue 639 if requirement=="int64": 640 if self.conf['PETSC_SIZEOF_INT']==8: 641 if isNull: 642 testDict['SKIP'].append("NOT int64 required") 643 continue 644 continue # Success 645 elif not isNull: 646 testDict['SKIP'].append("int64 required") 647 continue 648 # Datafilespath 649 if requirement=="datafilespath" and not isNull: 650 testDict['SKIP'].append("Requires DATAFILESPATH") 651 continue 652 # Defines -- not sure I have comments matching 653 if "define(" in requirement.lower(): 654 reqdef=requirement.split("(")[1].split(")")[0] 655 if reqdef in self.conf: 656 if isNull: 657 testDict['SKIP'].append("Null requirement not met: "+requirement) 658 continue 659 continue # Success 660 elif not isNull: 661 testDict['SKIP'].append("Required: "+requirement) 662 continue 663 664 # Rest should be packages that we can just get from conf 665 if requirement == "complex": 666 petscconfvar="PETSC_USE_COMPLEX" 667 else: 668 petscconfvar="PETSC_HAVE_"+requirement.upper() 669 if self.conf.get(petscconfvar): 670 if isNull: 671 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 672 continue 673 continue # Success 674 elif not isNull: 675 if debug: print "requirement not found: ", requirement 676 testDict['SKIP'].append(petscconfvar+" requirement not met") 677 continue 678 679 return testDict['SKIP'] == [] 680 681 def genPetscTests_summarize(self,dataDict): 682 """ 683 Required method to state what happened 684 """ 685 if not self.summarize: return 686 indent=" " 687 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 688 fh=open(fhname,"w") 689 #print "See ", fhname 690 for root in dataDict: 691 relroot=self.srcrelpath(root) 692 pkg=relroot.split("/")[1] 693 fh.write(relroot+"\n") 694 allSrcs=[] 695 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 696 for exfile in dataDict[root]: 697 # Basic information 698 rfile=os.path.join(relroot,exfile) 699 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 700 fh.write(indent+exfile+indent*4+builtStatus+"\n") 701 702 for test in dataDict[root][exfile]: 703 if test in self.buildkeys: continue 704 line=indent*2+test 705 fh.write(line+"\n") 706 # Looks nice to have the keys in order 707 #for key in dataDict[root][exfile][test]: 708 for key in "isrun abstracted nsize args requires script".split(): 709 if key not in dataDict[root][exfile][test]: continue 710 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 711 fh.write(line+"\n") 712 fh.write("\n") 713 fh.write("\n") 714 fh.write("\n") 715 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 716 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 717 fh.close() 718 return 719 720 def genPetscTests(self,root,dirs,files,dataDict): 721 """ 722 Go through and parse the source files in the directory to generate 723 the examples based on the metadata contained in the source files 724 """ 725 debug=False 726 # Use examplesAnalyze to get what the makefles think are sources 727 #self.examplesAnalyze(root,dirs,files,anlzDict) 728 729 dataDict[root]={} 730 731 for exfile in files: 732 #TST: Until we replace files, still leaving the orginals as is 733 #if not exfile.startswith("new_"+"ex"): continue 734 #if not exfile.startswith("ex"): continue 735 736 # Convenience 737 fullex=os.path.join(root,exfile) 738 relpfile=os.path.join(self.srcrelpath(root),exfile) 739 if debug: print relpfile 740 dataDict[root].update(testparse.parseTestFile(fullex,0)) 741 # Need to check and make sure tests are in the file 742 # if verbosity>=1: print relpfile 743 if exfile in dataDict[root]: 744 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 745 746 return 747 748 def walktree(self,top,action="printFiles"): 749 """ 750 Walk a directory tree, starting from 'top' 751 """ 752 #print "action", action 753 # Goal of action is to fill this dictionary 754 dataDict={} 755 for root, dirs, files in os.walk(top, topdown=False): 756 if not "examples" in root: continue 757 if not os.path.isfile(os.path.join(root,"makefile")): continue 758 bname=os.path.basename(root.rstrip("/")) 759 if bname=="tests" or bname=="tutorials": 760 eval("self."+action+"(root,dirs,files,dataDict)") 761 if type(top) != types.StringType: 762 raise TypeError("top must be a string") 763 # Now summarize this dictionary 764 eval("self."+action+"_summarize(dataDict)") 765 return dataDict 766 767 def gen_gnumake(self, fd): 768 """ 769 Overwrite of the method in the base PETSc class 770 """ 771 def write(stem, srcs): 772 for lang in LANGS: 773 if self.inInstallDir: 774 if len(srcs[lang]['srcs'])>0: 775 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 776 else: 777 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 778 else: 779 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 780 for pkg in PKGS: 781 srcs = self.gen_pkg(pkg) 782 write('testsrcs-' + pkg, srcs) 783 return self.gendeps 784 785 def gen_pkg(self, pkg): 786 """ 787 Overwrite of the method in the base PETSc class 788 """ 789 return self.sources[pkg] 790 791 def write_gnumake(self,dataDict): 792 """ 793 Write out something similar to files from gmakegen.py 794 795 Test depends on script which also depends on source 796 file, but since I don't have a good way generating 797 acting on a single file (oops) just depend on 798 executable which in turn will depend on src file 799 """ 800 # Different options for how to set up the targets 801 compileExecsFirst=False 802 803 # Open file 804 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 805 fd = open(arch_files, 'w') 806 807 # Write out the sources 808 gendeps = self.gen_gnumake(fd) 809 810 # Write out the tests and execname targets 811 fd.write("\n#Tests and executables\n") # Delimiter 812 813 for pkg in PKGS: 814 # These grab the ones that are built 815 for lang in LANGS: 816 testdeps=[] 817 for ftest in self.tests[pkg][lang]: 818 test=os.path.basename(ftest) 819 basedir=os.path.dirname(ftest) 820 testdeps.append(self.nameSpace(test,basedir)) 821 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 822 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 823 824 # test targets 825 for ftest in self.tests[pkg][lang]: 826 test=os.path.basename(ftest) 827 basedir=os.path.dirname(ftest) 828 testdir="${TESTDIR}/"+basedir+"/" 829 nmtest=self.nameSpace(test,basedir) 830 rundir=os.path.join(testdir,test) 831 #print test, nmtest 832 script=test+".sh" 833 834 # Deps 835 exfile=self.tests[pkg][lang][ftest]['exfile'] 836 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 837 localexec=self.tests[pkg][lang][ftest]['exec'] 838 execname=os.path.join(testdir,localexec) 839 fullscript=os.path.join(testdir,script) 840 tmpfile=os.path.join(testdir,test,test+".tmp") 841 842 # *.counts depends on the script and either executable (will 843 # be run) or the example source file (SKIP or TODO) 844 fd.write('%s.counts : %s %s\n' 845 % (os.path.join('$(TESTDIR)/counts', nmtest), 846 fullscript, 847 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 848 # Now write the args: 849 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 850 851 fd.close() 852 return 853 854 def writeHarness(self,output,dataDict): 855 """ 856 This is set up to write out multiple harness even if only gnumake 857 is supported now 858 """ 859 eval("self.write_"+output+"(dataDict)") 860 return 861 862def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 863 if output is None: 864 output = 'gnumake' 865 866 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 867 if len(petsc_arch.split(os.path.sep))>1: 868 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 869 870 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 871 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 872 testdir=testdir) 873 dataDict=pEx.walktree(os.path.join(pEx.srcdir),action="genPetscTests") 874 pEx.writeHarness(output,dataDict) 875 876if __name__ == '__main__': 877 import optparse 878 parser = optparse.OptionParser() 879 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 880 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 881 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 882 parser.add_option('--output', help='Location to write output file', default=None) 883 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 884 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 885 opts, extra_args = parser.parse_args() 886 opts, extra_args = parser.parse_args() 887 if extra_args: 888 import sys 889 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 890 exit(1) 891 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 892 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 893