1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Determine if we are running from an install directory: 30 dirlist=thisscriptdir.split("/") 31 if len(dirlist)>4: 32 lastfour="/".join(dirlist[len(dirlist)-4:]) 33 installdir="/".join(dirlist[0:len(dirlist)-4]) 34 srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 35 self.inInstallDir=True if lastfour=='share/petsc/examples/config' else False 36 else: 37 self.inInstallDir=False 38 39 if not self.inInstallDir: 40 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 41 else: 42 self.arch_dir=installdir 43 44 # Do some initialization 45 if testdir: 46 # If full path given, then use it, otherwise assume relative to arch_dir 47 if testdir.strip().startswith("/"): 48 self.testroot_dir=testdir.strip() 49 else: 50 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 51 else: 52 self.testroot_dir=os.path.join(self.arch_dir,"tests") 53 54 if srcdir: 55 self.srcdir=srcdir 56 else: 57 if not self.inInstallDir: 58 self.srcdir=os.path.join(self.petsc_dir,'src') 59 else: 60 self.srcdir=srcdir 61 62 self.ptNaming=True 63 # Whether to write out a useful debugging 64 #if verbose: self.summarize=True 65 self.summarize=True 66 67 # For help in setting the requirements 68 self.precision_types="single double __float128 int32".split() 69 self.integer_types="int32 int64".split() 70 self.languages="fortran cuda cxx".split() # Always requires C so do not list 71 72 # Things that are not test 73 self.buildkeys=testparse.buildkeys 74 75 # Adding a dictionary for storing sources, objects, and tests 76 # to make building the dependency tree easier 77 self.sources={} 78 self.objects={} 79 self.tests={} 80 for pkg in PKGS: 81 self.sources[pkg]={} 82 self.objects[pkg]=[] 83 self.tests[pkg]={} 84 for lang in LANGS: 85 self.sources[pkg][lang]={} 86 self.sources[pkg][lang]['srcs']=[] 87 self.tests[pkg][lang]={} 88 89 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 90 91 self.indent=" " 92 return 93 94 def nameSpace(self,srcfile,srcdir): 95 """ 96 Because the scripts have a non-unique naming, the pretty-printing 97 needs to convey the srcdir and srcfile. There are two ways of doing this. 98 """ 99 if self.ptNaming: 100 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 101 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 102 nameString=prefix+srcfile 103 else: 104 #nameString=srcdir+": "+srcfile 105 nameString=srcfile 106 return nameString 107 108 def getLanguage(self,srcfile): 109 """ 110 Based on the source, determine associated language as found in gmakegen.LANGS 111 Can we just return srcext[1:\] now? 112 """ 113 langReq=None 114 srcext=os.path.splitext(srcfile)[-1] 115 if srcext in ".F90".split(): langReq="F90" 116 if srcext in ".F".split(): langReq="F" 117 if srcext in ".cxx".split(): langReq="cxx" 118 if srcext == ".cu": langReq="cu" 119 if srcext == ".c": langReq="c" 120 #if not langReq: print "ERROR: ", srcext, srcfile 121 return langReq 122 123 def _getLoopVars(self,inDict,testname, isSubtest=False): 124 """ 125 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 126 Return: 127 inDict['args']: -ksp_monitor 128 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 129 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 130 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 131 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 132 subst should be passed in instead of inDict 133 """ 134 loopVars={}; newargs="" 135 lkeys=inDict.keys() 136 lsuffix='_' 137 from testparse import parseLoopArgs 138 for key in lkeys: 139 if type(inDict[key])!=types.StringType: continue 140 keystr = str(inDict[key]) 141 akey=('subargs' if key=='args' else key) # what to assign 142 if akey not in inDict: inDict[akey]='' 143 varlist=[] 144 for varset in re.split('-(?=[a-zA-Z])',keystr): 145 if not varset.strip(): continue 146 if '{{' in varset: 147 keyvar,lvars,ftype=parseLoopArgs(varset) 148 if akey not in loopVars: loopVars[akey]={} 149 varlist.append(keyvar) 150 loopVars[akey][keyvar]=[keyvar,lvars] 151 if akey=='nsize': 152 inDict[akey] = '${' + keyvar + '}' 153 lsuffix+=akey+'-'+inDict[akey]+'_' 154 else: 155 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 156 lsuffix+=keyvar+'-${' + keyvar + '}_' 157 else: 158 if key=='args': newargs+=" -"+varset.strip() 159 if len(varlist)>0: loopVars[akey]['varlist']=varlist 160 161 162 # For subtests, args are always substituted in (not top level) 163 if isSubtest: 164 inDict['subargs']+=" "+newargs.strip() 165 inDict['args']='' 166 if 'label_suffix' in inDict: 167 inDict['label_suffix']+=lsuffix.rstrip('_') 168 else: 169 inDict['label_suffix']=lsuffix.rstrip('_') 170 else: 171 if len(loopVars.keys())>0: 172 inDict['args']=newargs.strip() 173 inDict['label_suffix']=lsuffix.rstrip('_') 174 if len(loopVars.keys())>0: 175 return loopVars 176 else: 177 return None 178 179 def getArgLabel(self,testDict): 180 """ 181 In all of the arguments in the test dictionary, create a simple 182 string for searching within the makefile system. For simplicity in 183 search, remove "-", for strings, etc. 184 Also, concatenate the arg commands 185 For now, ignore nsize -- seems hard to search for anyway 186 """ 187 # Collect all of the args associated with a test 188 argStr=("" if 'args' not in testDict else testDict['args']) 189 if 'subtests' in testDict: 190 for stest in testDict["subtests"]: 191 sd=testDict[stest] 192 argStr=argStr+("" if 'args' not in sd else sd['args']) 193 194 # Now go through and cleanup 195 argStr=re.sub('{{(.*?)}}',"",argStr) 196 argStr=re.sub('-'," ",argStr) 197 for digit in string.digits: argStr=re.sub(digit," ",argStr) 198 argStr=re.sub("\.","",argStr) 199 argStr=re.sub(",","",argStr) 200 argStr=re.sub('\+',' ',argStr) 201 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 202 return argStr.strip() 203 204 def addToSources(self,exfile,root,srcDict): 205 """ 206 Put into data structure that allows easy generation of makefile 207 """ 208 rpath=os.path.join('src',os.path.relpath(root,self.srcdir)) 209 pkg=rpath.split("/")[1] 210 fullfile=os.path.join(root,exfile) 211 relpfile=os.path.join(rpath,exfile) 212 lang=self.getLanguage(exfile) 213 if not lang: return 214 self.sources[pkg][lang]['srcs'].append(relpfile) 215 if 'depends' in srcDict: 216 depSrc=srcDict['depends'] 217 depObj=os.path.splitext(depSrc)[0]+".o" 218 self.sources[pkg][lang][exfile]=depObj 219 220 # In gmakefile, ${TESTDIR} var specifies the object compilation 221 testsdir=self.relpath(self.petsc_dir,root)+"/" 222 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 223 self.objects[pkg].append(objfile) 224 return 225 226 def addToTests(self,test,root,exfile,execname,testDict): 227 """ 228 Put into data structure that allows easy generation of makefile 229 Organized by languages to allow testing of languages 230 """ 231 rpath=os.path.join('src',os.path.relpath(root,self.srcdir)) 232 pkg=rpath.split("/")[1] 233 #nmtest=self.nameSpace(test,root) 234 nmtest=os.path.join(rpath,test) 235 lang=self.getLanguage(exfile) 236 if not lang: return 237 self.tests[pkg][lang][nmtest]={} 238 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 239 self.tests[pkg][lang][nmtest]['exec']=execname 240 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 241 return 242 243 def getExecname(self,exfile,root): 244 """ 245 Generate bash script using template found next to this file. 246 This file is read in at constructor time to avoid file I/O 247 """ 248 rpath=os.path.join('src',os.path.relpath(root,self.srcdir)) 249 if self.single_ex: 250 execname=rpath.split("/")[1]+"-ex" 251 else: 252 execname=os.path.splitext(exfile)[0] 253 return execname 254 255 def getSubstVars(self,testDict,rpath,testname): 256 """ 257 Create a dictionary with all of the variables that get substituted 258 into the template commands found in example_template.py 259 """ 260 subst={} 261 262 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 263 if 'nsize' not in testDict: testDict['nsize']=1 264 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 265 for ak in testparse.acceptedkeys: 266 if ak=='test': continue 267 subst[ak]=(testDict[ak] if ak in testDict else '') 268 269 # Now do other variables 270 subst['execname']=testDict['execname'] 271 if 'filter' in testDict: 272 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 273 274 # Others 275 subst['subargs']='' # Default. For variables override 276 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 277 subst['label_suffix']='' 278 subst['comments']="\n#".join(subst['comments'].split("\n")) 279 if subst['comments']: subst['comments']="#"+subst['comments'] 280 subst['exec']="../"+subst['execname'] 281 subst['testroot']=self.testroot_dir 282 subst['testname']=testname 283 dp = self.conf.get('DATAFILESPATH','') 284 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 285 286 # This is used to label some matrices 287 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 288 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 289 290 # These can have for loops and are treated separately later 291 subst['nsize']=str(subst['nsize']) 292 293 #Conf vars 294 if self.petsc_arch.find('valgrind')>=0: 295 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 296 else: 297 subst['mpiexec']=self.conf['MPIEXEC'] 298 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 299 subst['petsc_arch']=self.petsc_arch 300 if not self.inInstallDir: 301 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 302 else: 303 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 304 subst['diff']=self.conf['DIFF'] 305 subst['rm']=self.conf['RM'] 306 subst['grep']=self.conf['GREP'] 307 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 308 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 309 310 # Output file is special because of subtests override 311 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 312 if not "_" in defroot: defroot=defroot+"_1" 313 subst['defroot']=defroot 314 subst['label']=self.nameSpace(defroot,subst['srcdir']) 315 subst['redirect_file']=defroot+".tmp" 316 if 'output_file' not in testDict: 317 subst['output_file']="output/"+defroot+".out" 318 # Add in the full path here. 319 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 320 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 321 if not subst['TODO']: 322 print "Warning: "+subst['output_file']+" not found." 323 # Worry about alt files here -- see 324 # src/snes/examples/tutorials/output/ex22*.out 325 altlist=[subst['output_file']] 326 for i in range(1,3): 327 altroot=defroot+"_alt" 328 if i==2: altroot=altroot+"_2" 329 af="output/"+altroot+".out" 330 srcaf=os.path.join(subst['srcdir'],af) 331 fullaf=os.path.join(self.petsc_dir,srcaf) 332 if os.path.isfile(fullaf): altlist.append(srcaf) 333 if len(altlist)>1: subst['altfiles']=altlist 334 #if len(altlist)>1: print "Found alt files: ",altlist 335 336 return subst 337 338 def getCmds(self,subst,i): 339 """ 340 Generate bash script using template found next to this file. 341 This file is read in at constructor time to avoid file I/O 342 """ 343 nindnt=i # the start and has to be consistent with below 344 cmdindnt=self.indent*nindnt 345 cmdLines="" 346 347 # MPI is the default -- but we have a few odd commands 348 if not subst['command']: 349 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 350 else: 351 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 352 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 353 354 cmdLines+=cmdindnt+'if test $res = 0; then\n' 355 diffindnt=self.indent*(nindnt+1) 356 if not subst['filter_output']: 357 if 'altfiles' not in subst: 358 cmd=diffindnt+self._substVars(subst,example_template.difftest) 359 else: 360 # Have to do it by hand a bit because of variable number of alt files 361 rf=subst['redirect_file'] 362 cmd=diffindnt+example_template.difftest.split('@')[0] 363 for i in range(len(subst['altfiles'])): 364 af=subst['altfiles'][i] 365 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 366 if i!=len(subst['altfiles'])-1: 367 cmd+=' || ${diff_exe} ' 368 else: 369 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 370 cmd+=subst['label_suffix']+' ""' # Quotes are painful 371 else: 372 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 373 cmdLines+=cmd+"\n" 374 cmdLines+=cmdindnt+'else\n' 375 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 376 cmdLines+=cmdindnt+'fi\n' 377 return cmdLines 378 379 def _substVars(self,subst,origStr): 380 """ 381 Substitute variables 382 """ 383 Str=origStr 384 for subkey in subst: 385 if type(subst[subkey])!=types.StringType: continue 386 patt="@"+subkey.upper()+"@" 387 Str=re.sub(patt,subst[subkey],Str) 388 return Str 389 390 def _writeTodoSkip(self,fh,tors,reasons,footer): 391 """ 392 Write out the TODO and SKIP lines in the file 393 The TODO or SKIP variable, tors, should be lower case 394 """ 395 TORS=tors.upper() 396 template=eval("example_template."+tors+"line") 397 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 398 tab = '' 399 if reasons: 400 fh.write('if ! $force; then\n') 401 tab = tab + ' ' 402 if reasons == ["Requires DATAFILESPATH"]: 403 # The only reason not to run is DATAFILESPATH, which we check at run-time 404 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 405 tab = tab + ' ' 406 if reasons: 407 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 408 fh.write(tab+footer+"\n") 409 fh.write(tab+"exit\n") 410 if reasons == ["Requires DATAFILESPATH"]: 411 fh.write(' fi\n') 412 if reasons: 413 fh.write('fi\n') 414 fh.write('\n\n') 415 return 416 417 def getLoopVarsHead(self,loopVars,i): 418 """ 419 Generate a nicely indented string with the format loops 420 Here is what the data structure looks like 421 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 422 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 423 loopVars['subargs']['pc_type']=["j","cholesky sor"] 424 """ 425 outstr=''; indnt=self.indent 426 for key in loopVars: 427 for var in loopVars[key]['varlist']: 428 varval=loopVars[key][var] 429 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 430 i = i + 1 431 return (outstr,i) 432 433 def getLoopVarsFoot(self,loopVars,i): 434 outstr=''; indnt=self.indent 435 for key in loopVars: 436 for var in loopVars[key]['varlist']: 437 i = i - 1 438 outstr += indnt * i + "done\n" 439 return (outstr,i) 440 441 def genRunScript(self,testname,root,isRun,srcDict): 442 """ 443 Generate bash script using template found next to this file. 444 This file is read in at constructor time to avoid file I/O 445 """ 446 # runscript_dir directory has to be consistent with gmakefile 447 testDict=srcDict[testname] 448 rpath=os.path.join('src',os.path.relpath(root,self.srcdir)) 449 runscript_dir=os.path.join(self.testroot_dir,rpath) 450 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 451 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 452 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 453 454 # Get variables to go into shell scripts. last time testDict used 455 subst=self.getSubstVars(testDict,rpath,testname) 456 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 457 #if '33_' in testname: print subst['subargs'] 458 459 #Handle runfiles 460 for lfile in subst.get('localrunfiles','').split(): 461 fullfile=os.path.join(root,lfile) 462 shutil.copy(fullfile,runscript_dir) 463 # Check subtests for local runfiles 464 for stest in subst.get("subtests",[]): 465 for lfile in testDict[stest].get('localrunfiles','').split(): 466 fullfile=os.path.join(root,lfile) 467 shutil.copy(fullfile,self.runscript_dir) 468 469 # Now substitute the key variables into the header and footer 470 header=self._substVars(subst,example_template.header) 471 # The header is done twice to enable @...@ in header 472 header=self._substVars(subst,header) 473 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 474 475 # Start writing the file 476 fh.write(header+"\n") 477 478 # If there is a TODO or a SKIP then we do it before writing out the 479 # rest of the command (which is useful for working on the test) 480 # SKIP and TODO can be for the source file or for the runs 481 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 482 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 483 484 j=0 # for indentation 485 486 if loopVars: 487 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 488 if (loopHead): fh.write(loopHead+"\n") 489 490 # Subtests are special 491 if 'subtests' in testDict: 492 substP=subst # Subtests can inherit args but be careful 493 for stest in testDict["subtests"]: 494 subst=substP.copy() 495 subst.update(testDict[stest]) 496 subst['nsize']=str(subst['nsize']) 497 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 498 #if '10_9' in testname: print sLoopVars 499 if sLoopVars: 500 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 501 fh.write(sLoopHead+"\n") 502 fh.write(self.getCmds(subst,j)+"\n") 503 if sLoopVars: 504 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 505 fh.write(sLoopFoot+"\n") 506 else: 507 fh.write(self.getCmds(subst,j)+"\n") 508 509 if loopVars: 510 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 511 fh.write(loopFoot+"\n") 512 513 fh.write(footer+"\n") 514 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 515 #if '10_9' in testname: sys.exit() 516 return 517 518 def genScriptsAndInfo(self,exfile,root,srcDict): 519 """ 520 Generate scripts from the source file, determine if built, etc. 521 For every test in the exfile with info in the srcDict: 522 1. Determine if it needs to be run for this arch 523 2. Generate the script 524 3. Generate the data needed to write out the makefile in a 525 convenient way 526 All tests are *always* run, but some may be SKIP'd per the TAP standard 527 """ 528 debug=False 529 execname=self.getExecname(exfile,root) 530 isBuilt=self._isBuilt(exfile,srcDict) 531 for test in srcDict: 532 if test in self.buildkeys: continue 533 if debug: print self.nameSpace(exfile,root), test 534 srcDict[test]['execname']=execname # Convenience in generating scripts 535 isRun=self._isRun(srcDict[test]) 536 self.genRunScript(test,root,isRun,srcDict) 537 srcDict[test]['isrun']=isRun 538 self.addToTests(test,root,exfile,execname,srcDict[test]) 539 540 # This adds to datastructure for building deps 541 if isBuilt: self.addToSources(exfile,root,srcDict) 542 return 543 544 def _isBuilt(self,exfile,srcDict): 545 """ 546 Determine if this file should be built. 547 """ 548 # Get the language based on file extension 549 srcDict['SKIP'] = [] 550 lang=self.getLanguage(exfile) 551 if (lang=="F" or lang=="F90"): 552 if not self.have_fortran: 553 srcDict["SKIP"].append("Fortran required for this test") 554 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 555 srcDict["SKIP"].append("Fortran f90freeform required for this test") 556 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 557 srcDict["SKIP"].append("CUDA required for this test") 558 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 559 srcDict["SKIP"].append("C++ required for this test") 560 561 # Deprecated source files 562 if srcDict.get("TODO"): 563 return False 564 565 # isRun can work with srcDict to handle the requires 566 if "requires" in srcDict: 567 if len(srcDict["requires"])>0: 568 return self._isRun(srcDict) 569 570 return srcDict['SKIP'] == [] 571 572 573 def _isRun(self,testDict): 574 """ 575 Based on the requirements listed in the src file and the petscconf.h 576 info, determine whether this test should be run or not. 577 """ 578 indent=" " 579 debug=False 580 581 if 'SKIP' not in testDict: 582 testDict['SKIP'] = [] 583 # MPI requirements 584 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 585 if debug: print indent+"Cannot run parallel tests" 586 testDict['SKIP'].append("Parallel test with serial build") 587 588 # The requirements for the test are the sum of all the run subtests 589 if 'subtests' in testDict: 590 if 'requires' not in testDict: testDict['requires']="" 591 for stest in testDict['subtests']: 592 if 'requires' in testDict[stest]: 593 testDict['requires']+=" "+testDict[stest]['requires'] 594 595 596 # Now go through all requirements 597 if 'requires' in testDict: 598 for requirement in testDict['requires'].split(): 599 requirement=requirement.strip() 600 if not requirement: continue 601 if debug: print indent+"Requirement: ", requirement 602 isNull=False 603 if requirement.startswith("!"): 604 requirement=requirement[1:]; isNull=True 605 # Precision requirement for reals 606 if requirement in self.precision_types: 607 if self.conf['PETSC_PRECISION']==requirement: 608 if isNull: 609 testDict['SKIP'].append("not "+requirement+" required") 610 continue 611 continue # Success 612 elif not isNull: 613 testDict['SKIP'].append(requirement+" required") 614 continue 615 # Precision requirement for ints 616 if requirement in self.integer_types: 617 if requirement=="int32": 618 if self.conf['PETSC_SIZEOF_INT']==4: 619 if isNull: 620 testDict['SKIP'].append("not int32 required") 621 continue 622 continue # Success 623 elif not isNull: 624 testDict['SKIP'].append("int32 required") 625 continue 626 if requirement=="int64": 627 if self.conf['PETSC_SIZEOF_INT']==8: 628 if isNull: 629 testDict['SKIP'].append("NOT int64 required") 630 continue 631 continue # Success 632 elif not isNull: 633 testDict['SKIP'].append("int64 required") 634 continue 635 # Datafilespath 636 if requirement=="datafilespath" and not isNull: 637 testDict['SKIP'].append("Requires DATAFILESPATH") 638 continue 639 # Defines -- not sure I have comments matching 640 if "define(" in requirement.lower(): 641 reqdef=requirement.split("(")[1].split(")")[0] 642 if reqdef in self.conf: 643 if isNull: 644 testDict['SKIP'].append("Null requirement not met: "+requirement) 645 continue 646 continue # Success 647 elif not isNull: 648 testDict['SKIP'].append("Required: "+requirement) 649 continue 650 651 # Rest should be packages that we can just get from conf 652 if requirement == "complex": 653 petscconfvar="PETSC_USE_COMPLEX" 654 else: 655 petscconfvar="PETSC_HAVE_"+requirement.upper() 656 if self.conf.get(petscconfvar): 657 if isNull: 658 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 659 continue 660 continue # Success 661 elif not isNull: 662 if debug: print "requirement not found: ", requirement 663 testDict['SKIP'].append(petscconfvar+" requirement not met") 664 continue 665 666 return testDict['SKIP'] == [] 667 668 def genPetscTests_summarize(self,dataDict): 669 """ 670 Required method to state what happened 671 """ 672 if not self.summarize: return 673 indent=" " 674 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 675 fh=open(fhname,"w") 676 #print "See ", fhname 677 for root in dataDict: 678 relroot=os.path.join('src',os.path.relpath(root,self.srcdir)) 679 pkg=relroot.split("/")[1] 680 fh.write(relroot+"\n") 681 allSrcs=[] 682 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 683 for exfile in dataDict[root]: 684 # Basic information 685 fullfile=os.path.join(root,exfile) 686 rfile=os.path.join('src',os.path.relpath(root,self.srcdir)) 687 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 688 fh.write(indent+exfile+indent*4+builtStatus+"\n") 689 690 for test in dataDict[root][exfile]: 691 if test in self.buildkeys: continue 692 line=indent*2+test 693 fh.write(line+"\n") 694 # Looks nice to have the keys in order 695 #for key in dataDict[root][exfile][test]: 696 for key in "isrun abstracted nsize args requires script".split(): 697 if key not in dataDict[root][exfile][test]: continue 698 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 699 fh.write(line+"\n") 700 fh.write("\n") 701 fh.write("\n") 702 fh.write("\n") 703 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 704 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 705 fh.close() 706 return 707 708 def genPetscTests(self,root,dirs,files,dataDict): 709 """ 710 Go through and parse the source files in the directory to generate 711 the examples based on the metadata contained in the source files 712 """ 713 debug=False 714 # Use examplesAnalyze to get what the makefles think are sources 715 #self.examplesAnalyze(root,dirs,files,anlzDict) 716 717 dataDict[root]={} 718 719 for exfile in files: 720 #TST: Until we replace files, still leaving the orginals as is 721 #if not exfile.startswith("new_"+"ex"): continue 722 #if not exfile.startswith("ex"): continue 723 724 # Convenience 725 fullex=os.path.join(root,exfile) 726 relpfile=os.path.join('src',os.path.relpath(fullex,self.srcdir)) 727 if debug: print relpfile 728 dataDict[root].update(testparse.parseTestFile(fullex,0)) 729 # Need to check and make sure tests are in the file 730 # if verbosity>=1: print relpfile 731 if exfile in dataDict[root]: 732 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 733 734 return 735 736 def walktree(self,top,action="printFiles"): 737 """ 738 Walk a directory tree, starting from 'top' 739 """ 740 #print "action", action 741 # Goal of action is to fill this dictionary 742 dataDict={} 743 for root, dirs, files in os.walk(top, topdown=False): 744 if not "examples" in root: continue 745 if not os.path.isfile(os.path.join(root,"makefile")): continue 746 bname=os.path.basename(root.rstrip("/")) 747 if bname=="tests" or bname=="tutorials": 748 eval("self."+action+"(root,dirs,files,dataDict)") 749 if type(top) != types.StringType: 750 raise TypeError("top must be a string") 751 # Now summarize this dictionary 752 eval("self."+action+"_summarize(dataDict)") 753 return dataDict 754 755 def gen_gnumake(self, fd): 756 """ 757 Overwrite of the method in the base PETSc class 758 """ 759 def write(stem, srcs): 760 for lang in LANGS: 761 if self.inInstallDir: 762 if len(srcs[lang]['srcs'])>0: 763 fd.write('%(stem)s.%(lang)s := $(EXAMPLESDIR)/%(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' $(EXAMPLESDIR)/'.join(srcs[lang]['srcs']))) 764 else: 765 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 766 else: 767 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 768 for pkg in PKGS: 769 srcs = self.gen_pkg(pkg) 770 write('testsrcs-' + pkg, srcs) 771 return self.gendeps 772 773 def gen_pkg(self, pkg): 774 """ 775 Overwrite of the method in the base PETSc class 776 """ 777 return self.sources[pkg] 778 779 def write_gnumake(self,dataDict): 780 """ 781 Write out something similar to files from gmakegen.py 782 783 Test depends on script which also depends on source 784 file, but since I don't have a good way generating 785 acting on a single file (oops) just depend on 786 executable which in turn will depend on src file 787 """ 788 # Different options for how to set up the targets 789 compileExecsFirst=False 790 791 # Open file 792 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 793 fd = open(arch_files, 'w') 794 795 # Write out the sources 796 gendeps = self.gen_gnumake(fd) 797 798 # Write out the tests and execname targets 799 fd.write("\n#Tests and executables\n") # Delimiter 800 801 for pkg in PKGS: 802 # These grab the ones that are built 803 for lang in LANGS: 804 testdeps=[] 805 for ftest in self.tests[pkg][lang]: 806 test=os.path.basename(ftest) 807 basedir=os.path.dirname(ftest) 808 testdeps.append(self.nameSpace(test,basedir)) 809 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 810 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 811 812 # test targets 813 for ftest in self.tests[pkg][lang]: 814 test=os.path.basename(ftest) 815 basedir=os.path.dirname(ftest) 816 testdir="${TESTDIR}/"+basedir+"/" 817 nmtest=self.nameSpace(test,basedir) 818 rundir=os.path.join(testdir,test) 819 #print test, nmtest 820 script=test+".sh" 821 822 # Deps 823 exfile=self.tests[pkg][lang][ftest]['exfile'] 824 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 825 localexec=self.tests[pkg][lang][ftest]['exec'] 826 execname=os.path.join(testdir,localexec) 827 fullscript=os.path.join(testdir,script) 828 tmpfile=os.path.join(testdir,test,test+".tmp") 829 830 # *.counts depends on the script and either executable (will 831 # be run) or the example source file (SKIP or TODO) 832 fd.write('%s.counts : %s %s\n' 833 % (os.path.join('$(TESTDIR)/counts', nmtest), 834 fullscript, 835 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 836 # Now write the args: 837 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 838 839 fd.close() 840 return 841 842 def writeHarness(self,output,dataDict): 843 """ 844 This is set up to write out multiple harness even if only gnumake 845 is supported now 846 """ 847 eval("self.write_"+output+"(dataDict)") 848 return 849 850def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 851 if output is None: 852 output = 'gnumake' 853 854 855 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 856 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 857 testdir=testdir) 858 dataDict=pEx.walktree(os.path.join(pEx.srcdir),action="genPetscTests") 859 pEx.writeHarness(output,dataDict) 860 861if __name__ == '__main__': 862 import optparse 863 parser = optparse.OptionParser() 864 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 865 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 866 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 867 parser.add_option('--output', help='Location to write output file', default=None) 868 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 869 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 870 opts, extra_args = parser.parse_args() 871 opts, extra_args = parser.parse_args() 872 if extra_args: 873 import sys 874 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 875 exit(1) 876 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 877 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 878