1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 # Whether to write out a useful debugging 53 #if verbose: self.summarize=True 54 self.summarize=True 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 return 82 83 def srcrelpath(self,rdir): 84 """ 85 Get relative path to source directory 86 """ 87 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 88 89 def getInInstallDir(self,thisscriptdir): 90 """ 91 When petsc is installed then this file in installed in: 92 <PREFIX>/share/petsc/examples/config/gmakegentest.py 93 otherwise the path is: 94 <PETSC_DIR>/config/gmakegentest.py 95 We use this difference to determine if we are in installdir 96 """ 97 dirlist=thisscriptdir.split(os.path.sep) 98 if len(dirlist)>4: 99 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 100 if lastfour==os.path.join('share','petsc','examples','config'): 101 return True 102 else: 103 return False 104 else: 105 return False 106 107 def nameSpace(self,srcfile,srcdir): 108 """ 109 Because the scripts have a non-unique naming, the pretty-printing 110 needs to convey the srcdir and srcfile. There are two ways of doing this. 111 """ 112 if self.ptNaming: 113 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 114 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 115 nameString=prefix+srcfile 116 else: 117 #nameString=srcdir+": "+srcfile 118 nameString=srcfile 119 return nameString 120 121 def getLanguage(self,srcfile): 122 """ 123 Based on the source, determine associated language as found in gmakegen.LANGS 124 Can we just return srcext[1:\] now? 125 """ 126 langReq=None 127 srcext=os.path.splitext(srcfile)[-1] 128 if srcext in ".F90".split(): langReq="F90" 129 if srcext in ".F".split(): langReq="F" 130 if srcext in ".cxx".split(): langReq="cxx" 131 if srcext == ".cu": langReq="cu" 132 if srcext == ".c": langReq="c" 133 #if not langReq: print "ERROR: ", srcext, srcfile 134 return langReq 135 136 def _getLoopVars(self,inDict,testname, isSubtest=False): 137 """ 138 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 139 Return: 140 inDict['args']: -ksp_monitor 141 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 142 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 143 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 144 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 145 subst should be passed in instead of inDict 146 """ 147 loopVars={}; newargs="" 148 lkeys=inDict.keys() 149 lsuffix='_' 150 from testparse import parseLoopArgs 151 for key in lkeys: 152 if type(inDict[key])!=types.StringType: continue 153 keystr = str(inDict[key]) 154 akey=('subargs' if key=='args' else key) # what to assign 155 if akey not in inDict: inDict[akey]='' 156 varlist=[] 157 for varset in re.split('-(?=[a-zA-Z])',keystr): 158 if not varset.strip(): continue 159 if '{{' in varset: 160 keyvar,lvars,ftype=parseLoopArgs(varset) 161 if akey not in loopVars: loopVars[akey]={} 162 varlist.append(keyvar) 163 loopVars[akey][keyvar]=[keyvar,lvars] 164 if akey=='nsize': 165 inDict[akey] = '${' + keyvar + '}' 166 lsuffix+=akey+'-'+inDict[akey]+'_' 167 else: 168 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 169 lsuffix+=keyvar+'-${' + keyvar + '}_' 170 else: 171 if key=='args': newargs+=" -"+varset.strip() 172 if len(varlist)>0: loopVars[akey]['varlist']=varlist 173 174 175 # For subtests, args are always substituted in (not top level) 176 if isSubtest: 177 inDict['subargs']+=" "+newargs.strip() 178 inDict['args']='' 179 if 'label_suffix' in inDict: 180 inDict['label_suffix']+=lsuffix.rstrip('_') 181 else: 182 inDict['label_suffix']=lsuffix.rstrip('_') 183 else: 184 if len(loopVars.keys())>0: 185 inDict['args']=newargs.strip() 186 inDict['label_suffix']=lsuffix.rstrip('_') 187 if len(loopVars.keys())>0: 188 return loopVars 189 else: 190 return None 191 192 def getArgLabel(self,testDict): 193 """ 194 In all of the arguments in the test dictionary, create a simple 195 string for searching within the makefile system. For simplicity in 196 search, remove "-", for strings, etc. 197 Also, concatenate the arg commands 198 For now, ignore nsize -- seems hard to search for anyway 199 """ 200 # Collect all of the args associated with a test 201 argStr=("" if 'args' not in testDict else testDict['args']) 202 if 'subtests' in testDict: 203 for stest in testDict["subtests"]: 204 sd=testDict[stest] 205 argStr=argStr+("" if 'args' not in sd else sd['args']) 206 207 # Now go through and cleanup 208 argStr=re.sub('{{(.*?)}}',"",argStr) 209 argStr=re.sub('-'," ",argStr) 210 for digit in string.digits: argStr=re.sub(digit," ",argStr) 211 argStr=re.sub("\.","",argStr) 212 argStr=re.sub(",","",argStr) 213 argStr=re.sub('\+',' ',argStr) 214 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 215 return argStr.strip() 216 217 def addToSources(self,exfile,root,srcDict): 218 """ 219 Put into data structure that allows easy generation of makefile 220 """ 221 rpath=self.srcrelpath(root) 222 pkg=rpath.split(os.path.sep)[1] 223 relpfile=os.path.join(rpath,exfile) 224 lang=self.getLanguage(exfile) 225 if not lang: return 226 self.sources[pkg][lang]['srcs'].append(relpfile) 227 if 'depends' in srcDict: 228 depSrc=srcDict['depends'] 229 depObj=os.path.splitext(depSrc)[0]+".o" 230 self.sources[pkg][lang][exfile]=depObj 231 232 # In gmakefile, ${TESTDIR} var specifies the object compilation 233 testsdir=self.srcrelpath(root)+"/" 234 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 235 self.objects[pkg].append(objfile) 236 return 237 238 def addToTests(self,test,root,exfile,execname,testDict): 239 """ 240 Put into data structure that allows easy generation of makefile 241 Organized by languages to allow testing of languages 242 """ 243 rpath=self.srcrelpath(root) 244 pkg=rpath.split("/")[1] 245 #nmtest=self.nameSpace(test,root) 246 nmtest=os.path.join(rpath,test) 247 lang=self.getLanguage(exfile) 248 if not lang: return 249 self.tests[pkg][lang][nmtest]={} 250 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 251 self.tests[pkg][lang][nmtest]['exec']=execname 252 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 253 return 254 255 def getExecname(self,exfile,root): 256 """ 257 Generate bash script using template found next to this file. 258 This file is read in at constructor time to avoid file I/O 259 """ 260 rpath=self.srcrelpath(root) 261 if self.single_ex: 262 execname=rpath.split("/")[1]+"-ex" 263 else: 264 execname=os.path.splitext(exfile)[0] 265 return execname 266 267 def getSubstVars(self,testDict,rpath,testname): 268 """ 269 Create a dictionary with all of the variables that get substituted 270 into the template commands found in example_template.py 271 """ 272 subst={} 273 274 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 275 if 'nsize' not in testDict: testDict['nsize']=1 276 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 277 for ak in testparse.acceptedkeys: 278 if ak=='test': continue 279 subst[ak]=(testDict[ak] if ak in testDict else '') 280 281 # Now do other variables 282 subst['execname']=testDict['execname'] 283 if 'filter' in testDict: 284 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 285 286 # Others 287 subst['subargs']='' # Default. For variables override 288 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 289 subst['label_suffix']='' 290 subst['comments']="\n#".join(subst['comments'].split("\n")) 291 if subst['comments']: subst['comments']="#"+subst['comments'] 292 subst['exec']="../"+subst['execname'] 293 subst['testroot']=self.testroot_dir 294 subst['testname']=testname 295 dp = self.conf.get('DATAFILESPATH','') 296 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 297 298 # This is used to label some matrices 299 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 300 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 301 302 # These can have for loops and are treated separately later 303 subst['nsize']=str(subst['nsize']) 304 305 #Conf vars 306 if self.petsc_arch.find('valgrind')>=0: 307 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 308 else: 309 subst['mpiexec']=self.conf['MPIEXEC'] 310 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 311 subst['petsc_arch']=self.petsc_arch 312 if not self.inInstallDir: 313 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 314 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 315 else: 316 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 317 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 318 subst['diff']=self.conf['DIFF'] 319 subst['rm']=self.conf['RM'] 320 subst['grep']=self.conf['GREP'] 321 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 322 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 323 324 # Output file is special because of subtests override 325 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 326 if not "_" in defroot: defroot=defroot+"_1" 327 subst['defroot']=defroot 328 subst['label']=self.nameSpace(defroot,subst['srcdir']) 329 subst['redirect_file']=defroot+".tmp" 330 if 'output_file' not in testDict: 331 subst['output_file']="output/"+defroot+".out" 332 # Add in the full path here. 333 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 334 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 335 if not subst['TODO']: 336 print "Warning: "+subst['output_file']+" not found." 337 # Worry about alt files here -- see 338 # src/snes/examples/tutorials/output/ex22*.out 339 altlist=[subst['output_file']] 340 for i in range(1,3): 341 altroot=defroot+"_alt" 342 if i==2: altroot=altroot+"_2" 343 af="output/"+altroot+".out" 344 srcaf=os.path.join(subst['srcdir'],af) 345 fullaf=os.path.join(self.petsc_dir,srcaf) 346 if os.path.isfile(fullaf): altlist.append(srcaf) 347 if len(altlist)>1: subst['altfiles']=altlist 348 #if len(altlist)>1: print "Found alt files: ",altlist 349 350 return subst 351 352 def getCmds(self,subst,i): 353 """ 354 Generate bash script using template found next to this file. 355 This file is read in at constructor time to avoid file I/O 356 """ 357 nindnt=i # the start and has to be consistent with below 358 cmdindnt=self.indent*nindnt 359 cmdLines="" 360 361 # MPI is the default -- but we have a few odd commands 362 if not subst['command']: 363 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 364 else: 365 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 366 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 367 368 cmdLines+=cmdindnt+'if test $res = 0; then\n' 369 diffindnt=self.indent*(nindnt+1) 370 if not subst['filter_output']: 371 if 'altfiles' not in subst: 372 cmd=diffindnt+self._substVars(subst,example_template.difftest) 373 else: 374 # Have to do it by hand a bit because of variable number of alt files 375 rf=subst['redirect_file'] 376 cmd=diffindnt+example_template.difftest.split('@')[0] 377 for i in range(len(subst['altfiles'])): 378 af=subst['altfiles'][i] 379 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 380 if i!=len(subst['altfiles'])-1: 381 cmd+=' || ${diff_exe} ' 382 else: 383 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 384 cmd+=subst['label_suffix']+' ""' # Quotes are painful 385 else: 386 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 387 cmdLines+=cmd+"\n" 388 cmdLines+=cmdindnt+'else\n' 389 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 390 cmdLines+=cmdindnt+'fi\n' 391 return cmdLines 392 393 def _substVars(self,subst,origStr): 394 """ 395 Substitute variables 396 """ 397 Str=origStr 398 for subkey in subst: 399 if type(subst[subkey])!=types.StringType: continue 400 patt="@"+subkey.upper()+"@" 401 Str=re.sub(patt,subst[subkey],Str) 402 return Str 403 404 def _writeTodoSkip(self,fh,tors,reasons,footer): 405 """ 406 Write out the TODO and SKIP lines in the file 407 The TODO or SKIP variable, tors, should be lower case 408 """ 409 TORS=tors.upper() 410 template=eval("example_template."+tors+"line") 411 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 412 tab = '' 413 if reasons: 414 fh.write('if ! $force; then\n') 415 tab = tab + ' ' 416 if reasons == ["Requires DATAFILESPATH"]: 417 # The only reason not to run is DATAFILESPATH, which we check at run-time 418 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 419 tab = tab + ' ' 420 if reasons: 421 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 422 fh.write(tab+footer+"\n") 423 fh.write(tab+"exit\n") 424 if reasons == ["Requires DATAFILESPATH"]: 425 fh.write(' fi\n') 426 if reasons: 427 fh.write('fi\n') 428 fh.write('\n\n') 429 return 430 431 def getLoopVarsHead(self,loopVars,i): 432 """ 433 Generate a nicely indented string with the format loops 434 Here is what the data structure looks like 435 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 436 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 437 loopVars['subargs']['pc_type']=["j","cholesky sor"] 438 """ 439 outstr=''; indnt=self.indent 440 for key in loopVars: 441 for var in loopVars[key]['varlist']: 442 varval=loopVars[key][var] 443 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 444 i = i + 1 445 return (outstr,i) 446 447 def getLoopVarsFoot(self,loopVars,i): 448 outstr=''; indnt=self.indent 449 for key in loopVars: 450 for var in loopVars[key]['varlist']: 451 i = i - 1 452 outstr += indnt * i + "done\n" 453 return (outstr,i) 454 455 def genRunScript(self,testname,root,isRun,srcDict): 456 """ 457 Generate bash script using template found next to this file. 458 This file is read in at constructor time to avoid file I/O 459 """ 460 # runscript_dir directory has to be consistent with gmakefile 461 testDict=srcDict[testname] 462 rpath=self.srcrelpath(root) 463 runscript_dir=os.path.join(self.testroot_dir,rpath) 464 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 465 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 466 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 467 468 # Get variables to go into shell scripts. last time testDict used 469 subst=self.getSubstVars(testDict,rpath,testname) 470 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 471 #if '33_' in testname: print subst['subargs'] 472 473 #Handle runfiles 474 for lfile in subst.get('localrunfiles','').split(): 475 fullfile=os.path.join(root,lfile) 476 shutil.copy(fullfile,runscript_dir) 477 # Check subtests for local runfiles 478 for stest in subst.get("subtests",[]): 479 for lfile in testDict[stest].get('localrunfiles','').split(): 480 fullfile=os.path.join(root,lfile) 481 shutil.copy(fullfile,self.runscript_dir) 482 483 # Now substitute the key variables into the header and footer 484 header=self._substVars(subst,example_template.header) 485 # The header is done twice to enable @...@ in header 486 header=self._substVars(subst,header) 487 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 488 489 # Start writing the file 490 fh.write(header+"\n") 491 492 # If there is a TODO or a SKIP then we do it before writing out the 493 # rest of the command (which is useful for working on the test) 494 # SKIP and TODO can be for the source file or for the runs 495 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 496 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 497 498 j=0 # for indentation 499 500 if loopVars: 501 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 502 if (loopHead): fh.write(loopHead+"\n") 503 504 # Subtests are special 505 if 'subtests' in testDict: 506 substP=subst # Subtests can inherit args but be careful 507 k=0 # for label suffixes 508 for stest in testDict["subtests"]: 509 subst=substP.copy() 510 subst.update(testDict[stest]) 511 # nsize is special because it is usually overwritten 512 if 'nsize' in testDict[stest]: 513 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 514 else: 515 fh.write("nsize=1\n") 516 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 517 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 518 #if '10_9' in testname: print sLoopVars 519 if sLoopVars: 520 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 521 fh.write(sLoopHead+"\n") 522 fh.write(self.getCmds(subst,j)+"\n") 523 if sLoopVars: 524 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 525 fh.write(sLoopFoot+"\n") 526 else: 527 fh.write(self.getCmds(subst,j)+"\n") 528 529 if loopVars: 530 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 531 fh.write(loopFoot+"\n") 532 533 fh.write(footer+"\n") 534 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 535 #if '10_9' in testname: sys.exit() 536 return 537 538 def genScriptsAndInfo(self,exfile,root,srcDict): 539 """ 540 Generate scripts from the source file, determine if built, etc. 541 For every test in the exfile with info in the srcDict: 542 1. Determine if it needs to be run for this arch 543 2. Generate the script 544 3. Generate the data needed to write out the makefile in a 545 convenient way 546 All tests are *always* run, but some may be SKIP'd per the TAP standard 547 """ 548 debug=False 549 execname=self.getExecname(exfile,root) 550 isBuilt=self._isBuilt(exfile,srcDict) 551 for test in srcDict: 552 if test in self.buildkeys: continue 553 if debug: print self.nameSpace(exfile,root), test 554 srcDict[test]['execname']=execname # Convenience in generating scripts 555 isRun=self._isRun(srcDict[test]) 556 self.genRunScript(test,root,isRun,srcDict) 557 srcDict[test]['isrun']=isRun 558 self.addToTests(test,root,exfile,execname,srcDict[test]) 559 560 # This adds to datastructure for building deps 561 if isBuilt: self.addToSources(exfile,root,srcDict) 562 return 563 564 def _isBuilt(self,exfile,srcDict): 565 """ 566 Determine if this file should be built. 567 """ 568 # Get the language based on file extension 569 srcDict['SKIP'] = [] 570 lang=self.getLanguage(exfile) 571 if (lang=="F" or lang=="F90"): 572 if not self.have_fortran: 573 srcDict["SKIP"].append("Fortran required for this test") 574 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 575 srcDict["SKIP"].append("Fortran f90freeform required for this test") 576 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 577 srcDict["SKIP"].append("CUDA required for this test") 578 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 579 srcDict["SKIP"].append("C++ required for this test") 580 581 # Deprecated source files 582 if srcDict.get("TODO"): 583 return False 584 585 # isRun can work with srcDict to handle the requires 586 if "requires" in srcDict: 587 if len(srcDict["requires"])>0: 588 return self._isRun(srcDict) 589 590 return srcDict['SKIP'] == [] 591 592 593 def _isRun(self,testDict): 594 """ 595 Based on the requirements listed in the src file and the petscconf.h 596 info, determine whether this test should be run or not. 597 """ 598 indent=" " 599 debug=False 600 601 if 'SKIP' not in testDict: 602 testDict['SKIP'] = [] 603 # MPI requirements 604 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 605 if debug: print indent+"Cannot run parallel tests" 606 testDict['SKIP'].append("Parallel test with serial build") 607 608 # The requirements for the test are the sum of all the run subtests 609 if 'subtests' in testDict: 610 if 'requires' not in testDict: testDict['requires']="" 611 for stest in testDict['subtests']: 612 if 'requires' in testDict[stest]: 613 testDict['requires']+=" "+testDict[stest]['requires'] 614 615 616 # Now go through all requirements 617 if 'requires' in testDict: 618 for requirement in testDict['requires'].split(): 619 requirement=requirement.strip() 620 if not requirement: continue 621 if debug: print indent+"Requirement: ", requirement 622 isNull=False 623 if requirement.startswith("!"): 624 requirement=requirement[1:]; isNull=True 625 # Precision requirement for reals 626 if requirement in self.precision_types: 627 if self.conf['PETSC_PRECISION']==requirement: 628 if isNull: 629 testDict['SKIP'].append("not "+requirement+" required") 630 continue 631 continue # Success 632 elif not isNull: 633 testDict['SKIP'].append(requirement+" required") 634 continue 635 # Precision requirement for ints 636 if requirement in self.integer_types: 637 if requirement=="int32": 638 if self.conf['PETSC_SIZEOF_INT']==4: 639 if isNull: 640 testDict['SKIP'].append("not int32 required") 641 continue 642 continue # Success 643 elif not isNull: 644 testDict['SKIP'].append("int32 required") 645 continue 646 if requirement=="int64": 647 if self.conf['PETSC_SIZEOF_INT']==8: 648 if isNull: 649 testDict['SKIP'].append("NOT int64 required") 650 continue 651 continue # Success 652 elif not isNull: 653 testDict['SKIP'].append("int64 required") 654 continue 655 # Datafilespath 656 if requirement=="datafilespath" and not isNull: 657 testDict['SKIP'].append("Requires DATAFILESPATH") 658 continue 659 # Defines -- not sure I have comments matching 660 if "define(" in requirement.lower(): 661 reqdef=requirement.split("(")[1].split(")")[0] 662 if reqdef in self.conf: 663 if isNull: 664 testDict['SKIP'].append("Null requirement not met: "+requirement) 665 continue 666 continue # Success 667 elif not isNull: 668 testDict['SKIP'].append("Required: "+requirement) 669 continue 670 671 # Rest should be packages that we can just get from conf 672 if requirement == "complex": 673 petscconfvar="PETSC_USE_COMPLEX" 674 else: 675 petscconfvar="PETSC_HAVE_"+requirement.upper() 676 if self.conf.get(petscconfvar): 677 if isNull: 678 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 679 continue 680 continue # Success 681 elif not isNull: 682 if debug: print "requirement not found: ", requirement 683 testDict['SKIP'].append(petscconfvar+" requirement not met") 684 continue 685 686 return testDict['SKIP'] == [] 687 688 def genPetscTests_summarize(self,dataDict): 689 """ 690 Required method to state what happened 691 """ 692 if not self.summarize: return 693 indent=" " 694 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 695 fh=open(fhname,"w") 696 #print "See ", fhname 697 for root in dataDict: 698 relroot=self.srcrelpath(root) 699 pkg=relroot.split("/")[1] 700 fh.write(relroot+"\n") 701 allSrcs=[] 702 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 703 for exfile in dataDict[root]: 704 # Basic information 705 rfile=os.path.join(relroot,exfile) 706 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 707 fh.write(indent+exfile+indent*4+builtStatus+"\n") 708 709 for test in dataDict[root][exfile]: 710 if test in self.buildkeys: continue 711 line=indent*2+test 712 fh.write(line+"\n") 713 # Looks nice to have the keys in order 714 #for key in dataDict[root][exfile][test]: 715 for key in "isrun abstracted nsize args requires script".split(): 716 if key not in dataDict[root][exfile][test]: continue 717 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 718 fh.write(line+"\n") 719 fh.write("\n") 720 fh.write("\n") 721 fh.write("\n") 722 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 723 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 724 fh.close() 725 return 726 727 def genPetscTests(self,root,dirs,files,dataDict): 728 """ 729 Go through and parse the source files in the directory to generate 730 the examples based on the metadata contained in the source files 731 """ 732 debug=False 733 # Use examplesAnalyze to get what the makefles think are sources 734 #self.examplesAnalyze(root,dirs,files,anlzDict) 735 736 dataDict[root]={} 737 738 for exfile in files: 739 #TST: Until we replace files, still leaving the orginals as is 740 #if not exfile.startswith("new_"+"ex"): continue 741 #if not exfile.startswith("ex"): continue 742 743 # Ignore emacs files 744 if exfile.startswith("#") or exfile.startswith(".#"): continue 745 746 # Convenience 747 fullex=os.path.join(root,exfile) 748 relpfile=os.path.join(self.srcrelpath(root),exfile) 749 if debug: print relpfile 750 dataDict[root].update(testparse.parseTestFile(fullex,0)) 751 # Need to check and make sure tests are in the file 752 # if verbosity>=1: print relpfile 753 if exfile in dataDict[root]: 754 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 755 756 return 757 758 def walktree(self,top,action="printFiles"): 759 """ 760 Walk a directory tree, starting from 'top' 761 """ 762 #print "action", action 763 # Goal of action is to fill this dictionary 764 dataDict={} 765 for root, dirs, files in os.walk(top, topdown=False): 766 if not "examples" in root: continue 767 if not os.path.isfile(os.path.join(root,"makefile")): continue 768 bname=os.path.basename(root.rstrip("/")) 769 self.genPetscTests(root,dirs,files,dataDict) 770 # Now summarize this dictionary 771 eval("self."+action+"_summarize(dataDict)") 772 return dataDict 773 774 def gen_gnumake(self, fd): 775 """ 776 Overwrite of the method in the base PETSc class 777 """ 778 def write(stem, srcs): 779 for lang in LANGS: 780 if self.inInstallDir: 781 if len(srcs[lang]['srcs'])>0: 782 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 783 else: 784 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 785 else: 786 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 787 for pkg in PKGS: 788 srcs = self.gen_pkg(pkg) 789 write('testsrcs-' + pkg, srcs) 790 return self.gendeps 791 792 def gen_pkg(self, pkg): 793 """ 794 Overwrite of the method in the base PETSc class 795 """ 796 return self.sources[pkg] 797 798 def write_gnumake(self,dataDict): 799 """ 800 Write out something similar to files from gmakegen.py 801 802 Test depends on script which also depends on source 803 file, but since I don't have a good way generating 804 acting on a single file (oops) just depend on 805 executable which in turn will depend on src file 806 """ 807 # Different options for how to set up the targets 808 compileExecsFirst=False 809 810 # Open file 811 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 812 fd = open(arch_files, 'w') 813 814 # Write out the sources 815 gendeps = self.gen_gnumake(fd) 816 817 # Write out the tests and execname targets 818 fd.write("\n#Tests and executables\n") # Delimiter 819 820 for pkg in PKGS: 821 # These grab the ones that are built 822 for lang in LANGS: 823 testdeps=[] 824 for ftest in self.tests[pkg][lang]: 825 test=os.path.basename(ftest) 826 basedir=os.path.dirname(ftest) 827 testdeps.append(self.nameSpace(test,basedir)) 828 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 829 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 830 831 # test targets 832 for ftest in self.tests[pkg][lang]: 833 test=os.path.basename(ftest) 834 basedir=os.path.dirname(ftest) 835 testdir="${TESTDIR}/"+basedir+"/" 836 nmtest=self.nameSpace(test,basedir) 837 rundir=os.path.join(testdir,test) 838 #print test, nmtest 839 script=test+".sh" 840 841 # Deps 842 exfile=self.tests[pkg][lang][ftest]['exfile'] 843 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 844 localexec=self.tests[pkg][lang][ftest]['exec'] 845 execname=os.path.join(testdir,localexec) 846 fullscript=os.path.join(testdir,script) 847 tmpfile=os.path.join(testdir,test,test+".tmp") 848 849 # *.counts depends on the script and either executable (will 850 # be run) or the example source file (SKIP or TODO) 851 if exfile in self.sources[pkg][lang]: 852 fd.write('%s.counts : %s %s %s\n' 853 % (os.path.join('$(TESTDIR)/counts', nmtest), 854 fullscript, 855 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex, 856 os.path.join('$(TESTDIR)',self.sources[pkg][lang][exfile])) 857 ) 858 else: 859 fd.write('%s.counts : %s %s\n' 860 % (os.path.join('$(TESTDIR)/counts', nmtest), 861 fullscript, 862 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 863 ) 864 865 # Now write the args: 866 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 867 868 fd.close() 869 return 870 871 def writeHarness(self,output,dataDict): 872 """ 873 This is set up to write out multiple harness even if only gnumake 874 is supported now 875 """ 876 eval("self.write_"+output+"(dataDict)") 877 return 878 879def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 880 if output is None: 881 output = 'gnumake' 882 883 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 884 if len(petsc_arch.split(os.path.sep))>1: 885 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 886 887 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 888 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 889 testdir=testdir) 890 dataDict=pEx.walktree(os.path.join(pEx.srcdir),action="genPetscTests") 891 pEx.writeHarness(output,dataDict) 892 893if __name__ == '__main__': 894 import optparse 895 parser = optparse.OptionParser() 896 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 897 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 898 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 899 parser.add_option('--output', help='Location to write output file', default=None) 900 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 901 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 902 opts, extra_args = parser.parse_args() 903 opts, extra_args = parser.parse_args() 904 if extra_args: 905 import sys 906 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 907 exit(1) 908 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 909 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 910