1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 self.verbose=verbose 53 # Whether to write out a useful debugging 54 self.summarize=True if verbose else False 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 if self.verbose: print('Finishing the constructor') 82 return 83 84 def srcrelpath(self,rdir): 85 """ 86 Get relative path to source directory 87 """ 88 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 89 90 def getInInstallDir(self,thisscriptdir): 91 """ 92 When petsc is installed then this file in installed in: 93 <PREFIX>/share/petsc/examples/config/gmakegentest.py 94 otherwise the path is: 95 <PETSC_DIR>/config/gmakegentest.py 96 We use this difference to determine if we are in installdir 97 """ 98 dirlist=thisscriptdir.split(os.path.sep) 99 if len(dirlist)>4: 100 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 101 if lastfour==os.path.join('share','petsc','examples','config'): 102 return True 103 else: 104 return False 105 else: 106 return False 107 108 def nameSpace(self,srcfile,srcdir): 109 """ 110 Because the scripts have a non-unique naming, the pretty-printing 111 needs to convey the srcdir and srcfile. There are two ways of doing this. 112 """ 113 if self.ptNaming: 114 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 115 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 116 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 117 nameString=prefix+srcfile 118 else: 119 #nameString=srcdir+": "+srcfile 120 nameString=srcfile 121 return nameString 122 123 def getLanguage(self,srcfile): 124 """ 125 Based on the source, determine associated language as found in gmakegen.LANGS 126 Can we just return srcext[1:\] now? 127 """ 128 langReq=None 129 srcext=os.path.splitext(srcfile)[-1] 130 if srcext in ".F90".split(): langReq="F90" 131 if srcext in ".F".split(): langReq="F" 132 if srcext in ".cxx".split(): langReq="cxx" 133 if srcext == ".cu": langReq="cu" 134 if srcext == ".c": langReq="c" 135 #if not langReq: print "ERROR: ", srcext, srcfile 136 return langReq 137 138 def _getLoopVars(self,inDict,testname, isSubtest=False): 139 """ 140 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 141 Return: 142 inDict['args']: -ksp_monitor 143 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 144 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 145 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 146 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 147 subst should be passed in instead of inDict 148 """ 149 loopVars={}; newargs="" 150 lkeys=inDict.keys() 151 lsuffix='_' 152 from testparse import parseLoopArgs 153 for key in lkeys: 154 if type(inDict[key])!=types.StringType: continue 155 keystr = str(inDict[key]) 156 akey=('subargs' if key=='args' else key) # what to assign 157 if akey not in inDict: inDict[akey]='' 158 varlist=[] 159 for varset in re.split('-(?=[a-zA-Z])',keystr): 160 if not varset.strip(): continue 161 if '{{' in varset: 162 keyvar,lvars,ftype=parseLoopArgs(varset) 163 if akey not in loopVars: loopVars[akey]={} 164 varlist.append(keyvar) 165 loopVars[akey][keyvar]=[keyvar,lvars] 166 if akey=='nsize': 167 inDict[akey] = '${' + keyvar + '}' 168 lsuffix+=akey+'-'+inDict[akey]+'_' 169 else: 170 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 171 lsuffix+=keyvar+'-${' + keyvar + '}_' 172 else: 173 if key=='args': newargs+=" -"+varset.strip() 174 if varlist: loopVars[akey]['varlist']=varlist 175 176 177 # For subtests, args are always substituted in (not top level) 178 if isSubtest: 179 inDict['subargs']+=" "+newargs.strip() 180 inDict['args']='' 181 if 'label_suffix' in inDict: 182 inDict['label_suffix']+=lsuffix.rstrip('_') 183 else: 184 inDict['label_suffix']=lsuffix.rstrip('_') 185 else: 186 if loopVars.keys(): 187 inDict['args']=newargs.strip() 188 inDict['label_suffix']=lsuffix.rstrip('_') 189 if loopVars.keys(): 190 return loopVars 191 else: 192 return None 193 194 def getArgLabel(self,testDict): 195 """ 196 In all of the arguments in the test dictionary, create a simple 197 string for searching within the makefile system. For simplicity in 198 search, remove "-", for strings, etc. 199 Also, concatenate the arg commands 200 For now, ignore nsize -- seems hard to search for anyway 201 """ 202 # Collect all of the args associated with a test 203 argStr=("" if 'args' not in testDict else testDict['args']) 204 if 'subtests' in testDict: 205 for stest in testDict["subtests"]: 206 sd=testDict[stest] 207 argStr=argStr+("" if 'args' not in sd else sd['args']) 208 209 # Now go through and cleanup 210 argStr=re.sub('{{(.*?)}}',"",argStr) 211 argStr=re.sub('-'," ",argStr) 212 for digit in string.digits: argStr=re.sub(digit," ",argStr) 213 argStr=re.sub("\.","",argStr) 214 argStr=re.sub(",","",argStr) 215 argStr=re.sub('\+',' ',argStr) 216 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 217 return argStr.strip() 218 219 def addToSources(self,exfile,root,srcDict): 220 """ 221 Put into data structure that allows easy generation of makefile 222 """ 223 rpath=self.srcrelpath(root) 224 pkg=rpath.split(os.path.sep)[1] 225 relpfile=os.path.join(rpath,exfile) 226 lang=self.getLanguage(exfile) 227 if not lang: return 228 self.sources[pkg][lang]['srcs'].append(relpfile) 229 self.sources[pkg][lang][relpfile] = [] 230 if 'depends' in srcDict: 231 depSrcList=srcDict['depends'].split() 232 for depSrc in depSrcList: 233 depObj=os.path.splitext(depSrc)[0]+".o" 234 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 235 236 # In gmakefile, ${TESTDIR} var specifies the object compilation 237 testsdir=self.srcrelpath(root)+"/" 238 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 239 self.objects[pkg].append(objfile) 240 return 241 242 def addToTests(self,test,root,exfile,execname,testDict): 243 """ 244 Put into data structure that allows easy generation of makefile 245 Organized by languages to allow testing of languages 246 """ 247 rpath=self.srcrelpath(root) 248 pkg=rpath.split("/")[1] 249 #nmtest=self.nameSpace(test,root) 250 nmtest=os.path.join(rpath,test) 251 lang=self.getLanguage(exfile) 252 if not lang: return 253 self.tests[pkg][lang][nmtest]={} 254 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 255 self.tests[pkg][lang][nmtest]['exec']=execname 256 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 257 return 258 259 def getExecname(self,exfile,root): 260 """ 261 Generate bash script using template found next to this file. 262 This file is read in at constructor time to avoid file I/O 263 """ 264 rpath=self.srcrelpath(root) 265 if self.single_ex: 266 execname=rpath.split("/")[1]+"-ex" 267 else: 268 execname=os.path.splitext(exfile)[0] 269 return execname 270 271 def getSubstVars(self,testDict,rpath,testname): 272 """ 273 Create a dictionary with all of the variables that get substituted 274 into the template commands found in example_template.py 275 """ 276 subst={} 277 278 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 279 if 'nsize' not in testDict: testDict['nsize']=1 280 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 281 for ak in testparse.acceptedkeys: 282 if ak=='test': continue 283 subst[ak]=(testDict[ak] if ak in testDict else '') 284 285 # Now do other variables 286 subst['execname']=testDict['execname'] 287 if 'filter' in testDict: 288 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 289 290 # Others 291 subst['subargs']='' # Default. For variables override 292 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 293 subst['label_suffix']='' 294 subst['comments']="\n#".join(subst['comments'].split("\n")) 295 if subst['comments']: subst['comments']="#"+subst['comments'] 296 subst['exec']="../"+subst['execname'] 297 subst['testroot']=self.testroot_dir 298 subst['testname']=testname 299 dp = self.conf.get('DATAFILESPATH','') 300 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 301 302 # This is used to label some matrices 303 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 304 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 305 306 # These can have for loops and are treated separately later 307 subst['nsize']=str(subst['nsize']) 308 309 #Conf vars 310 if self.petsc_arch.find('valgrind')>=0: 311 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 312 else: 313 subst['mpiexec']=self.conf['MPIEXEC'] 314 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 315 subst['petsc_arch']=self.petsc_arch 316 if not self.inInstallDir: 317 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 318 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 319 else: 320 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 321 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 322 subst['diff']=self.conf['DIFF'] 323 subst['rm']=self.conf['RM'] 324 subst['grep']=self.conf['GREP'] 325 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 326 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 327 328 # Output file is special because of subtests override 329 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 330 if not "_" in defroot: defroot=defroot+"_1" 331 subst['defroot']=defroot 332 subst['label']=self.nameSpace(defroot,subst['srcdir']) 333 subst['redirect_file']=defroot+".tmp" 334 if 'output_file' not in testDict: 335 subst['output_file']="output/"+defroot+".out" 336 # Add in the full path here. 337 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 338 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 339 if not subst['TODO']: 340 print "Warning: "+subst['output_file']+" not found." 341 # Worry about alt files here -- see 342 # src/snes/examples/tutorials/output/ex22*.out 343 altlist=[subst['output_file']] 344 for i in range(1,3): 345 altroot=defroot+"_alt" 346 if i==2: altroot=altroot+"_2" 347 af="output/"+altroot+".out" 348 srcaf=os.path.join(subst['srcdir'],af) 349 fullaf=os.path.join(self.petsc_dir,srcaf) 350 if os.path.isfile(fullaf): altlist.append(srcaf) 351 if len(altlist)>1: subst['altfiles']=altlist 352 #if len(altlist)>1: print "Found alt files: ",altlist 353 354 return subst 355 356 def getCmds(self,subst,i): 357 """ 358 Generate bash script using template found next to this file. 359 This file is read in at constructor time to avoid file I/O 360 """ 361 nindnt=i # the start and has to be consistent with below 362 cmdindnt=self.indent*nindnt 363 cmdLines="" 364 365 # MPI is the default -- but we have a few odd commands 366 if not subst['command']: 367 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 368 else: 369 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 370 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 371 372 cmdLines+=cmdindnt+'if test $res = 0; then\n' 373 diffindnt=self.indent*(nindnt+1) 374 if not subst['filter_output']: 375 if 'altfiles' not in subst: 376 cmd=diffindnt+self._substVars(subst,example_template.difftest) 377 else: 378 # Have to do it by hand a bit because of variable number of alt files 379 rf=subst['redirect_file'] 380 cmd=diffindnt+example_template.difftest.split('@')[0] 381 for i in range(len(subst['altfiles'])): 382 af=subst['altfiles'][i] 383 cmd+=af+' '+rf 384 if i!=len(subst['altfiles'])-1: 385 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 386 cmd+=' || ${diff_exe} ' 387 else: 388 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 389 cmd+=subst['label_suffix']+' ""' # Quotes are painful 390 else: 391 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 392 cmdLines+=cmd+"\n" 393 cmdLines+=cmdindnt+'else\n' 394 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 395 cmdLines+=cmdindnt+'fi\n' 396 return cmdLines 397 398 def _substVars(self,subst,origStr): 399 """ 400 Substitute variables 401 """ 402 Str=origStr 403 for subkey in subst: 404 if type(subst[subkey])!=types.StringType: continue 405 patt="@"+subkey.upper()+"@" 406 Str=re.sub(patt,subst[subkey],Str) 407 return Str 408 409 def _writeTodoSkip(self,fh,tors,reasons,footer): 410 """ 411 Write out the TODO and SKIP lines in the file 412 The TODO or SKIP variable, tors, should be lower case 413 """ 414 TORS=tors.upper() 415 template=eval("example_template."+tors+"line") 416 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 417 tab = '' 418 if reasons: 419 fh.write('if ! $force; then\n') 420 tab = tab + ' ' 421 if reasons == ["Requires DATAFILESPATH"]: 422 # The only reason not to run is DATAFILESPATH, which we check at run-time 423 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 424 tab = tab + ' ' 425 if reasons: 426 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 427 fh.write(tab+footer+"\n") 428 fh.write(tab+"exit\n") 429 if reasons == ["Requires DATAFILESPATH"]: 430 fh.write(' fi\n') 431 if reasons: 432 fh.write('fi\n') 433 fh.write('\n\n') 434 return 435 436 def getLoopVarsHead(self,loopVars,i): 437 """ 438 Generate a nicely indented string with the format loops 439 Here is what the data structure looks like 440 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 441 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 442 loopVars['subargs']['pc_type']=["j","cholesky sor"] 443 """ 444 outstr=''; indnt=self.indent 445 for key in loopVars: 446 for var in loopVars[key]['varlist']: 447 varval=loopVars[key][var] 448 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 449 i = i + 1 450 return (outstr,i) 451 452 def getLoopVarsFoot(self,loopVars,i): 453 outstr=''; indnt=self.indent 454 for key in loopVars: 455 for var in loopVars[key]['varlist']: 456 i = i - 1 457 outstr += indnt * i + "done\n" 458 return (outstr,i) 459 460 def genRunScript(self,testname,root,isRun,srcDict): 461 """ 462 Generate bash script using template found next to this file. 463 This file is read in at constructor time to avoid file I/O 464 """ 465 # runscript_dir directory has to be consistent with gmakefile 466 testDict=srcDict[testname] 467 rpath=self.srcrelpath(root) 468 runscript_dir=os.path.join(self.testroot_dir,rpath) 469 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 470 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 471 472 # Get variables to go into shell scripts. last time testDict used 473 subst=self.getSubstVars(testDict,rpath,testname) 474 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 475 #if '33_' in testname: print subst['subargs'] 476 477 #Handle runfiles 478 for lfile in subst.get('localrunfiles','').split(): 479 fullfile=os.path.join(root,lfile) 480 shutil.copy(fullfile,runscript_dir) 481 # Check subtests for local runfiles 482 for stest in subst.get("subtests",[]): 483 for lfile in testDict[stest].get('localrunfiles','').split(): 484 fullfile=os.path.join(root,lfile) 485 shutil.copy(fullfile,self.runscript_dir) 486 487 # Now substitute the key variables into the header and footer 488 header=self._substVars(subst,example_template.header) 489 # The header is done twice to enable @...@ in header 490 header=self._substVars(subst,header) 491 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 492 493 # Start writing the file 494 fh.write(header+"\n") 495 496 # If there is a TODO or a SKIP then we do it before writing out the 497 # rest of the command (which is useful for working on the test) 498 # SKIP and TODO can be for the source file or for the runs 499 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 500 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 501 502 j=0 # for indentation 503 504 if loopVars: 505 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 506 if (loopHead): fh.write(loopHead+"\n") 507 508 # Subtests are special 509 if 'subtests' in testDict: 510 substP=subst # Subtests can inherit args but be careful 511 k=0 # for label suffixes 512 for stest in testDict["subtests"]: 513 subst=substP.copy() 514 subst.update(testDict[stest]) 515 # nsize is special because it is usually overwritten 516 if 'nsize' in testDict[stest]: 517 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 518 else: 519 fh.write("nsize=1\n") 520 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 521 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 522 #if '10_9' in testname: print sLoopVars 523 if sLoopVars: 524 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 525 fh.write(sLoopHead+"\n") 526 fh.write(self.getCmds(subst,j)+"\n") 527 if sLoopVars: 528 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 529 fh.write(sLoopFoot+"\n") 530 else: 531 fh.write(self.getCmds(subst,j)+"\n") 532 533 if loopVars: 534 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 535 fh.write(loopFoot+"\n") 536 537 fh.write(footer+"\n") 538 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 539 #if '10_9' in testname: sys.exit() 540 return 541 542 def genScriptsAndInfo(self,exfile,root,srcDict): 543 """ 544 Generate scripts from the source file, determine if built, etc. 545 For every test in the exfile with info in the srcDict: 546 1. Determine if it needs to be run for this arch 547 2. Generate the script 548 3. Generate the data needed to write out the makefile in a 549 convenient way 550 All tests are *always* run, but some may be SKIP'd per the TAP standard 551 """ 552 debug=False 553 execname=self.getExecname(exfile,root) 554 isBuilt=self._isBuilt(exfile,srcDict) 555 for test in srcDict: 556 if test in self.buildkeys: continue 557 if debug: print self.nameSpace(exfile,root), test 558 srcDict[test]['execname']=execname # Convenience in generating scripts 559 isRun=self._isRun(srcDict[test]) 560 self.genRunScript(test,root,isRun,srcDict) 561 srcDict[test]['isrun']=isRun 562 self.addToTests(test,root,exfile,execname,srcDict[test]) 563 564 # This adds to datastructure for building deps 565 if isBuilt: self.addToSources(exfile,root,srcDict) 566 return 567 568 def _isBuilt(self,exfile,srcDict): 569 """ 570 Determine if this file should be built. 571 """ 572 # Get the language based on file extension 573 srcDict['SKIP'] = [] 574 lang=self.getLanguage(exfile) 575 if (lang=="F" or lang=="F90"): 576 if not self.have_fortran: 577 srcDict["SKIP"].append("Fortran required for this test") 578 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 579 srcDict["SKIP"].append("Fortran f90freeform required for this test") 580 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 581 srcDict["SKIP"].append("CUDA required for this test") 582 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 583 srcDict["SKIP"].append("C++ required for this test") 584 585 # Deprecated source files 586 if srcDict.get("TODO"): 587 return False 588 589 # isRun can work with srcDict to handle the requires 590 if "requires" in srcDict: 591 if srcDict["requires"]: 592 return self._isRun(srcDict) 593 594 return srcDict['SKIP'] == [] 595 596 597 def _isRun(self,testDict): 598 """ 599 Based on the requirements listed in the src file and the petscconf.h 600 info, determine whether this test should be run or not. 601 """ 602 indent=" " 603 debug=False 604 605 if 'SKIP' not in testDict: 606 testDict['SKIP'] = [] 607 # MPI requirements 608 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 609 if debug: print indent+"Cannot run parallel tests" 610 testDict['SKIP'].append("Parallel test with serial build") 611 612 # The requirements for the test are the sum of all the run subtests 613 if 'subtests' in testDict: 614 if 'requires' not in testDict: testDict['requires']="" 615 for stest in testDict['subtests']: 616 if 'requires' in testDict[stest]: 617 testDict['requires']+=" "+testDict[stest]['requires'] 618 619 620 # Now go through all requirements 621 if 'requires' in testDict: 622 for requirement in testDict['requires'].split(): 623 requirement=requirement.strip() 624 if not requirement: continue 625 if debug: print indent+"Requirement: ", requirement 626 isNull=False 627 if requirement.startswith("!"): 628 requirement=requirement[1:]; isNull=True 629 # Precision requirement for reals 630 if requirement in self.precision_types: 631 if self.conf['PETSC_PRECISION']==requirement: 632 if isNull: 633 testDict['SKIP'].append("not "+requirement+" required") 634 continue 635 continue # Success 636 elif not isNull: 637 testDict['SKIP'].append(requirement+" required") 638 continue 639 # Precision requirement for ints 640 if requirement in self.integer_types: 641 if requirement=="int32": 642 if self.conf['PETSC_SIZEOF_INT']==4: 643 if isNull: 644 testDict['SKIP'].append("not int32 required") 645 continue 646 continue # Success 647 elif not isNull: 648 testDict['SKIP'].append("int32 required") 649 continue 650 if requirement=="int64": 651 if self.conf['PETSC_SIZEOF_INT']==8: 652 if isNull: 653 testDict['SKIP'].append("NOT int64 required") 654 continue 655 continue # Success 656 elif not isNull: 657 testDict['SKIP'].append("int64 required") 658 continue 659 # Datafilespath 660 if requirement=="datafilespath" and not isNull: 661 testDict['SKIP'].append("Requires DATAFILESPATH") 662 continue 663 # Defines -- not sure I have comments matching 664 if "define(" in requirement.lower(): 665 reqdef=requirement.split("(")[1].split(")")[0] 666 if reqdef in self.conf: 667 if isNull: 668 testDict['SKIP'].append("Null requirement not met: "+requirement) 669 continue 670 continue # Success 671 elif not isNull: 672 testDict['SKIP'].append("Required: "+requirement) 673 continue 674 675 # Rest should be packages that we can just get from conf 676 if requirement == "complex": 677 petscconfvar="PETSC_USE_COMPLEX" 678 else: 679 petscconfvar="PETSC_HAVE_"+requirement.upper() 680 if self.conf.get(petscconfvar): 681 if isNull: 682 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 683 continue 684 continue # Success 685 elif not isNull: 686 if debug: print "requirement not found: ", requirement 687 testDict['SKIP'].append(petscconfvar+" requirement not met") 688 continue 689 690 return testDict['SKIP'] == [] 691 692 def genPetscTests_summarize(self,dataDict): 693 """ 694 Required method to state what happened 695 """ 696 if not self.summarize: return 697 indent=" " 698 print(self.testroot_dir, os.path.realpath(os.curdir)) 699 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 700 fh=open(fhname,"w") 701 #print "See ", fhname 702 for root in dataDict: 703 relroot=self.srcrelpath(root) 704 pkg=relroot.split("/")[1] 705 fh.write(relroot+"\n") 706 allSrcs=[] 707 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 708 for exfile in dataDict[root]: 709 # Basic information 710 rfile=os.path.join(relroot,exfile) 711 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 712 fh.write(indent+exfile+indent*4+builtStatus+"\n") 713 714 for test in dataDict[root][exfile]: 715 if test in self.buildkeys: continue 716 line=indent*2+test 717 fh.write(line+"\n") 718 # Looks nice to have the keys in order 719 #for key in dataDict[root][exfile][test]: 720 for key in "isrun abstracted nsize args requires script".split(): 721 if key not in dataDict[root][exfile][test]: continue 722 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 723 fh.write(line+"\n") 724 fh.write("\n") 725 fh.write("\n") 726 fh.write("\n") 727 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 728 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 729 fh.close() 730 return 731 732 def genPetscTests(self,root,dirs,files,dataDict): 733 """ 734 Go through and parse the source files in the directory to generate 735 the examples based on the metadata contained in the source files 736 """ 737 debug=False 738 # Use examplesAnalyze to get what the makefles think are sources 739 #self.examplesAnalyze(root,dirs,files,anlzDict) 740 741 dataDict[root]={} 742 743 for exfile in files: 744 #TST: Until we replace files, still leaving the orginals as is 745 #if not exfile.startswith("new_"+"ex"): continue 746 #if not exfile.startswith("ex"): continue 747 748 # Ignore emacs files 749 if exfile.startswith("#") or exfile.startswith(".#"): continue 750 751 # Convenience 752 fullex=os.path.join(root,exfile) 753 if self.verbose: print(' --> '+fullex) 754 dataDict[root].update(testparse.parseTestFile(fullex,0)) 755 if exfile in dataDict[root]: 756 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 757 758 return 759 760 def walktree(self,top): 761 """ 762 Walk a directory tree, starting from 'top' 763 """ 764 #print "action", action 765 # Goal of action is to fill this dictionary 766 dataDict={} 767 for root, dirs, files in os.walk(top, topdown=True): 768 if not "examples" in root: continue 769 if "dSYM" in root: continue 770 if os.path.basename(root.rstrip("/")) == 'output': continue 771 if self.verbose: print(root) 772 self.genPetscTests(root,dirs,files,dataDict) 773 # Now summarize this dictionary 774 self.genPetscTests_summarize(dataDict) 775 return dataDict 776 777 def gen_gnumake(self, fd): 778 """ 779 Overwrite of the method in the base PETSc class 780 """ 781 def write(stem, srcs): 782 for lang in LANGS: 783 if srcs[lang]['srcs']: 784 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 785 for pkg in PKGS: 786 srcs = self.gen_pkg(pkg) 787 write('testsrcs-' + pkg, srcs) 788 # Handle dependencies 789 for lang in LANGS: 790 for exfile in srcs[lang]['srcs']: 791 if exfile in srcs[lang]: 792 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 793 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 794 for dep in srcs[lang][exfile]: 795 fd.write(exfile+": $(TESTDIR)/"+ dep+'\n') 796 fd.write(ex +": "+exfo+" $(TESTDIR)/"+dep +'\n') 797 798 return self.gendeps 799 800 def gen_pkg(self, pkg): 801 """ 802 Overwrite of the method in the base PETSc class 803 """ 804 return self.sources[pkg] 805 806 def write_gnumake(self,dataDict): 807 """ 808 Write out something similar to files from gmakegen.py 809 810 Test depends on script which also depends on source 811 file, but since I don't have a good way generating 812 acting on a single file (oops) just depend on 813 executable which in turn will depend on src file 814 """ 815 # Different options for how to set up the targets 816 compileExecsFirst=False 817 818 # Open file 819 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 820 fd = open(arch_files, 'w') 821 822 # Write out the sources 823 gendeps = self.gen_gnumake(fd) 824 825 # Write out the tests and execname targets 826 fd.write("\n#Tests and executables\n") # Delimiter 827 828 for pkg in PKGS: 829 # These grab the ones that are built 830 for lang in LANGS: 831 testdeps=[] 832 for ftest in self.tests[pkg][lang]: 833 test=os.path.basename(ftest) 834 basedir=os.path.dirname(ftest) 835 testdeps.append(self.nameSpace(test,basedir)) 836 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 837 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 838 839 # test targets 840 for ftest in self.tests[pkg][lang]: 841 test=os.path.basename(ftest) 842 basedir=os.path.dirname(ftest) 843 testdir="${TESTDIR}/"+basedir+"/" 844 nmtest=self.nameSpace(test,basedir) 845 rundir=os.path.join(testdir,test) 846 #print test, nmtest 847 script=test+".sh" 848 849 # Deps 850 exfile=self.tests[pkg][lang][ftest]['exfile'] 851 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 852 localexec=self.tests[pkg][lang][ftest]['exec'] 853 execname=os.path.join(testdir,localexec) 854 fullscript=os.path.join(testdir,script) 855 tmpfile=os.path.join(testdir,test,test+".tmp") 856 857 # *.counts depends on the script and either executable (will 858 # be run) or the example source file (SKIP or TODO) 859 fd.write('%s.counts : %s %s' 860 % (os.path.join('$(TESTDIR)/counts', nmtest), 861 fullscript, 862 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 863 ) 864 if exfile in self.sources[pkg][lang]: 865 for dep in self.sources[pkg][lang][exfile]: 866 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 867 fd.write('\n') 868 869 # Now write the args: 870 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 871 872 fd.close() 873 return 874 875 def writeHarness(self,output,dataDict): 876 """ 877 This is set up to write out multiple harness even if only gnumake 878 is supported now 879 """ 880 eval("self.write_"+output+"(dataDict)") 881 return 882 883def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 884 if output is None: 885 output = 'gnumake' 886 887 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 888 if petsc_arch: 889 if len(petsc_arch.split(os.path.sep))>1: 890 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 891 892 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 893 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 894 testdir=testdir) 895 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 896 pEx.writeHarness(output,dataDict) 897 898if __name__ == '__main__': 899 import optparse 900 parser = optparse.OptionParser() 901 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 902 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 903 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 904 parser.add_option('--output', help='Location to write output file', default=None) 905 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 906 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 907 opts, extra_args = parser.parse_args() 908 opts, extra_args = parser.parse_args() 909 if extra_args: 910 import sys 911 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 912 exit(1) 913 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 914 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 915