1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 self.verbose=verbose 53 # Whether to write out a useful debugging 54 self.summarize=True if verbose else False 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 if self.verbose: print('Finishing the constructor') 82 return 83 84 def srcrelpath(self,rdir): 85 """ 86 Get relative path to source directory 87 """ 88 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 89 90 def getInInstallDir(self,thisscriptdir): 91 """ 92 When petsc is installed then this file in installed in: 93 <PREFIX>/share/petsc/examples/config/gmakegentest.py 94 otherwise the path is: 95 <PETSC_DIR>/config/gmakegentest.py 96 We use this difference to determine if we are in installdir 97 """ 98 dirlist=thisscriptdir.split(os.path.sep) 99 if len(dirlist)>4: 100 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 101 if lastfour==os.path.join('share','petsc','examples','config'): 102 return True 103 else: 104 return False 105 else: 106 return False 107 108 def nameSpace(self,srcfile,srcdir): 109 """ 110 Because the scripts have a non-unique naming, the pretty-printing 111 needs to convey the srcdir and srcfile. There are two ways of doing this. 112 """ 113 if self.ptNaming: 114 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 115 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 116 nameString=prefix+srcfile 117 else: 118 #nameString=srcdir+": "+srcfile 119 nameString=srcfile 120 return nameString 121 122 def getLanguage(self,srcfile): 123 """ 124 Based on the source, determine associated language as found in gmakegen.LANGS 125 Can we just return srcext[1:\] now? 126 """ 127 langReq=None 128 srcext=os.path.splitext(srcfile)[-1] 129 if srcext in ".F90".split(): langReq="F90" 130 if srcext in ".F".split(): langReq="F" 131 if srcext in ".cxx".split(): langReq="cxx" 132 if srcext == ".cu": langReq="cu" 133 if srcext == ".c": langReq="c" 134 #if not langReq: print "ERROR: ", srcext, srcfile 135 return langReq 136 137 def _getLoopVars(self,inDict,testname, isSubtest=False): 138 """ 139 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 140 Return: 141 inDict['args']: -ksp_monitor 142 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 143 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 144 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 145 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 146 subst should be passed in instead of inDict 147 """ 148 loopVars={}; newargs="" 149 lkeys=inDict.keys() 150 lsuffix='_' 151 from testparse import parseLoopArgs 152 for key in lkeys: 153 if type(inDict[key])!=types.StringType: continue 154 keystr = str(inDict[key]) 155 akey=('subargs' if key=='args' else key) # what to assign 156 if akey not in inDict: inDict[akey]='' 157 varlist=[] 158 for varset in re.split('-(?=[a-zA-Z])',keystr): 159 if not varset.strip(): continue 160 if '{{' in varset: 161 keyvar,lvars,ftype=parseLoopArgs(varset) 162 if akey not in loopVars: loopVars[akey]={} 163 varlist.append(keyvar) 164 loopVars[akey][keyvar]=[keyvar,lvars] 165 if akey=='nsize': 166 inDict[akey] = '${' + keyvar + '}' 167 lsuffix+=akey+'-'+inDict[akey]+'_' 168 else: 169 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 170 lsuffix+=keyvar+'-${' + keyvar + '}_' 171 else: 172 if key=='args': newargs+=" -"+varset.strip() 173 if varlist: loopVars[akey]['varlist']=varlist 174 175 176 # For subtests, args are always substituted in (not top level) 177 if isSubtest: 178 inDict['subargs']+=" "+newargs.strip() 179 inDict['args']='' 180 if 'label_suffix' in inDict: 181 inDict['label_suffix']+=lsuffix.rstrip('_') 182 else: 183 inDict['label_suffix']=lsuffix.rstrip('_') 184 else: 185 if loopVars.keys(): 186 inDict['args']=newargs.strip() 187 inDict['label_suffix']=lsuffix.rstrip('_') 188 if loopVars.keys(): 189 return loopVars 190 else: 191 return None 192 193 def getArgLabel(self,testDict): 194 """ 195 In all of the arguments in the test dictionary, create a simple 196 string for searching within the makefile system. For simplicity in 197 search, remove "-", for strings, etc. 198 Also, concatenate the arg commands 199 For now, ignore nsize -- seems hard to search for anyway 200 """ 201 # Collect all of the args associated with a test 202 argStr=("" if 'args' not in testDict else testDict['args']) 203 if 'subtests' in testDict: 204 for stest in testDict["subtests"]: 205 sd=testDict[stest] 206 argStr=argStr+("" if 'args' not in sd else sd['args']) 207 208 # Now go through and cleanup 209 argStr=re.sub('{{(.*?)}}',"",argStr) 210 argStr=re.sub('-'," ",argStr) 211 for digit in string.digits: argStr=re.sub(digit," ",argStr) 212 argStr=re.sub("\.","",argStr) 213 argStr=re.sub(",","",argStr) 214 argStr=re.sub('\+',' ',argStr) 215 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 216 return argStr.strip() 217 218 def addToSources(self,exfile,root,srcDict): 219 """ 220 Put into data structure that allows easy generation of makefile 221 """ 222 rpath=self.srcrelpath(root) 223 pkg=rpath.split(os.path.sep)[1] 224 relpfile=os.path.join(rpath,exfile) 225 lang=self.getLanguage(exfile) 226 if not lang: return 227 self.sources[pkg][lang]['srcs'].append(relpfile) 228 self.sources[pkg][lang][relpfile] = [] 229 if 'depends' in srcDict: 230 depSrcList=srcDict['depends'].split() 231 for depSrc in depSrcList: 232 depObj=os.path.splitext(depSrc)[0]+".o" 233 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 234 235 # In gmakefile, ${TESTDIR} var specifies the object compilation 236 testsdir=self.srcrelpath(root)+"/" 237 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 238 self.objects[pkg].append(objfile) 239 return 240 241 def addToTests(self,test,root,exfile,execname,testDict): 242 """ 243 Put into data structure that allows easy generation of makefile 244 Organized by languages to allow testing of languages 245 """ 246 rpath=self.srcrelpath(root) 247 pkg=rpath.split("/")[1] 248 #nmtest=self.nameSpace(test,root) 249 nmtest=os.path.join(rpath,test) 250 lang=self.getLanguage(exfile) 251 if not lang: return 252 self.tests[pkg][lang][nmtest]={} 253 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 254 self.tests[pkg][lang][nmtest]['exec']=execname 255 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 256 return 257 258 def getExecname(self,exfile,root): 259 """ 260 Generate bash script using template found next to this file. 261 This file is read in at constructor time to avoid file I/O 262 """ 263 rpath=self.srcrelpath(root) 264 if self.single_ex: 265 execname=rpath.split("/")[1]+"-ex" 266 else: 267 execname=os.path.splitext(exfile)[0] 268 return execname 269 270 def getSubstVars(self,testDict,rpath,testname): 271 """ 272 Create a dictionary with all of the variables that get substituted 273 into the template commands found in example_template.py 274 """ 275 subst={} 276 277 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 278 if 'nsize' not in testDict: testDict['nsize']=1 279 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 280 for ak in testparse.acceptedkeys: 281 if ak=='test': continue 282 subst[ak]=(testDict[ak] if ak in testDict else '') 283 284 # Now do other variables 285 subst['execname']=testDict['execname'] 286 if 'filter' in testDict: 287 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 288 289 # Others 290 subst['subargs']='' # Default. For variables override 291 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 292 subst['label_suffix']='' 293 subst['comments']="\n#".join(subst['comments'].split("\n")) 294 if subst['comments']: subst['comments']="#"+subst['comments'] 295 subst['exec']="../"+subst['execname'] 296 subst['testroot']=self.testroot_dir 297 subst['testname']=testname 298 dp = self.conf.get('DATAFILESPATH','') 299 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 300 301 # This is used to label some matrices 302 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 303 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 304 305 # These can have for loops and are treated separately later 306 subst['nsize']=str(subst['nsize']) 307 308 #Conf vars 309 if self.petsc_arch.find('valgrind')>=0: 310 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 311 else: 312 subst['mpiexec']=self.conf['MPIEXEC'] 313 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 314 subst['petsc_arch']=self.petsc_arch 315 if not self.inInstallDir: 316 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 317 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 318 else: 319 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 320 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 321 subst['diff']=self.conf['DIFF'] 322 subst['rm']=self.conf['RM'] 323 subst['grep']=self.conf['GREP'] 324 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 325 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 326 327 # Output file is special because of subtests override 328 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 329 if not "_" in defroot: defroot=defroot+"_1" 330 subst['defroot']=defroot 331 subst['label']=self.nameSpace(defroot,subst['srcdir']) 332 subst['redirect_file']=defroot+".tmp" 333 if 'output_file' not in testDict: 334 subst['output_file']="output/"+defroot+".out" 335 # Add in the full path here. 336 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 337 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 338 if not subst['TODO']: 339 print "Warning: "+subst['output_file']+" not found." 340 # Worry about alt files here -- see 341 # src/snes/examples/tutorials/output/ex22*.out 342 altlist=[subst['output_file']] 343 basefile,ext = os.path.splitext(subst['output_file']) 344 for i in range(1,9): 345 altroot=basefile+"_alt" 346 if i > 1: altroot=altroot+"_"+str(i) 347 af=altroot+".out" 348 srcaf=os.path.join(subst['srcdir'],af) 349 fullaf=os.path.join(self.petsc_dir,srcaf) 350 if os.path.isfile(fullaf): altlist.append(srcaf) 351 if len(altlist)>1: subst['altfiles']=altlist 352 #if len(altlist)>1: print "Found alt files: ",altlist 353 354 return subst 355 356 def getCmds(self,subst,i): 357 """ 358 Generate bash script using template found next to this file. 359 This file is read in at constructor time to avoid file I/O 360 """ 361 nindnt=i # the start and has to be consistent with below 362 cmdindnt=self.indent*nindnt 363 cmdLines="" 364 365 # MPI is the default -- but we have a few odd commands 366 if not subst['command']: 367 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 368 else: 369 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 370 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 371 372 cmdLines+=cmdindnt+'if test $res = 0; then\n' 373 diffindnt=self.indent*(nindnt+1) 374 if not subst['filter_output']: 375 if 'altfiles' not in subst: 376 cmd=diffindnt+self._substVars(subst,example_template.difftest) 377 else: 378 # Have to do it by hand a bit because of variable number of alt files 379 rf=subst['redirect_file'] 380 cmd=diffindnt+example_template.difftest.split('@')[0] 381 for i in range(len(subst['altfiles'])): 382 af=subst['altfiles'][i] 383 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 384 if i!=len(subst['altfiles'])-1: 385 cmd+=' || ${diff_exe} ' 386 else: 387 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 388 cmd+=subst['label_suffix']+' ""' # Quotes are painful 389 else: 390 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 391 cmdLines+=cmd+"\n" 392 cmdLines+=cmdindnt+'else\n' 393 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 394 cmdLines+=cmdindnt+'fi\n' 395 return cmdLines 396 397 def _substVars(self,subst,origStr): 398 """ 399 Substitute variables 400 """ 401 Str=origStr 402 for subkey in subst: 403 if type(subst[subkey])!=types.StringType: continue 404 patt="@"+subkey.upper()+"@" 405 Str=re.sub(patt,subst[subkey],Str) 406 return Str 407 408 def _writeTodoSkip(self,fh,tors,reasons,footer): 409 """ 410 Write out the TODO and SKIP lines in the file 411 The TODO or SKIP variable, tors, should be lower case 412 """ 413 TORS=tors.upper() 414 template=eval("example_template."+tors+"line") 415 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 416 tab = '' 417 if reasons: 418 fh.write('if ! $force; then\n') 419 tab = tab + ' ' 420 if reasons == ["Requires DATAFILESPATH"]: 421 # The only reason not to run is DATAFILESPATH, which we check at run-time 422 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 423 tab = tab + ' ' 424 if reasons: 425 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 426 fh.write(tab+footer+"\n") 427 fh.write(tab+"exit\n") 428 if reasons == ["Requires DATAFILESPATH"]: 429 fh.write(' fi\n') 430 if reasons: 431 fh.write('fi\n') 432 fh.write('\n\n') 433 return 434 435 def getLoopVarsHead(self,loopVars,i): 436 """ 437 Generate a nicely indented string with the format loops 438 Here is what the data structure looks like 439 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 440 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 441 loopVars['subargs']['pc_type']=["j","cholesky sor"] 442 """ 443 outstr=''; indnt=self.indent 444 for key in loopVars: 445 for var in loopVars[key]['varlist']: 446 varval=loopVars[key][var] 447 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 448 i = i + 1 449 return (outstr,i) 450 451 def getLoopVarsFoot(self,loopVars,i): 452 outstr=''; indnt=self.indent 453 for key in loopVars: 454 for var in loopVars[key]['varlist']: 455 i = i - 1 456 outstr += indnt * i + "done\n" 457 return (outstr,i) 458 459 def genRunScript(self,testname,root,isRun,srcDict): 460 """ 461 Generate bash script using template found next to this file. 462 This file is read in at constructor time to avoid file I/O 463 """ 464 # runscript_dir directory has to be consistent with gmakefile 465 testDict=srcDict[testname] 466 rpath=self.srcrelpath(root) 467 runscript_dir=os.path.join(self.testroot_dir,rpath) 468 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 469 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 470 471 # Get variables to go into shell scripts. last time testDict used 472 subst=self.getSubstVars(testDict,rpath,testname) 473 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 474 #if '33_' in testname: print subst['subargs'] 475 476 #Handle runfiles 477 for lfile in subst.get('localrunfiles','').split(): 478 fullfile=os.path.join(root,lfile) 479 shutil.copy(fullfile,runscript_dir) 480 # Check subtests for local runfiles 481 for stest in subst.get("subtests",[]): 482 for lfile in testDict[stest].get('localrunfiles','').split(): 483 fullfile=os.path.join(root,lfile) 484 shutil.copy(fullfile,self.runscript_dir) 485 486 # Now substitute the key variables into the header and footer 487 header=self._substVars(subst,example_template.header) 488 # The header is done twice to enable @...@ in header 489 header=self._substVars(subst,header) 490 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 491 492 # Start writing the file 493 fh.write(header+"\n") 494 495 # If there is a TODO or a SKIP then we do it before writing out the 496 # rest of the command (which is useful for working on the test) 497 # SKIP and TODO can be for the source file or for the runs 498 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 499 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 500 501 j=0 # for indentation 502 503 if loopVars: 504 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 505 if (loopHead): fh.write(loopHead+"\n") 506 507 # Subtests are special 508 if 'subtests' in testDict: 509 substP=subst # Subtests can inherit args but be careful 510 k=0 # for label suffixes 511 for stest in testDict["subtests"]: 512 subst=substP.copy() 513 subst.update(testDict[stest]) 514 # nsize is special because it is usually overwritten 515 if 'nsize' in testDict[stest]: 516 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 517 else: 518 fh.write("nsize=1\n") 519 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 520 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 521 #if '10_9' in testname: print sLoopVars 522 if sLoopVars: 523 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 524 fh.write(sLoopHead+"\n") 525 fh.write(self.getCmds(subst,j)+"\n") 526 if sLoopVars: 527 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 528 fh.write(sLoopFoot+"\n") 529 else: 530 fh.write(self.getCmds(subst,j)+"\n") 531 532 if loopVars: 533 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 534 fh.write(loopFoot+"\n") 535 536 fh.write(footer+"\n") 537 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 538 #if '10_9' in testname: sys.exit() 539 return 540 541 def genScriptsAndInfo(self,exfile,root,srcDict): 542 """ 543 Generate scripts from the source file, determine if built, etc. 544 For every test in the exfile with info in the srcDict: 545 1. Determine if it needs to be run for this arch 546 2. Generate the script 547 3. Generate the data needed to write out the makefile in a 548 convenient way 549 All tests are *always* run, but some may be SKIP'd per the TAP standard 550 """ 551 debug=False 552 execname=self.getExecname(exfile,root) 553 isBuilt=self._isBuilt(exfile,srcDict) 554 for test in srcDict: 555 if test in self.buildkeys: continue 556 if debug: print self.nameSpace(exfile,root), test 557 srcDict[test]['execname']=execname # Convenience in generating scripts 558 isRun=self._isRun(srcDict[test]) 559 self.genRunScript(test,root,isRun,srcDict) 560 srcDict[test]['isrun']=isRun 561 self.addToTests(test,root,exfile,execname,srcDict[test]) 562 563 # This adds to datastructure for building deps 564 if isBuilt: self.addToSources(exfile,root,srcDict) 565 return 566 567 def _isBuilt(self,exfile,srcDict): 568 """ 569 Determine if this file should be built. 570 """ 571 # Get the language based on file extension 572 srcDict['SKIP'] = [] 573 lang=self.getLanguage(exfile) 574 if (lang=="F" or lang=="F90"): 575 if not self.have_fortran: 576 srcDict["SKIP"].append("Fortran required for this test") 577 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 578 srcDict["SKIP"].append("Fortran f90freeform required for this test") 579 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 580 srcDict["SKIP"].append("CUDA required for this test") 581 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 582 srcDict["SKIP"].append("C++ required for this test") 583 584 # Deprecated source files 585 if srcDict.get("TODO"): 586 return False 587 588 # isRun can work with srcDict to handle the requires 589 if "requires" in srcDict: 590 if srcDict["requires"]: 591 return self._isRun(srcDict) 592 593 return srcDict['SKIP'] == [] 594 595 596 def _isRun(self,testDict): 597 """ 598 Based on the requirements listed in the src file and the petscconf.h 599 info, determine whether this test should be run or not. 600 """ 601 indent=" " 602 debug=False 603 604 if 'SKIP' not in testDict: 605 testDict['SKIP'] = [] 606 # MPI requirements 607 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 608 if debug: print indent+"Cannot run parallel tests" 609 testDict['SKIP'].append("Parallel test with serial build") 610 611 # The requirements for the test are the sum of all the run subtests 612 if 'subtests' in testDict: 613 if 'requires' not in testDict: testDict['requires']="" 614 for stest in testDict['subtests']: 615 if 'requires' in testDict[stest]: 616 testDict['requires']+=" "+testDict[stest]['requires'] 617 618 619 # Now go through all requirements 620 if 'requires' in testDict: 621 for requirement in testDict['requires'].split(): 622 requirement=requirement.strip() 623 if not requirement: continue 624 if debug: print indent+"Requirement: ", requirement 625 isNull=False 626 if requirement.startswith("!"): 627 requirement=requirement[1:]; isNull=True 628 # Precision requirement for reals 629 if requirement in self.precision_types: 630 if self.conf['PETSC_PRECISION']==requirement: 631 if isNull: 632 testDict['SKIP'].append("not "+requirement+" required") 633 continue 634 continue # Success 635 elif not isNull: 636 testDict['SKIP'].append(requirement+" required") 637 continue 638 # Precision requirement for ints 639 if requirement in self.integer_types: 640 if requirement=="int32": 641 if self.conf['PETSC_SIZEOF_INT']==4: 642 if isNull: 643 testDict['SKIP'].append("not int32 required") 644 continue 645 continue # Success 646 elif not isNull: 647 testDict['SKIP'].append("int32 required") 648 continue 649 if requirement=="int64": 650 if self.conf['PETSC_SIZEOF_INT']==8: 651 if isNull: 652 testDict['SKIP'].append("NOT int64 required") 653 continue 654 continue # Success 655 elif not isNull: 656 testDict['SKIP'].append("int64 required") 657 continue 658 # Datafilespath 659 if requirement=="datafilespath" and not isNull: 660 testDict['SKIP'].append("Requires DATAFILESPATH") 661 continue 662 # Defines -- not sure I have comments matching 663 if "define(" in requirement.lower(): 664 reqdef=requirement.split("(")[1].split(")")[0] 665 if reqdef in self.conf: 666 if isNull: 667 testDict['SKIP'].append("Null requirement not met: "+requirement) 668 continue 669 continue # Success 670 elif not isNull: 671 testDict['SKIP'].append("Required: "+requirement) 672 continue 673 674 # Rest should be packages that we can just get from conf 675 if requirement == "complex": 676 petscconfvar="PETSC_USE_COMPLEX" 677 else: 678 petscconfvar="PETSC_HAVE_"+requirement.upper() 679 if self.conf.get(petscconfvar): 680 if isNull: 681 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 682 continue 683 continue # Success 684 elif not isNull: 685 if debug: print "requirement not found: ", requirement 686 testDict['SKIP'].append(petscconfvar+" requirement not met") 687 continue 688 689 return testDict['SKIP'] == [] 690 691 def genPetscTests_summarize(self,dataDict): 692 """ 693 Required method to state what happened 694 """ 695 if not self.summarize: return 696 indent=" " 697 print(self.testroot_dir, os.path.realpath(os.curdir)) 698 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 699 fh=open(fhname,"w") 700 #print "See ", fhname 701 for root in dataDict: 702 relroot=self.srcrelpath(root) 703 pkg=relroot.split("/")[1] 704 fh.write(relroot+"\n") 705 allSrcs=[] 706 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 707 for exfile in dataDict[root]: 708 # Basic information 709 rfile=os.path.join(relroot,exfile) 710 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 711 fh.write(indent+exfile+indent*4+builtStatus+"\n") 712 713 for test in dataDict[root][exfile]: 714 if test in self.buildkeys: continue 715 line=indent*2+test 716 fh.write(line+"\n") 717 # Looks nice to have the keys in order 718 #for key in dataDict[root][exfile][test]: 719 for key in "isrun abstracted nsize args requires script".split(): 720 if key not in dataDict[root][exfile][test]: continue 721 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 722 fh.write(line+"\n") 723 fh.write("\n") 724 fh.write("\n") 725 fh.write("\n") 726 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 727 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 728 fh.close() 729 return 730 731 def genPetscTests(self,root,dirs,files,dataDict): 732 """ 733 Go through and parse the source files in the directory to generate 734 the examples based on the metadata contained in the source files 735 """ 736 debug=False 737 # Use examplesAnalyze to get what the makefles think are sources 738 #self.examplesAnalyze(root,dirs,files,anlzDict) 739 740 dataDict[root]={} 741 742 for exfile in files: 743 #TST: Until we replace files, still leaving the orginals as is 744 #if not exfile.startswith("new_"+"ex"): continue 745 #if not exfile.startswith("ex"): continue 746 747 # Ignore emacs files 748 if exfile.startswith("#") or exfile.startswith(".#"): continue 749 750 # Convenience 751 fullex=os.path.join(root,exfile) 752 if self.verbose: print(' --> '+fullex) 753 dataDict[root].update(testparse.parseTestFile(fullex,0)) 754 if exfile in dataDict[root]: 755 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 756 757 return 758 759 def walktree(self,top): 760 """ 761 Walk a directory tree, starting from 'top' 762 """ 763 #print "action", action 764 # Goal of action is to fill this dictionary 765 dataDict={} 766 for root, dirs, files in os.walk(top, topdown=False): 767 if not "examples" in root: continue 768 if not os.path.isfile(os.path.join(root,"makefile")): continue 769 if self.verbose: print(root) 770 bname=os.path.basename(root.rstrip("/")) 771 self.genPetscTests(root,dirs,files,dataDict) 772 # Now summarize this dictionary 773 self.genPetscTests_summarize(dataDict) 774 return dataDict 775 776 def gen_gnumake(self, fd): 777 """ 778 Overwrite of the method in the base PETSc class 779 """ 780 def write(stem, srcs): 781 for lang in LANGS: 782 if srcs[lang]['srcs']: 783 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 784 for pkg in PKGS: 785 srcs = self.gen_pkg(pkg) 786 write('testsrcs-' + pkg, srcs) 787 # Handle dependencies 788 for lang in LANGS: 789 for exfile in srcs[lang]['srcs']: 790 if exfile in srcs[lang]: 791 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 792 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 793 for dep in srcs[lang][exfile]: 794 fd.write(exfile+": $(TESTDIR)/"+ dep+'\n') 795 fd.write(ex +": "+exfo+" $(TESTDIR)/"+dep +'\n') 796 797 return self.gendeps 798 799 def gen_pkg(self, pkg): 800 """ 801 Overwrite of the method in the base PETSc class 802 """ 803 return self.sources[pkg] 804 805 def write_gnumake(self,dataDict): 806 """ 807 Write out something similar to files from gmakegen.py 808 809 Test depends on script which also depends on source 810 file, but since I don't have a good way generating 811 acting on a single file (oops) just depend on 812 executable which in turn will depend on src file 813 """ 814 # Different options for how to set up the targets 815 compileExecsFirst=False 816 817 # Open file 818 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 819 fd = open(arch_files, 'w') 820 821 # Write out the sources 822 gendeps = self.gen_gnumake(fd) 823 824 # Write out the tests and execname targets 825 fd.write("\n#Tests and executables\n") # Delimiter 826 827 for pkg in PKGS: 828 # These grab the ones that are built 829 for lang in LANGS: 830 testdeps=[] 831 for ftest in self.tests[pkg][lang]: 832 test=os.path.basename(ftest) 833 basedir=os.path.dirname(ftest) 834 testdeps.append(self.nameSpace(test,basedir)) 835 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 836 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 837 838 # test targets 839 for ftest in self.tests[pkg][lang]: 840 test=os.path.basename(ftest) 841 basedir=os.path.dirname(ftest) 842 testdir="${TESTDIR}/"+basedir+"/" 843 nmtest=self.nameSpace(test,basedir) 844 rundir=os.path.join(testdir,test) 845 #print test, nmtest 846 script=test+".sh" 847 848 # Deps 849 exfile=self.tests[pkg][lang][ftest]['exfile'] 850 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 851 localexec=self.tests[pkg][lang][ftest]['exec'] 852 execname=os.path.join(testdir,localexec) 853 fullscript=os.path.join(testdir,script) 854 tmpfile=os.path.join(testdir,test,test+".tmp") 855 856 # *.counts depends on the script and either executable (will 857 # be run) or the example source file (SKIP or TODO) 858 fd.write('%s.counts : %s %s' 859 % (os.path.join('$(TESTDIR)/counts', nmtest), 860 fullscript, 861 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 862 ) 863 if exfile in self.sources[pkg][lang]: 864 for dep in self.sources[pkg][lang][exfile]: 865 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 866 fd.write('\n') 867 868 # Now write the args: 869 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 870 871 fd.close() 872 return 873 874 def writeHarness(self,output,dataDict): 875 """ 876 This is set up to write out multiple harness even if only gnumake 877 is supported now 878 """ 879 eval("self.write_"+output+"(dataDict)") 880 return 881 882def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 883 if output is None: 884 output = 'gnumake' 885 886 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 887 if petsc_arch: 888 if len(petsc_arch.split(os.path.sep))>1: 889 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 890 891 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 892 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 893 testdir=testdir) 894 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 895 pEx.writeHarness(output,dataDict) 896 897if __name__ == '__main__': 898 import optparse 899 parser = optparse.OptionParser() 900 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 901 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 902 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 903 parser.add_option('--output', help='Location to write output file', default=None) 904 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 905 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 906 opts, extra_args = parser.parse_args() 907 opts, extra_args = parser.parse_args() 908 if extra_args: 909 import sys 910 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 911 exit(1) 912 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 913 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 914