1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 self.verbose=verbose 53 # Whether to write out a useful debugging 54 self.summarize=True if verbose else False 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 if self.verbose: print('Finishing the constructor') 82 return 83 84 def srcrelpath(self,rdir): 85 """ 86 Get relative path to source directory 87 """ 88 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 89 90 def getInInstallDir(self,thisscriptdir): 91 """ 92 When petsc is installed then this file in installed in: 93 <PREFIX>/share/petsc/examples/config/gmakegentest.py 94 otherwise the path is: 95 <PETSC_DIR>/config/gmakegentest.py 96 We use this difference to determine if we are in installdir 97 """ 98 dirlist=thisscriptdir.split(os.path.sep) 99 if len(dirlist)>4: 100 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 101 if lastfour==os.path.join('share','petsc','examples','config'): 102 return True 103 else: 104 return False 105 else: 106 return False 107 108 def nameSpace(self,srcfile,srcdir): 109 """ 110 Because the scripts have a non-unique naming, the pretty-printing 111 needs to convey the srcdir and srcfile. There are two ways of doing this. 112 """ 113 if self.ptNaming: 114 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 115 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 116 nameString=prefix+srcfile 117 else: 118 #nameString=srcdir+": "+srcfile 119 nameString=srcfile 120 return nameString 121 122 def getLanguage(self,srcfile): 123 """ 124 Based on the source, determine associated language as found in gmakegen.LANGS 125 Can we just return srcext[1:\] now? 126 """ 127 langReq=None 128 srcext=os.path.splitext(srcfile)[-1] 129 if srcext in ".F90".split(): langReq="F90" 130 if srcext in ".F".split(): langReq="F" 131 if srcext in ".cxx".split(): langReq="cxx" 132 if srcext == ".cu": langReq="cu" 133 if srcext == ".c": langReq="c" 134 #if not langReq: print "ERROR: ", srcext, srcfile 135 return langReq 136 137 def _getLoopVars(self,inDict,testname, isSubtest=False): 138 """ 139 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 140 Return: 141 inDict['args']: -ksp_monitor 142 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 143 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 144 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 145 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 146 subst should be passed in instead of inDict 147 """ 148 loopVars={}; newargs="" 149 lkeys=inDict.keys() 150 lsuffix='_' 151 from testparse import parseLoopArgs 152 for key in lkeys: 153 if type(inDict[key])!=types.StringType: continue 154 keystr = str(inDict[key]) 155 akey=('subargs' if key=='args' else key) # what to assign 156 if akey not in inDict: inDict[akey]='' 157 varlist=[] 158 for varset in re.split('-(?=[a-zA-Z])',keystr): 159 if not varset.strip(): continue 160 if '{{' in varset: 161 keyvar,lvars,ftype=parseLoopArgs(varset) 162 if akey not in loopVars: loopVars[akey]={} 163 varlist.append(keyvar) 164 loopVars[akey][keyvar]=[keyvar,lvars] 165 if akey=='nsize': 166 inDict[akey] = '${' + keyvar + '}' 167 lsuffix+=akey+'-'+inDict[akey]+'_' 168 else: 169 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 170 lsuffix+=keyvar+'-${' + keyvar + '}_' 171 else: 172 if key=='args': newargs+=" -"+varset.strip() 173 if varlist: loopVars[akey]['varlist']=varlist 174 175 176 # For subtests, args are always substituted in (not top level) 177 if isSubtest: 178 inDict['subargs']+=" "+newargs.strip() 179 inDict['args']='' 180 if 'label_suffix' in inDict: 181 inDict['label_suffix']+=lsuffix.rstrip('_') 182 else: 183 inDict['label_suffix']=lsuffix.rstrip('_') 184 else: 185 if loopVars.keys(): 186 inDict['args']=newargs.strip() 187 inDict['label_suffix']=lsuffix.rstrip('_') 188 if loopVars.keys(): 189 return loopVars 190 else: 191 return None 192 193 def getArgLabel(self,testDict): 194 """ 195 In all of the arguments in the test dictionary, create a simple 196 string for searching within the makefile system. For simplicity in 197 search, remove "-", for strings, etc. 198 Also, concatenate the arg commands 199 For now, ignore nsize -- seems hard to search for anyway 200 """ 201 # Collect all of the args associated with a test 202 argStr=("" if 'args' not in testDict else testDict['args']) 203 if 'subtests' in testDict: 204 for stest in testDict["subtests"]: 205 sd=testDict[stest] 206 argStr=argStr+("" if 'args' not in sd else sd['args']) 207 208 # Now go through and cleanup 209 argStr=re.sub('{{(.*?)}}',"",argStr) 210 argStr=re.sub('-'," ",argStr) 211 for digit in string.digits: argStr=re.sub(digit," ",argStr) 212 argStr=re.sub("\.","",argStr) 213 argStr=re.sub(",","",argStr) 214 argStr=re.sub('\+',' ',argStr) 215 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 216 return argStr.strip() 217 218 def addToSources(self,exfile,root,srcDict): 219 """ 220 Put into data structure that allows easy generation of makefile 221 """ 222 rpath=self.srcrelpath(root) 223 pkg=rpath.split(os.path.sep)[1] 224 relpfile=os.path.join(rpath,exfile) 225 lang=self.getLanguage(exfile) 226 if not lang: return 227 self.sources[pkg][lang]['srcs'].append(relpfile) 228 self.sources[pkg][lang][relpfile] = [] 229 if 'depends' in srcDict: 230 depSrcList=srcDict['depends'].split() 231 for depSrc in depSrcList: 232 depObj=os.path.splitext(depSrc)[0]+".o" 233 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 234 235 # In gmakefile, ${TESTDIR} var specifies the object compilation 236 testsdir=self.srcrelpath(root)+"/" 237 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 238 self.objects[pkg].append(objfile) 239 return 240 241 def addToTests(self,test,root,exfile,execname,testDict): 242 """ 243 Put into data structure that allows easy generation of makefile 244 Organized by languages to allow testing of languages 245 """ 246 rpath=self.srcrelpath(root) 247 pkg=rpath.split("/")[1] 248 #nmtest=self.nameSpace(test,root) 249 nmtest=os.path.join(rpath,test) 250 lang=self.getLanguage(exfile) 251 if not lang: return 252 self.tests[pkg][lang][nmtest]={} 253 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 254 self.tests[pkg][lang][nmtest]['exec']=execname 255 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 256 return 257 258 def getExecname(self,exfile,root): 259 """ 260 Generate bash script using template found next to this file. 261 This file is read in at constructor time to avoid file I/O 262 """ 263 rpath=self.srcrelpath(root) 264 if self.single_ex: 265 execname=rpath.split("/")[1]+"-ex" 266 else: 267 execname=os.path.splitext(exfile)[0] 268 return execname 269 270 def getSubstVars(self,testDict,rpath,testname): 271 """ 272 Create a dictionary with all of the variables that get substituted 273 into the template commands found in example_template.py 274 """ 275 subst={} 276 277 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 278 if 'nsize' not in testDict: testDict['nsize']=1 279 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 280 for ak in testparse.acceptedkeys: 281 if ak=='test': continue 282 subst[ak]=(testDict[ak] if ak in testDict else '') 283 284 # Now do other variables 285 subst['execname']=testDict['execname'] 286 if 'filter' in testDict: 287 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 288 289 # Others 290 subst['subargs']='' # Default. For variables override 291 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 292 subst['label_suffix']='' 293 subst['comments']="\n#".join(subst['comments'].split("\n")) 294 if subst['comments']: subst['comments']="#"+subst['comments'] 295 subst['exec']="../"+subst['execname'] 296 subst['testroot']=self.testroot_dir 297 subst['testname']=testname 298 dp = self.conf.get('DATAFILESPATH','') 299 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 300 301 # This is used to label some matrices 302 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 303 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 304 305 # These can have for loops and are treated separately later 306 subst['nsize']=str(subst['nsize']) 307 308 #Conf vars 309 if self.petsc_arch.find('valgrind')>=0: 310 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 311 else: 312 subst['mpiexec']=self.conf['MPIEXEC'] 313 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 314 subst['petsc_arch']=self.petsc_arch 315 if not self.inInstallDir: 316 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 317 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 318 else: 319 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 320 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 321 subst['diff']=self.conf['DIFF'] 322 subst['rm']=self.conf['RM'] 323 subst['grep']=self.conf['GREP'] 324 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 325 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 326 327 # Output file is special because of subtests override 328 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 329 if not "_" in defroot: defroot=defroot+"_1" 330 subst['defroot']=defroot 331 subst['label']=self.nameSpace(defroot,subst['srcdir']) 332 subst['redirect_file']=defroot+".tmp" 333 if 'output_file' not in testDict: 334 subst['output_file']="output/"+defroot+".out" 335 # Add in the full path here. 336 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 337 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 338 if not subst['TODO']: 339 print "Warning: "+subst['output_file']+" not found." 340 # Worry about alt files here -- see 341 # src/snes/examples/tutorials/output/ex22*.out 342 altlist=[subst['output_file']] 343 for i in range(1,3): 344 altroot=defroot+"_alt" 345 if i==2: altroot=altroot+"_2" 346 af="output/"+altroot+".out" 347 srcaf=os.path.join(subst['srcdir'],af) 348 fullaf=os.path.join(self.petsc_dir,srcaf) 349 if os.path.isfile(fullaf): altlist.append(srcaf) 350 if len(altlist)>1: subst['altfiles']=altlist 351 #if len(altlist)>1: print "Found alt files: ",altlist 352 353 return subst 354 355 def getCmds(self,subst,i): 356 """ 357 Generate bash script using template found next to this file. 358 This file is read in at constructor time to avoid file I/O 359 """ 360 nindnt=i # the start and has to be consistent with below 361 cmdindnt=self.indent*nindnt 362 cmdLines="" 363 364 # MPI is the default -- but we have a few odd commands 365 if not subst['command']: 366 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 367 else: 368 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 369 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 370 371 cmdLines+=cmdindnt+'if test $res = 0; then\n' 372 diffindnt=self.indent*(nindnt+1) 373 if not subst['filter_output']: 374 if 'altfiles' not in subst: 375 cmd=diffindnt+self._substVars(subst,example_template.difftest) 376 else: 377 # Have to do it by hand a bit because of variable number of alt files 378 rf=subst['redirect_file'] 379 cmd=diffindnt+example_template.difftest.split('@')[0] 380 for i in range(len(subst['altfiles'])): 381 af=subst['altfiles'][i] 382 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 383 if i!=len(subst['altfiles'])-1: 384 cmd+=' || ${diff_exe} ' 385 else: 386 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 387 cmd+=subst['label_suffix']+' ""' # Quotes are painful 388 else: 389 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 390 cmdLines+=cmd+"\n" 391 cmdLines+=cmdindnt+'else\n' 392 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 393 cmdLines+=cmdindnt+'fi\n' 394 return cmdLines 395 396 def _substVars(self,subst,origStr): 397 """ 398 Substitute variables 399 """ 400 Str=origStr 401 for subkey in subst: 402 if type(subst[subkey])!=types.StringType: continue 403 patt="@"+subkey.upper()+"@" 404 Str=re.sub(patt,subst[subkey],Str) 405 return Str 406 407 def _writeTodoSkip(self,fh,tors,reasons,footer): 408 """ 409 Write out the TODO and SKIP lines in the file 410 The TODO or SKIP variable, tors, should be lower case 411 """ 412 TORS=tors.upper() 413 template=eval("example_template."+tors+"line") 414 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 415 tab = '' 416 if reasons: 417 fh.write('if ! $force; then\n') 418 tab = tab + ' ' 419 if reasons == ["Requires DATAFILESPATH"]: 420 # The only reason not to run is DATAFILESPATH, which we check at run-time 421 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 422 tab = tab + ' ' 423 if reasons: 424 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 425 fh.write(tab+footer+"\n") 426 fh.write(tab+"exit\n") 427 if reasons == ["Requires DATAFILESPATH"]: 428 fh.write(' fi\n') 429 if reasons: 430 fh.write('fi\n') 431 fh.write('\n\n') 432 return 433 434 def getLoopVarsHead(self,loopVars,i): 435 """ 436 Generate a nicely indented string with the format loops 437 Here is what the data structure looks like 438 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 439 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 440 loopVars['subargs']['pc_type']=["j","cholesky sor"] 441 """ 442 outstr=''; indnt=self.indent 443 for key in loopVars: 444 for var in loopVars[key]['varlist']: 445 varval=loopVars[key][var] 446 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 447 i = i + 1 448 return (outstr,i) 449 450 def getLoopVarsFoot(self,loopVars,i): 451 outstr=''; indnt=self.indent 452 for key in loopVars: 453 for var in loopVars[key]['varlist']: 454 i = i - 1 455 outstr += indnt * i + "done\n" 456 return (outstr,i) 457 458 def genRunScript(self,testname,root,isRun,srcDict): 459 """ 460 Generate bash script using template found next to this file. 461 This file is read in at constructor time to avoid file I/O 462 """ 463 # runscript_dir directory has to be consistent with gmakefile 464 testDict=srcDict[testname] 465 rpath=self.srcrelpath(root) 466 runscript_dir=os.path.join(self.testroot_dir,rpath) 467 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 468 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 469 470 # Get variables to go into shell scripts. last time testDict used 471 subst=self.getSubstVars(testDict,rpath,testname) 472 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 473 #if '33_' in testname: print subst['subargs'] 474 475 #Handle runfiles 476 for lfile in subst.get('localrunfiles','').split(): 477 fullfile=os.path.join(root,lfile) 478 if os.path.isdir(fullfile): 479 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 480 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 481 else: 482 shutil.copy(fullfile,runscript_dir) 483 # Check subtests for local runfiles 484 for stest in subst.get("subtests",[]): 485 for lfile in testDict[stest].get('localrunfiles','').split(): 486 fullfile=os.path.join(root,lfile) 487 if os.path.isdir(fullfile): 488 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 489 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 490 else: 491 shutil.copy(fullfile,self.runscript_dir) 492 493 # Now substitute the key variables into the header and footer 494 header=self._substVars(subst,example_template.header) 495 # The header is done twice to enable @...@ in header 496 header=self._substVars(subst,header) 497 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 498 499 # Start writing the file 500 fh.write(header+"\n") 501 502 # If there is a TODO or a SKIP then we do it before writing out the 503 # rest of the command (which is useful for working on the test) 504 # SKIP and TODO can be for the source file or for the runs 505 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 506 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 507 508 j=0 # for indentation 509 510 if loopVars: 511 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 512 if (loopHead): fh.write(loopHead+"\n") 513 514 # Subtests are special 515 if 'subtests' in testDict: 516 substP=subst # Subtests can inherit args but be careful 517 k=0 # for label suffixes 518 for stest in testDict["subtests"]: 519 subst=substP.copy() 520 subst.update(testDict[stest]) 521 # nsize is special because it is usually overwritten 522 if 'nsize' in testDict[stest]: 523 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 524 else: 525 fh.write("nsize=1\n") 526 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 527 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 528 #if '10_9' in testname: print sLoopVars 529 if sLoopVars: 530 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 531 fh.write(sLoopHead+"\n") 532 fh.write(self.getCmds(subst,j)+"\n") 533 if sLoopVars: 534 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 535 fh.write(sLoopFoot+"\n") 536 else: 537 fh.write(self.getCmds(subst,j)+"\n") 538 539 if loopVars: 540 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 541 fh.write(loopFoot+"\n") 542 543 fh.write(footer+"\n") 544 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 545 #if '10_9' in testname: sys.exit() 546 return 547 548 def genScriptsAndInfo(self,exfile,root,srcDict): 549 """ 550 Generate scripts from the source file, determine if built, etc. 551 For every test in the exfile with info in the srcDict: 552 1. Determine if it needs to be run for this arch 553 2. Generate the script 554 3. Generate the data needed to write out the makefile in a 555 convenient way 556 All tests are *always* run, but some may be SKIP'd per the TAP standard 557 """ 558 debug=False 559 execname=self.getExecname(exfile,root) 560 isBuilt=self._isBuilt(exfile,srcDict) 561 for test in srcDict: 562 if test in self.buildkeys: continue 563 if debug: print self.nameSpace(exfile,root), test 564 srcDict[test]['execname']=execname # Convenience in generating scripts 565 isRun=self._isRun(srcDict[test]) 566 self.genRunScript(test,root,isRun,srcDict) 567 srcDict[test]['isrun']=isRun 568 self.addToTests(test,root,exfile,execname,srcDict[test]) 569 570 # This adds to datastructure for building deps 571 if isBuilt: self.addToSources(exfile,root,srcDict) 572 return 573 574 def _isBuilt(self,exfile,srcDict): 575 """ 576 Determine if this file should be built. 577 """ 578 # Get the language based on file extension 579 srcDict['SKIP'] = [] 580 lang=self.getLanguage(exfile) 581 if (lang=="F" or lang=="F90"): 582 if not self.have_fortran: 583 srcDict["SKIP"].append("Fortran required for this test") 584 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 585 srcDict["SKIP"].append("Fortran f90freeform required for this test") 586 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 587 srcDict["SKIP"].append("CUDA required for this test") 588 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 589 srcDict["SKIP"].append("C++ required for this test") 590 591 # Deprecated source files 592 if srcDict.get("TODO"): 593 return False 594 595 # isRun can work with srcDict to handle the requires 596 if "requires" in srcDict: 597 if srcDict["requires"]: 598 return self._isRun(srcDict) 599 600 return srcDict['SKIP'] == [] 601 602 603 def _isRun(self,testDict): 604 """ 605 Based on the requirements listed in the src file and the petscconf.h 606 info, determine whether this test should be run or not. 607 """ 608 indent=" " 609 debug=False 610 611 if 'SKIP' not in testDict: 612 testDict['SKIP'] = [] 613 # MPI requirements 614 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 615 if debug: print indent+"Cannot run parallel tests" 616 testDict['SKIP'].append("Parallel test with serial build") 617 618 # The requirements for the test are the sum of all the run subtests 619 if 'subtests' in testDict: 620 if 'requires' not in testDict: testDict['requires']="" 621 for stest in testDict['subtests']: 622 if 'requires' in testDict[stest]: 623 testDict['requires']+=" "+testDict[stest]['requires'] 624 625 626 # Now go through all requirements 627 if 'requires' in testDict: 628 for requirement in testDict['requires'].split(): 629 requirement=requirement.strip() 630 if not requirement: continue 631 if debug: print indent+"Requirement: ", requirement 632 isNull=False 633 if requirement.startswith("!"): 634 requirement=requirement[1:]; isNull=True 635 # Precision requirement for reals 636 if requirement in self.precision_types: 637 if self.conf['PETSC_PRECISION']==requirement: 638 if isNull: 639 testDict['SKIP'].append("not "+requirement+" required") 640 continue 641 continue # Success 642 elif not isNull: 643 testDict['SKIP'].append(requirement+" required") 644 continue 645 # Precision requirement for ints 646 if requirement in self.integer_types: 647 if requirement=="int32": 648 if self.conf['PETSC_SIZEOF_INT']==4: 649 if isNull: 650 testDict['SKIP'].append("not int32 required") 651 continue 652 continue # Success 653 elif not isNull: 654 testDict['SKIP'].append("int32 required") 655 continue 656 if requirement=="int64": 657 if self.conf['PETSC_SIZEOF_INT']==8: 658 if isNull: 659 testDict['SKIP'].append("NOT int64 required") 660 continue 661 continue # Success 662 elif not isNull: 663 testDict['SKIP'].append("int64 required") 664 continue 665 # Datafilespath 666 if requirement=="datafilespath" and not isNull: 667 testDict['SKIP'].append("Requires DATAFILESPATH") 668 continue 669 # Defines -- not sure I have comments matching 670 if "define(" in requirement.lower(): 671 reqdef=requirement.split("(")[1].split(")")[0] 672 if reqdef in self.conf: 673 if isNull: 674 testDict['SKIP'].append("Null requirement not met: "+requirement) 675 continue 676 continue # Success 677 elif not isNull: 678 testDict['SKIP'].append("Required: "+requirement) 679 continue 680 681 # Rest should be packages that we can just get from conf 682 if requirement == "complex": 683 petscconfvar="PETSC_USE_COMPLEX" 684 else: 685 petscconfvar="PETSC_HAVE_"+requirement.upper() 686 if self.conf.get(petscconfvar): 687 if isNull: 688 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 689 continue 690 continue # Success 691 elif not isNull: 692 if debug: print "requirement not found: ", requirement 693 testDict['SKIP'].append(petscconfvar+" requirement not met") 694 continue 695 696 return testDict['SKIP'] == [] 697 698 def genPetscTests_summarize(self,dataDict): 699 """ 700 Required method to state what happened 701 """ 702 if not self.summarize: return 703 indent=" " 704 print(self.testroot_dir, os.path.realpath(os.curdir)) 705 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 706 fh=open(fhname,"w") 707 #print "See ", fhname 708 for root in dataDict: 709 relroot=self.srcrelpath(root) 710 pkg=relroot.split("/")[1] 711 fh.write(relroot+"\n") 712 allSrcs=[] 713 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 714 for exfile in dataDict[root]: 715 # Basic information 716 rfile=os.path.join(relroot,exfile) 717 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 718 fh.write(indent+exfile+indent*4+builtStatus+"\n") 719 720 for test in dataDict[root][exfile]: 721 if test in self.buildkeys: continue 722 line=indent*2+test 723 fh.write(line+"\n") 724 # Looks nice to have the keys in order 725 #for key in dataDict[root][exfile][test]: 726 for key in "isrun abstracted nsize args requires script".split(): 727 if key not in dataDict[root][exfile][test]: continue 728 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 729 fh.write(line+"\n") 730 fh.write("\n") 731 fh.write("\n") 732 fh.write("\n") 733 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 734 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 735 fh.close() 736 return 737 738 def genPetscTests(self,root,dirs,files,dataDict): 739 """ 740 Go through and parse the source files in the directory to generate 741 the examples based on the metadata contained in the source files 742 """ 743 debug=False 744 # Use examplesAnalyze to get what the makefles think are sources 745 #self.examplesAnalyze(root,dirs,files,anlzDict) 746 747 dataDict[root]={} 748 749 for exfile in files: 750 #TST: Until we replace files, still leaving the orginals as is 751 #if not exfile.startswith("new_"+"ex"): continue 752 #if not exfile.startswith("ex"): continue 753 754 # Ignore emacs files 755 if exfile.startswith("#") or exfile.startswith(".#"): continue 756 757 # Convenience 758 fullex=os.path.join(root,exfile) 759 if self.verbose: print(' --> '+fullex) 760 dataDict[root].update(testparse.parseTestFile(fullex,0)) 761 if exfile in dataDict[root]: 762 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 763 764 return 765 766 def walktree(self,top): 767 """ 768 Walk a directory tree, starting from 'top' 769 """ 770 #print "action", action 771 # Goal of action is to fill this dictionary 772 dataDict={} 773 for root, dirs, files in os.walk(top, topdown=False): 774 if not "examples" in root: continue 775 if not os.path.isfile(os.path.join(root,"makefile")): continue 776 if self.verbose: print(root) 777 bname=os.path.basename(root.rstrip("/")) 778 self.genPetscTests(root,dirs,files,dataDict) 779 # Now summarize this dictionary 780 self.genPetscTests_summarize(dataDict) 781 return dataDict 782 783 def gen_gnumake(self, fd): 784 """ 785 Overwrite of the method in the base PETSc class 786 """ 787 def write(stem, srcs): 788 for lang in LANGS: 789 if srcs[lang]['srcs']: 790 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 791 for pkg in PKGS: 792 srcs = self.gen_pkg(pkg) 793 write('testsrcs-' + pkg, srcs) 794 # Handle dependencies 795 for lang in LANGS: 796 for exfile in srcs[lang]['srcs']: 797 if exfile in srcs[lang]: 798 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 799 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 800 for dep in srcs[lang][exfile]: 801 fd.write(exfile+": $(TESTDIR)/"+ dep+'\n') 802 fd.write(ex +": "+exfo+" $(TESTDIR)/"+dep +'\n') 803 804 return self.gendeps 805 806 def gen_pkg(self, pkg): 807 """ 808 Overwrite of the method in the base PETSc class 809 """ 810 return self.sources[pkg] 811 812 def write_gnumake(self,dataDict): 813 """ 814 Write out something similar to files from gmakegen.py 815 816 Test depends on script which also depends on source 817 file, but since I don't have a good way generating 818 acting on a single file (oops) just depend on 819 executable which in turn will depend on src file 820 """ 821 # Different options for how to set up the targets 822 compileExecsFirst=False 823 824 # Open file 825 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 826 fd = open(arch_files, 'w') 827 828 # Write out the sources 829 gendeps = self.gen_gnumake(fd) 830 831 # Write out the tests and execname targets 832 fd.write("\n#Tests and executables\n") # Delimiter 833 834 for pkg in PKGS: 835 # These grab the ones that are built 836 for lang in LANGS: 837 testdeps=[] 838 for ftest in self.tests[pkg][lang]: 839 test=os.path.basename(ftest) 840 basedir=os.path.dirname(ftest) 841 testdeps.append(self.nameSpace(test,basedir)) 842 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 843 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 844 845 # test targets 846 for ftest in self.tests[pkg][lang]: 847 test=os.path.basename(ftest) 848 basedir=os.path.dirname(ftest) 849 testdir="${TESTDIR}/"+basedir+"/" 850 nmtest=self.nameSpace(test,basedir) 851 rundir=os.path.join(testdir,test) 852 #print test, nmtest 853 script=test+".sh" 854 855 # Deps 856 exfile=self.tests[pkg][lang][ftest]['exfile'] 857 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 858 localexec=self.tests[pkg][lang][ftest]['exec'] 859 execname=os.path.join(testdir,localexec) 860 fullscript=os.path.join(testdir,script) 861 tmpfile=os.path.join(testdir,test,test+".tmp") 862 863 # *.counts depends on the script and either executable (will 864 # be run) or the example source file (SKIP or TODO) 865 fd.write('%s.counts : %s %s' 866 % (os.path.join('$(TESTDIR)/counts', nmtest), 867 fullscript, 868 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 869 ) 870 if exfile in self.sources[pkg][lang]: 871 for dep in self.sources[pkg][lang][exfile]: 872 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 873 fd.write('\n') 874 875 # Now write the args: 876 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 877 878 fd.close() 879 return 880 881 def writeHarness(self,output,dataDict): 882 """ 883 This is set up to write out multiple harness even if only gnumake 884 is supported now 885 """ 886 eval("self.write_"+output+"(dataDict)") 887 return 888 889def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 890 if output is None: 891 output = 'gnumake' 892 893 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 894 if petsc_arch: 895 if len(petsc_arch.split(os.path.sep))>1: 896 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 897 898 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 899 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 900 testdir=testdir) 901 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 902 pEx.writeHarness(output,dataDict) 903 904if __name__ == '__main__': 905 import optparse 906 parser = optparse.OptionParser() 907 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 908 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 909 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 910 parser.add_option('--output', help='Location to write output file', default=None) 911 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 912 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 913 opts, extra_args = parser.parse_args() 914 opts, extra_args = parser.parse_args() 915 if extra_args: 916 import sys 917 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 918 exit(1) 919 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 920 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 921