1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77class generateExamples(Petsc): 78 """ 79 gmakegen.py has basic structure for finding the files, writing out 80 the dependencies, etc. 81 """ 82 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 83 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 84 85 self.single_ex=single_ex 86 87 # Set locations to handle movement 88 self.inInstallDir=self.getInInstallDir(thisscriptdir) 89 90 if self.inInstallDir: 91 # Case 2 discussed above 92 # set PETSC_ARCH to install directory to allow script to work in both 93 dirlist=thisscriptdir.split(os.path.sep) 94 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 95 self.arch_dir=installdir 96 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 97 else: 98 if petsc_arch == '': 99 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 100 # Case 1 discussed above 101 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 102 self.srcdir=os.path.join(self.petsc_dir,'src') 103 104 self.testroot_dir=os.path.abspath(testdir) 105 106 self.ptNaming=True 107 self.verbose=verbose 108 # Whether to write out a useful debugging 109 self.summarize=True if verbose else False 110 111 # For help in setting the requirements 112 self.precision_types="single double __float128 int32".split() 113 self.integer_types="int32 int64".split() 114 self.languages="fortran cuda cxx".split() # Always requires C so do not list 115 116 # Things that are not test 117 self.buildkeys=testparse.buildkeys 118 119 # Adding a dictionary for storing sources, objects, and tests 120 # to make building the dependency tree easier 121 self.sources={} 122 self.objects={} 123 self.tests={} 124 for pkg in PKGS: 125 self.sources[pkg]={} 126 self.objects[pkg]=[] 127 self.tests[pkg]={} 128 for lang in LANGS: 129 self.sources[pkg][lang]={} 130 self.sources[pkg][lang]['srcs']=[] 131 self.tests[pkg][lang]={} 132 133 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 134 135 self.indent=" " 136 if self.verbose: print('Finishing the constructor') 137 return 138 139 def srcrelpath(self,rdir): 140 """ 141 Get relative path to source directory 142 """ 143 return os.path.relpath(rdir,self.srcdir) 144 145 def getInInstallDir(self,thisscriptdir): 146 """ 147 When petsc is installed then this file in installed in: 148 <PREFIX>/share/petsc/examples/config/gmakegentest.py 149 otherwise the path is: 150 <PETSC_DIR>/config/gmakegentest.py 151 We use this difference to determine if we are in installdir 152 """ 153 dirlist=thisscriptdir.split(os.path.sep) 154 if len(dirlist)>4: 155 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 156 if lastfour==os.path.join('share','petsc','examples','config'): 157 return True 158 else: 159 return False 160 else: 161 return False 162 163 def nameSpace(self,srcfile,srcdir): 164 """ 165 Because the scripts have a non-unique naming, the pretty-printing 166 needs to convey the srcdir and srcfile. There are two ways of doing this. 167 """ 168 if self.ptNaming: 169 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 170 cdir=srcdir 171 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 172 nameString=prefix+srcfile 173 else: 174 #nameString=srcdir+": "+srcfile 175 nameString=srcfile 176 return nameString 177 178 def getLanguage(self,srcfile): 179 """ 180 Based on the source, determine associated language as found in gmakegen.LANGS 181 Can we just return srcext[1:\] now? 182 """ 183 langReq=None 184 srcext=os.path.splitext(srcfile)[-1] 185 if srcext in ".F90".split(): langReq="F90" 186 if srcext in ".F".split(): langReq="F" 187 if srcext in ".cxx".split(): langReq="cxx" 188 if srcext == ".cu": langReq="cu" 189 if srcext == ".c": langReq="c" 190 #if not langReq: print "ERROR: ", srcext, srcfile 191 return langReq 192 193 def _getLoopVars(self,inDict,testname, isSubtest=False): 194 """ 195 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 196 Return: 197 inDict['args']: -ksp_monitor 198 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 199 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 200 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 201 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 202 subst should be passed in instead of inDict 203 """ 204 loopVars={}; newargs="" 205 lkeys=inDict.keys() 206 lsuffix='_' 207 from testparse import parseLoopArgs 208 for key in lkeys: 209 if type(inDict[key])!=types.StringType: continue 210 keystr = str(inDict[key]) 211 akey=('subargs' if key=='args' else key) # what to assign 212 if akey not in inDict: inDict[akey]='' 213 varlist=[] 214 for varset in re.split('-(?=[a-zA-Z])',keystr): 215 if not varset.strip(): continue 216 if '{{' in varset: 217 keyvar,lvars,ftype=parseLoopArgs(varset) 218 if akey not in loopVars: loopVars[akey]={} 219 varlist.append(keyvar) 220 loopVars[akey][keyvar]=[keyvar,lvars] 221 if akey=='nsize': 222 inDict[akey] = '${' + keyvar + '}' 223 lsuffix+=akey+'-'+inDict[akey]+'_' 224 else: 225 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 226 lsuffix+=keyvar+'-${' + keyvar + '}_' 227 else: 228 if key=='args': newargs+=" -"+varset.strip() 229 if varlist: loopVars[akey]['varlist']=varlist 230 231 232 # For subtests, args are always substituted in (not top level) 233 if isSubtest: 234 inDict['subargs']+=" "+newargs.strip() 235 inDict['args']='' 236 if 'label_suffix' in inDict: 237 inDict['label_suffix']+=lsuffix.rstrip('_') 238 else: 239 inDict['label_suffix']=lsuffix.rstrip('_') 240 else: 241 if loopVars.keys(): 242 inDict['args']=newargs.strip() 243 inDict['label_suffix']=lsuffix.rstrip('_') 244 if loopVars.keys(): 245 return loopVars 246 else: 247 return None 248 249 def getArgLabel(self,testDict): 250 """ 251 In all of the arguments in the test dictionary, create a simple 252 string for searching within the makefile system. For simplicity in 253 search, remove "-", for strings, etc. 254 Also, concatenate the arg commands 255 For now, ignore nsize -- seems hard to search for anyway 256 """ 257 # Collect all of the args associated with a test 258 argStr=("" if 'args' not in testDict else testDict['args']) 259 if 'subtests' in testDict: 260 for stest in testDict["subtests"]: 261 sd=testDict[stest] 262 argStr=argStr+("" if 'args' not in sd else sd['args']) 263 264 # Now go through and cleanup 265 argStr=re.sub('{{(.*?)}}',"",argStr) 266 argStr=re.sub('-'," ",argStr) 267 for digit in string.digits: argStr=re.sub(digit," ",argStr) 268 argStr=re.sub("\.","",argStr) 269 argStr=re.sub(",","",argStr) 270 argStr=re.sub('\+',' ',argStr) 271 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 272 return argStr.strip() 273 274 def addToSources(self,exfile,root,srcDict): 275 """ 276 Put into data structure that allows easy generation of makefile 277 """ 278 rpath=self.srcrelpath(root) 279 pkg=rpath.split(os.path.sep)[0] 280 relpfile=os.path.join(rpath,exfile) 281 lang=self.getLanguage(exfile) 282 if not lang: return 283 self.sources[pkg][lang]['srcs'].append(relpfile) 284 self.sources[pkg][lang][relpfile] = [] 285 if 'depends' in srcDict: 286 depSrcList=srcDict['depends'].split() 287 for depSrc in depSrcList: 288 depObj=os.path.splitext(depSrc)[0]+".o" 289 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 290 291 # In gmakefile, ${TESTDIR} var specifies the object compilation 292 testsdir=self.srcrelpath(root)+"/" 293 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 294 self.objects[pkg].append(objfile) 295 return 296 297 def addToTests(self,test,root,exfile,execname,testDict): 298 """ 299 Put into data structure that allows easy generation of makefile 300 Organized by languages to allow testing of languages 301 """ 302 rpath=self.srcrelpath(root) 303 pkg=rpath.split("/")[0] 304 #nmtest=self.nameSpace(test,root) 305 nmtest=os.path.join(rpath,test) 306 lang=self.getLanguage(exfile) 307 if not lang: return 308 self.tests[pkg][lang][nmtest]={} 309 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 310 self.tests[pkg][lang][nmtest]['exec']=execname 311 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 312 return 313 314 def getExecname(self,exfile,root): 315 """ 316 Generate bash script using template found next to this file. 317 This file is read in at constructor time to avoid file I/O 318 """ 319 rpath=self.srcrelpath(root) 320 if self.single_ex: 321 execname=rpath.split("/")[1]+"-ex" 322 else: 323 execname=os.path.splitext(exfile)[0] 324 return execname 325 326 def getSubstVars(self,testDict,rpath,testname): 327 """ 328 Create a dictionary with all of the variables that get substituted 329 into the template commands found in example_template.py 330 """ 331 subst={} 332 333 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 334 if 'nsize' not in testDict: testDict['nsize']=1 335 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 336 for ak in testparse.acceptedkeys: 337 if ak=='test': continue 338 subst[ak]=(testDict[ak] if ak in testDict else '') 339 340 # Now do other variables 341 subst['execname']=testDict['execname'] 342 if 'filter' in testDict: 343 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 344 345 # Others 346 subst['subargs']='' # Default. For variables override 347 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 348 subst['label_suffix']='' 349 subst['comments']="\n#".join(subst['comments'].split("\n")) 350 if subst['comments']: subst['comments']="#"+subst['comments'] 351 subst['exec']="../"+subst['execname'] 352 subst['testroot']=self.testroot_dir 353 subst['testname']=testname 354 dp = self.conf.get('DATAFILESPATH','') 355 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 356 357 # This is used to label some matrices 358 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 359 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 360 361 # These can have for loops and are treated separately later 362 subst['nsize']=str(subst['nsize']) 363 364 #Conf vars 365 if self.petsc_arch.find('valgrind')>=0: 366 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 367 else: 368 subst['mpiexec']=self.conf['MPIEXEC'] 369 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 370 subst['petsc_arch']=self.petsc_arch 371 if self.inInstallDir: 372 # Case 2 373 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 374 else: 375 # Case 1 376 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 377 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 378 subst['diff']=self.conf['DIFF'] 379 subst['rm']=self.conf['RM'] 380 subst['grep']=self.conf['GREP'] 381 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 382 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 383 384 # Output file is special because of subtests override 385 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 386 if not "_" in defroot: defroot=defroot+"_1" 387 subst['defroot']=defroot 388 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 389 subst['redirect_file']=defroot+".tmp" 390 if 'output_file' not in testDict: 391 subst['output_file']="output/"+defroot+".out" 392 # Add in the full path here. 393 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 394 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 395 if not subst['TODO']: 396 print "Warning: "+subst['output_file']+" not found." 397 # Worry about alt files here -- see 398 # src/snes/examples/tutorials/output/ex22*.out 399 altlist=[subst['output_file']] 400 basefile,ext = os.path.splitext(subst['output_file']) 401 for i in range(1,9): 402 altroot=basefile+"_alt" 403 if i > 1: altroot=altroot+"_"+str(i) 404 af=altroot+".out" 405 srcaf=os.path.join(subst['srcdir'],af) 406 fullaf=os.path.join(self.petsc_dir,srcaf) 407 if os.path.isfile(fullaf): altlist.append(srcaf) 408 if len(altlist)>1: subst['altfiles']=altlist 409 #if len(altlist)>1: print "Found alt files: ",altlist 410 411 return subst 412 413 def getCmds(self,subst,i): 414 """ 415 Generate bash script using template found next to this file. 416 This file is read in at constructor time to avoid file I/O 417 """ 418 nindnt=i # the start and has to be consistent with below 419 cmdindnt=self.indent*nindnt 420 cmdLines="" 421 422 # MPI is the default -- but we have a few odd commands 423 if not subst['command']: 424 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 425 else: 426 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 427 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 428 429 cmdLines+=cmdindnt+'if test $res = 0; then\n' 430 diffindnt=self.indent*(nindnt+1) 431 if not subst['filter_output']: 432 if 'altfiles' not in subst: 433 cmd=diffindnt+self._substVars(subst,example_template.difftest) 434 else: 435 # Have to do it by hand a bit because of variable number of alt files 436 rf=subst['redirect_file'] 437 cmd=diffindnt+example_template.difftest.split('@')[0] 438 for i in range(len(subst['altfiles'])): 439 af=subst['altfiles'][i] 440 cmd+=af+' '+rf 441 if i!=len(subst['altfiles'])-1: 442 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 443 cmd+=' || ${diff_exe} ' 444 else: 445 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 446 cmd+=subst['label_suffix']+' ""' # Quotes are painful 447 else: 448 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 449 cmdLines+=cmd+"\n" 450 cmdLines+=cmdindnt+'else\n' 451 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 452 cmdLines+=cmdindnt+'fi\n' 453 return cmdLines 454 455 def _substVars(self,subst,origStr): 456 """ 457 Substitute variables 458 """ 459 Str=origStr 460 for subkey in subst: 461 if type(subst[subkey])!=types.StringType: continue 462 patt="@"+subkey.upper()+"@" 463 Str=re.sub(patt,subst[subkey],Str) 464 return Str 465 466 def _writeTodoSkip(self,fh,tors,reasons,footer): 467 """ 468 Write out the TODO and SKIP lines in the file 469 The TODO or SKIP variable, tors, should be lower case 470 """ 471 TORS=tors.upper() 472 template=eval("example_template."+tors+"line") 473 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 474 tab = '' 475 if reasons: 476 fh.write('if ! $force; then\n') 477 tab = tab + ' ' 478 if reasons == ["Requires DATAFILESPATH"]: 479 # The only reason not to run is DATAFILESPATH, which we check at run-time 480 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 481 tab = tab + ' ' 482 if reasons: 483 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 484 fh.write(tab+footer+"\n") 485 fh.write(tab+"exit\n") 486 if reasons == ["Requires DATAFILESPATH"]: 487 fh.write(' fi\n') 488 if reasons: 489 fh.write('fi\n') 490 fh.write('\n\n') 491 return 492 493 def getLoopVarsHead(self,loopVars,i): 494 """ 495 Generate a nicely indented string with the format loops 496 Here is what the data structure looks like 497 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 498 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 499 loopVars['subargs']['pc_type']=["j","cholesky sor"] 500 """ 501 outstr=''; indnt=self.indent 502 for key in loopVars: 503 for var in loopVars[key]['varlist']: 504 varval=loopVars[key][var] 505 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 506 i = i + 1 507 return (outstr,i) 508 509 def getLoopVarsFoot(self,loopVars,i): 510 outstr=''; indnt=self.indent 511 for key in loopVars: 512 for var in loopVars[key]['varlist']: 513 i = i - 1 514 outstr += indnt * i + "done\n" 515 return (outstr,i) 516 517 def genRunScript(self,testname,root,isRun,srcDict): 518 """ 519 Generate bash script using template found next to this file. 520 This file is read in at constructor time to avoid file I/O 521 """ 522 # runscript_dir directory has to be consistent with gmakefile 523 testDict=srcDict[testname] 524 rpath=self.srcrelpath(root) 525 runscript_dir=os.path.join(self.testroot_dir,rpath) 526 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 527 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 528 529 # Get variables to go into shell scripts. last time testDict used 530 subst=self.getSubstVars(testDict,rpath,testname) 531 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 532 #if '33_' in testname: print subst['subargs'] 533 534 #Handle runfiles 535 for lfile in subst.get('localrunfiles','').split(): 536 install_files(os.path.join(root, lfile), 537 os.path.join(runscript_dir, os.path.dirname(lfile))) 538 # Check subtests for local runfiles 539 for stest in subst.get("subtests",[]): 540 for lfile in testDict[stest].get('localrunfiles','').split(): 541 install_files(os.path.join(root, lfile), 542 os.path.join(runscript_dir, os.path.dirname(lfile))) 543 544 # Now substitute the key variables into the header and footer 545 header=self._substVars(subst,example_template.header) 546 # The header is done twice to enable @...@ in header 547 header=self._substVars(subst,header) 548 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 549 550 # Start writing the file 551 fh.write(header+"\n") 552 553 # If there is a TODO or a SKIP then we do it before writing out the 554 # rest of the command (which is useful for working on the test) 555 # SKIP and TODO can be for the source file or for the runs 556 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 557 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 558 559 j=0 # for indentation 560 561 if loopVars: 562 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 563 if (loopHead): fh.write(loopHead+"\n") 564 565 # Subtests are special 566 if 'subtests' in testDict: 567 substP=subst # Subtests can inherit args but be careful 568 k=0 # for label suffixes 569 for stest in testDict["subtests"]: 570 subst=substP.copy() 571 subst.update(testDict[stest]) 572 # nsize is special because it is usually overwritten 573 if 'nsize' in testDict[stest]: 574 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 575 else: 576 fh.write("nsize=1\n") 577 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 578 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 579 #if '10_9' in testname: print sLoopVars 580 if sLoopVars: 581 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 582 fh.write(sLoopHead+"\n") 583 fh.write(self.getCmds(subst,j)+"\n") 584 if sLoopVars: 585 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 586 fh.write(sLoopFoot+"\n") 587 else: 588 fh.write(self.getCmds(subst,j)+"\n") 589 590 if loopVars: 591 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 592 fh.write(loopFoot+"\n") 593 594 fh.write(footer+"\n") 595 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 596 #if '10_9' in testname: sys.exit() 597 return 598 599 def genScriptsAndInfo(self,exfile,root,srcDict): 600 """ 601 Generate scripts from the source file, determine if built, etc. 602 For every test in the exfile with info in the srcDict: 603 1. Determine if it needs to be run for this arch 604 2. Generate the script 605 3. Generate the data needed to write out the makefile in a 606 convenient way 607 All tests are *always* run, but some may be SKIP'd per the TAP standard 608 """ 609 debug=False 610 execname=self.getExecname(exfile,root) 611 isBuilt=self._isBuilt(exfile,srcDict) 612 for test in srcDict: 613 if test in self.buildkeys: continue 614 if debug: print self.nameSpace(exfile,root), test 615 srcDict[test]['execname']=execname # Convenience in generating scripts 616 isRun=self._isRun(srcDict[test]) 617 self.genRunScript(test,root,isRun,srcDict) 618 srcDict[test]['isrun']=isRun 619 self.addToTests(test,root,exfile,execname,srcDict[test]) 620 621 # This adds to datastructure for building deps 622 if isBuilt: self.addToSources(exfile,root,srcDict) 623 return 624 625 def _isBuilt(self,exfile,srcDict): 626 """ 627 Determine if this file should be built. 628 """ 629 # Get the language based on file extension 630 srcDict['SKIP'] = [] 631 lang=self.getLanguage(exfile) 632 if (lang=="F" or lang=="F90"): 633 if not self.have_fortran: 634 srcDict["SKIP"].append("Fortran required for this test") 635 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 636 srcDict["SKIP"].append("Fortran f90freeform required for this test") 637 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 638 srcDict["SKIP"].append("CUDA required for this test") 639 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 640 srcDict["SKIP"].append("C++ required for this test") 641 642 # Deprecated source files 643 if srcDict.get("TODO"): 644 return False 645 646 # isRun can work with srcDict to handle the requires 647 if "requires" in srcDict: 648 if srcDict["requires"]: 649 return self._isRun(srcDict) 650 651 return srcDict['SKIP'] == [] 652 653 654 def _isRun(self,testDict, debug=False): 655 """ 656 Based on the requirements listed in the src file and the petscconf.h 657 info, determine whether this test should be run or not. 658 """ 659 indent=" " 660 661 if 'SKIP' not in testDict: 662 testDict['SKIP'] = [] 663 # MPI requirements 664 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 665 if debug: print indent+"Cannot run parallel tests" 666 testDict['SKIP'].append("Parallel test with serial build") 667 668 # The requirements for the test are the sum of all the run subtests 669 if 'subtests' in testDict: 670 if 'requires' not in testDict: testDict['requires']="" 671 for stest in testDict['subtests']: 672 if 'requires' in testDict[stest]: 673 testDict['requires']+=" "+testDict[stest]['requires'] 674 if 'nsize' in testDict[stest]: 675 if testDict[stest].get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 676 testDict['SKIP'].append("Parallel test with serial build") 677 678 679 # Now go through all requirements 680 if 'requires' in testDict: 681 for requirement in testDict['requires'].split(): 682 requirement=requirement.strip() 683 if not requirement: continue 684 if debug: print indent+"Requirement: ", requirement 685 isNull=False 686 if requirement.startswith("!"): 687 requirement=requirement[1:]; isNull=True 688 # Precision requirement for reals 689 if requirement in self.precision_types: 690 if self.conf['PETSC_PRECISION']==requirement: 691 if isNull: 692 testDict['SKIP'].append("not "+requirement+" required") 693 continue 694 continue # Success 695 elif not isNull: 696 testDict['SKIP'].append(requirement+" required") 697 continue 698 # Precision requirement for ints 699 if requirement in self.integer_types: 700 if requirement=="int32": 701 if self.conf['PETSC_SIZEOF_INT']==4: 702 if isNull: 703 testDict['SKIP'].append("not int32 required") 704 continue 705 continue # Success 706 elif not isNull: 707 testDict['SKIP'].append("int32 required") 708 continue 709 if requirement=="int64": 710 if self.conf['PETSC_SIZEOF_INT']==8: 711 if isNull: 712 testDict['SKIP'].append("NOT int64 required") 713 continue 714 continue # Success 715 elif not isNull: 716 testDict['SKIP'].append("int64 required") 717 continue 718 # Datafilespath 719 if requirement=="datafilespath" and not isNull: 720 testDict['SKIP'].append("Requires DATAFILESPATH") 721 continue 722 # Defines -- not sure I have comments matching 723 if "define(" in requirement.lower(): 724 reqdef=requirement.split("(")[1].split(")")[0] 725 if reqdef in self.conf: 726 if isNull: 727 testDict['SKIP'].append("Null requirement not met: "+requirement) 728 continue 729 continue # Success 730 elif not isNull: 731 testDict['SKIP'].append("Required: "+requirement) 732 continue 733 734 # Rest should be packages that we can just get from conf 735 if requirement == "complex": 736 petscconfvar="PETSC_USE_COMPLEX" 737 else: 738 petscconfvar="PETSC_HAVE_"+requirement.upper() 739 if self.conf.get(petscconfvar): 740 if isNull: 741 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 742 continue 743 continue # Success 744 elif not isNull: 745 if debug: print "requirement not found: ", requirement 746 testDict['SKIP'].append(petscconfvar+" requirement not met") 747 continue 748 749 return testDict['SKIP'] == [] 750 751 def genPetscTests_summarize(self,dataDict): 752 """ 753 Required method to state what happened 754 """ 755 if not self.summarize: return 756 indent=" " 757 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 758 fh=open(fhname,"w") 759 #print "See ", fhname 760 for root in dataDict: 761 relroot=self.srcrelpath(root) 762 pkg=relroot.split("/")[1] 763 fh.write(relroot+"\n") 764 allSrcs=[] 765 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 766 for exfile in dataDict[root]: 767 # Basic information 768 rfile=os.path.join(relroot,exfile) 769 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 770 fh.write(indent+exfile+indent*4+builtStatus+"\n") 771 772 for test in dataDict[root][exfile]: 773 if test in self.buildkeys: continue 774 line=indent*2+test 775 fh.write(line+"\n") 776 # Looks nice to have the keys in order 777 #for key in dataDict[root][exfile][test]: 778 for key in "isrun abstracted nsize args requires script".split(): 779 if key not in dataDict[root][exfile][test]: continue 780 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 781 fh.write(line+"\n") 782 fh.write("\n") 783 fh.write("\n") 784 fh.write("\n") 785 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 786 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 787 fh.close() 788 return 789 790 def genPetscTests(self,root,dirs,files,dataDict): 791 """ 792 Go through and parse the source files in the directory to generate 793 the examples based on the metadata contained in the source files 794 """ 795 debug=False 796 # Use examplesAnalyze to get what the makefles think are sources 797 #self.examplesAnalyze(root,dirs,files,anlzDict) 798 799 dataDict[root]={} 800 801 for exfile in files: 802 #TST: Until we replace files, still leaving the orginals as is 803 #if not exfile.startswith("new_"+"ex"): continue 804 #if not exfile.startswith("ex"): continue 805 806 # Ignore emacs and other temporary files 807 if exfile.startswith("."): continue 808 if exfile.startswith("#"): continue 809 810 # Convenience 811 fullex=os.path.join(root,exfile) 812 if self.verbose: print(' --> '+fullex) 813 dataDict[root].update(testparse.parseTestFile(fullex,0)) 814 if exfile in dataDict[root]: 815 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 816 817 return 818 819 def walktree(self,top): 820 """ 821 Walk a directory tree, starting from 'top' 822 """ 823 #print "action", action 824 # Goal of action is to fill this dictionary 825 dataDict={} 826 for root, dirs, files in os.walk(top, topdown=True): 827 if not "examples" in root: continue 828 if "dSYM" in root: continue 829 if os.path.basename(root.rstrip("/")) == 'output': continue 830 if self.verbose: print(root) 831 self.genPetscTests(root,dirs,files,dataDict) 832 # Now summarize this dictionary 833 self.genPetscTests_summarize(dataDict) 834 return dataDict 835 836 def gen_gnumake(self, fd): 837 """ 838 Overwrite of the method in the base PETSc class 839 """ 840 def write(stem, srcs): 841 for lang in LANGS: 842 if srcs[lang]['srcs']: 843 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 844 for pkg in PKGS: 845 srcs = self.gen_pkg(pkg) 846 write('testsrcs-' + pkg, srcs) 847 # Handle dependencies 848 for lang in LANGS: 849 for exfile in srcs[lang]['srcs']: 850 if exfile in srcs[lang]: 851 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 852 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 853 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 854 if deps: 855 # The executable literally depends on the object file because it is linked 856 fd.write(ex +": " + " ".join(deps) +'\n') 857 # The object file containing 'main' does not normally depend on other object 858 # files, but it does when it includes their modules. This dependency is 859 # overly blunt and could be reduced to only depend on object files for 860 # modules that are used, like "*f90aux.o". 861 fd.write(exfo +": " + " ".join(deps) +'\n') 862 863 return self.gendeps 864 865 def gen_pkg(self, pkg): 866 """ 867 Overwrite of the method in the base PETSc class 868 """ 869 return self.sources[pkg] 870 871 def write_gnumake(self, dataDict, output=None): 872 """ 873 Write out something similar to files from gmakegen.py 874 875 Test depends on script which also depends on source 876 file, but since I don't have a good way generating 877 acting on a single file (oops) just depend on 878 executable which in turn will depend on src file 879 """ 880 # Different options for how to set up the targets 881 compileExecsFirst=False 882 883 # Open file 884 fd = open(output, 'w') 885 886 # Write out the sources 887 gendeps = self.gen_gnumake(fd) 888 889 # Write out the tests and execname targets 890 fd.write("\n#Tests and executables\n") # Delimiter 891 892 for pkg in PKGS: 893 # These grab the ones that are built 894 for lang in LANGS: 895 testdeps=[] 896 for ftest in self.tests[pkg][lang]: 897 test=os.path.basename(ftest) 898 basedir=os.path.dirname(ftest) 899 testdeps.append(self.nameSpace(test,basedir)) 900 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 901 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 902 903 # test targets 904 for ftest in self.tests[pkg][lang]: 905 test=os.path.basename(ftest) 906 basedir=os.path.dirname(ftest) 907 testdir="${TESTDIR}/"+basedir+"/" 908 nmtest=self.nameSpace(test,basedir) 909 rundir=os.path.join(testdir,test) 910 #print test, nmtest 911 script=test+".sh" 912 913 # Deps 914 exfile=self.tests[pkg][lang][ftest]['exfile'] 915 fullex=os.path.join(self.srcdir,exfile) 916 localexec=self.tests[pkg][lang][ftest]['exec'] 917 execname=os.path.join(testdir,localexec) 918 fullscript=os.path.join(testdir,script) 919 tmpfile=os.path.join(testdir,test,test+".tmp") 920 921 # *.counts depends on the script and either executable (will 922 # be run) or the example source file (SKIP or TODO) 923 fd.write('%s.counts : %s %s' 924 % (os.path.join('$(TESTDIR)/counts', nmtest), 925 fullscript, 926 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 927 ) 928 if exfile in self.sources[pkg][lang]: 929 for dep in self.sources[pkg][lang][exfile]: 930 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 931 fd.write('\n') 932 933 # Now write the args: 934 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 935 936 fd.close() 937 return 938 939def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 940 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 941 if petsc_arch: 942 if len(petsc_arch.split(os.path.sep))>1: 943 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 944 output = os.path.join(testdir, 'testfiles') 945 946 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 947 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 948 testdir=testdir) 949 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 950 pEx.write_gnumake(dataDict, output) 951 952if __name__ == '__main__': 953 import optparse 954 parser = optparse.OptionParser() 955 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 956 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 957 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 958 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 959 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 960 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 961 962 opts, extra_args = parser.parse_args() 963 if extra_args: 964 import sys 965 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 966 exit(1) 967 if opts.testdir is None: 968 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 969 970 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 971 verbose=opts.verbose, 972 single_ex=opts.single_executable, srcdir=opts.srcdir, 973 testdir=opts.testdir) 974