1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77class generateExamples(Petsc): 78 """ 79 gmakegen.py has basic structure for finding the files, writing out 80 the dependencies, etc. 81 """ 82 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 83 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 84 85 self.single_ex=single_ex 86 87 # Set locations to handle movement 88 self.inInstallDir=self.getInInstallDir(thisscriptdir) 89 90 if self.inInstallDir: 91 # Case 2 discussed above 92 # set PETSC_ARCH to install directory to allow script to work in both 93 dirlist=thisscriptdir.split(os.path.sep) 94 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 95 self.arch_dir=installdir 96 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 97 else: 98 if petsc_arch == '': 99 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 100 # Case 1 discussed above 101 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 102 self.srcdir=os.path.join(self.petsc_dir,'src') 103 104 self.testroot_dir=os.path.abspath(testdir) 105 106 self.ptNaming=True 107 self.verbose=verbose 108 # Whether to write out a useful debugging 109 self.summarize=True if verbose else False 110 111 # For help in setting the requirements 112 self.precision_types="single double __float128 int32".split() 113 self.integer_types="int32 int64".split() 114 self.languages="fortran cuda cxx".split() # Always requires C so do not list 115 116 # Things that are not test 117 self.buildkeys=testparse.buildkeys 118 119 # Adding a dictionary for storing sources, objects, and tests 120 # to make building the dependency tree easier 121 self.sources={} 122 self.objects={} 123 self.tests={} 124 for pkg in PKGS: 125 self.sources[pkg]={} 126 self.objects[pkg]=[] 127 self.tests[pkg]={} 128 for lang in LANGS: 129 self.sources[pkg][lang]={} 130 self.sources[pkg][lang]['srcs']=[] 131 self.tests[pkg][lang]={} 132 133 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 134 135 self.indent=" " 136 if self.verbose: print('Finishing the constructor') 137 return 138 139 def srcrelpath(self,rdir): 140 """ 141 Get relative path to source directory 142 """ 143 return os.path.relpath(rdir,self.srcdir) 144 145 def getInInstallDir(self,thisscriptdir): 146 """ 147 When petsc is installed then this file in installed in: 148 <PREFIX>/share/petsc/examples/config/gmakegentest.py 149 otherwise the path is: 150 <PETSC_DIR>/config/gmakegentest.py 151 We use this difference to determine if we are in installdir 152 """ 153 dirlist=thisscriptdir.split(os.path.sep) 154 if len(dirlist)>4: 155 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 156 if lastfour==os.path.join('share','petsc','examples','config'): 157 return True 158 else: 159 return False 160 else: 161 return False 162 163 def nameSpace(self,srcfile,srcdir): 164 """ 165 Because the scripts have a non-unique naming, the pretty-printing 166 needs to convey the srcdir and srcfile. There are two ways of doing this. 167 """ 168 if self.ptNaming: 169 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 170 cdir=srcdir 171 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 172 nameString=prefix+srcfile 173 else: 174 #nameString=srcdir+": "+srcfile 175 nameString=srcfile 176 return nameString 177 178 def getLanguage(self,srcfile): 179 """ 180 Based on the source, determine associated language as found in gmakegen.LANGS 181 Can we just return srcext[1:\] now? 182 """ 183 langReq=None 184 srcext=os.path.splitext(srcfile)[-1] 185 if srcext in ".F90".split(): langReq="F90" 186 if srcext in ".F".split(): langReq="F" 187 if srcext in ".cxx".split(): langReq="cxx" 188 if srcext == ".cu": langReq="cu" 189 if srcext == ".c": langReq="c" 190 #if not langReq: print "ERROR: ", srcext, srcfile 191 return langReq 192 193 def _getLoopVars(self,inDict,testname, isSubtest=False): 194 """ 195 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 196 Return: 197 inDict['args']: -ksp_monitor 198 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 199 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 200 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 201 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 202 subst should be passed in instead of inDict 203 """ 204 loopVars={}; newargs="" 205 lkeys=inDict.keys() 206 lsuffix='_' 207 argregex=re.compile('(?<![a-zA-Z])-(?=[a-zA-Z])') 208 from testparse import parseLoopArgs 209 for key in lkeys: 210 if type(inDict[key])!=types.StringType: continue 211 keystr = str(inDict[key]) 212 akey=('subargs' if key=='args' else key) # what to assign 213 if akey not in inDict: inDict[akey]='' 214 varlist=[] 215 for varset in argregex.split(keystr): 216 if not varset.strip(): continue 217 if '{{' in varset: 218 keyvar,lvars,ftype=parseLoopArgs(varset) 219 if akey not in loopVars: loopVars[akey]={} 220 varlist.append(keyvar) 221 loopVars[akey][keyvar]=[keyvar,lvars] 222 if akey=='nsize': 223 inDict[akey] = '${' + keyvar + '}' 224 lsuffix+=akey+'-'+inDict[akey]+'_' 225 else: 226 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 227 lsuffix+=keyvar+'-${' + keyvar + '}_' 228 else: 229 if key=='args': newargs+=" -"+varset.strip() 230 if varlist: loopVars[akey]['varlist']=varlist 231 232 233 # For subtests, args are always substituted in (not top level) 234 if isSubtest: 235 inDict['subargs']+=" "+newargs.strip() 236 inDict['args']='' 237 if 'label_suffix' in inDict: 238 inDict['label_suffix']+=lsuffix.rstrip('_') 239 else: 240 inDict['label_suffix']=lsuffix.rstrip('_') 241 else: 242 if loopVars.keys(): 243 inDict['args']=newargs.strip() 244 inDict['label_suffix']=lsuffix.rstrip('_') 245 if loopVars.keys(): 246 return loopVars 247 else: 248 return None 249 250 def getArgLabel(self,testDict): 251 """ 252 In all of the arguments in the test dictionary, create a simple 253 string for searching within the makefile system. For simplicity in 254 search, remove "-", for strings, etc. 255 Also, concatenate the arg commands 256 For now, ignore nsize -- seems hard to search for anyway 257 """ 258 # Collect all of the args associated with a test 259 argStr=("" if 'args' not in testDict else testDict['args']) 260 if 'subtests' in testDict: 261 for stest in testDict["subtests"]: 262 sd=testDict[stest] 263 argStr=argStr+("" if 'args' not in sd else sd['args']) 264 265 # Now go through and cleanup 266 argStr=re.sub('{{(.*?)}}',"",argStr) 267 argStr=re.sub('-'," ",argStr) 268 for digit in string.digits: argStr=re.sub(digit," ",argStr) 269 argStr=re.sub("\.","",argStr) 270 argStr=re.sub(",","",argStr) 271 argStr=re.sub('\+',' ',argStr) 272 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 273 return argStr.strip() 274 275 def addToSources(self,exfile,root,srcDict): 276 """ 277 Put into data structure that allows easy generation of makefile 278 """ 279 rpath=self.srcrelpath(root) 280 pkg=rpath.split(os.path.sep)[0] 281 relpfile=os.path.join(rpath,exfile) 282 lang=self.getLanguage(exfile) 283 if not lang: return 284 self.sources[pkg][lang]['srcs'].append(relpfile) 285 self.sources[pkg][lang][relpfile] = [] 286 if 'depends' in srcDict: 287 depSrcList=srcDict['depends'].split() 288 for depSrc in depSrcList: 289 depObj=os.path.splitext(depSrc)[0]+".o" 290 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 291 292 # In gmakefile, ${TESTDIR} var specifies the object compilation 293 testsdir=self.srcrelpath(root)+"/" 294 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 295 self.objects[pkg].append(objfile) 296 return 297 298 def addToTests(self,test,root,exfile,execname,testDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 Organized by languages to allow testing of languages 302 """ 303 rpath=self.srcrelpath(root) 304 pkg=rpath.split("/")[0] 305 #nmtest=self.nameSpace(test,root) 306 nmtest=os.path.join(rpath,test) 307 lang=self.getLanguage(exfile) 308 if not lang: return 309 self.tests[pkg][lang][nmtest]={} 310 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 311 self.tests[pkg][lang][nmtest]['exec']=execname 312 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 313 return 314 315 def getExecname(self,exfile,root): 316 """ 317 Generate bash script using template found next to this file. 318 This file is read in at constructor time to avoid file I/O 319 """ 320 rpath=self.srcrelpath(root) 321 if self.single_ex: 322 execname=rpath.split("/")[1]+"-ex" 323 else: 324 execname=os.path.splitext(exfile)[0] 325 return execname 326 327 def getSubstVars(self,testDict,rpath,testname): 328 """ 329 Create a dictionary with all of the variables that get substituted 330 into the template commands found in example_template.py 331 """ 332 subst={} 333 334 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 335 if 'nsize' not in testDict: testDict['nsize']=1 336 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 337 for ak in testparse.acceptedkeys: 338 if ak=='test': continue 339 subst[ak]=(testDict[ak] if ak in testDict else '') 340 341 # Now do other variables 342 subst['execname']=testDict['execname'] 343 if 'filter' in testDict: 344 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 345 346 # Others 347 subst['subargs']='' # Default. For variables override 348 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 349 subst['label_suffix']='' 350 subst['comments']="\n#".join(subst['comments'].split("\n")) 351 if subst['comments']: subst['comments']="#"+subst['comments'] 352 subst['exec']="../"+subst['execname'] 353 subst['testroot']=self.testroot_dir 354 subst['testname']=testname 355 dp = self.conf.get('DATAFILESPATH','') 356 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 357 358 # This is used to label some matrices 359 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 360 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 361 362 # These can have for loops and are treated separately later 363 subst['nsize']=str(subst['nsize']) 364 365 #Conf vars 366 if self.petsc_arch.find('valgrind')>=0: 367 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 368 else: 369 subst['mpiexec']=self.conf['MPIEXEC'] 370 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 371 subst['petsc_arch']=self.petsc_arch 372 if self.inInstallDir: 373 # Case 2 374 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 375 else: 376 # Case 1 377 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 378 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 379 subst['diff']=self.conf['DIFF'] 380 subst['rm']=self.conf['RM'] 381 subst['grep']=self.conf['GREP'] 382 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 383 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 384 385 # Output file is special because of subtests override 386 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 387 if not "_" in defroot: defroot=defroot+"_1" 388 subst['defroot']=defroot 389 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 390 subst['redirect_file']=defroot+".tmp" 391 if 'output_file' not in testDict: 392 subst['output_file']="output/"+defroot+".out" 393 # Add in the full path here. 394 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 395 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 396 if not subst['TODO']: 397 print "Warning: "+subst['output_file']+" not found." 398 # Worry about alt files here -- see 399 # src/snes/examples/tutorials/output/ex22*.out 400 altlist=[subst['output_file']] 401 basefile,ext = os.path.splitext(subst['output_file']) 402 for i in range(1,9): 403 altroot=basefile+"_alt" 404 if i > 1: altroot=altroot+"_"+str(i) 405 af=altroot+".out" 406 srcaf=os.path.join(subst['srcdir'],af) 407 fullaf=os.path.join(self.petsc_dir,srcaf) 408 if os.path.isfile(fullaf): altlist.append(srcaf) 409 if len(altlist)>1: subst['altfiles']=altlist 410 #if len(altlist)>1: print "Found alt files: ",altlist 411 412 return subst 413 414 def getCmds(self,subst,i): 415 """ 416 Generate bash script using template found next to this file. 417 This file is read in at constructor time to avoid file I/O 418 """ 419 nindnt=i # the start and has to be consistent with below 420 cmdindnt=self.indent*nindnt 421 cmdLines="" 422 423 # MPI is the default -- but we have a few odd commands 424 if not subst['command']: 425 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 426 else: 427 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 428 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 429 430 cmdLines+=cmdindnt+'if test $res = 0; then\n' 431 diffindnt=self.indent*(nindnt+1) 432 if not subst['filter_output']: 433 if 'altfiles' not in subst: 434 cmd=diffindnt+self._substVars(subst,example_template.difftest) 435 else: 436 # Have to do it by hand a bit because of variable number of alt files 437 rf=subst['redirect_file'] 438 cmd=diffindnt+example_template.difftest.split('@')[0] 439 for i in range(len(subst['altfiles'])): 440 af=subst['altfiles'][i] 441 cmd+=af+' '+rf 442 if i!=len(subst['altfiles'])-1: 443 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 444 cmd+=' || ${diff_exe} ' 445 else: 446 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 447 cmd+=subst['label_suffix']+' ""' # Quotes are painful 448 else: 449 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 450 cmdLines+=cmd+"\n" 451 cmdLines+=cmdindnt+'else\n' 452 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 453 cmdLines+=cmdindnt+'fi\n' 454 return cmdLines 455 456 def _substVars(self,subst,origStr): 457 """ 458 Substitute variables 459 """ 460 Str=origStr 461 for subkey in subst: 462 if type(subst[subkey])!=types.StringType: continue 463 patt="@"+subkey.upper()+"@" 464 Str=re.sub(patt,subst[subkey],Str) 465 return Str 466 467 def _writeTodoSkip(self,fh,tors,reasons,footer): 468 """ 469 Write out the TODO and SKIP lines in the file 470 The TODO or SKIP variable, tors, should be lower case 471 """ 472 TORS=tors.upper() 473 template=eval("example_template."+tors+"line") 474 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 475 tab = '' 476 if reasons: 477 fh.write('if ! $force; then\n') 478 tab = tab + ' ' 479 if reasons == ["Requires DATAFILESPATH"]: 480 # The only reason not to run is DATAFILESPATH, which we check at run-time 481 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 482 tab = tab + ' ' 483 if reasons: 484 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 485 fh.write(tab+footer+"\n") 486 fh.write(tab+"exit\n") 487 if reasons == ["Requires DATAFILESPATH"]: 488 fh.write(' fi\n') 489 if reasons: 490 fh.write('fi\n') 491 fh.write('\n\n') 492 return 493 494 def getLoopVarsHead(self,loopVars,i): 495 """ 496 Generate a nicely indented string with the format loops 497 Here is what the data structure looks like 498 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 499 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 500 loopVars['subargs']['pc_type']=["j","cholesky sor"] 501 """ 502 outstr=''; indnt=self.indent 503 for key in loopVars: 504 for var in loopVars[key]['varlist']: 505 varval=loopVars[key][var] 506 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 507 i = i + 1 508 return (outstr,i) 509 510 def getLoopVarsFoot(self,loopVars,i): 511 outstr=''; indnt=self.indent 512 for key in loopVars: 513 for var in loopVars[key]['varlist']: 514 i = i - 1 515 outstr += indnt * i + "done\n" 516 return (outstr,i) 517 518 def genRunScript(self,testname,root,isRun,srcDict): 519 """ 520 Generate bash script using template found next to this file. 521 This file is read in at constructor time to avoid file I/O 522 """ 523 # runscript_dir directory has to be consistent with gmakefile 524 testDict=srcDict[testname] 525 rpath=self.srcrelpath(root) 526 runscript_dir=os.path.join(self.testroot_dir,rpath) 527 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 528 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 529 530 # Get variables to go into shell scripts. last time testDict used 531 subst=self.getSubstVars(testDict,rpath,testname) 532 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 533 #if '33_' in testname: print subst['subargs'] 534 535 #Handle runfiles 536 for lfile in subst.get('localrunfiles','').split(): 537 install_files(os.path.join(root, lfile), 538 os.path.join(runscript_dir, os.path.dirname(lfile))) 539 # Check subtests for local runfiles 540 for stest in subst.get("subtests",[]): 541 for lfile in testDict[stest].get('localrunfiles','').split(): 542 install_files(os.path.join(root, lfile), 543 os.path.join(runscript_dir, os.path.dirname(lfile))) 544 545 # Now substitute the key variables into the header and footer 546 header=self._substVars(subst,example_template.header) 547 # The header is done twice to enable @...@ in header 548 header=self._substVars(subst,header) 549 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 550 551 # Start writing the file 552 fh.write(header+"\n") 553 554 # If there is a TODO or a SKIP then we do it before writing out the 555 # rest of the command (which is useful for working on the test) 556 # SKIP and TODO can be for the source file or for the runs 557 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 558 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 559 560 j=0 # for indentation 561 562 if loopVars: 563 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 564 if (loopHead): fh.write(loopHead+"\n") 565 566 # Subtests are special 567 if 'subtests' in testDict: 568 substP=subst # Subtests can inherit args but be careful 569 k=0 # for label suffixes 570 for stest in testDict["subtests"]: 571 subst=substP.copy() 572 subst.update(testDict[stest]) 573 # nsize is special because it is usually overwritten 574 if 'nsize' in testDict[stest]: 575 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 576 else: 577 fh.write("nsize=1\n") 578 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 579 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 580 #if '10_9' in testname: print sLoopVars 581 if sLoopVars: 582 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 583 fh.write(sLoopHead+"\n") 584 fh.write(self.getCmds(subst,j)+"\n") 585 if sLoopVars: 586 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 587 fh.write(sLoopFoot+"\n") 588 else: 589 fh.write(self.getCmds(subst,j)+"\n") 590 591 if loopVars: 592 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 593 fh.write(loopFoot+"\n") 594 595 fh.write(footer+"\n") 596 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 597 #if '10_9' in testname: sys.exit() 598 return 599 600 def genScriptsAndInfo(self,exfile,root,srcDict): 601 """ 602 Generate scripts from the source file, determine if built, etc. 603 For every test in the exfile with info in the srcDict: 604 1. Determine if it needs to be run for this arch 605 2. Generate the script 606 3. Generate the data needed to write out the makefile in a 607 convenient way 608 All tests are *always* run, but some may be SKIP'd per the TAP standard 609 """ 610 debug=False 611 execname=self.getExecname(exfile,root) 612 isBuilt=self._isBuilt(exfile,srcDict) 613 for test in srcDict: 614 if test in self.buildkeys: continue 615 if debug: print self.nameSpace(exfile,root), test 616 srcDict[test]['execname']=execname # Convenience in generating scripts 617 isRun=self._isRun(srcDict[test]) 618 self.genRunScript(test,root,isRun,srcDict) 619 srcDict[test]['isrun']=isRun 620 self.addToTests(test,root,exfile,execname,srcDict[test]) 621 622 # This adds to datastructure for building deps 623 if isBuilt: self.addToSources(exfile,root,srcDict) 624 return 625 626 def _isBuilt(self,exfile,srcDict): 627 """ 628 Determine if this file should be built. 629 """ 630 # Get the language based on file extension 631 srcDict['SKIP'] = [] 632 lang=self.getLanguage(exfile) 633 if (lang=="F" or lang=="F90"): 634 if not self.have_fortran: 635 srcDict["SKIP"].append("Fortran required for this test") 636 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 637 srcDict["SKIP"].append("Fortran f90freeform required for this test") 638 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 639 srcDict["SKIP"].append("CUDA required for this test") 640 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 641 srcDict["SKIP"].append("C++ required for this test") 642 643 # Deprecated source files 644 if srcDict.get("TODO"): 645 return False 646 647 # isRun can work with srcDict to handle the requires 648 if "requires" in srcDict: 649 if srcDict["requires"]: 650 return self._isRun(srcDict) 651 652 return srcDict['SKIP'] == [] 653 654 655 def _isRun(self,testDict, debug=False): 656 """ 657 Based on the requirements listed in the src file and the petscconf.h 658 info, determine whether this test should be run or not. 659 """ 660 indent=" " 661 662 if 'SKIP' not in testDict: 663 testDict['SKIP'] = [] 664 # MPI requirements 665 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 666 if debug: print indent+"Cannot run parallel tests" 667 testDict['SKIP'].append("Parallel test with serial build") 668 669 # The requirements for the test are the sum of all the run subtests 670 if 'subtests' in testDict: 671 if 'requires' not in testDict: testDict['requires']="" 672 for stest in testDict['subtests']: 673 if 'requires' in testDict[stest]: 674 testDict['requires']+=" "+testDict[stest]['requires'] 675 if 'nsize' in testDict[stest]: 676 if testDict[stest].get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 677 testDict['SKIP'].append("Parallel test with serial build") 678 679 680 # Now go through all requirements 681 if 'requires' in testDict: 682 for requirement in testDict['requires'].split(): 683 requirement=requirement.strip() 684 if not requirement: continue 685 if debug: print indent+"Requirement: ", requirement 686 isNull=False 687 if requirement.startswith("!"): 688 requirement=requirement[1:]; isNull=True 689 # Precision requirement for reals 690 if requirement in self.precision_types: 691 if self.conf['PETSC_PRECISION']==requirement: 692 if isNull: 693 testDict['SKIP'].append("not "+requirement+" required") 694 continue 695 continue # Success 696 elif not isNull: 697 testDict['SKIP'].append(requirement+" required") 698 continue 699 # Precision requirement for ints 700 if requirement in self.integer_types: 701 if requirement=="int32": 702 if self.conf['PETSC_SIZEOF_INT']==4: 703 if isNull: 704 testDict['SKIP'].append("not int32 required") 705 continue 706 continue # Success 707 elif not isNull: 708 testDict['SKIP'].append("int32 required") 709 continue 710 if requirement=="int64": 711 if self.conf['PETSC_SIZEOF_INT']==8: 712 if isNull: 713 testDict['SKIP'].append("NOT int64 required") 714 continue 715 continue # Success 716 elif not isNull: 717 testDict['SKIP'].append("int64 required") 718 continue 719 # Datafilespath 720 if requirement=="datafilespath" and not isNull: 721 testDict['SKIP'].append("Requires DATAFILESPATH") 722 continue 723 # Defines -- not sure I have comments matching 724 if "define(" in requirement.lower(): 725 reqdef=requirement.split("(")[1].split(")")[0] 726 if reqdef in self.conf: 727 if isNull: 728 testDict['SKIP'].append("Null requirement not met: "+requirement) 729 continue 730 continue # Success 731 elif not isNull: 732 testDict['SKIP'].append("Required: "+requirement) 733 continue 734 735 # Rest should be packages that we can just get from conf 736 if requirement == "complex": 737 petscconfvar="PETSC_USE_COMPLEX" 738 else: 739 petscconfvar="PETSC_HAVE_"+requirement.upper() 740 if self.conf.get(petscconfvar): 741 if isNull: 742 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 743 continue 744 continue # Success 745 elif not isNull: 746 if debug: print "requirement not found: ", requirement 747 testDict['SKIP'].append(petscconfvar+" requirement not met") 748 continue 749 750 return testDict['SKIP'] == [] 751 752 def genPetscTests_summarize(self,dataDict): 753 """ 754 Required method to state what happened 755 """ 756 if not self.summarize: return 757 indent=" " 758 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 759 fh=open(fhname,"w") 760 #print "See ", fhname 761 for root in dataDict: 762 relroot=self.srcrelpath(root) 763 pkg=relroot.split("/")[1] 764 fh.write(relroot+"\n") 765 allSrcs=[] 766 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 767 for exfile in dataDict[root]: 768 # Basic information 769 rfile=os.path.join(relroot,exfile) 770 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 771 fh.write(indent+exfile+indent*4+builtStatus+"\n") 772 773 for test in dataDict[root][exfile]: 774 if test in self.buildkeys: continue 775 line=indent*2+test 776 fh.write(line+"\n") 777 # Looks nice to have the keys in order 778 #for key in dataDict[root][exfile][test]: 779 for key in "isrun abstracted nsize args requires script".split(): 780 if key not in dataDict[root][exfile][test]: continue 781 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 782 fh.write(line+"\n") 783 fh.write("\n") 784 fh.write("\n") 785 fh.write("\n") 786 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 787 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 788 fh.close() 789 return 790 791 def genPetscTests(self,root,dirs,files,dataDict): 792 """ 793 Go through and parse the source files in the directory to generate 794 the examples based on the metadata contained in the source files 795 """ 796 debug=False 797 # Use examplesAnalyze to get what the makefles think are sources 798 #self.examplesAnalyze(root,dirs,files,anlzDict) 799 800 dataDict[root]={} 801 802 for exfile in files: 803 #TST: Until we replace files, still leaving the orginals as is 804 #if not exfile.startswith("new_"+"ex"): continue 805 #if not exfile.startswith("ex"): continue 806 807 # Ignore emacs and other temporary files 808 if exfile.startswith("."): continue 809 if exfile.startswith("#"): continue 810 if exfile.endswith("~"): continue 811 812 # Convenience 813 fullex=os.path.join(root,exfile) 814 if self.verbose: print(' --> '+fullex) 815 dataDict[root].update(testparse.parseTestFile(fullex,0)) 816 if exfile in dataDict[root]: 817 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 818 819 return 820 821 def walktree(self,top): 822 """ 823 Walk a directory tree, starting from 'top' 824 """ 825 #print "action", action 826 # Goal of action is to fill this dictionary 827 dataDict={} 828 for root, dirs, files in os.walk(top, topdown=True): 829 dirs.sort() 830 files.sort() 831 if not "examples" in root: continue 832 if "dSYM" in root: continue 833 if os.path.basename(root.rstrip("/")) == 'output': continue 834 if self.verbose: print(root) 835 self.genPetscTests(root,dirs,files,dataDict) 836 # Now summarize this dictionary 837 self.genPetscTests_summarize(dataDict) 838 return dataDict 839 840 def gen_gnumake(self, fd): 841 """ 842 Overwrite of the method in the base PETSc class 843 """ 844 def write(stem, srcs): 845 for lang in LANGS: 846 if srcs[lang]['srcs']: 847 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 848 for pkg in PKGS: 849 srcs = self.gen_pkg(pkg) 850 write('testsrcs-' + pkg, srcs) 851 # Handle dependencies 852 for lang in LANGS: 853 for exfile in srcs[lang]['srcs']: 854 if exfile in srcs[lang]: 855 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 856 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 857 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 858 if deps: 859 # The executable literally depends on the object file because it is linked 860 fd.write(ex +": " + " ".join(deps) +'\n') 861 # The object file containing 'main' does not normally depend on other object 862 # files, but it does when it includes their modules. This dependency is 863 # overly blunt and could be reduced to only depend on object files for 864 # modules that are used, like "*f90aux.o". 865 fd.write(exfo +": " + " ".join(deps) +'\n') 866 867 return self.gendeps 868 869 def gen_pkg(self, pkg): 870 """ 871 Overwrite of the method in the base PETSc class 872 """ 873 return self.sources[pkg] 874 875 def write_gnumake(self, dataDict, output=None): 876 """ 877 Write out something similar to files from gmakegen.py 878 879 Test depends on script which also depends on source 880 file, but since I don't have a good way generating 881 acting on a single file (oops) just depend on 882 executable which in turn will depend on src file 883 """ 884 # Different options for how to set up the targets 885 compileExecsFirst=False 886 887 # Open file 888 fd = open(output, 'w') 889 890 # Write out the sources 891 gendeps = self.gen_gnumake(fd) 892 893 # Write out the tests and execname targets 894 fd.write("\n#Tests and executables\n") # Delimiter 895 896 for pkg in PKGS: 897 # These grab the ones that are built 898 for lang in LANGS: 899 testdeps=[] 900 for ftest in self.tests[pkg][lang]: 901 test=os.path.basename(ftest) 902 basedir=os.path.dirname(ftest) 903 testdeps.append(self.nameSpace(test,basedir)) 904 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 905 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 906 907 # test targets 908 for ftest in self.tests[pkg][lang]: 909 test=os.path.basename(ftest) 910 basedir=os.path.dirname(ftest) 911 testdir="${TESTDIR}/"+basedir+"/" 912 nmtest=self.nameSpace(test,basedir) 913 rundir=os.path.join(testdir,test) 914 #print test, nmtest 915 script=test+".sh" 916 917 # Deps 918 exfile=self.tests[pkg][lang][ftest]['exfile'] 919 fullex=os.path.join(self.srcdir,exfile) 920 localexec=self.tests[pkg][lang][ftest]['exec'] 921 execname=os.path.join(testdir,localexec) 922 fullscript=os.path.join(testdir,script) 923 tmpfile=os.path.join(testdir,test,test+".tmp") 924 925 # *.counts depends on the script and either executable (will 926 # be run) or the example source file (SKIP or TODO) 927 fd.write('%s.counts : %s %s' 928 % (os.path.join('$(TESTDIR)/counts', nmtest), 929 fullscript, 930 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 931 ) 932 if exfile in self.sources[pkg][lang]: 933 for dep in self.sources[pkg][lang][exfile]: 934 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 935 fd.write('\n') 936 937 # Now write the args: 938 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 939 940 fd.close() 941 return 942 943def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 944 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 945 if petsc_arch: 946 if len(petsc_arch.split(os.path.sep))>1: 947 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 948 output = os.path.join(testdir, 'testfiles') 949 950 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 951 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 952 testdir=testdir) 953 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 954 pEx.write_gnumake(dataDict, output) 955 956if __name__ == '__main__': 957 import optparse 958 parser = optparse.OptionParser() 959 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 960 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 961 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 962 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 963 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 964 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 965 966 opts, extra_args = parser.parse_args() 967 if extra_args: 968 import sys 969 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 970 exit(1) 971 if opts.testdir is None: 972 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 973 974 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 975 verbose=opts.verbose, 976 single_ex=opts.single_executable, srcdir=opts.srcdir, 977 testdir=opts.testdir) 978