1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77class generateExamples(Petsc): 78 """ 79 gmakegen.py has basic structure for finding the files, writing out 80 the dependencies, etc. 81 """ 82 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 83 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 84 85 self.single_ex=single_ex 86 87 # Set locations to handle movement 88 self.inInstallDir=self.getInInstallDir(thisscriptdir) 89 90 if self.inInstallDir: 91 # Case 2 discussed above 92 # set PETSC_ARCH to install directory to allow script to work in both 93 dirlist=thisscriptdir.split(os.path.sep) 94 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 95 self.arch_dir=installdir 96 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 97 else: 98 if petsc_arch == '': 99 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 100 # Case 1 discussed above 101 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 102 self.srcdir=os.path.join(self.petsc_dir,'src') 103 104 # If full path given, then use it, otherwise assume relative to arch_dir 105 if testdir.strip().startswith(os.path.sep): 106 self.testroot_dir=testdir.strip() 107 else: 108 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 109 110 self.ptNaming=True 111 self.verbose=verbose 112 # Whether to write out a useful debugging 113 self.summarize=True if verbose else False 114 115 # For help in setting the requirements 116 self.precision_types="single double __float128 int32".split() 117 self.integer_types="int32 int64".split() 118 self.languages="fortran cuda cxx".split() # Always requires C so do not list 119 120 # Things that are not test 121 self.buildkeys=testparse.buildkeys 122 123 # Adding a dictionary for storing sources, objects, and tests 124 # to make building the dependency tree easier 125 self.sources={} 126 self.objects={} 127 self.tests={} 128 for pkg in PKGS: 129 self.sources[pkg]={} 130 self.objects[pkg]=[] 131 self.tests[pkg]={} 132 for lang in LANGS: 133 self.sources[pkg][lang]={} 134 self.sources[pkg][lang]['srcs']=[] 135 self.tests[pkg][lang]={} 136 137 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 138 139 self.indent=" " 140 if self.verbose: print('Finishing the constructor') 141 return 142 143 def srcrelpath(self,rdir): 144 """ 145 Get relative path to source directory 146 """ 147 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 148 149 def getInInstallDir(self,thisscriptdir): 150 """ 151 When petsc is installed then this file in installed in: 152 <PREFIX>/share/petsc/examples/config/gmakegentest.py 153 otherwise the path is: 154 <PETSC_DIR>/config/gmakegentest.py 155 We use this difference to determine if we are in installdir 156 """ 157 dirlist=thisscriptdir.split(os.path.sep) 158 if len(dirlist)>4: 159 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 160 if lastfour==os.path.join('share','petsc','examples','config'): 161 return True 162 else: 163 return False 164 else: 165 return False 166 167 def nameSpace(self,srcfile,srcdir): 168 """ 169 Because the scripts have a non-unique naming, the pretty-printing 170 needs to convey the srcdir and srcfile. There are two ways of doing this. 171 """ 172 if self.ptNaming: 173 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 174 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 175 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 176 nameString=prefix+srcfile 177 else: 178 #nameString=srcdir+": "+srcfile 179 nameString=srcfile 180 return nameString 181 182 def getLanguage(self,srcfile): 183 """ 184 Based on the source, determine associated language as found in gmakegen.LANGS 185 Can we just return srcext[1:\] now? 186 """ 187 langReq=None 188 srcext=os.path.splitext(srcfile)[-1] 189 if srcext in ".F90".split(): langReq="F90" 190 if srcext in ".F".split(): langReq="F" 191 if srcext in ".cxx".split(): langReq="cxx" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print "ERROR: ", srcext, srcfile 195 return langReq 196 197 def _getLoopVars(self,inDict,testname, isSubtest=False): 198 """ 199 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 200 Return: 201 inDict['args']: -ksp_monitor 202 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 203 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 204 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 205 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 206 subst should be passed in instead of inDict 207 """ 208 loopVars={}; newargs="" 209 lkeys=inDict.keys() 210 lsuffix='_' 211 from testparse import parseLoopArgs 212 for key in lkeys: 213 if type(inDict[key])!=types.StringType: continue 214 keystr = str(inDict[key]) 215 akey=('subargs' if key=='args' else key) # what to assign 216 if akey not in inDict: inDict[akey]='' 217 varlist=[] 218 for varset in re.split('-(?=[a-zA-Z])',keystr): 219 if not varset.strip(): continue 220 if '{{' in varset: 221 keyvar,lvars,ftype=parseLoopArgs(varset) 222 if akey not in loopVars: loopVars[akey]={} 223 varlist.append(keyvar) 224 loopVars[akey][keyvar]=[keyvar,lvars] 225 if akey=='nsize': 226 inDict[akey] = '${' + keyvar + '}' 227 lsuffix+=akey+'-'+inDict[akey]+'_' 228 else: 229 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 230 lsuffix+=keyvar+'-${' + keyvar + '}_' 231 else: 232 if key=='args': newargs+=" -"+varset.strip() 233 if varlist: loopVars[akey]['varlist']=varlist 234 235 236 # For subtests, args are always substituted in (not top level) 237 if isSubtest: 238 inDict['subargs']+=" "+newargs.strip() 239 inDict['args']='' 240 if 'label_suffix' in inDict: 241 inDict['label_suffix']+=lsuffix.rstrip('_') 242 else: 243 inDict['label_suffix']=lsuffix.rstrip('_') 244 else: 245 if loopVars.keys(): 246 inDict['args']=newargs.strip() 247 inDict['label_suffix']=lsuffix.rstrip('_') 248 if loopVars.keys(): 249 return loopVars 250 else: 251 return None 252 253 def getArgLabel(self,testDict): 254 """ 255 In all of the arguments in the test dictionary, create a simple 256 string for searching within the makefile system. For simplicity in 257 search, remove "-", for strings, etc. 258 Also, concatenate the arg commands 259 For now, ignore nsize -- seems hard to search for anyway 260 """ 261 # Collect all of the args associated with a test 262 argStr=("" if 'args' not in testDict else testDict['args']) 263 if 'subtests' in testDict: 264 for stest in testDict["subtests"]: 265 sd=testDict[stest] 266 argStr=argStr+("" if 'args' not in sd else sd['args']) 267 268 # Now go through and cleanup 269 argStr=re.sub('{{(.*?)}}',"",argStr) 270 argStr=re.sub('-'," ",argStr) 271 for digit in string.digits: argStr=re.sub(digit," ",argStr) 272 argStr=re.sub("\.","",argStr) 273 argStr=re.sub(",","",argStr) 274 argStr=re.sub('\+',' ',argStr) 275 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 276 return argStr.strip() 277 278 def addToSources(self,exfile,root,srcDict): 279 """ 280 Put into data structure that allows easy generation of makefile 281 """ 282 rpath=self.srcrelpath(root) 283 pkg=rpath.split(os.path.sep)[1] 284 relpfile=os.path.join(rpath,exfile) 285 lang=self.getLanguage(exfile) 286 if not lang: return 287 self.sources[pkg][lang]['srcs'].append(relpfile) 288 self.sources[pkg][lang][relpfile] = [] 289 if 'depends' in srcDict: 290 depSrcList=srcDict['depends'].split() 291 for depSrc in depSrcList: 292 depObj=os.path.splitext(depSrc)[0]+".o" 293 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 294 295 # In gmakefile, ${TESTDIR} var specifies the object compilation 296 testsdir=self.srcrelpath(root)+"/" 297 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 298 self.objects[pkg].append(objfile) 299 return 300 301 def addToTests(self,test,root,exfile,execname,testDict): 302 """ 303 Put into data structure that allows easy generation of makefile 304 Organized by languages to allow testing of languages 305 """ 306 rpath=self.srcrelpath(root) 307 pkg=rpath.split("/")[1] 308 #nmtest=self.nameSpace(test,root) 309 nmtest=os.path.join(rpath,test) 310 lang=self.getLanguage(exfile) 311 if not lang: return 312 self.tests[pkg][lang][nmtest]={} 313 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 314 self.tests[pkg][lang][nmtest]['exec']=execname 315 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 316 return 317 318 def getExecname(self,exfile,root): 319 """ 320 Generate bash script using template found next to this file. 321 This file is read in at constructor time to avoid file I/O 322 """ 323 rpath=self.srcrelpath(root) 324 if self.single_ex: 325 execname=rpath.split("/")[1]+"-ex" 326 else: 327 execname=os.path.splitext(exfile)[0] 328 return execname 329 330 def getSubstVars(self,testDict,rpath,testname): 331 """ 332 Create a dictionary with all of the variables that get substituted 333 into the template commands found in example_template.py 334 """ 335 subst={} 336 337 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 338 if 'nsize' not in testDict: testDict['nsize']=1 339 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 340 for ak in testparse.acceptedkeys: 341 if ak=='test': continue 342 subst[ak]=(testDict[ak] if ak in testDict else '') 343 344 # Now do other variables 345 subst['execname']=testDict['execname'] 346 if 'filter' in testDict: 347 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 348 349 # Others 350 subst['subargs']='' # Default. For variables override 351 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 352 subst['label_suffix']='' 353 subst['comments']="\n#".join(subst['comments'].split("\n")) 354 if subst['comments']: subst['comments']="#"+subst['comments'] 355 subst['exec']="../"+subst['execname'] 356 subst['testroot']=self.testroot_dir 357 subst['testname']=testname 358 dp = self.conf.get('DATAFILESPATH','') 359 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 360 361 # This is used to label some matrices 362 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 363 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 364 365 # These can have for loops and are treated separately later 366 subst['nsize']=str(subst['nsize']) 367 368 #Conf vars 369 if self.petsc_arch.find('valgrind')>=0: 370 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 371 else: 372 subst['mpiexec']=self.conf['MPIEXEC'] 373 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 374 subst['petsc_arch']=self.petsc_arch 375 if self.inInstallDir: 376 # Case 2 377 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 378 else: 379 # Case 1 380 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 381 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 382 subst['diff']=self.conf['DIFF'] 383 subst['rm']=self.conf['RM'] 384 subst['grep']=self.conf['GREP'] 385 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 386 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 387 388 # Output file is special because of subtests override 389 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 390 if not "_" in defroot: defroot=defroot+"_1" 391 subst['defroot']=defroot 392 subst['label']=self.nameSpace(defroot,subst['srcdir']) 393 subst['redirect_file']=defroot+".tmp" 394 if 'output_file' not in testDict: 395 subst['output_file']="output/"+defroot+".out" 396 # Add in the full path here. 397 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 398 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 399 if not subst['TODO']: 400 print "Warning: "+subst['output_file']+" not found." 401 # Worry about alt files here -- see 402 # src/snes/examples/tutorials/output/ex22*.out 403 altlist=[subst['output_file']] 404 basefile,ext = os.path.splitext(subst['output_file']) 405 for i in range(1,9): 406 altroot=basefile+"_alt" 407 if i > 1: altroot=altroot+"_"+str(i) 408 af=altroot+".out" 409 srcaf=os.path.join(subst['srcdir'],af) 410 fullaf=os.path.join(self.petsc_dir,srcaf) 411 if os.path.isfile(fullaf): altlist.append(srcaf) 412 if len(altlist)>1: subst['altfiles']=altlist 413 #if len(altlist)>1: print "Found alt files: ",altlist 414 415 return subst 416 417 def getCmds(self,subst,i): 418 """ 419 Generate bash script using template found next to this file. 420 This file is read in at constructor time to avoid file I/O 421 """ 422 nindnt=i # the start and has to be consistent with below 423 cmdindnt=self.indent*nindnt 424 cmdLines="" 425 426 # MPI is the default -- but we have a few odd commands 427 if not subst['command']: 428 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 429 else: 430 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 431 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 432 433 cmdLines+=cmdindnt+'if test $res = 0; then\n' 434 diffindnt=self.indent*(nindnt+1) 435 if not subst['filter_output']: 436 if 'altfiles' not in subst: 437 cmd=diffindnt+self._substVars(subst,example_template.difftest) 438 else: 439 # Have to do it by hand a bit because of variable number of alt files 440 rf=subst['redirect_file'] 441 cmd=diffindnt+example_template.difftest.split('@')[0] 442 for i in range(len(subst['altfiles'])): 443 af=subst['altfiles'][i] 444 cmd+=af+' '+rf 445 if i!=len(subst['altfiles'])-1: 446 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 447 cmd+=' || ${diff_exe} ' 448 else: 449 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 450 cmd+=subst['label_suffix']+' ""' # Quotes are painful 451 else: 452 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 453 cmdLines+=cmd+"\n" 454 cmdLines+=cmdindnt+'else\n' 455 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 456 cmdLines+=cmdindnt+'fi\n' 457 return cmdLines 458 459 def _substVars(self,subst,origStr): 460 """ 461 Substitute variables 462 """ 463 Str=origStr 464 for subkey in subst: 465 if type(subst[subkey])!=types.StringType: continue 466 patt="@"+subkey.upper()+"@" 467 Str=re.sub(patt,subst[subkey],Str) 468 return Str 469 470 def _writeTodoSkip(self,fh,tors,reasons,footer): 471 """ 472 Write out the TODO and SKIP lines in the file 473 The TODO or SKIP variable, tors, should be lower case 474 """ 475 TORS=tors.upper() 476 template=eval("example_template."+tors+"line") 477 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 478 tab = '' 479 if reasons: 480 fh.write('if ! $force; then\n') 481 tab = tab + ' ' 482 if reasons == ["Requires DATAFILESPATH"]: 483 # The only reason not to run is DATAFILESPATH, which we check at run-time 484 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 485 tab = tab + ' ' 486 if reasons: 487 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 488 fh.write(tab+footer+"\n") 489 fh.write(tab+"exit\n") 490 if reasons == ["Requires DATAFILESPATH"]: 491 fh.write(' fi\n') 492 if reasons: 493 fh.write('fi\n') 494 fh.write('\n\n') 495 return 496 497 def getLoopVarsHead(self,loopVars,i): 498 """ 499 Generate a nicely indented string with the format loops 500 Here is what the data structure looks like 501 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 502 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 503 loopVars['subargs']['pc_type']=["j","cholesky sor"] 504 """ 505 outstr=''; indnt=self.indent 506 for key in loopVars: 507 for var in loopVars[key]['varlist']: 508 varval=loopVars[key][var] 509 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 510 i = i + 1 511 return (outstr,i) 512 513 def getLoopVarsFoot(self,loopVars,i): 514 outstr=''; indnt=self.indent 515 for key in loopVars: 516 for var in loopVars[key]['varlist']: 517 i = i - 1 518 outstr += indnt * i + "done\n" 519 return (outstr,i) 520 521 def genRunScript(self,testname,root,isRun,srcDict): 522 """ 523 Generate bash script using template found next to this file. 524 This file is read in at constructor time to avoid file I/O 525 """ 526 # runscript_dir directory has to be consistent with gmakefile 527 testDict=srcDict[testname] 528 rpath=self.srcrelpath(root) 529 runscript_dir=os.path.join(self.testroot_dir,rpath) 530 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 531 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 532 533 # Get variables to go into shell scripts. last time testDict used 534 subst=self.getSubstVars(testDict,rpath,testname) 535 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 536 #if '33_' in testname: print subst['subargs'] 537 538 #Handle runfiles 539 for lfile in subst.get('localrunfiles','').split(): 540 install_files(os.path.join(root, lfile), runscript_dir) 541 # Check subtests for local runfiles 542 for stest in subst.get("subtests",[]): 543 for lfile in testDict[stest].get('localrunfiles','').split(): 544 install_files(os.path.join(root, lfile), runscript_dir) 545 546 # Now substitute the key variables into the header and footer 547 header=self._substVars(subst,example_template.header) 548 # The header is done twice to enable @...@ in header 549 header=self._substVars(subst,header) 550 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 551 552 # Start writing the file 553 fh.write(header+"\n") 554 555 # If there is a TODO or a SKIP then we do it before writing out the 556 # rest of the command (which is useful for working on the test) 557 # SKIP and TODO can be for the source file or for the runs 558 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 559 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 560 561 j=0 # for indentation 562 563 if loopVars: 564 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 565 if (loopHead): fh.write(loopHead+"\n") 566 567 # Subtests are special 568 if 'subtests' in testDict: 569 substP=subst # Subtests can inherit args but be careful 570 k=0 # for label suffixes 571 for stest in testDict["subtests"]: 572 subst=substP.copy() 573 subst.update(testDict[stest]) 574 # nsize is special because it is usually overwritten 575 if 'nsize' in testDict[stest]: 576 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 577 else: 578 fh.write("nsize=1\n") 579 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 580 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 581 #if '10_9' in testname: print sLoopVars 582 if sLoopVars: 583 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 584 fh.write(sLoopHead+"\n") 585 fh.write(self.getCmds(subst,j)+"\n") 586 if sLoopVars: 587 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 588 fh.write(sLoopFoot+"\n") 589 else: 590 fh.write(self.getCmds(subst,j)+"\n") 591 592 if loopVars: 593 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 594 fh.write(loopFoot+"\n") 595 596 fh.write(footer+"\n") 597 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 598 #if '10_9' in testname: sys.exit() 599 return 600 601 def genScriptsAndInfo(self,exfile,root,srcDict): 602 """ 603 Generate scripts from the source file, determine if built, etc. 604 For every test in the exfile with info in the srcDict: 605 1. Determine if it needs to be run for this arch 606 2. Generate the script 607 3. Generate the data needed to write out the makefile in a 608 convenient way 609 All tests are *always* run, but some may be SKIP'd per the TAP standard 610 """ 611 debug=False 612 execname=self.getExecname(exfile,root) 613 isBuilt=self._isBuilt(exfile,srcDict) 614 for test in srcDict: 615 if test in self.buildkeys: continue 616 if debug: print self.nameSpace(exfile,root), test 617 srcDict[test]['execname']=execname # Convenience in generating scripts 618 isRun=self._isRun(srcDict[test]) 619 self.genRunScript(test,root,isRun,srcDict) 620 srcDict[test]['isrun']=isRun 621 self.addToTests(test,root,exfile,execname,srcDict[test]) 622 623 # This adds to datastructure for building deps 624 if isBuilt: self.addToSources(exfile,root,srcDict) 625 return 626 627 def _isBuilt(self,exfile,srcDict): 628 """ 629 Determine if this file should be built. 630 """ 631 # Get the language based on file extension 632 srcDict['SKIP'] = [] 633 lang=self.getLanguage(exfile) 634 if (lang=="F" or lang=="F90"): 635 if not self.have_fortran: 636 srcDict["SKIP"].append("Fortran required for this test") 637 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 638 srcDict["SKIP"].append("Fortran f90freeform required for this test") 639 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 640 srcDict["SKIP"].append("CUDA required for this test") 641 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 642 srcDict["SKIP"].append("C++ required for this test") 643 644 # Deprecated source files 645 if srcDict.get("TODO"): 646 return False 647 648 # isRun can work with srcDict to handle the requires 649 if "requires" in srcDict: 650 if srcDict["requires"]: 651 return self._isRun(srcDict) 652 653 return srcDict['SKIP'] == [] 654 655 656 def _isRun(self,testDict, debug=False): 657 """ 658 Based on the requirements listed in the src file and the petscconf.h 659 info, determine whether this test should be run or not. 660 """ 661 indent=" " 662 663 if 'SKIP' not in testDict: 664 testDict['SKIP'] = [] 665 # MPI requirements 666 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 667 if debug: print indent+"Cannot run parallel tests" 668 testDict['SKIP'].append("Parallel test with serial build") 669 670 # The requirements for the test are the sum of all the run subtests 671 if 'subtests' in testDict: 672 if 'requires' not in testDict: testDict['requires']="" 673 for stest in testDict['subtests']: 674 if 'requires' in testDict[stest]: 675 testDict['requires']+=" "+testDict[stest]['requires'] 676 if 'nsize' in testDict[stest]: 677 if testDict[stest].get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 678 testDict['SKIP'].append("Parallel test with serial build") 679 680 681 # Now go through all requirements 682 if 'requires' in testDict: 683 for requirement in testDict['requires'].split(): 684 requirement=requirement.strip() 685 if not requirement: continue 686 if debug: print indent+"Requirement: ", requirement 687 isNull=False 688 if requirement.startswith("!"): 689 requirement=requirement[1:]; isNull=True 690 # Precision requirement for reals 691 if requirement in self.precision_types: 692 if self.conf['PETSC_PRECISION']==requirement: 693 if isNull: 694 testDict['SKIP'].append("not "+requirement+" required") 695 continue 696 continue # Success 697 elif not isNull: 698 testDict['SKIP'].append(requirement+" required") 699 continue 700 # Precision requirement for ints 701 if requirement in self.integer_types: 702 if requirement=="int32": 703 if self.conf['PETSC_SIZEOF_INT']==4: 704 if isNull: 705 testDict['SKIP'].append("not int32 required") 706 continue 707 continue # Success 708 elif not isNull: 709 testDict['SKIP'].append("int32 required") 710 continue 711 if requirement=="int64": 712 if self.conf['PETSC_SIZEOF_INT']==8: 713 if isNull: 714 testDict['SKIP'].append("NOT int64 required") 715 continue 716 continue # Success 717 elif not isNull: 718 testDict['SKIP'].append("int64 required") 719 continue 720 # Datafilespath 721 if requirement=="datafilespath" and not isNull: 722 testDict['SKIP'].append("Requires DATAFILESPATH") 723 continue 724 # Defines -- not sure I have comments matching 725 if "define(" in requirement.lower(): 726 reqdef=requirement.split("(")[1].split(")")[0] 727 if reqdef in self.conf: 728 if isNull: 729 testDict['SKIP'].append("Null requirement not met: "+requirement) 730 continue 731 continue # Success 732 elif not isNull: 733 testDict['SKIP'].append("Required: "+requirement) 734 continue 735 736 # Rest should be packages that we can just get from conf 737 if requirement == "complex": 738 petscconfvar="PETSC_USE_COMPLEX" 739 else: 740 petscconfvar="PETSC_HAVE_"+requirement.upper() 741 if self.conf.get(petscconfvar): 742 if isNull: 743 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 744 continue 745 continue # Success 746 elif not isNull: 747 if debug: print "requirement not found: ", requirement 748 testDict['SKIP'].append(petscconfvar+" requirement not met") 749 continue 750 751 return testDict['SKIP'] == [] 752 753 def genPetscTests_summarize(self,dataDict): 754 """ 755 Required method to state what happened 756 """ 757 if not self.summarize: return 758 indent=" " 759 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 760 fh=open(fhname,"w") 761 #print "See ", fhname 762 for root in dataDict: 763 relroot=self.srcrelpath(root) 764 pkg=relroot.split("/")[1] 765 fh.write(relroot+"\n") 766 allSrcs=[] 767 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 768 for exfile in dataDict[root]: 769 # Basic information 770 rfile=os.path.join(relroot,exfile) 771 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 772 fh.write(indent+exfile+indent*4+builtStatus+"\n") 773 774 for test in dataDict[root][exfile]: 775 if test in self.buildkeys: continue 776 line=indent*2+test 777 fh.write(line+"\n") 778 # Looks nice to have the keys in order 779 #for key in dataDict[root][exfile][test]: 780 for key in "isrun abstracted nsize args requires script".split(): 781 if key not in dataDict[root][exfile][test]: continue 782 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 783 fh.write(line+"\n") 784 fh.write("\n") 785 fh.write("\n") 786 fh.write("\n") 787 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 788 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 789 fh.close() 790 return 791 792 def genPetscTests(self,root,dirs,files,dataDict): 793 """ 794 Go through and parse the source files in the directory to generate 795 the examples based on the metadata contained in the source files 796 """ 797 debug=False 798 # Use examplesAnalyze to get what the makefles think are sources 799 #self.examplesAnalyze(root,dirs,files,anlzDict) 800 801 dataDict[root]={} 802 803 for exfile in files: 804 #TST: Until we replace files, still leaving the orginals as is 805 #if not exfile.startswith("new_"+"ex"): continue 806 #if not exfile.startswith("ex"): continue 807 808 # Ignore emacs and other temporary files 809 if exfile.startswith("."): continue 810 if exfile.startswith("#"): continue 811 812 # Convenience 813 fullex=os.path.join(root,exfile) 814 if self.verbose: print(' --> '+fullex) 815 dataDict[root].update(testparse.parseTestFile(fullex,0)) 816 if exfile in dataDict[root]: 817 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 818 819 return 820 821 def walktree(self,top): 822 """ 823 Walk a directory tree, starting from 'top' 824 """ 825 #print "action", action 826 # Goal of action is to fill this dictionary 827 dataDict={} 828 for root, dirs, files in os.walk(top, topdown=True): 829 if not "examples" in root: continue 830 if "dSYM" in root: continue 831 if os.path.basename(root.rstrip("/")) == 'output': continue 832 if self.verbose: print(root) 833 self.genPetscTests(root,dirs,files,dataDict) 834 # Now summarize this dictionary 835 self.genPetscTests_summarize(dataDict) 836 return dataDict 837 838 def gen_gnumake(self, fd): 839 """ 840 Overwrite of the method in the base PETSc class 841 """ 842 def write(stem, srcs): 843 for lang in LANGS: 844 if srcs[lang]['srcs']: 845 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 846 for pkg in PKGS: 847 srcs = self.gen_pkg(pkg) 848 write('testsrcs-' + pkg, srcs) 849 # Handle dependencies 850 for lang in LANGS: 851 for exfile in srcs[lang]['srcs']: 852 if exfile in srcs[lang]: 853 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 854 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 855 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 856 if deps: 857 # The executable literally depends on the object file because it is linked 858 fd.write(ex +": " + " ".join(deps) +'\n') 859 # The object file containing 'main' does not normally depend on other object 860 # files, but it does when it includes their modules. This dependency is 861 # overly blunt and could be reduced to only depend on object files for 862 # modules that are used, like "*f90aux.o". 863 fd.write(exfo +": " + " ".join(deps) +'\n') 864 865 return self.gendeps 866 867 def gen_pkg(self, pkg): 868 """ 869 Overwrite of the method in the base PETSc class 870 """ 871 return self.sources[pkg] 872 873 def write_gnumake(self, dataDict, output=None): 874 """ 875 Write out something similar to files from gmakegen.py 876 877 Test depends on script which also depends on source 878 file, but since I don't have a good way generating 879 acting on a single file (oops) just depend on 880 executable which in turn will depend on src file 881 """ 882 # Different options for how to set up the targets 883 compileExecsFirst=False 884 885 # Open file 886 fd = open(output, 'w') 887 888 # Write out the sources 889 gendeps = self.gen_gnumake(fd) 890 891 # Write out the tests and execname targets 892 fd.write("\n#Tests and executables\n") # Delimiter 893 894 for pkg in PKGS: 895 # These grab the ones that are built 896 for lang in LANGS: 897 testdeps=[] 898 for ftest in self.tests[pkg][lang]: 899 test=os.path.basename(ftest) 900 basedir=os.path.dirname(ftest) 901 testdeps.append(self.nameSpace(test,basedir)) 902 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 903 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 904 905 # test targets 906 for ftest in self.tests[pkg][lang]: 907 test=os.path.basename(ftest) 908 basedir=os.path.dirname(ftest) 909 testdir="${TESTDIR}/"+basedir+"/" 910 nmtest=self.nameSpace(test,basedir) 911 rundir=os.path.join(testdir,test) 912 #print test, nmtest 913 script=test+".sh" 914 915 # Deps 916 exfile=self.tests[pkg][lang][ftest]['exfile'] 917 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 918 localexec=self.tests[pkg][lang][ftest]['exec'] 919 execname=os.path.join(testdir,localexec) 920 fullscript=os.path.join(testdir,script) 921 tmpfile=os.path.join(testdir,test,test+".tmp") 922 923 # *.counts depends on the script and either executable (will 924 # be run) or the example source file (SKIP or TODO) 925 fd.write('%s.counts : %s %s' 926 % (os.path.join('$(TESTDIR)/counts', nmtest), 927 fullscript, 928 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 929 ) 930 if exfile in self.sources[pkg][lang]: 931 for dep in self.sources[pkg][lang][exfile]: 932 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 933 fd.write('\n') 934 935 # Now write the args: 936 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 937 938 fd.close() 939 return 940 941def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 942 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 943 if petsc_arch: 944 if len(petsc_arch.split(os.path.sep))>1: 945 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 946 output = os.path.join(testdir, 'testfiles') 947 948 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 949 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 950 testdir=testdir) 951 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 952 pEx.write_gnumake(dataDict, output) 953 954if __name__ == '__main__': 955 import optparse 956 parser = optparse.OptionParser() 957 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 958 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 959 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 960 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 961 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 962 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 963 964 opts, extra_args = parser.parse_args() 965 if extra_args: 966 import sys 967 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 968 exit(1) 969 if opts.testdir is None: 970 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 971 972 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 973 verbose=opts.verbose, 974 single_ex=opts.single_executable, srcdir=opts.srcdir, 975 testdir=opts.testdir) 976