1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77class generateExamples(Petsc): 78 """ 79 gmakegen.py has basic structure for finding the files, writing out 80 the dependencies, etc. 81 """ 82 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 83 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 84 85 self.single_ex=single_ex 86 87 # Set locations to handle movement 88 self.inInstallDir=self.getInInstallDir(thisscriptdir) 89 90 if self.inInstallDir: 91 # Case 2 discussed above 92 # set PETSC_ARCH to install directory to allow script to work in both 93 dirlist=thisscriptdir.split(os.path.sep) 94 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 95 self.arch_dir=installdir 96 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 97 else: 98 if petsc_arch == '': 99 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 100 # Case 1 discussed above 101 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 102 self.srcdir=os.path.join(self.petsc_dir,'src') 103 104 self.testroot_dir=os.path.abspath(testdir) 105 106 self.ptNaming=True 107 self.verbose=verbose 108 # Whether to write out a useful debugging 109 self.summarize=True if verbose else False 110 111 # For help in setting the requirements 112 self.precision_types="single double __float128 int32".split() 113 self.integer_types="int32 int64".split() 114 self.languages="fortran cuda cxx".split() # Always requires C so do not list 115 116 # Things that are not test 117 self.buildkeys=testparse.buildkeys 118 119 # Adding a dictionary for storing sources, objects, and tests 120 # to make building the dependency tree easier 121 self.sources={} 122 self.objects={} 123 self.tests={} 124 for pkg in PKGS: 125 self.sources[pkg]={} 126 self.objects[pkg]=[] 127 self.tests[pkg]={} 128 for lang in LANGS: 129 self.sources[pkg][lang]={} 130 self.sources[pkg][lang]['srcs']=[] 131 self.tests[pkg][lang]={} 132 133 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 134 135 self.indent=" " 136 if self.verbose: print('Finishing the constructor') 137 return 138 139 def srcrelpath(self,rdir): 140 """ 141 Get relative path to source directory 142 """ 143 return os.path.relpath(rdir,self.srcdir) 144 145 def getInInstallDir(self,thisscriptdir): 146 """ 147 When petsc is installed then this file in installed in: 148 <PREFIX>/share/petsc/examples/config/gmakegentest.py 149 otherwise the path is: 150 <PETSC_DIR>/config/gmakegentest.py 151 We use this difference to determine if we are in installdir 152 """ 153 dirlist=thisscriptdir.split(os.path.sep) 154 if len(dirlist)>4: 155 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 156 if lastfour==os.path.join('share','petsc','examples','config'): 157 return True 158 else: 159 return False 160 else: 161 return False 162 163 def nameSpace(self,srcfile,srcdir): 164 """ 165 Because the scripts have a non-unique naming, the pretty-printing 166 needs to convey the srcdir and srcfile. There are two ways of doing this. 167 """ 168 if self.ptNaming: 169 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 170 cdir=srcdir 171 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 172 nameString=prefix+srcfile 173 else: 174 #nameString=srcdir+": "+srcfile 175 nameString=srcfile 176 return nameString 177 178 def getLanguage(self,srcfile): 179 """ 180 Based on the source, determine associated language as found in gmakegen.LANGS 181 Can we just return srcext[1:\] now? 182 """ 183 langReq=None 184 srcext=os.path.splitext(srcfile)[-1] 185 if srcext in ".F90".split(): langReq="F90" 186 if srcext in ".F".split(): langReq="F" 187 if srcext in ".cxx".split(): langReq="cxx" 188 if srcext == ".cu": langReq="cu" 189 if srcext == ".c": langReq="c" 190 #if not langReq: print "ERROR: ", srcext, srcfile 191 return langReq 192 193 def _getLoopVars(self,inDict,testname, isSubtest=False): 194 """ 195 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 196 Return: 197 inDict['args']: -ksp_monitor 198 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 199 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 200 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 201 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 202 subst should be passed in instead of inDict 203 """ 204 loopVars={}; newargs="" 205 lkeys=inDict.keys() 206 lsuffix='_' 207 from testparse import parseLoopArgs 208 for key in lkeys: 209 if type(inDict[key])!=types.StringType: continue 210 keystr = str(inDict[key]) 211 akey=('subargs' if key=='args' else key) # what to assign 212 if akey not in inDict: inDict[akey]='' 213 varlist=[] 214 for varset in re.split('-(?=[a-zA-Z])',keystr): 215 if not varset.strip(): continue 216 if '{{' in varset: 217 keyvar,lvars,ftype=parseLoopArgs(varset) 218 if akey not in loopVars: loopVars[akey]={} 219 varlist.append(keyvar) 220 loopVars[akey][keyvar]=[keyvar,lvars] 221 if akey=='nsize': 222 inDict[akey] = '${' + keyvar + '}' 223 lsuffix+=akey+'-'+inDict[akey]+'_' 224 else: 225 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 226 lsuffix+=keyvar+'-${' + keyvar + '}_' 227 else: 228 if key=='args': newargs+=" -"+varset.strip() 229 if varlist: loopVars[akey]['varlist']=varlist 230 231 232 # For subtests, args are always substituted in (not top level) 233 if isSubtest: 234 inDict['subargs']+=" "+newargs.strip() 235 inDict['args']='' 236 if 'label_suffix' in inDict: 237 inDict['label_suffix']+=lsuffix.rstrip('_') 238 else: 239 inDict['label_suffix']=lsuffix.rstrip('_') 240 else: 241 if loopVars.keys(): 242 inDict['args']=newargs.strip() 243 inDict['label_suffix']=lsuffix.rstrip('_') 244 if loopVars.keys(): 245 return loopVars 246 else: 247 return None 248 249 def getArgLabel(self,testDict): 250 """ 251 In all of the arguments in the test dictionary, create a simple 252 string for searching within the makefile system. For simplicity in 253 search, remove "-", for strings, etc. 254 Also, concatenate the arg commands 255 For now, ignore nsize -- seems hard to search for anyway 256 """ 257 # Collect all of the args associated with a test 258 argStr=("" if 'args' not in testDict else testDict['args']) 259 if 'subtests' in testDict: 260 for stest in testDict["subtests"]: 261 sd=testDict[stest] 262 argStr=argStr+("" if 'args' not in sd else sd['args']) 263 264 # Now go through and cleanup 265 argStr=re.sub('{{(.*?)}}',"",argStr) 266 argStr=re.sub('-'," ",argStr) 267 for digit in string.digits: argStr=re.sub(digit," ",argStr) 268 argStr=re.sub("\.","",argStr) 269 argStr=re.sub(",","",argStr) 270 argStr=re.sub('\+',' ',argStr) 271 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 272 return argStr.strip() 273 274 def addToSources(self,exfile,root,srcDict): 275 """ 276 Put into data structure that allows easy generation of makefile 277 """ 278 rpath=self.srcrelpath(root) 279 pkg=rpath.split(os.path.sep)[0] 280 relpfile=os.path.join(rpath,exfile) 281 lang=self.getLanguage(exfile) 282 if not lang: return 283 self.sources[pkg][lang]['srcs'].append(relpfile) 284 self.sources[pkg][lang][relpfile] = [] 285 if 'depends' in srcDict: 286 depSrcList=srcDict['depends'].split() 287 for depSrc in depSrcList: 288 depObj=os.path.splitext(depSrc)[0]+".o" 289 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 290 291 # In gmakefile, ${TESTDIR} var specifies the object compilation 292 testsdir=self.srcrelpath(root)+"/" 293 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 294 self.objects[pkg].append(objfile) 295 return 296 297 def addToTests(self,test,root,exfile,execname,testDict): 298 """ 299 Put into data structure that allows easy generation of makefile 300 Organized by languages to allow testing of languages 301 """ 302 rpath=self.srcrelpath(root) 303 pkg=rpath.split("/")[0] 304 #nmtest=self.nameSpace(test,root) 305 nmtest=os.path.join(rpath,test) 306 lang=self.getLanguage(exfile) 307 if not lang: return 308 self.tests[pkg][lang][nmtest]={} 309 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 310 self.tests[pkg][lang][nmtest]['exec']=execname 311 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 312 return 313 314 def getExecname(self,exfile,root): 315 """ 316 Generate bash script using template found next to this file. 317 This file is read in at constructor time to avoid file I/O 318 """ 319 rpath=self.srcrelpath(root) 320 if self.single_ex: 321 execname=rpath.split("/")[1]+"-ex" 322 else: 323 execname=os.path.splitext(exfile)[0] 324 return execname 325 326 def getSubstVars(self,testDict,rpath,testname): 327 """ 328 Create a dictionary with all of the variables that get substituted 329 into the template commands found in example_template.py 330 """ 331 subst={} 332 333 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 334 if 'nsize' not in testDict: testDict['nsize']=1 335 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 336 for ak in testparse.acceptedkeys: 337 if ak=='test': continue 338 subst[ak]=(testDict[ak] if ak in testDict else '') 339 340 # Now do other variables 341 subst['execname']=testDict['execname'] 342 if 'filter' in testDict: 343 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 344 345 # Others 346 subst['subargs']='' # Default. For variables override 347 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 348 subst['label_suffix']='' 349 subst['comments']="\n#".join(subst['comments'].split("\n")) 350 if subst['comments']: subst['comments']="#"+subst['comments'] 351 subst['exec']="../"+subst['execname'] 352 subst['testroot']=self.testroot_dir 353 subst['testname']=testname 354 dp = self.conf.get('DATAFILESPATH','') 355 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 356 357 # This is used to label some matrices 358 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 359 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 360 361 # These can have for loops and are treated separately later 362 subst['nsize']=str(subst['nsize']) 363 364 #Conf vars 365 if self.petsc_arch.find('valgrind')>=0: 366 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 367 else: 368 subst['mpiexec']=self.conf['MPIEXEC'] 369 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 370 subst['petsc_arch']=self.petsc_arch 371 if self.inInstallDir: 372 # Case 2 373 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 374 else: 375 # Case 1 376 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 377 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 378 subst['diff']=self.conf['DIFF'] 379 subst['rm']=self.conf['RM'] 380 subst['grep']=self.conf['GREP'] 381 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 382 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 383 384 # Output file is special because of subtests override 385 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 386 if not "_" in defroot: defroot=defroot+"_1" 387 subst['defroot']=defroot 388 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 389 subst['redirect_file']=defroot+".tmp" 390 if 'output_file' not in testDict: 391 subst['output_file']="output/"+defroot+".out" 392 # Add in the full path here. 393 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 394 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 395 if not subst['TODO']: 396 print "Warning: "+subst['output_file']+" not found." 397 # Worry about alt files here -- see 398 # src/snes/examples/tutorials/output/ex22*.out 399 altlist=[subst['output_file']] 400 basefile,ext = os.path.splitext(subst['output_file']) 401 for i in range(1,9): 402 altroot=basefile+"_alt" 403 if i > 1: altroot=altroot+"_"+str(i) 404 af=altroot+".out" 405 srcaf=os.path.join(subst['srcdir'],af) 406 fullaf=os.path.join(self.petsc_dir,srcaf) 407 if os.path.isfile(fullaf): altlist.append(srcaf) 408 if len(altlist)>1: subst['altfiles']=altlist 409 #if len(altlist)>1: print "Found alt files: ",altlist 410 411 return subst 412 413 def getCmds(self,subst,i): 414 """ 415 Generate bash script using template found next to this file. 416 This file is read in at constructor time to avoid file I/O 417 """ 418 nindnt=i # the start and has to be consistent with below 419 cmdindnt=self.indent*nindnt 420 cmdLines="" 421 422 # MPI is the default -- but we have a few odd commands 423 if not subst['command']: 424 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 425 else: 426 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 427 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 428 429 cmdLines+=cmdindnt+'if test $res = 0; then\n' 430 diffindnt=self.indent*(nindnt+1) 431 if not subst['filter_output']: 432 if 'altfiles' not in subst: 433 cmd=diffindnt+self._substVars(subst,example_template.difftest) 434 else: 435 # Have to do it by hand a bit because of variable number of alt files 436 rf=subst['redirect_file'] 437 cmd=diffindnt+example_template.difftest.split('@')[0] 438 for i in range(len(subst['altfiles'])): 439 af=subst['altfiles'][i] 440 cmd+=af+' '+rf 441 if i!=len(subst['altfiles'])-1: 442 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 443 cmd+=' || ${diff_exe} ' 444 else: 445 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 446 cmd+=subst['label_suffix']+' ""' # Quotes are painful 447 else: 448 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 449 cmdLines+=cmd+"\n" 450 cmdLines+=cmdindnt+'else\n' 451 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 452 cmdLines+=cmdindnt+'fi\n' 453 return cmdLines 454 455 def _substVars(self,subst,origStr): 456 """ 457 Substitute variables 458 """ 459 Str=origStr 460 for subkey in subst: 461 if type(subst[subkey])!=types.StringType: continue 462 patt="@"+subkey.upper()+"@" 463 Str=re.sub(patt,subst[subkey],Str) 464 return Str 465 466 def _writeTodoSkip(self,fh,tors,reasons,footer): 467 """ 468 Write out the TODO and SKIP lines in the file 469 The TODO or SKIP variable, tors, should be lower case 470 """ 471 TORS=tors.upper() 472 template=eval("example_template."+tors+"line") 473 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 474 tab = '' 475 if reasons: 476 fh.write('if ! $force; then\n') 477 tab = tab + ' ' 478 if reasons == ["Requires DATAFILESPATH"]: 479 # The only reason not to run is DATAFILESPATH, which we check at run-time 480 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 481 tab = tab + ' ' 482 if reasons: 483 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 484 fh.write(tab+footer+"\n") 485 fh.write(tab+"exit\n") 486 if reasons == ["Requires DATAFILESPATH"]: 487 fh.write(' fi\n') 488 if reasons: 489 fh.write('fi\n') 490 fh.write('\n\n') 491 return 492 493 def getLoopVarsHead(self,loopVars,i): 494 """ 495 Generate a nicely indented string with the format loops 496 Here is what the data structure looks like 497 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 498 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 499 loopVars['subargs']['pc_type']=["j","cholesky sor"] 500 """ 501 outstr=''; indnt=self.indent 502 for key in loopVars: 503 for var in loopVars[key]['varlist']: 504 varval=loopVars[key][var] 505 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 506 i = i + 1 507 return (outstr,i) 508 509 def getLoopVarsFoot(self,loopVars,i): 510 outstr=''; indnt=self.indent 511 for key in loopVars: 512 for var in loopVars[key]['varlist']: 513 i = i - 1 514 outstr += indnt * i + "done\n" 515 return (outstr,i) 516 517 def genRunScript(self,testname,root,isRun,srcDict): 518 """ 519 Generate bash script using template found next to this file. 520 This file is read in at constructor time to avoid file I/O 521 """ 522 # runscript_dir directory has to be consistent with gmakefile 523 testDict=srcDict[testname] 524 rpath=self.srcrelpath(root) 525 runscript_dir=os.path.join(self.testroot_dir,rpath) 526 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 527 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 528 529 # Get variables to go into shell scripts. last time testDict used 530 subst=self.getSubstVars(testDict,rpath,testname) 531 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 532 #if '33_' in testname: print subst['subargs'] 533 534 #Handle runfiles 535 for lfile in subst.get('localrunfiles','').split(): 536 install_files(os.path.join(root, lfile), runscript_dir) 537 # Check subtests for local runfiles 538 for stest in subst.get("subtests",[]): 539 for lfile in testDict[stest].get('localrunfiles','').split(): 540 install_files(os.path.join(root, lfile), runscript_dir) 541 542 # Now substitute the key variables into the header and footer 543 header=self._substVars(subst,example_template.header) 544 # The header is done twice to enable @...@ in header 545 header=self._substVars(subst,header) 546 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 547 548 # Start writing the file 549 fh.write(header+"\n") 550 551 # If there is a TODO or a SKIP then we do it before writing out the 552 # rest of the command (which is useful for working on the test) 553 # SKIP and TODO can be for the source file or for the runs 554 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 555 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 556 557 j=0 # for indentation 558 559 if loopVars: 560 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 561 if (loopHead): fh.write(loopHead+"\n") 562 563 # Subtests are special 564 if 'subtests' in testDict: 565 substP=subst # Subtests can inherit args but be careful 566 k=0 # for label suffixes 567 for stest in testDict["subtests"]: 568 subst=substP.copy() 569 subst.update(testDict[stest]) 570 # nsize is special because it is usually overwritten 571 if 'nsize' in testDict[stest]: 572 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 573 else: 574 fh.write("nsize=1\n") 575 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 576 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 577 #if '10_9' in testname: print sLoopVars 578 if sLoopVars: 579 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 580 fh.write(sLoopHead+"\n") 581 fh.write(self.getCmds(subst,j)+"\n") 582 if sLoopVars: 583 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 584 fh.write(sLoopFoot+"\n") 585 else: 586 fh.write(self.getCmds(subst,j)+"\n") 587 588 if loopVars: 589 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 590 fh.write(loopFoot+"\n") 591 592 fh.write(footer+"\n") 593 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 594 #if '10_9' in testname: sys.exit() 595 return 596 597 def genScriptsAndInfo(self,exfile,root,srcDict): 598 """ 599 Generate scripts from the source file, determine if built, etc. 600 For every test in the exfile with info in the srcDict: 601 1. Determine if it needs to be run for this arch 602 2. Generate the script 603 3. Generate the data needed to write out the makefile in a 604 convenient way 605 All tests are *always* run, but some may be SKIP'd per the TAP standard 606 """ 607 debug=False 608 execname=self.getExecname(exfile,root) 609 isBuilt=self._isBuilt(exfile,srcDict) 610 for test in srcDict: 611 if test in self.buildkeys: continue 612 if debug: print self.nameSpace(exfile,root), test 613 srcDict[test]['execname']=execname # Convenience in generating scripts 614 isRun=self._isRun(srcDict[test]) 615 self.genRunScript(test,root,isRun,srcDict) 616 srcDict[test]['isrun']=isRun 617 self.addToTests(test,root,exfile,execname,srcDict[test]) 618 619 # This adds to datastructure for building deps 620 if isBuilt: self.addToSources(exfile,root,srcDict) 621 return 622 623 def _isBuilt(self,exfile,srcDict): 624 """ 625 Determine if this file should be built. 626 """ 627 # Get the language based on file extension 628 srcDict['SKIP'] = [] 629 lang=self.getLanguage(exfile) 630 if (lang=="F" or lang=="F90"): 631 if not self.have_fortran: 632 srcDict["SKIP"].append("Fortran required for this test") 633 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 634 srcDict["SKIP"].append("Fortran f90freeform required for this test") 635 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 636 srcDict["SKIP"].append("CUDA required for this test") 637 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 638 srcDict["SKIP"].append("C++ required for this test") 639 640 # Deprecated source files 641 if srcDict.get("TODO"): 642 return False 643 644 # isRun can work with srcDict to handle the requires 645 if "requires" in srcDict: 646 if srcDict["requires"]: 647 return self._isRun(srcDict) 648 649 return srcDict['SKIP'] == [] 650 651 652 def _isRun(self,testDict, debug=False): 653 """ 654 Based on the requirements listed in the src file and the petscconf.h 655 info, determine whether this test should be run or not. 656 """ 657 indent=" " 658 659 if 'SKIP' not in testDict: 660 testDict['SKIP'] = [] 661 # MPI requirements 662 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 663 if debug: print indent+"Cannot run parallel tests" 664 testDict['SKIP'].append("Parallel test with serial build") 665 666 # The requirements for the test are the sum of all the run subtests 667 if 'subtests' in testDict: 668 if 'requires' not in testDict: testDict['requires']="" 669 for stest in testDict['subtests']: 670 if 'requires' in testDict[stest]: 671 testDict['requires']+=" "+testDict[stest]['requires'] 672 if 'nsize' in testDict[stest]: 673 if testDict[stest].get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 674 testDict['SKIP'].append("Parallel test with serial build") 675 676 677 # Now go through all requirements 678 if 'requires' in testDict: 679 for requirement in testDict['requires'].split(): 680 requirement=requirement.strip() 681 if not requirement: continue 682 if debug: print indent+"Requirement: ", requirement 683 isNull=False 684 if requirement.startswith("!"): 685 requirement=requirement[1:]; isNull=True 686 # Precision requirement for reals 687 if requirement in self.precision_types: 688 if self.conf['PETSC_PRECISION']==requirement: 689 if isNull: 690 testDict['SKIP'].append("not "+requirement+" required") 691 continue 692 continue # Success 693 elif not isNull: 694 testDict['SKIP'].append(requirement+" required") 695 continue 696 # Precision requirement for ints 697 if requirement in self.integer_types: 698 if requirement=="int32": 699 if self.conf['PETSC_SIZEOF_INT']==4: 700 if isNull: 701 testDict['SKIP'].append("not int32 required") 702 continue 703 continue # Success 704 elif not isNull: 705 testDict['SKIP'].append("int32 required") 706 continue 707 if requirement=="int64": 708 if self.conf['PETSC_SIZEOF_INT']==8: 709 if isNull: 710 testDict['SKIP'].append("NOT int64 required") 711 continue 712 continue # Success 713 elif not isNull: 714 testDict['SKIP'].append("int64 required") 715 continue 716 # Datafilespath 717 if requirement=="datafilespath" and not isNull: 718 testDict['SKIP'].append("Requires DATAFILESPATH") 719 continue 720 # Defines -- not sure I have comments matching 721 if "define(" in requirement.lower(): 722 reqdef=requirement.split("(")[1].split(")")[0] 723 if reqdef in self.conf: 724 if isNull: 725 testDict['SKIP'].append("Null requirement not met: "+requirement) 726 continue 727 continue # Success 728 elif not isNull: 729 testDict['SKIP'].append("Required: "+requirement) 730 continue 731 732 # Rest should be packages that we can just get from conf 733 if requirement == "complex": 734 petscconfvar="PETSC_USE_COMPLEX" 735 else: 736 petscconfvar="PETSC_HAVE_"+requirement.upper() 737 if self.conf.get(petscconfvar): 738 if isNull: 739 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 740 continue 741 continue # Success 742 elif not isNull: 743 if debug: print "requirement not found: ", requirement 744 testDict['SKIP'].append(petscconfvar+" requirement not met") 745 continue 746 747 return testDict['SKIP'] == [] 748 749 def genPetscTests_summarize(self,dataDict): 750 """ 751 Required method to state what happened 752 """ 753 if not self.summarize: return 754 indent=" " 755 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 756 fh=open(fhname,"w") 757 #print "See ", fhname 758 for root in dataDict: 759 relroot=self.srcrelpath(root) 760 pkg=relroot.split("/")[1] 761 fh.write(relroot+"\n") 762 allSrcs=[] 763 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 764 for exfile in dataDict[root]: 765 # Basic information 766 rfile=os.path.join(relroot,exfile) 767 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 768 fh.write(indent+exfile+indent*4+builtStatus+"\n") 769 770 for test in dataDict[root][exfile]: 771 if test in self.buildkeys: continue 772 line=indent*2+test 773 fh.write(line+"\n") 774 # Looks nice to have the keys in order 775 #for key in dataDict[root][exfile][test]: 776 for key in "isrun abstracted nsize args requires script".split(): 777 if key not in dataDict[root][exfile][test]: continue 778 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 779 fh.write(line+"\n") 780 fh.write("\n") 781 fh.write("\n") 782 fh.write("\n") 783 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 784 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 785 fh.close() 786 return 787 788 def genPetscTests(self,root,dirs,files,dataDict): 789 """ 790 Go through and parse the source files in the directory to generate 791 the examples based on the metadata contained in the source files 792 """ 793 debug=False 794 # Use examplesAnalyze to get what the makefles think are sources 795 #self.examplesAnalyze(root,dirs,files,anlzDict) 796 797 dataDict[root]={} 798 799 for exfile in files: 800 #TST: Until we replace files, still leaving the orginals as is 801 #if not exfile.startswith("new_"+"ex"): continue 802 #if not exfile.startswith("ex"): continue 803 804 # Ignore emacs and other temporary files 805 if exfile.startswith("."): continue 806 if exfile.startswith("#"): continue 807 808 # Convenience 809 fullex=os.path.join(root,exfile) 810 if self.verbose: print(' --> '+fullex) 811 dataDict[root].update(testparse.parseTestFile(fullex,0)) 812 if exfile in dataDict[root]: 813 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 814 815 return 816 817 def walktree(self,top): 818 """ 819 Walk a directory tree, starting from 'top' 820 """ 821 #print "action", action 822 # Goal of action is to fill this dictionary 823 dataDict={} 824 for root, dirs, files in os.walk(top, topdown=True): 825 if not "examples" in root: continue 826 if "dSYM" in root: continue 827 if os.path.basename(root.rstrip("/")) == 'output': continue 828 if self.verbose: print(root) 829 self.genPetscTests(root,dirs,files,dataDict) 830 # Now summarize this dictionary 831 self.genPetscTests_summarize(dataDict) 832 return dataDict 833 834 def gen_gnumake(self, fd): 835 """ 836 Overwrite of the method in the base PETSc class 837 """ 838 def write(stem, srcs): 839 for lang in LANGS: 840 if srcs[lang]['srcs']: 841 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 842 for pkg in PKGS: 843 srcs = self.gen_pkg(pkg) 844 write('testsrcs-' + pkg, srcs) 845 # Handle dependencies 846 for lang in LANGS: 847 for exfile in srcs[lang]['srcs']: 848 if exfile in srcs[lang]: 849 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 850 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 851 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 852 if deps: 853 # The executable literally depends on the object file because it is linked 854 fd.write(ex +": " + " ".join(deps) +'\n') 855 # The object file containing 'main' does not normally depend on other object 856 # files, but it does when it includes their modules. This dependency is 857 # overly blunt and could be reduced to only depend on object files for 858 # modules that are used, like "*f90aux.o". 859 fd.write(exfo +": " + " ".join(deps) +'\n') 860 861 return self.gendeps 862 863 def gen_pkg(self, pkg): 864 """ 865 Overwrite of the method in the base PETSc class 866 """ 867 return self.sources[pkg] 868 869 def write_gnumake(self, dataDict, output=None): 870 """ 871 Write out something similar to files from gmakegen.py 872 873 Test depends on script which also depends on source 874 file, but since I don't have a good way generating 875 acting on a single file (oops) just depend on 876 executable which in turn will depend on src file 877 """ 878 # Different options for how to set up the targets 879 compileExecsFirst=False 880 881 # Open file 882 fd = open(output, 'w') 883 884 # Write out the sources 885 gendeps = self.gen_gnumake(fd) 886 887 # Write out the tests and execname targets 888 fd.write("\n#Tests and executables\n") # Delimiter 889 890 for pkg in PKGS: 891 # These grab the ones that are built 892 for lang in LANGS: 893 testdeps=[] 894 for ftest in self.tests[pkg][lang]: 895 test=os.path.basename(ftest) 896 basedir=os.path.dirname(ftest) 897 testdeps.append(self.nameSpace(test,basedir)) 898 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 899 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 900 901 # test targets 902 for ftest in self.tests[pkg][lang]: 903 test=os.path.basename(ftest) 904 basedir=os.path.dirname(ftest) 905 testdir="${TESTDIR}/"+basedir+"/" 906 nmtest=self.nameSpace(test,basedir) 907 rundir=os.path.join(testdir,test) 908 #print test, nmtest 909 script=test+".sh" 910 911 # Deps 912 exfile=self.tests[pkg][lang][ftest]['exfile'] 913 fullex=os.path.join(self.srcdir,exfile) 914 localexec=self.tests[pkg][lang][ftest]['exec'] 915 execname=os.path.join(testdir,localexec) 916 fullscript=os.path.join(testdir,script) 917 tmpfile=os.path.join(testdir,test,test+".tmp") 918 919 # *.counts depends on the script and either executable (will 920 # be run) or the example source file (SKIP or TODO) 921 fd.write('%s.counts : %s %s' 922 % (os.path.join('$(TESTDIR)/counts', nmtest), 923 fullscript, 924 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 925 ) 926 if exfile in self.sources[pkg][lang]: 927 for dep in self.sources[pkg][lang][exfile]: 928 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 929 fd.write('\n') 930 931 # Now write the args: 932 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 933 934 fd.close() 935 return 936 937def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 938 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 939 if petsc_arch: 940 if len(petsc_arch.split(os.path.sep))>1: 941 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 942 output = os.path.join(testdir, 'testfiles') 943 944 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 945 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 946 testdir=testdir) 947 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 948 pEx.write_gnumake(dataDict, output) 949 950if __name__ == '__main__': 951 import optparse 952 parser = optparse.OptionParser() 953 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 954 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 955 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 956 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 957 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 958 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 959 960 opts, extra_args = parser.parse_args() 961 if extra_args: 962 import sys 963 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 964 exit(1) 965 if opts.testdir is None: 966 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 967 968 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 969 verbose=opts.verbose, 970 single_ex=opts.single_executable, srcdir=opts.srcdir, 971 testdir=opts.testdir) 972