1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 3 modes of running tests: Normal builds, test installs from builds, 23test installs from within install dir. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. Test immediately after build through makefile & lib/petsc/conf/test: 46 (Not in installDir, but using prefix dir. PETSC_ARCH='') 47 48 +---------------------+----------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+----------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+----------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+----------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_SRC_DIR/src | 56 +---------------------+----------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+----------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 60 +---------------------+----------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 62 +---------------------+----------------------------------+ 63 64Case 3. From install dir: 65 66 +---------------------+-------------------------------------------------------+ 67 | PETSC_DIR | <prefix dir> | 68 +---------------------+-------------------------------------------------------+ 69 | PETSC_ARCH | '' | 70 +---------------------+-------------------------------------------------------+ 71 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 72 +---------------------+-------------------------------------------------------+ 73 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 74 +---------------------+-------------------------------------------------------+ 75 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 76 +---------------------+-------------------------------------------------------+ 77 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 78 +---------------------+-------------------------------------------------------+ 79 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 80 +---------------------+-------------------------------------------------------+ 81 82""" 83class generateExamples(Petsc): 84 """ 85 gmakegen.py has basic structure for finding the files, writing out 86 the dependencies, etc. 87 """ 88 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 89 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 90 91 self.single_ex=single_ex 92 93 # Set locations to handle movement 94 self.inInstallDir=self.getInInstallDir(thisscriptdir) 95 96 if not self.inInstallDir: 97 if not petsc_arch == '': 98 # Case 1 discussed above 99 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 100 self.srcdir=os.path.join(self.petsc_dir,'src') 101 else: 102 # Case 2 discussed above 103 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 104 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 105 else: 106 # Case 3 discussed above 107 # set PETSC_ARCH to install directory to allow script to work in both 108 dirlist=thisscriptdir.split(os.path.sep) 109 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 110 self.arch_dir=installdir 111 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 112 113 # Do some initialization 114 if testdir: 115 # If full path given, then use it, otherwise assume relative to arch_dir 116 if testdir.strip().startswith(os.path.sep): 117 self.testroot_dir=testdir.strip() 118 else: 119 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 120 else: 121 self.testroot_dir=os.path.join(self.arch_dir,"tests") 122 123 self.ptNaming=True 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64".split() 131 self.languages="fortran cuda cxx".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in PKGS: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def nameSpace(self,srcfile,srcdir): 181 """ 182 Because the scripts have a non-unique naming, the pretty-printing 183 needs to convey the srcdir and srcfile. There are two ways of doing this. 184 """ 185 if self.ptNaming: 186 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 187 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 188 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 189 nameString=prefix+srcfile 190 else: 191 #nameString=srcdir+": "+srcfile 192 nameString=srcfile 193 return nameString 194 195 def getLanguage(self,srcfile): 196 """ 197 Based on the source, determine associated language as found in gmakegen.LANGS 198 Can we just return srcext[1:\] now? 199 """ 200 langReq=None 201 srcext=os.path.splitext(srcfile)[-1] 202 if srcext in ".F90".split(): langReq="F90" 203 if srcext in ".F".split(): langReq="F" 204 if srcext in ".cxx".split(): langReq="cxx" 205 if srcext == ".cu": langReq="cu" 206 if srcext == ".c": langReq="c" 207 #if not langReq: print "ERROR: ", srcext, srcfile 208 return langReq 209 210 def _getLoopVars(self,inDict,testname, isSubtest=False): 211 """ 212 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 213 Return: 214 inDict['args']: -ksp_monitor 215 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 216 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 217 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 218 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 219 subst should be passed in instead of inDict 220 """ 221 loopVars={}; newargs="" 222 lkeys=inDict.keys() 223 lsuffix='_' 224 from testparse import parseLoopArgs 225 for key in lkeys: 226 if type(inDict[key])!=types.StringType: continue 227 keystr = str(inDict[key]) 228 akey=('subargs' if key=='args' else key) # what to assign 229 if akey not in inDict: inDict[akey]='' 230 varlist=[] 231 for varset in re.split('-(?=[a-zA-Z])',keystr): 232 if not varset.strip(): continue 233 if '{{' in varset: 234 keyvar,lvars,ftype=parseLoopArgs(varset) 235 if akey not in loopVars: loopVars[akey]={} 236 varlist.append(keyvar) 237 loopVars[akey][keyvar]=[keyvar,lvars] 238 if akey=='nsize': 239 inDict[akey] = '${' + keyvar + '}' 240 lsuffix+=akey+'-'+inDict[akey]+'_' 241 else: 242 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 243 lsuffix+=keyvar+'-${' + keyvar + '}_' 244 else: 245 if key=='args': newargs+=" -"+varset.strip() 246 if varlist: loopVars[akey]['varlist']=varlist 247 248 249 # For subtests, args are always substituted in (not top level) 250 if isSubtest: 251 inDict['subargs']+=" "+newargs.strip() 252 inDict['args']='' 253 if 'label_suffix' in inDict: 254 inDict['label_suffix']+=lsuffix.rstrip('_') 255 else: 256 inDict['label_suffix']=lsuffix.rstrip('_') 257 else: 258 if loopVars.keys(): 259 inDict['args']=newargs.strip() 260 inDict['label_suffix']=lsuffix.rstrip('_') 261 if loopVars.keys(): 262 return loopVars 263 else: 264 return None 265 266 def getArgLabel(self,testDict): 267 """ 268 In all of the arguments in the test dictionary, create a simple 269 string for searching within the makefile system. For simplicity in 270 search, remove "-", for strings, etc. 271 Also, concatenate the arg commands 272 For now, ignore nsize -- seems hard to search for anyway 273 """ 274 # Collect all of the args associated with a test 275 argStr=("" if 'args' not in testDict else testDict['args']) 276 if 'subtests' in testDict: 277 for stest in testDict["subtests"]: 278 sd=testDict[stest] 279 argStr=argStr+("" if 'args' not in sd else sd['args']) 280 281 # Now go through and cleanup 282 argStr=re.sub('{{(.*?)}}',"",argStr) 283 argStr=re.sub('-'," ",argStr) 284 for digit in string.digits: argStr=re.sub(digit," ",argStr) 285 argStr=re.sub("\.","",argStr) 286 argStr=re.sub(",","",argStr) 287 argStr=re.sub('\+',' ',argStr) 288 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 289 return argStr.strip() 290 291 def addToSources(self,exfile,root,srcDict): 292 """ 293 Put into data structure that allows easy generation of makefile 294 """ 295 rpath=self.srcrelpath(root) 296 pkg=rpath.split(os.path.sep)[1] 297 relpfile=os.path.join(rpath,exfile) 298 lang=self.getLanguage(exfile) 299 if not lang: return 300 self.sources[pkg][lang]['srcs'].append(relpfile) 301 self.sources[pkg][lang][relpfile] = [] 302 if 'depends' in srcDict: 303 depSrcList=srcDict['depends'].split() 304 for depSrc in depSrcList: 305 depObj=os.path.splitext(depSrc)[0]+".o" 306 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 307 308 # In gmakefile, ${TESTDIR} var specifies the object compilation 309 testsdir=self.srcrelpath(root)+"/" 310 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 311 self.objects[pkg].append(objfile) 312 return 313 314 def addToTests(self,test,root,exfile,execname,testDict): 315 """ 316 Put into data structure that allows easy generation of makefile 317 Organized by languages to allow testing of languages 318 """ 319 rpath=self.srcrelpath(root) 320 pkg=rpath.split("/")[1] 321 #nmtest=self.nameSpace(test,root) 322 nmtest=os.path.join(rpath,test) 323 lang=self.getLanguage(exfile) 324 if not lang: return 325 self.tests[pkg][lang][nmtest]={} 326 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 327 self.tests[pkg][lang][nmtest]['exec']=execname 328 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 329 return 330 331 def getExecname(self,exfile,root): 332 """ 333 Generate bash script using template found next to this file. 334 This file is read in at constructor time to avoid file I/O 335 """ 336 rpath=self.srcrelpath(root) 337 if self.single_ex: 338 execname=rpath.split("/")[1]+"-ex" 339 else: 340 execname=os.path.splitext(exfile)[0] 341 return execname 342 343 def getSubstVars(self,testDict,rpath,testname): 344 """ 345 Create a dictionary with all of the variables that get substituted 346 into the template commands found in example_template.py 347 """ 348 subst={} 349 350 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 351 if 'nsize' not in testDict: testDict['nsize']=1 352 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 353 for ak in testparse.acceptedkeys: 354 if ak=='test': continue 355 subst[ak]=(testDict[ak] if ak in testDict else '') 356 357 # Now do other variables 358 subst['execname']=testDict['execname'] 359 if 'filter' in testDict: 360 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 361 362 # Others 363 subst['subargs']='' # Default. For variables override 364 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 365 subst['label_suffix']='' 366 subst['comments']="\n#".join(subst['comments'].split("\n")) 367 if subst['comments']: subst['comments']="#"+subst['comments'] 368 subst['exec']="../"+subst['execname'] 369 subst['testroot']=self.testroot_dir 370 subst['testname']=testname 371 dp = self.conf.get('DATAFILESPATH','') 372 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 373 374 # This is used to label some matrices 375 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 376 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 377 378 # These can have for loops and are treated separately later 379 subst['nsize']=str(subst['nsize']) 380 381 #Conf vars 382 if self.petsc_arch.find('valgrind')>=0: 383 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 384 else: 385 subst['mpiexec']=self.conf['MPIEXEC'] 386 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 387 subst['petsc_arch']=self.petsc_arch 388 if not self.inInstallDir: 389 if not self.petsc_arch == '': 390 # Case 1 391 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 392 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 393 else: 394 # Case 2 395 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 396 subst['PETSC_BINDIR']=os.path.join(os.path.dirname(self.srcdir),'bin') 397 else: 398 # Case 3 399 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 400 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 401 subst['diff']=self.conf['DIFF'] 402 subst['rm']=self.conf['RM'] 403 subst['grep']=self.conf['GREP'] 404 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 405 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 406 407 # Output file is special because of subtests override 408 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 409 if not "_" in defroot: defroot=defroot+"_1" 410 subst['defroot']=defroot 411 subst['label']=self.nameSpace(defroot,subst['srcdir']) 412 subst['redirect_file']=defroot+".tmp" 413 if 'output_file' not in testDict: 414 subst['output_file']="output/"+defroot+".out" 415 # Add in the full path here. 416 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 417 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 418 if not subst['TODO']: 419 print "Warning: "+subst['output_file']+" not found." 420 # Worry about alt files here -- see 421 # src/snes/examples/tutorials/output/ex22*.out 422 altlist=[subst['output_file']] 423 basefile,ext = os.path.splitext(subst['output_file']) 424 for i in range(1,9): 425 altroot=basefile+"_alt" 426 if i > 1: altroot=altroot+"_"+str(i) 427 af=altroot+".out" 428 srcaf=os.path.join(subst['srcdir'],af) 429 fullaf=os.path.join(self.petsc_dir,srcaf) 430 if os.path.isfile(fullaf): altlist.append(srcaf) 431 if len(altlist)>1: subst['altfiles']=altlist 432 #if len(altlist)>1: print "Found alt files: ",altlist 433 434 return subst 435 436 def getCmds(self,subst,i): 437 """ 438 Generate bash script using template found next to this file. 439 This file is read in at constructor time to avoid file I/O 440 """ 441 nindnt=i # the start and has to be consistent with below 442 cmdindnt=self.indent*nindnt 443 cmdLines="" 444 445 # MPI is the default -- but we have a few odd commands 446 if not subst['command']: 447 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 448 else: 449 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 450 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 451 452 cmdLines+=cmdindnt+'if test $res = 0; then\n' 453 diffindnt=self.indent*(nindnt+1) 454 if not subst['filter_output']: 455 if 'altfiles' not in subst: 456 cmd=diffindnt+self._substVars(subst,example_template.difftest) 457 else: 458 # Have to do it by hand a bit because of variable number of alt files 459 rf=subst['redirect_file'] 460 cmd=diffindnt+example_template.difftest.split('@')[0] 461 for i in range(len(subst['altfiles'])): 462 af=subst['altfiles'][i] 463 cmd+=af+' '+rf 464 if i!=len(subst['altfiles'])-1: 465 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 466 cmd+=' || ${diff_exe} ' 467 else: 468 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 469 cmd+=subst['label_suffix']+' ""' # Quotes are painful 470 else: 471 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 472 cmdLines+=cmd+"\n" 473 cmdLines+=cmdindnt+'else\n' 474 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 475 cmdLines+=cmdindnt+'fi\n' 476 return cmdLines 477 478 def _substVars(self,subst,origStr): 479 """ 480 Substitute variables 481 """ 482 Str=origStr 483 for subkey in subst: 484 if type(subst[subkey])!=types.StringType: continue 485 patt="@"+subkey.upper()+"@" 486 Str=re.sub(patt,subst[subkey],Str) 487 return Str 488 489 def _writeTodoSkip(self,fh,tors,reasons,footer): 490 """ 491 Write out the TODO and SKIP lines in the file 492 The TODO or SKIP variable, tors, should be lower case 493 """ 494 TORS=tors.upper() 495 template=eval("example_template."+tors+"line") 496 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 497 tab = '' 498 if reasons: 499 fh.write('if ! $force; then\n') 500 tab = tab + ' ' 501 if reasons == ["Requires DATAFILESPATH"]: 502 # The only reason not to run is DATAFILESPATH, which we check at run-time 503 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 504 tab = tab + ' ' 505 if reasons: 506 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 507 fh.write(tab+footer+"\n") 508 fh.write(tab+"exit\n") 509 if reasons == ["Requires DATAFILESPATH"]: 510 fh.write(' fi\n') 511 if reasons: 512 fh.write('fi\n') 513 fh.write('\n\n') 514 return 515 516 def getLoopVarsHead(self,loopVars,i): 517 """ 518 Generate a nicely indented string with the format loops 519 Here is what the data structure looks like 520 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 521 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 522 loopVars['subargs']['pc_type']=["j","cholesky sor"] 523 """ 524 outstr=''; indnt=self.indent 525 for key in loopVars: 526 for var in loopVars[key]['varlist']: 527 varval=loopVars[key][var] 528 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 529 i = i + 1 530 return (outstr,i) 531 532 def getLoopVarsFoot(self,loopVars,i): 533 outstr=''; indnt=self.indent 534 for key in loopVars: 535 for var in loopVars[key]['varlist']: 536 i = i - 1 537 outstr += indnt * i + "done\n" 538 return (outstr,i) 539 540 def genRunScript(self,testname,root,isRun,srcDict): 541 """ 542 Generate bash script using template found next to this file. 543 This file is read in at constructor time to avoid file I/O 544 """ 545 # runscript_dir directory has to be consistent with gmakefile 546 testDict=srcDict[testname] 547 rpath=self.srcrelpath(root) 548 runscript_dir=os.path.join(self.testroot_dir,rpath) 549 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 550 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 551 552 # Get variables to go into shell scripts. last time testDict used 553 subst=self.getSubstVars(testDict,rpath,testname) 554 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 555 #if '33_' in testname: print subst['subargs'] 556 557 #Handle runfiles 558 for lfile in subst.get('localrunfiles','').split(): 559 fullfile=os.path.join(root,lfile) 560 if os.path.isdir(fullfile): 561 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 562 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 563 else: 564 shutil.copy(fullfile,runscript_dir) 565 # Check subtests for local runfiles 566 for stest in subst.get("subtests",[]): 567 for lfile in testDict[stest].get('localrunfiles','').split(): 568 fullfile=os.path.join(root,lfile) 569 if os.path.isdir(fullfile): 570 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 571 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 572 else: 573 shutil.copy(fullfile,self.runscript_dir) 574 575 # Now substitute the key variables into the header and footer 576 header=self._substVars(subst,example_template.header) 577 # The header is done twice to enable @...@ in header 578 header=self._substVars(subst,header) 579 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 580 581 # Start writing the file 582 fh.write(header+"\n") 583 584 # If there is a TODO or a SKIP then we do it before writing out the 585 # rest of the command (which is useful for working on the test) 586 # SKIP and TODO can be for the source file or for the runs 587 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 588 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 589 590 j=0 # for indentation 591 592 if loopVars: 593 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 594 if (loopHead): fh.write(loopHead+"\n") 595 596 # Subtests are special 597 if 'subtests' in testDict: 598 substP=subst # Subtests can inherit args but be careful 599 k=0 # for label suffixes 600 for stest in testDict["subtests"]: 601 subst=substP.copy() 602 subst.update(testDict[stest]) 603 # nsize is special because it is usually overwritten 604 if 'nsize' in testDict[stest]: 605 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 606 else: 607 fh.write("nsize=1\n") 608 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 609 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 610 #if '10_9' in testname: print sLoopVars 611 if sLoopVars: 612 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 613 fh.write(sLoopHead+"\n") 614 fh.write(self.getCmds(subst,j)+"\n") 615 if sLoopVars: 616 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 617 fh.write(sLoopFoot+"\n") 618 else: 619 fh.write(self.getCmds(subst,j)+"\n") 620 621 if loopVars: 622 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 623 fh.write(loopFoot+"\n") 624 625 fh.write(footer+"\n") 626 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 627 #if '10_9' in testname: sys.exit() 628 return 629 630 def genScriptsAndInfo(self,exfile,root,srcDict): 631 """ 632 Generate scripts from the source file, determine if built, etc. 633 For every test in the exfile with info in the srcDict: 634 1. Determine if it needs to be run for this arch 635 2. Generate the script 636 3. Generate the data needed to write out the makefile in a 637 convenient way 638 All tests are *always* run, but some may be SKIP'd per the TAP standard 639 """ 640 debug=False 641 execname=self.getExecname(exfile,root) 642 isBuilt=self._isBuilt(exfile,srcDict) 643 for test in srcDict: 644 if test in self.buildkeys: continue 645 if debug: print self.nameSpace(exfile,root), test 646 srcDict[test]['execname']=execname # Convenience in generating scripts 647 isRun=self._isRun(srcDict[test]) 648 self.genRunScript(test,root,isRun,srcDict) 649 srcDict[test]['isrun']=isRun 650 self.addToTests(test,root,exfile,execname,srcDict[test]) 651 652 # This adds to datastructure for building deps 653 if isBuilt: self.addToSources(exfile,root,srcDict) 654 return 655 656 def _isBuilt(self,exfile,srcDict): 657 """ 658 Determine if this file should be built. 659 """ 660 # Get the language based on file extension 661 srcDict['SKIP'] = [] 662 lang=self.getLanguage(exfile) 663 if (lang=="F" or lang=="F90"): 664 if not self.have_fortran: 665 srcDict["SKIP"].append("Fortran required for this test") 666 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 667 srcDict["SKIP"].append("Fortran f90freeform required for this test") 668 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 669 srcDict["SKIP"].append("CUDA required for this test") 670 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 671 srcDict["SKIP"].append("C++ required for this test") 672 673 # Deprecated source files 674 if srcDict.get("TODO"): 675 return False 676 677 # isRun can work with srcDict to handle the requires 678 if "requires" in srcDict: 679 if srcDict["requires"]: 680 return self._isRun(srcDict) 681 682 return srcDict['SKIP'] == [] 683 684 685 def _isRun(self,testDict, debug=False): 686 """ 687 Based on the requirements listed in the src file and the petscconf.h 688 info, determine whether this test should be run or not. 689 """ 690 indent=" " 691 692 if 'SKIP' not in testDict: 693 testDict['SKIP'] = [] 694 # MPI requirements 695 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 696 if debug: print indent+"Cannot run parallel tests" 697 testDict['SKIP'].append("Parallel test with serial build") 698 699 # The requirements for the test are the sum of all the run subtests 700 if 'subtests' in testDict: 701 if 'requires' not in testDict: testDict['requires']="" 702 for stest in testDict['subtests']: 703 if 'requires' in testDict[stest]: 704 testDict['requires']+=" "+testDict[stest]['requires'] 705 706 707 # Now go through all requirements 708 if 'requires' in testDict: 709 for requirement in testDict['requires'].split(): 710 requirement=requirement.strip() 711 if not requirement: continue 712 if debug: print indent+"Requirement: ", requirement 713 isNull=False 714 if requirement.startswith("!"): 715 requirement=requirement[1:]; isNull=True 716 # Precision requirement for reals 717 if requirement in self.precision_types: 718 if self.conf['PETSC_PRECISION']==requirement: 719 if isNull: 720 testDict['SKIP'].append("not "+requirement+" required") 721 continue 722 continue # Success 723 elif not isNull: 724 testDict['SKIP'].append(requirement+" required") 725 continue 726 # Precision requirement for ints 727 if requirement in self.integer_types: 728 if requirement=="int32": 729 if self.conf['PETSC_SIZEOF_INT']==4: 730 if isNull: 731 testDict['SKIP'].append("not int32 required") 732 continue 733 continue # Success 734 elif not isNull: 735 testDict['SKIP'].append("int32 required") 736 continue 737 if requirement=="int64": 738 if self.conf['PETSC_SIZEOF_INT']==8: 739 if isNull: 740 testDict['SKIP'].append("NOT int64 required") 741 continue 742 continue # Success 743 elif not isNull: 744 testDict['SKIP'].append("int64 required") 745 continue 746 # Datafilespath 747 if requirement=="datafilespath" and not isNull: 748 testDict['SKIP'].append("Requires DATAFILESPATH") 749 continue 750 # Defines -- not sure I have comments matching 751 if "define(" in requirement.lower(): 752 reqdef=requirement.split("(")[1].split(")")[0] 753 if reqdef in self.conf: 754 if isNull: 755 testDict['SKIP'].append("Null requirement not met: "+requirement) 756 continue 757 continue # Success 758 elif not isNull: 759 testDict['SKIP'].append("Required: "+requirement) 760 continue 761 762 # Rest should be packages that we can just get from conf 763 if requirement == "complex": 764 petscconfvar="PETSC_USE_COMPLEX" 765 else: 766 petscconfvar="PETSC_HAVE_"+requirement.upper() 767 if self.conf.get(petscconfvar): 768 if isNull: 769 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 770 continue 771 continue # Success 772 elif not isNull: 773 if debug: print "requirement not found: ", requirement 774 testDict['SKIP'].append(petscconfvar+" requirement not met") 775 continue 776 777 return testDict['SKIP'] == [] 778 779 def genPetscTests_summarize(self,dataDict): 780 """ 781 Required method to state what happened 782 """ 783 if not self.summarize: return 784 indent=" " 785 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 786 fh=open(fhname,"w") 787 #print "See ", fhname 788 for root in dataDict: 789 relroot=self.srcrelpath(root) 790 pkg=relroot.split("/")[1] 791 fh.write(relroot+"\n") 792 allSrcs=[] 793 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 794 for exfile in dataDict[root]: 795 # Basic information 796 rfile=os.path.join(relroot,exfile) 797 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 798 fh.write(indent+exfile+indent*4+builtStatus+"\n") 799 800 for test in dataDict[root][exfile]: 801 if test in self.buildkeys: continue 802 line=indent*2+test 803 fh.write(line+"\n") 804 # Looks nice to have the keys in order 805 #for key in dataDict[root][exfile][test]: 806 for key in "isrun abstracted nsize args requires script".split(): 807 if key not in dataDict[root][exfile][test]: continue 808 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 809 fh.write(line+"\n") 810 fh.write("\n") 811 fh.write("\n") 812 fh.write("\n") 813 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 814 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 815 fh.close() 816 return 817 818 def genPetscTests(self,root,dirs,files,dataDict): 819 """ 820 Go through and parse the source files in the directory to generate 821 the examples based on the metadata contained in the source files 822 """ 823 debug=False 824 # Use examplesAnalyze to get what the makefles think are sources 825 #self.examplesAnalyze(root,dirs,files,anlzDict) 826 827 dataDict[root]={} 828 829 for exfile in files: 830 #TST: Until we replace files, still leaving the orginals as is 831 #if not exfile.startswith("new_"+"ex"): continue 832 #if not exfile.startswith("ex"): continue 833 834 # Ignore emacs and other temporary files 835 if exfile.startswith("."): continue 836 if exfile.startswith("#"): continue 837 838 # Convenience 839 fullex=os.path.join(root,exfile) 840 if self.verbose: print(' --> '+fullex) 841 dataDict[root].update(testparse.parseTestFile(fullex,0)) 842 if exfile in dataDict[root]: 843 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 844 845 return 846 847 def walktree(self,top): 848 """ 849 Walk a directory tree, starting from 'top' 850 """ 851 #print "action", action 852 # Goal of action is to fill this dictionary 853 dataDict={} 854 for root, dirs, files in os.walk(top, topdown=True): 855 if not "examples" in root: continue 856 if "dSYM" in root: continue 857 if os.path.basename(root.rstrip("/")) == 'output': continue 858 if self.verbose: print(root) 859 self.genPetscTests(root,dirs,files,dataDict) 860 # Now summarize this dictionary 861 self.genPetscTests_summarize(dataDict) 862 return dataDict 863 864 def gen_gnumake(self, fd): 865 """ 866 Overwrite of the method in the base PETSc class 867 """ 868 def write(stem, srcs): 869 for lang in LANGS: 870 if srcs[lang]['srcs']: 871 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 872 for pkg in PKGS: 873 srcs = self.gen_pkg(pkg) 874 write('testsrcs-' + pkg, srcs) 875 # Handle dependencies 876 for lang in LANGS: 877 for exfile in srcs[lang]['srcs']: 878 if exfile in srcs[lang]: 879 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 880 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 881 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 882 if deps: 883 # The executable literally depends on the object file because it is linked 884 fd.write(ex +": " + " ".join(deps) +'\n') 885 # The object file containing 'main' does not normally depend on other object 886 # files, but it does when it includes their modules. This dependency is 887 # overly blunt and could be reduced to only depend on object files for 888 # modules that are used, like "*f90aux.o". 889 fd.write(exfo +": " + " ".join(deps) +'\n') 890 891 return self.gendeps 892 893 def gen_pkg(self, pkg): 894 """ 895 Overwrite of the method in the base PETSc class 896 """ 897 return self.sources[pkg] 898 899 def write_gnumake(self,dataDict): 900 """ 901 Write out something similar to files from gmakegen.py 902 903 Test depends on script which also depends on source 904 file, but since I don't have a good way generating 905 acting on a single file (oops) just depend on 906 executable which in turn will depend on src file 907 """ 908 # Different options for how to set up the targets 909 compileExecsFirst=False 910 911 # Open file 912 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 913 fd = open(arch_files, 'w') 914 915 # Write out the sources 916 gendeps = self.gen_gnumake(fd) 917 918 # Write out the tests and execname targets 919 fd.write("\n#Tests and executables\n") # Delimiter 920 921 for pkg in PKGS: 922 # These grab the ones that are built 923 for lang in LANGS: 924 testdeps=[] 925 for ftest in self.tests[pkg][lang]: 926 test=os.path.basename(ftest) 927 basedir=os.path.dirname(ftest) 928 testdeps.append(self.nameSpace(test,basedir)) 929 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 930 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 931 932 # test targets 933 for ftest in self.tests[pkg][lang]: 934 test=os.path.basename(ftest) 935 basedir=os.path.dirname(ftest) 936 testdir="${TESTDIR}/"+basedir+"/" 937 nmtest=self.nameSpace(test,basedir) 938 rundir=os.path.join(testdir,test) 939 #print test, nmtest 940 script=test+".sh" 941 942 # Deps 943 exfile=self.tests[pkg][lang][ftest]['exfile'] 944 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 945 localexec=self.tests[pkg][lang][ftest]['exec'] 946 execname=os.path.join(testdir,localexec) 947 fullscript=os.path.join(testdir,script) 948 tmpfile=os.path.join(testdir,test,test+".tmp") 949 950 # *.counts depends on the script and either executable (will 951 # be run) or the example source file (SKIP or TODO) 952 fd.write('%s.counts : %s %s' 953 % (os.path.join('$(TESTDIR)/counts', nmtest), 954 fullscript, 955 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 956 ) 957 if exfile in self.sources[pkg][lang]: 958 for dep in self.sources[pkg][lang][exfile]: 959 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 960 fd.write('\n') 961 962 # Now write the args: 963 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 964 965 fd.close() 966 return 967 968 def writeHarness(self,output,dataDict): 969 """ 970 This is set up to write out multiple harness even if only gnumake 971 is supported now 972 """ 973 eval("self.write_"+output+"(dataDict)") 974 return 975 976def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 977 if output is None: 978 output = 'gnumake' 979 980 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 981 if petsc_arch: 982 if len(petsc_arch.split(os.path.sep))>1: 983 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 984 985 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 986 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 987 testdir=testdir) 988 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 989 pEx.writeHarness(output,dataDict) 990 991if __name__ == '__main__': 992 import optparse 993 parser = optparse.OptionParser() 994 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 995 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 996 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 997 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 998 parser.add_option('--output', help='Location to write output file', default=None) 999 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1000 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 1001 1002 opts, extra_args = parser.parse_args() 1003 if extra_args: 1004 import sys 1005 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1006 exit(1) 1007 1008 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1009 output=opts.output, verbose=opts.verbose, 1010 single_ex=opts.single_executable, srcdir=opts.srcdir, 1011 testdir=opts.testdir) 1012