1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 85 86 self.single_ex=single_ex 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 self.srcdir=os.path.join(self.petsc_dir,'src') 104 105 self.testroot_dir=os.path.abspath(testdir) 106 107 self.ptNaming=True 108 self.verbose=verbose 109 # Whether to write out a useful debugging 110 self.summarize=True if verbose else False 111 112 # For help in setting the requirements 113 self.precision_types="single double __float128 int32".split() 114 self.integer_types="int32 int64".split() 115 self.languages="fortran cuda cxx".split() # Always requires C so do not list 116 117 # Things that are not test 118 self.buildkeys=testparse.buildkeys 119 120 # Adding a dictionary for storing sources, objects, and tests 121 # to make building the dependency tree easier 122 self.sources={} 123 self.objects={} 124 self.tests={} 125 for pkg in PKGS: 126 self.sources[pkg]={} 127 self.objects[pkg]=[] 128 self.tests[pkg]={} 129 for lang in LANGS: 130 self.sources[pkg][lang]={} 131 self.sources[pkg][lang]['srcs']=[] 132 self.tests[pkg][lang]={} 133 134 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 135 136 self.indent=" " 137 if self.verbose: print('Finishing the constructor') 138 return 139 140 def srcrelpath(self,rdir): 141 """ 142 Get relative path to source directory 143 """ 144 return os.path.relpath(rdir,self.srcdir) 145 146 def getInInstallDir(self,thisscriptdir): 147 """ 148 When petsc is installed then this file in installed in: 149 <PREFIX>/share/petsc/examples/config/gmakegentest.py 150 otherwise the path is: 151 <PETSC_DIR>/config/gmakegentest.py 152 We use this difference to determine if we are in installdir 153 """ 154 dirlist=thisscriptdir.split(os.path.sep) 155 if len(dirlist)>4: 156 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 157 if lastfour==os.path.join('share','petsc','examples','config'): 158 return True 159 else: 160 return False 161 else: 162 return False 163 164 def nameSpace(self,srcfile,srcdir): 165 """ 166 Because the scripts have a non-unique naming, the pretty-printing 167 needs to convey the srcdir and srcfile. There are two ways of doing this. 168 """ 169 if self.ptNaming: 170 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 171 cdir=srcdir 172 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 173 nameString=prefix+srcfile 174 else: 175 #nameString=srcdir+": "+srcfile 176 nameString=srcfile 177 return nameString 178 179 def getLanguage(self,srcfile): 180 """ 181 Based on the source, determine associated language as found in gmakegen.LANGS 182 Can we just return srcext[1:\] now? 183 """ 184 langReq=None 185 srcext=os.path.splitext(srcfile)[-1] 186 if srcext in ".F90".split(): langReq="F90" 187 if srcext in ".F".split(): langReq="F" 188 if srcext in ".cxx".split(): langReq="cxx" 189 if srcext == ".cu": langReq="cu" 190 if srcext == ".c": langReq="c" 191 #if not langReq: print("ERROR: ", srcext, srcfile) 192 return langReq 193 194 def _getLoopVars(self,inDict,testname, isSubtest=False): 195 """ 196 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 197 Return: 198 inDict['args']: -ksp_monitor 199 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 200 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 201 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 202 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 203 subst should be passed in instead of inDict 204 """ 205 loopVars={}; newargs=[] 206 lsuffix='_' 207 argregex = re.compile(' (?=-[a-zA-Z])') 208 from testparse import parseLoopArgs 209 for key in inDict: 210 if key in ('SKIP', 'regexes'): 211 continue 212 akey=('subargs' if key=='args' else key) # what to assign 213 if akey not in inDict: inDict[akey]='' 214 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 215 # Always generate a loop over nsize, even if there is only one value 216 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 217 keystr = str(inDict[key]) 218 varlist = [] 219 for varset in argregex.split(keystr): 220 if not varset.strip(): continue 221 if '{{' in varset: 222 keyvar,lvars,ftype=parseLoopArgs(varset) 223 if akey not in loopVars: loopVars[akey]={} 224 varlist.append(keyvar) 225 loopVars[akey][keyvar]=[keyvar,lvars] 226 if akey=='nsize': 227 if len(lvars.split()) > 1: 228 lsuffix += akey +'-${' + keyvar + '}' 229 else: 230 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 231 lsuffix+=keyvar+'-${' + keyvar + '}_' 232 else: 233 if key=='args': 234 newargs.append(varset.strip()) 235 if varlist: 236 loopVars[akey]['varlist']=varlist 237 238 # For subtests, args are always substituted in (not top level) 239 if isSubtest: 240 inDict['subargs'] += " ".join(newargs) 241 inDict['args']='' 242 if 'label_suffix' in inDict: 243 inDict['label_suffix']+=lsuffix.rstrip('_') 244 else: 245 inDict['label_suffix']=lsuffix.rstrip('_') 246 else: 247 if loopVars: 248 inDict['args'] = ' '.join(newargs) 249 inDict['label_suffix']=lsuffix.rstrip('_') 250 return loopVars 251 252 def getArgLabel(self,testDict): 253 """ 254 In all of the arguments in the test dictionary, create a simple 255 string for searching within the makefile system. For simplicity in 256 search, remove "-", for strings, etc. 257 Also, concatenate the arg commands 258 For now, ignore nsize -- seems hard to search for anyway 259 """ 260 # Collect all of the args associated with a test 261 argStr=("" if 'args' not in testDict else testDict['args']) 262 if 'subtests' in testDict: 263 for stest in testDict["subtests"]: 264 sd=testDict[stest] 265 argStr=argStr+("" if 'args' not in sd else sd['args']) 266 267 # Now go through and cleanup 268 argStr=re.sub('{{(.*?)}}',"",argStr) 269 argStr=re.sub('-'," ",argStr) 270 for digit in string.digits: argStr=re.sub(digit," ",argStr) 271 argStr=re.sub("\.","",argStr) 272 argStr=re.sub(",","",argStr) 273 argStr=re.sub('\+',' ',argStr) 274 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 275 return argStr.strip() 276 277 def addToSources(self,exfile,rpath,srcDict): 278 """ 279 Put into data structure that allows easy generation of makefile 280 """ 281 pkg=rpath.split(os.path.sep)[0] 282 relpfile=os.path.join(rpath,exfile) 283 lang=self.getLanguage(exfile) 284 if not lang: return 285 self.sources[pkg][lang]['srcs'].append(relpfile) 286 self.sources[pkg][lang][relpfile] = [] 287 if 'depends' in srcDict: 288 depSrcList=srcDict['depends'].split() 289 for depSrc in depSrcList: 290 depObj=os.path.splitext(depSrc)[0]+".o" 291 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 292 293 # In gmakefile, ${TESTDIR} var specifies the object compilation 294 testsdir=rpath+"/" 295 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 296 self.objects[pkg].append(objfile) 297 return 298 299 def addToTests(self,test,rpath,exfile,execname,testDict): 300 """ 301 Put into data structure that allows easy generation of makefile 302 Organized by languages to allow testing of languages 303 """ 304 pkg=rpath.split("/")[0] 305 nmtest=os.path.join(rpath,test) 306 lang=self.getLanguage(exfile) 307 if not lang: return 308 self.tests[pkg][lang][nmtest]={} 309 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 310 self.tests[pkg][lang][nmtest]['exec']=execname 311 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 312 return 313 314 def getExecname(self,exfile,rpath): 315 """ 316 Generate bash script using template found next to this file. 317 This file is read in at constructor time to avoid file I/O 318 """ 319 if self.single_ex: 320 execname=rpath.split("/")[1]+"-ex" 321 else: 322 execname=os.path.splitext(exfile)[0] 323 return execname 324 325 def getSubstVars(self,testDict,rpath,testname): 326 """ 327 Create a dictionary with all of the variables that get substituted 328 into the template commands found in example_template.py 329 """ 330 subst={} 331 332 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 333 if 'nsize' not in testDict: testDict['nsize'] = '1' 334 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 335 for ak in testparse.acceptedkeys: 336 if ak=='test': continue 337 subst[ak]=(testDict[ak] if ak in testDict else '') 338 339 # Now do other variables 340 subst['execname']=testDict['execname'] 341 if 'filter' in testDict: 342 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 343 344 # Others 345 subst['subargs']='' # Default. For variables override 346 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 347 subst['label_suffix']='' 348 subst['comments']="\n#".join(subst['comments'].split("\n")) 349 if subst['comments']: subst['comments']="#"+subst['comments'] 350 subst['exec']="../"+subst['execname'] 351 subst['testroot']=self.testroot_dir 352 subst['testname']=testname 353 dp = self.conf.get('DATAFILESPATH','') 354 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 355 356 # This is used to label some matrices 357 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 358 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 359 360 #Conf vars 361 if self.petsc_arch.find('valgrind')>=0: 362 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 363 else: 364 subst['mpiexec']=self.conf['MPIEXEC'] 365 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 366 subst['petsc_arch']=self.petsc_arch 367 if self.inInstallDir: 368 # Case 2 369 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 370 else: 371 # Case 1 372 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 373 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 374 subst['diff']=self.conf['DIFF'] 375 subst['rm']=self.conf['RM'] 376 subst['grep']=self.conf['GREP'] 377 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 378 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 379 380 # Output file is special because of subtests override 381 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 382 if not "_" in defroot: defroot=defroot+"_1" 383 subst['defroot']=defroot 384 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 385 subst['redirect_file']=defroot+".tmp" 386 if 'output_file' not in testDict: 387 subst['output_file']="output/"+defroot+".out" 388 # Add in the full path here. 389 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 390 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 391 if not subst['TODO']: 392 print("Warning: "+subst['output_file']+" not found.") 393 # Worry about alt files here -- see 394 # src/snes/examples/tutorials/output/ex22*.out 395 altlist=[subst['output_file']] 396 basefile,ext = os.path.splitext(subst['output_file']) 397 for i in range(1,9): 398 altroot=basefile+"_alt" 399 if i > 1: altroot=altroot+"_"+str(i) 400 af=altroot+".out" 401 srcaf=os.path.join(subst['srcdir'],af) 402 fullaf=os.path.join(self.petsc_dir,srcaf) 403 if os.path.isfile(fullaf): altlist.append(srcaf) 404 if len(altlist)>1: subst['altfiles']=altlist 405 #if len(altlist)>1: print("Found alt files: ",altlist) 406 407 subst['regexes']={} 408 for subkey in subst: 409 if subkey=='regexes': continue 410 if not isinstance(subst[subkey],str): continue 411 patt="@"+subkey.upper()+"@" 412 subst['regexes'][subkey]=re.compile(patt) 413 414 return subst 415 416 def _substVars(self,subst,origStr): 417 """ 418 Substitute variables 419 """ 420 Str=origStr 421 for subkey in subst: 422 if subkey=='regexes': continue 423 if not isinstance(subst[subkey],str): continue 424 if subkey.upper() not in Str: continue 425 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 426 return Str 427 428 def getCmds(self,subst,i): 429 """ 430 Generate bash script using template found next to this file. 431 This file is read in at constructor time to avoid file I/O 432 """ 433 nindnt=i # the start and has to be consistent with below 434 cmdindnt=self.indent*nindnt 435 cmdLines="" 436 437 # MPI is the default -- but we have a few odd commands 438 if not subst['command']: 439 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 440 else: 441 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 442 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 443 444 cmdLines+=cmdindnt+'if test $res = 0; then\n' 445 diffindnt=self.indent*(nindnt+1) 446 if not subst['filter_output']: 447 if 'altfiles' not in subst: 448 cmd=diffindnt+self._substVars(subst,example_template.difftest) 449 else: 450 # Have to do it by hand a bit because of variable number of alt files 451 rf=subst['redirect_file'] 452 cmd=diffindnt+example_template.difftest.split('@')[0] 453 for i in range(len(subst['altfiles'])): 454 af=subst['altfiles'][i] 455 cmd+=af+' '+rf 456 if i!=len(subst['altfiles'])-1: 457 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 458 cmd+=' || ${diff_exe} ' 459 else: 460 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 461 cmd+=subst['label_suffix']+' ""' # Quotes are painful 462 else: 463 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 464 cmdLines+=cmd+"\n" 465 cmdLines+=cmdindnt+'else\n' 466 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 467 cmdLines+=cmdindnt+'fi\n' 468 return cmdLines 469 470 def _writeTodoSkip(self,fh,tors,reasons,footer): 471 """ 472 Write out the TODO and SKIP lines in the file 473 The TODO or SKIP variable, tors, should be lower case 474 """ 475 TORS=tors.upper() 476 template=eval("example_template."+tors+"line") 477 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 478 tab = '' 479 if reasons: 480 fh.write('if ! $force; then\n') 481 tab = tab + ' ' 482 if reasons == ["Requires DATAFILESPATH"]: 483 # The only reason not to run is DATAFILESPATH, which we check at run-time 484 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 485 tab = tab + ' ' 486 if reasons: 487 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 488 fh.write(tab+footer+"\n") 489 fh.write(tab+"exit\n") 490 if reasons == ["Requires DATAFILESPATH"]: 491 fh.write(' fi\n') 492 if reasons: 493 fh.write('fi\n') 494 fh.write('\n\n') 495 return 496 497 def getLoopVarsHead(self,loopVars,i): 498 """ 499 Generate a nicely indented string with the format loops 500 Here is what the data structure looks like 501 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 502 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 503 loopVars['subargs']['pc_type']=["j","cholesky sor"] 504 """ 505 outstr=''; indnt=self.indent 506 507 for key in loopVars: 508 for var in loopVars[key]['varlist']: 509 varval=loopVars[key][var] 510 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 511 outstr += "\n\n" 512 513 for key in loopVars: 514 for var in loopVars[key]['varlist']: 515 varval=loopVars[key][var] 516 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 517 i = i + 1 518 return (outstr,i) 519 520 def getLoopVarsFoot(self,loopVars,i): 521 outstr=''; indnt=self.indent 522 for key in loopVars: 523 for var in loopVars[key]['varlist']: 524 i = i - 1 525 outstr += indnt * i + "done\n" 526 return (outstr,i) 527 528 def genRunScript(self,testname,root,isRun,srcDict): 529 """ 530 Generate bash script using template found next to this file. 531 This file is read in at constructor time to avoid file I/O 532 """ 533 # runscript_dir directory has to be consistent with gmakefile 534 testDict=srcDict[testname] 535 rpath=self.srcrelpath(root) 536 runscript_dir=os.path.join(self.testroot_dir,rpath) 537 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 538 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 539 540 # Get variables to go into shell scripts. last time testDict used 541 subst=self.getSubstVars(testDict,rpath,testname) 542 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 543 if 'subtests' in testDict: 544 # The subtests inherit inDict, so we don't need top-level loops. 545 loopVars = {} 546 547 #Handle runfiles 548 for lfile in subst.get('localrunfiles','').split(): 549 install_files(os.path.join(root, lfile), 550 os.path.join(runscript_dir, os.path.dirname(lfile))) 551 # Check subtests for local runfiles 552 for stest in subst.get("subtests",[]): 553 for lfile in testDict[stest].get('localrunfiles','').split(): 554 install_files(os.path.join(root, lfile), 555 os.path.join(runscript_dir, os.path.dirname(lfile))) 556 557 # Now substitute the key variables into the header and footer 558 header=self._substVars(subst,example_template.header) 559 # The header is done twice to enable @...@ in header 560 header=self._substVars(subst,header) 561 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 562 563 # Start writing the file 564 fh.write(header+"\n") 565 566 # If there is a TODO or a SKIP then we do it before writing out the 567 # rest of the command (which is useful for working on the test) 568 # SKIP and TODO can be for the source file or for the runs 569 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 570 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 571 572 j=0 # for indentation 573 574 if loopVars: 575 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 576 if (loopHead): fh.write(loopHead+"\n") 577 578 # Subtests are special 579 if 'subtests' in testDict: 580 substP=subst # Subtests can inherit args but be careful 581 k=0 # for label suffixes 582 for stest in testDict["subtests"]: 583 subst=substP.copy() 584 subst.update(testDict[stest]) 585 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 586 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 587 if sLoopVars: 588 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 589 fh.write(sLoopHead+"\n") 590 fh.write(self.getCmds(subst,j)+"\n") 591 if sLoopVars: 592 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 593 fh.write(sLoopFoot+"\n") 594 else: 595 fh.write(self.getCmds(subst,j)+"\n") 596 597 if loopVars: 598 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 599 fh.write(loopFoot+"\n") 600 601 fh.write(footer+"\n") 602 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 603 #if '10_9' in testname: sys.exit() 604 return 605 606 def genScriptsAndInfo(self,exfile,root,srcDict): 607 """ 608 Generate scripts from the source file, determine if built, etc. 609 For every test in the exfile with info in the srcDict: 610 1. Determine if it needs to be run for this arch 611 2. Generate the script 612 3. Generate the data needed to write out the makefile in a 613 convenient way 614 All tests are *always* run, but some may be SKIP'd per the TAP standard 615 """ 616 debug=False 617 rpath=self.srcrelpath(root) 618 execname=self.getExecname(exfile,rpath) 619 isBuilt=self._isBuilt(exfile,srcDict) 620 for test in srcDict: 621 if test in self.buildkeys: continue 622 if debug: print(self.nameSpace(exfile,root), test) 623 srcDict[test]['execname']=execname # Convenience in generating scripts 624 isRun=self._isRun(srcDict[test]) 625 self.genRunScript(test,root,isRun,srcDict) 626 srcDict[test]['isrun']=isRun 627 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 628 629 # This adds to datastructure for building deps 630 if isBuilt: self.addToSources(exfile,rpath,srcDict) 631 return 632 633 def _isBuilt(self,exfile,srcDict): 634 """ 635 Determine if this file should be built. 636 """ 637 # Get the language based on file extension 638 srcDict['SKIP'] = [] 639 lang=self.getLanguage(exfile) 640 if (lang=="F" or lang=="F90"): 641 if not self.have_fortran: 642 srcDict["SKIP"].append("Fortran required for this test") 643 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 644 srcDict["SKIP"].append("Fortran f90freeform required for this test") 645 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 646 srcDict["SKIP"].append("CUDA required for this test") 647 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 648 srcDict["SKIP"].append("C++ required for this test") 649 650 # Deprecated source files 651 if srcDict.get("TODO"): 652 return False 653 654 # isRun can work with srcDict to handle the requires 655 if "requires" in srcDict: 656 if srcDict["requires"]: 657 return self._isRun(srcDict) 658 659 return srcDict['SKIP'] == [] 660 661 662 def _isRun(self,testDict, debug=False): 663 """ 664 Based on the requirements listed in the src file and the petscconf.h 665 info, determine whether this test should be run or not. 666 """ 667 indent=" " 668 669 if 'SKIP' not in testDict: 670 testDict['SKIP'] = [] 671 # MPI requirements 672 if 'MPI_IS_MPIUNI' in self.conf: 673 if testDict.get('nsize', '1') != '1': 674 testDict['SKIP'].append("Parallel test with serial build") 675 676 # The requirements for the test are the sum of all the run subtests 677 if 'subtests' in testDict: 678 if 'requires' not in testDict: testDict['requires']="" 679 for stest in testDict['subtests']: 680 if 'requires' in testDict[stest]: 681 testDict['requires']+=" "+testDict[stest]['requires'] 682 if testDict.get('nsize', '1') != '1': 683 testDict['SKIP'].append("Parallel test with serial build") 684 break 685 686 # Now go through all requirements 687 if 'requires' in testDict: 688 for requirement in testDict['requires'].split(): 689 requirement=requirement.strip() 690 if not requirement: continue 691 if debug: print(indent+"Requirement: ", requirement) 692 isNull=False 693 if requirement.startswith("!"): 694 requirement=requirement[1:]; isNull=True 695 # Precision requirement for reals 696 if requirement in self.precision_types: 697 if self.conf['PETSC_PRECISION']==requirement: 698 if isNull: 699 testDict['SKIP'].append("not "+requirement+" required") 700 continue 701 continue # Success 702 elif not isNull: 703 testDict['SKIP'].append(requirement+" required") 704 continue 705 # Precision requirement for ints 706 if requirement in self.integer_types: 707 if requirement=="int32": 708 if self.conf['PETSC_SIZEOF_INT']==4: 709 if isNull: 710 testDict['SKIP'].append("not int32 required") 711 continue 712 continue # Success 713 elif not isNull: 714 testDict['SKIP'].append("int32 required") 715 continue 716 if requirement=="int64": 717 if self.conf['PETSC_SIZEOF_INT']==8: 718 if isNull: 719 testDict['SKIP'].append("NOT int64 required") 720 continue 721 continue # Success 722 elif not isNull: 723 testDict['SKIP'].append("int64 required") 724 continue 725 # Datafilespath 726 if requirement=="datafilespath" and not isNull: 727 testDict['SKIP'].append("Requires DATAFILESPATH") 728 continue 729 # Defines -- not sure I have comments matching 730 if "define(" in requirement.lower(): 731 reqdef=requirement.split("(")[1].split(")")[0] 732 if reqdef in self.conf: 733 if isNull: 734 testDict['SKIP'].append("Null requirement not met: "+requirement) 735 continue 736 continue # Success 737 elif not isNull: 738 testDict['SKIP'].append("Required: "+requirement) 739 continue 740 741 # Rest should be packages that we can just get from conf 742 if requirement == "complex": 743 petscconfvar="PETSC_USE_COMPLEX" 744 else: 745 petscconfvar="PETSC_HAVE_"+requirement.upper() 746 if self.conf.get(petscconfvar): 747 if isNull: 748 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 749 continue 750 continue # Success 751 elif not isNull: 752 if debug: print("requirement not found: ", requirement) 753 testDict['SKIP'].append(petscconfvar+" requirement not met") 754 continue 755 756 return testDict['SKIP'] == [] 757 758 def genPetscTests_summarize(self,dataDict): 759 """ 760 Required method to state what happened 761 """ 762 if not self.summarize: return 763 indent=" " 764 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 765 fh=open(fhname,"w") 766 for root in dataDict: 767 relroot=self.srcrelpath(root) 768 pkg=relroot.split("/")[1] 769 fh.write(relroot+"\n") 770 allSrcs=[] 771 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 772 for exfile in dataDict[root]: 773 # Basic information 774 rfile=os.path.join(relroot,exfile) 775 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 776 fh.write(indent+exfile+indent*4+builtStatus+"\n") 777 778 for test in dataDict[root][exfile]: 779 if test in self.buildkeys: continue 780 line=indent*2+test 781 fh.write(line+"\n") 782 # Looks nice to have the keys in order 783 #for key in dataDict[root][exfile][test]: 784 for key in "isrun abstracted nsize args requires script".split(): 785 if key not in dataDict[root][exfile][test]: continue 786 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 787 fh.write(line+"\n") 788 fh.write("\n") 789 fh.write("\n") 790 fh.write("\n") 791 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 792 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 793 fh.close() 794 return 795 796 def genPetscTests(self,root,dirs,files,dataDict): 797 """ 798 Go through and parse the source files in the directory to generate 799 the examples based on the metadata contained in the source files 800 """ 801 debug=False 802 # Use examplesAnalyze to get what the makefles think are sources 803 #self.examplesAnalyze(root,dirs,files,anlzDict) 804 805 dataDict[root]={} 806 807 for exfile in files: 808 #TST: Until we replace files, still leaving the orginals as is 809 #if not exfile.startswith("new_"+"ex"): continue 810 #if not exfile.startswith("ex"): continue 811 812 # Ignore emacs and other temporary files 813 if exfile.startswith("."): continue 814 if exfile.startswith("#"): continue 815 if exfile.endswith("~"): continue 816 # Only parse source files 817 ext=os.path.splitext(exfile)[-1].lstrip('.') 818 if ext not in LANGS: continue 819 820 # Convenience 821 fullex=os.path.join(root,exfile) 822 if self.verbose: print(' --> '+fullex) 823 dataDict[root].update(testparse.parseTestFile(fullex,0)) 824 if exfile in dataDict[root]: 825 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 826 827 return 828 829 def walktree(self,top): 830 """ 831 Walk a directory tree, starting from 'top' 832 """ 833 # Goal of action is to fill this dictionary 834 dataDict={} 835 for root, dirs, files in os.walk(top, topdown=True): 836 dirs.sort() 837 files.sort() 838 if not "examples" in root: continue 839 if "dSYM" in root: continue 840 if os.path.basename(root.rstrip("/")) == 'output': continue 841 if self.verbose: print(root) 842 self.genPetscTests(root,dirs,files,dataDict) 843 # Now summarize this dictionary 844 if self.verbose: self.genPetscTests_summarize(dataDict) 845 return dataDict 846 847 def gen_gnumake(self, fd): 848 """ 849 Overwrite of the method in the base PETSc class 850 """ 851 def write(stem, srcs): 852 for lang in LANGS: 853 if srcs[lang]['srcs']: 854 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 855 for pkg in PKGS: 856 srcs = self.gen_pkg(pkg) 857 write('testsrcs-' + pkg, srcs) 858 # Handle dependencies 859 for lang in LANGS: 860 for exfile in srcs[lang]['srcs']: 861 if exfile in srcs[lang]: 862 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 863 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 864 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 865 if deps: 866 # The executable literally depends on the object file because it is linked 867 fd.write(ex +": " + " ".join(deps) +'\n') 868 # The object file containing 'main' does not normally depend on other object 869 # files, but it does when it includes their modules. This dependency is 870 # overly blunt and could be reduced to only depend on object files for 871 # modules that are used, like "*f90aux.o". 872 fd.write(exfo +": " + " ".join(deps) +'\n') 873 874 return self.gendeps 875 876 def gen_pkg(self, pkg): 877 """ 878 Overwrite of the method in the base PETSc class 879 """ 880 return self.sources[pkg] 881 882 def write_gnumake(self, dataDict, output=None): 883 """ 884 Write out something similar to files from gmakegen.py 885 886 Test depends on script which also depends on source 887 file, but since I don't have a good way generating 888 acting on a single file (oops) just depend on 889 executable which in turn will depend on src file 890 """ 891 # Different options for how to set up the targets 892 compileExecsFirst=False 893 894 # Open file 895 fd = open(output, 'w') 896 897 # Write out the sources 898 gendeps = self.gen_gnumake(fd) 899 900 # Write out the tests and execname targets 901 fd.write("\n#Tests and executables\n") # Delimiter 902 903 for pkg in PKGS: 904 # These grab the ones that are built 905 for lang in LANGS: 906 testdeps=[] 907 for ftest in self.tests[pkg][lang]: 908 test=os.path.basename(ftest) 909 basedir=os.path.dirname(ftest) 910 testdeps.append(self.nameSpace(test,basedir)) 911 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 912 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 913 914 # test targets 915 for ftest in self.tests[pkg][lang]: 916 test=os.path.basename(ftest) 917 basedir=os.path.dirname(ftest) 918 testdir="${TESTDIR}/"+basedir+"/" 919 nmtest=self.nameSpace(test,basedir) 920 rundir=os.path.join(testdir,test) 921 script=test+".sh" 922 923 # Deps 924 exfile=self.tests[pkg][lang][ftest]['exfile'] 925 fullex=os.path.join(self.srcdir,exfile) 926 localexec=self.tests[pkg][lang][ftest]['exec'] 927 execname=os.path.join(testdir,localexec) 928 fullscript=os.path.join(testdir,script) 929 tmpfile=os.path.join(testdir,test,test+".tmp") 930 931 # *.counts depends on the script and either executable (will 932 # be run) or the example source file (SKIP or TODO) 933 fd.write('%s.counts : %s %s' 934 % (os.path.join('$(TESTDIR)/counts', nmtest), 935 fullscript, 936 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 937 ) 938 if exfile in self.sources[pkg][lang]: 939 for dep in self.sources[pkg][lang][exfile]: 940 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 941 fd.write('\n') 942 943 # Now write the args: 944 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 945 946 fd.close() 947 return 948 949def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 950 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 951 if petsc_arch: 952 if len(petsc_arch.split(os.path.sep))>1: 953 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 954 output = os.path.join(testdir, 'testfiles') 955 956 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 957 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 958 testdir=testdir) 959 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 960 pEx.write_gnumake(dataDict, output) 961 962if __name__ == '__main__': 963 import optparse 964 parser = optparse.OptionParser() 965 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 966 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 967 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 968 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 969 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 970 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 971 972 opts, extra_args = parser.parse_args() 973 if extra_args: 974 import sys 975 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 976 exit(1) 977 if opts.testdir is None: 978 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 979 980 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 981 verbose=opts.verbose, 982 single_ex=opts.single_executable, srcdir=opts.srcdir, 983 testdir=opts.testdir) 984