1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose) 85 86 self.single_ex=single_ex 87 self.srcdir=srcdir 88 89 # Set locations to handle movement 90 self.inInstallDir=self.getInInstallDir(thisscriptdir) 91 92 if self.inInstallDir: 93 # Case 2 discussed above 94 # set PETSC_ARCH to install directory to allow script to work in both 95 dirlist=thisscriptdir.split(os.path.sep) 96 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 97 self.arch_dir=installdir 98 if self.srcdir is None: 99 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 100 else: 101 if petsc_arch == '': 102 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 103 # Case 1 discussed above 104 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 105 if self.srcdir is None: 106 self.srcdir=os.path.join(self.petsc_dir,'src') 107 108 self.testroot_dir=os.path.abspath(testdir) 109 110 self.ptNaming=True 111 self.verbose=verbose 112 # Whether to write out a useful debugging 113 self.summarize=True if verbose else False 114 115 # For help in setting the requirements 116 self.precision_types="single double __float128 int32".split() 117 self.integer_types="int32 int64".split() 118 self.languages="fortran cuda cxx".split() # Always requires C so do not list 119 120 # Things that are not test 121 self.buildkeys=testparse.buildkeys 122 123 # Adding a dictionary for storing sources, objects, and tests 124 # to make building the dependency tree easier 125 self.sources={} 126 self.objects={} 127 self.tests={} 128 for pkg in PKGS: 129 self.sources[pkg]={} 130 self.objects[pkg]=[] 131 self.tests[pkg]={} 132 for lang in LANGS: 133 self.sources[pkg][lang]={} 134 self.sources[pkg][lang]['srcs']=[] 135 self.tests[pkg][lang]={} 136 137 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 138 139 self.indent=" " 140 if self.verbose: print('Finishing the constructor') 141 return 142 143 def srcrelpath(self,rdir): 144 """ 145 Get relative path to source directory 146 """ 147 return os.path.relpath(rdir,self.srcdir) 148 149 def getInInstallDir(self,thisscriptdir): 150 """ 151 When petsc is installed then this file in installed in: 152 <PREFIX>/share/petsc/examples/config/gmakegentest.py 153 otherwise the path is: 154 <PETSC_DIR>/config/gmakegentest.py 155 We use this difference to determine if we are in installdir 156 """ 157 dirlist=thisscriptdir.split(os.path.sep) 158 if len(dirlist)>4: 159 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 160 if lastfour==os.path.join('share','petsc','examples','config'): 161 return True 162 else: 163 return False 164 else: 165 return False 166 167 def nameSpace(self,srcfile,srcdir): 168 """ 169 Because the scripts have a non-unique naming, the pretty-printing 170 needs to convey the srcdir and srcfile. There are two ways of doing this. 171 """ 172 if self.ptNaming: 173 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 174 cdir=srcdir 175 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 176 nameString=prefix+srcfile 177 else: 178 #nameString=srcdir+": "+srcfile 179 nameString=srcfile 180 return nameString 181 182 def getLanguage(self,srcfile): 183 """ 184 Based on the source, determine associated language as found in gmakegen.LANGS 185 Can we just return srcext[1:\] now? 186 """ 187 langReq=None 188 srcext=os.path.splitext(srcfile)[-1] 189 if srcext in ".F90".split(): langReq="F90" 190 if srcext in ".F".split(): langReq="F" 191 if srcext in ".cxx".split(): langReq="cxx" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getLoopVars(self,inDict,testname, isSubtest=False): 198 """ 199 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 200 Return: 201 inDict['args']: -ksp_monitor 202 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 203 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 204 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 205 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 206 subst should be passed in instead of inDict 207 """ 208 loopVars={}; newargs=[] 209 lsuffix='_' 210 argregex = re.compile(' (?=-[a-zA-Z])') 211 from testparse import parseLoopArgs 212 for key in inDict: 213 if key in ('SKIP', 'regexes'): 214 continue 215 akey=('subargs' if key=='args' else key) # what to assign 216 if akey not in inDict: inDict[akey]='' 217 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 218 # Always generate a loop over nsize, even if there is only one value 219 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 220 keystr = str(inDict[key]) 221 varlist = [] 222 for varset in argregex.split(keystr): 223 if not varset.strip(): continue 224 if '{{' in varset: 225 keyvar,lvars,ftype=parseLoopArgs(varset) 226 if akey not in loopVars: loopVars[akey]={} 227 varlist.append(keyvar) 228 loopVars[akey][keyvar]=[keyvar,lvars] 229 if akey=='nsize': 230 if len(lvars.split()) > 1: 231 lsuffix += akey +'-${' + keyvar + '}' 232 else: 233 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 234 lsuffix+=keyvar+'-${' + keyvar + '}_' 235 else: 236 if key=='args': 237 newargs.append(varset.strip()) 238 if varlist: 239 loopVars[akey]['varlist']=varlist 240 241 # For subtests, args are always substituted in (not top level) 242 if isSubtest: 243 inDict['subargs'] += " ".join(newargs) 244 inDict['args']='' 245 if 'label_suffix' in inDict: 246 inDict['label_suffix']+=lsuffix.rstrip('_') 247 else: 248 inDict['label_suffix']=lsuffix.rstrip('_') 249 else: 250 if loopVars: 251 inDict['args'] = ' '.join(newargs) 252 inDict['label_suffix']=lsuffix.rstrip('_') 253 return loopVars 254 255 def getArgLabel(self,testDict): 256 """ 257 In all of the arguments in the test dictionary, create a simple 258 string for searching within the makefile system. For simplicity in 259 search, remove "-", for strings, etc. 260 Also, concatenate the arg commands 261 For now, ignore nsize -- seems hard to search for anyway 262 """ 263 # Collect all of the args associated with a test 264 argStr=("" if 'args' not in testDict else testDict['args']) 265 if 'subtests' in testDict: 266 for stest in testDict["subtests"]: 267 sd=testDict[stest] 268 argStr=argStr+("" if 'args' not in sd else sd['args']) 269 270 # Now go through and cleanup 271 argStr=re.sub('{{(.*?)}}',"",argStr) 272 argStr=re.sub('-'," ",argStr) 273 for digit in string.digits: argStr=re.sub(digit," ",argStr) 274 argStr=re.sub("\.","",argStr) 275 argStr=re.sub(",","",argStr) 276 argStr=re.sub('\+',' ',argStr) 277 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 278 return argStr.strip() 279 280 def addToSources(self,exfile,rpath,srcDict): 281 """ 282 Put into data structure that allows easy generation of makefile 283 """ 284 pkg=rpath.split(os.path.sep)[0] 285 relpfile=os.path.join(rpath,exfile) 286 lang=self.getLanguage(exfile) 287 if not lang: return 288 self.sources[pkg][lang]['srcs'].append(relpfile) 289 self.sources[pkg][lang][relpfile] = [] 290 if 'depends' in srcDict: 291 depSrcList=srcDict['depends'].split() 292 for depSrc in depSrcList: 293 depObj=os.path.splitext(depSrc)[0]+".o" 294 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 295 296 # In gmakefile, ${TESTDIR} var specifies the object compilation 297 testsdir=rpath+"/" 298 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 299 self.objects[pkg].append(objfile) 300 return 301 302 def addToTests(self,test,rpath,exfile,execname,testDict): 303 """ 304 Put into data structure that allows easy generation of makefile 305 Organized by languages to allow testing of languages 306 """ 307 pkg=rpath.split("/")[0] 308 nmtest=os.path.join(rpath,test) 309 lang=self.getLanguage(exfile) 310 if not lang: return 311 self.tests[pkg][lang][nmtest]={} 312 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 313 self.tests[pkg][lang][nmtest]['exec']=execname 314 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 315 return 316 317 def getExecname(self,exfile,rpath): 318 """ 319 Generate bash script using template found next to this file. 320 This file is read in at constructor time to avoid file I/O 321 """ 322 if self.single_ex: 323 execname=rpath.split("/")[1]+"-ex" 324 else: 325 execname=os.path.splitext(exfile)[0] 326 return execname 327 328 def getSubstVars(self,testDict,rpath,testname): 329 """ 330 Create a dictionary with all of the variables that get substituted 331 into the template commands found in example_template.py 332 """ 333 subst={} 334 335 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 336 if 'nsize' not in testDict: testDict['nsize'] = '1' 337 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 338 for ak in testparse.acceptedkeys: 339 if ak=='test': continue 340 subst[ak]=(testDict[ak] if ak in testDict else '') 341 342 # Now do other variables 343 subst['execname']=testDict['execname'] 344 if 'filter' in testDict: 345 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 346 347 # Others 348 subst['subargs']='' # Default. For variables override 349 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 350 subst['label_suffix']='' 351 subst['comments']="\n#".join(subst['comments'].split("\n")) 352 if subst['comments']: subst['comments']="#"+subst['comments'] 353 subst['exec']="../"+subst['execname'] 354 subst['testroot']=self.testroot_dir 355 subst['testname']=testname 356 dp = self.conf.get('DATAFILESPATH','') 357 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 358 359 # This is used to label some matrices 360 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 361 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 362 363 #Conf vars 364 if self.petsc_arch.find('valgrind')>=0: 365 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 366 else: 367 subst['mpiexec']=self.conf['MPIEXEC'] 368 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 369 subst['petsc_arch']=self.petsc_arch 370 if self.inInstallDir: 371 # Case 2 372 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 373 else: 374 # Case 1 375 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 376 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 377 subst['diff']=self.conf['DIFF'] 378 subst['rm']=self.conf['RM'] 379 subst['grep']=self.conf['GREP'] 380 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 381 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 382 383 # Output file is special because of subtests override 384 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 385 if not "_" in defroot: defroot=defroot+"_1" 386 subst['defroot']=defroot 387 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 388 subst['redirect_file']=defroot+".tmp" 389 if 'output_file' not in testDict: 390 subst['output_file']="output/"+defroot+".out" 391 # Add in the full path here. 392 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 393 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 394 if not subst['TODO']: 395 print("Warning: "+subst['output_file']+" not found.") 396 # Worry about alt files here -- see 397 # src/snes/examples/tutorials/output/ex22*.out 398 altlist=[subst['output_file']] 399 basefile,ext = os.path.splitext(subst['output_file']) 400 for i in range(1,9): 401 altroot=basefile+"_alt" 402 if i > 1: altroot=altroot+"_"+str(i) 403 af=altroot+".out" 404 srcaf=os.path.join(subst['srcdir'],af) 405 fullaf=os.path.join(self.petsc_dir,srcaf) 406 if os.path.isfile(fullaf): altlist.append(srcaf) 407 if len(altlist)>1: subst['altfiles']=altlist 408 #if len(altlist)>1: print("Found alt files: ",altlist) 409 410 subst['regexes']={} 411 for subkey in subst: 412 if subkey=='regexes': continue 413 if not isinstance(subst[subkey],str): continue 414 patt="@"+subkey.upper()+"@" 415 subst['regexes'][subkey]=re.compile(patt) 416 417 return subst 418 419 def _substVars(self,subst,origStr): 420 """ 421 Substitute variables 422 """ 423 Str=origStr 424 for subkey in subst: 425 if subkey=='regexes': continue 426 if not isinstance(subst[subkey],str): continue 427 if subkey.upper() not in Str: continue 428 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 429 return Str 430 431 def getCmds(self,subst,i): 432 """ 433 Generate bash script using template found next to this file. 434 This file is read in at constructor time to avoid file I/O 435 """ 436 nindnt=i # the start and has to be consistent with below 437 cmdindnt=self.indent*nindnt 438 cmdLines="" 439 440 # MPI is the default -- but we have a few odd commands 441 if not subst['command']: 442 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 443 else: 444 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 445 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 446 447 cmdLines+=cmdindnt+'if test $res = 0; then\n' 448 diffindnt=self.indent*(nindnt+1) 449 if not subst['filter_output']: 450 if 'altfiles' not in subst: 451 cmd=diffindnt+self._substVars(subst,example_template.difftest) 452 else: 453 # Have to do it by hand a bit because of variable number of alt files 454 rf=subst['redirect_file'] 455 cmd=diffindnt+example_template.difftest.split('@')[0] 456 for i in range(len(subst['altfiles'])): 457 af=subst['altfiles'][i] 458 cmd+=af+' '+rf 459 if i!=len(subst['altfiles'])-1: 460 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 461 cmd+=' || ${diff_exe} ' 462 else: 463 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 464 cmd+=subst['label_suffix']+' ""' # Quotes are painful 465 else: 466 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 467 cmdLines+=cmd+"\n" 468 cmdLines+=cmdindnt+'else\n' 469 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 470 cmdLines+=cmdindnt+'fi\n' 471 return cmdLines 472 473 def _writeTodoSkip(self,fh,tors,reasons,footer): 474 """ 475 Write out the TODO and SKIP lines in the file 476 The TODO or SKIP variable, tors, should be lower case 477 """ 478 TORS=tors.upper() 479 template=eval("example_template."+tors+"line") 480 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 481 tab = '' 482 if reasons: 483 fh.write('if ! $force; then\n') 484 tab = tab + ' ' 485 if reasons == ["Requires DATAFILESPATH"]: 486 # The only reason not to run is DATAFILESPATH, which we check at run-time 487 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 488 tab = tab + ' ' 489 if reasons: 490 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 491 fh.write(tab+footer+"\n") 492 fh.write(tab+"exit\n") 493 if reasons == ["Requires DATAFILESPATH"]: 494 fh.write(' fi\n') 495 if reasons: 496 fh.write('fi\n') 497 fh.write('\n\n') 498 return 499 500 def getLoopVarsHead(self,loopVars,i): 501 """ 502 Generate a nicely indented string with the format loops 503 Here is what the data structure looks like 504 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 505 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 506 loopVars['subargs']['pc_type']=["j","cholesky sor"] 507 """ 508 outstr=''; indnt=self.indent 509 510 for key in loopVars: 511 for var in loopVars[key]['varlist']: 512 varval=loopVars[key][var] 513 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 514 outstr += "\n\n" 515 516 for key in loopVars: 517 for var in loopVars[key]['varlist']: 518 varval=loopVars[key][var] 519 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 520 i = i + 1 521 return (outstr,i) 522 523 def getLoopVarsFoot(self,loopVars,i): 524 outstr=''; indnt=self.indent 525 for key in loopVars: 526 for var in loopVars[key]['varlist']: 527 i = i - 1 528 outstr += indnt * i + "done\n" 529 return (outstr,i) 530 531 def genRunScript(self,testname,root,isRun,srcDict): 532 """ 533 Generate bash script using template found next to this file. 534 This file is read in at constructor time to avoid file I/O 535 """ 536 # runscript_dir directory has to be consistent with gmakefile 537 testDict=srcDict[testname] 538 rpath=self.srcrelpath(root) 539 runscript_dir=os.path.join(self.testroot_dir,rpath) 540 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 541 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 542 543 # Get variables to go into shell scripts. last time testDict used 544 subst=self.getSubstVars(testDict,rpath,testname) 545 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 546 if 'subtests' in testDict: 547 # The subtests inherit inDict, so we don't need top-level loops. 548 loopVars = {} 549 550 #Handle runfiles 551 for lfile in subst.get('localrunfiles','').split(): 552 install_files(os.path.join(root, lfile), 553 os.path.join(runscript_dir, os.path.dirname(lfile))) 554 # Check subtests for local runfiles 555 for stest in subst.get("subtests",[]): 556 for lfile in testDict[stest].get('localrunfiles','').split(): 557 install_files(os.path.join(root, lfile), 558 os.path.join(runscript_dir, os.path.dirname(lfile))) 559 560 # Now substitute the key variables into the header and footer 561 header=self._substVars(subst,example_template.header) 562 # The header is done twice to enable @...@ in header 563 header=self._substVars(subst,header) 564 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 565 566 # Start writing the file 567 fh.write(header+"\n") 568 569 # If there is a TODO or a SKIP then we do it before writing out the 570 # rest of the command (which is useful for working on the test) 571 # SKIP and TODO can be for the source file or for the runs 572 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 573 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 574 575 j=0 # for indentation 576 577 if loopVars: 578 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 579 if (loopHead): fh.write(loopHead+"\n") 580 581 # Subtests are special 582 if 'subtests' in testDict: 583 substP=subst # Subtests can inherit args but be careful 584 k=0 # for label suffixes 585 for stest in testDict["subtests"]: 586 subst=substP.copy() 587 subst.update(testDict[stest]) 588 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 589 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 590 if sLoopVars: 591 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 592 fh.write(sLoopHead+"\n") 593 fh.write(self.getCmds(subst,j)+"\n") 594 if sLoopVars: 595 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 596 fh.write(sLoopFoot+"\n") 597 else: 598 fh.write(self.getCmds(subst,j)+"\n") 599 600 if loopVars: 601 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 602 fh.write(loopFoot+"\n") 603 604 fh.write(footer+"\n") 605 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 606 #if '10_9' in testname: sys.exit() 607 return 608 609 def genScriptsAndInfo(self,exfile,root,srcDict): 610 """ 611 Generate scripts from the source file, determine if built, etc. 612 For every test in the exfile with info in the srcDict: 613 1. Determine if it needs to be run for this arch 614 2. Generate the script 615 3. Generate the data needed to write out the makefile in a 616 convenient way 617 All tests are *always* run, but some may be SKIP'd per the TAP standard 618 """ 619 debug=False 620 rpath=self.srcrelpath(root) 621 execname=self.getExecname(exfile,rpath) 622 isBuilt=self._isBuilt(exfile,srcDict) 623 for test in srcDict: 624 if test in self.buildkeys: continue 625 if debug: print(self.nameSpace(exfile,root), test) 626 srcDict[test]['execname']=execname # Convenience in generating scripts 627 isRun=self._isRun(srcDict[test]) 628 self.genRunScript(test,root,isRun,srcDict) 629 srcDict[test]['isrun']=isRun 630 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 631 632 # This adds to datastructure for building deps 633 if isBuilt: self.addToSources(exfile,rpath,srcDict) 634 return 635 636 def _isBuilt(self,exfile,srcDict): 637 """ 638 Determine if this file should be built. 639 """ 640 # Get the language based on file extension 641 srcDict['SKIP'] = [] 642 lang=self.getLanguage(exfile) 643 if (lang=="F" or lang=="F90"): 644 if not self.have_fortran: 645 srcDict["SKIP"].append("Fortran required for this test") 646 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 647 srcDict["SKIP"].append("Fortran f90freeform required for this test") 648 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 649 srcDict["SKIP"].append("CUDA required for this test") 650 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 651 srcDict["SKIP"].append("C++ required for this test") 652 653 # Deprecated source files 654 if srcDict.get("TODO"): 655 return False 656 657 # isRun can work with srcDict to handle the requires 658 if "requires" in srcDict: 659 if srcDict["requires"]: 660 return self._isRun(srcDict) 661 662 return srcDict['SKIP'] == [] 663 664 665 def _isRun(self,testDict, debug=False): 666 """ 667 Based on the requirements listed in the src file and the petscconf.h 668 info, determine whether this test should be run or not. 669 """ 670 indent=" " 671 672 if 'SKIP' not in testDict: 673 testDict['SKIP'] = [] 674 # MPI requirements 675 if 'MPI_IS_MPIUNI' in self.conf: 676 if testDict.get('nsize', '1') != '1': 677 testDict['SKIP'].append("Parallel test with serial build") 678 679 # The requirements for the test are the sum of all the run subtests 680 if 'subtests' in testDict: 681 if 'requires' not in testDict: testDict['requires']="" 682 for stest in testDict['subtests']: 683 if 'requires' in testDict[stest]: 684 testDict['requires']+=" "+testDict[stest]['requires'] 685 if testDict.get('nsize', '1') != '1': 686 testDict['SKIP'].append("Parallel test with serial build") 687 break 688 689 # Now go through all requirements 690 if 'requires' in testDict: 691 for requirement in testDict['requires'].split(): 692 requirement=requirement.strip() 693 if not requirement: continue 694 if debug: print(indent+"Requirement: ", requirement) 695 isNull=False 696 if requirement.startswith("!"): 697 requirement=requirement[1:]; isNull=True 698 # Precision requirement for reals 699 if requirement in self.precision_types: 700 if self.conf['PETSC_PRECISION']==requirement: 701 if isNull: 702 testDict['SKIP'].append("not "+requirement+" required") 703 continue 704 continue # Success 705 elif not isNull: 706 testDict['SKIP'].append(requirement+" required") 707 continue 708 # Precision requirement for ints 709 if requirement in self.integer_types: 710 if requirement=="int32": 711 if self.conf['PETSC_SIZEOF_INT']==4: 712 if isNull: 713 testDict['SKIP'].append("not int32 required") 714 continue 715 continue # Success 716 elif not isNull: 717 testDict['SKIP'].append("int32 required") 718 continue 719 if requirement=="int64": 720 if self.conf['PETSC_SIZEOF_INT']==8: 721 if isNull: 722 testDict['SKIP'].append("NOT int64 required") 723 continue 724 continue # Success 725 elif not isNull: 726 testDict['SKIP'].append("int64 required") 727 continue 728 # Datafilespath 729 if requirement=="datafilespath" and not isNull: 730 testDict['SKIP'].append("Requires DATAFILESPATH") 731 continue 732 # Defines -- not sure I have comments matching 733 if "define(" in requirement.lower(): 734 reqdef=requirement.split("(")[1].split(")")[0] 735 if reqdef in self.conf: 736 if isNull: 737 testDict['SKIP'].append("Null requirement not met: "+requirement) 738 continue 739 continue # Success 740 elif not isNull: 741 testDict['SKIP'].append("Required: "+requirement) 742 continue 743 744 # Rest should be packages that we can just get from conf 745 if requirement == "complex": 746 petscconfvar="PETSC_USE_COMPLEX" 747 else: 748 petscconfvar="PETSC_HAVE_"+requirement.upper() 749 if self.conf.get(petscconfvar): 750 if isNull: 751 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 752 continue 753 continue # Success 754 elif not isNull: 755 if debug: print("requirement not found: ", requirement) 756 testDict['SKIP'].append(petscconfvar+" requirement not met") 757 continue 758 759 return testDict['SKIP'] == [] 760 761 def genPetscTests_summarize(self,dataDict): 762 """ 763 Required method to state what happened 764 """ 765 if not self.summarize: return 766 indent=" " 767 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 768 fh=open(fhname,"w") 769 for root in dataDict: 770 relroot=self.srcrelpath(root) 771 pkg=relroot.split("/")[1] 772 fh.write(relroot+"\n") 773 allSrcs=[] 774 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 775 for exfile in dataDict[root]: 776 # Basic information 777 rfile=os.path.join(relroot,exfile) 778 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 779 fh.write(indent+exfile+indent*4+builtStatus+"\n") 780 781 for test in dataDict[root][exfile]: 782 if test in self.buildkeys: continue 783 line=indent*2+test 784 fh.write(line+"\n") 785 # Looks nice to have the keys in order 786 #for key in dataDict[root][exfile][test]: 787 for key in "isrun abstracted nsize args requires script".split(): 788 if key not in dataDict[root][exfile][test]: continue 789 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 790 fh.write(line+"\n") 791 fh.write("\n") 792 fh.write("\n") 793 fh.write("\n") 794 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 795 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 796 fh.close() 797 return 798 799 def genPetscTests(self,root,dirs,files,dataDict): 800 """ 801 Go through and parse the source files in the directory to generate 802 the examples based on the metadata contained in the source files 803 """ 804 debug=False 805 # Use examplesAnalyze to get what the makefles think are sources 806 #self.examplesAnalyze(root,dirs,files,anlzDict) 807 808 dataDict[root]={} 809 810 for exfile in files: 811 #TST: Until we replace files, still leaving the orginals as is 812 #if not exfile.startswith("new_"+"ex"): continue 813 #if not exfile.startswith("ex"): continue 814 815 # Ignore emacs and other temporary files 816 if exfile.startswith("."): continue 817 if exfile.startswith("#"): continue 818 if exfile.endswith("~"): continue 819 # Only parse source files 820 ext=os.path.splitext(exfile)[-1].lstrip('.') 821 if ext not in LANGS: continue 822 823 # Convenience 824 fullex=os.path.join(root,exfile) 825 if self.verbose: print(' --> '+fullex) 826 dataDict[root].update(testparse.parseTestFile(fullex,0)) 827 if exfile in dataDict[root]: 828 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 829 830 return 831 832 def walktree(self,top): 833 """ 834 Walk a directory tree, starting from 'top' 835 """ 836 # Goal of action is to fill this dictionary 837 dataDict={} 838 for root, dirs, files in os.walk(top, topdown=True): 839 dirs.sort() 840 files.sort() 841 if not "examples" in root: continue 842 if "dSYM" in root: continue 843 if os.path.basename(root.rstrip("/")) == 'output': continue 844 if self.verbose: print(root) 845 self.genPetscTests(root,dirs,files,dataDict) 846 # Now summarize this dictionary 847 if self.verbose: self.genPetscTests_summarize(dataDict) 848 return dataDict 849 850 def gen_gnumake(self, fd): 851 """ 852 Overwrite of the method in the base PETSc class 853 """ 854 def write(stem, srcs): 855 for lang in LANGS: 856 if srcs[lang]['srcs']: 857 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 858 for pkg in PKGS: 859 srcs = self.gen_pkg(pkg) 860 write('testsrcs-' + pkg, srcs) 861 # Handle dependencies 862 for lang in LANGS: 863 for exfile in srcs[lang]['srcs']: 864 if exfile in srcs[lang]: 865 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 866 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 867 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 868 if deps: 869 # The executable literally depends on the object file because it is linked 870 fd.write(ex +": " + " ".join(deps) +'\n') 871 # The object file containing 'main' does not normally depend on other object 872 # files, but it does when it includes their modules. This dependency is 873 # overly blunt and could be reduced to only depend on object files for 874 # modules that are used, like "*f90aux.o". 875 fd.write(exfo +": " + " ".join(deps) +'\n') 876 877 return self.gendeps 878 879 def gen_pkg(self, pkg): 880 """ 881 Overwrite of the method in the base PETSc class 882 """ 883 return self.sources[pkg] 884 885 def write_gnumake(self, dataDict, output=None): 886 """ 887 Write out something similar to files from gmakegen.py 888 889 Test depends on script which also depends on source 890 file, but since I don't have a good way generating 891 acting on a single file (oops) just depend on 892 executable which in turn will depend on src file 893 """ 894 # Different options for how to set up the targets 895 compileExecsFirst=False 896 897 # Open file 898 fd = open(output, 'w') 899 900 # Write out the sources 901 gendeps = self.gen_gnumake(fd) 902 903 # Write out the tests and execname targets 904 fd.write("\n#Tests and executables\n") # Delimiter 905 906 for pkg in PKGS: 907 # These grab the ones that are built 908 for lang in LANGS: 909 testdeps=[] 910 for ftest in self.tests[pkg][lang]: 911 test=os.path.basename(ftest) 912 basedir=os.path.dirname(ftest) 913 testdeps.append(self.nameSpace(test,basedir)) 914 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 915 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 916 917 # test targets 918 for ftest in self.tests[pkg][lang]: 919 test=os.path.basename(ftest) 920 basedir=os.path.dirname(ftest) 921 testdir="${TESTDIR}/"+basedir+"/" 922 nmtest=self.nameSpace(test,basedir) 923 rundir=os.path.join(testdir,test) 924 script=test+".sh" 925 926 # Deps 927 exfile=self.tests[pkg][lang][ftest]['exfile'] 928 fullex=os.path.join(self.srcdir,exfile) 929 localexec=self.tests[pkg][lang][ftest]['exec'] 930 execname=os.path.join(testdir,localexec) 931 fullscript=os.path.join(testdir,script) 932 tmpfile=os.path.join(testdir,test,test+".tmp") 933 934 # *.counts depends on the script and either executable (will 935 # be run) or the example source file (SKIP or TODO) 936 fd.write('%s.counts : %s %s' 937 % (os.path.join('$(TESTDIR)/counts', nmtest), 938 fullscript, 939 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 940 ) 941 if exfile in self.sources[pkg][lang]: 942 for dep in self.sources[pkg][lang][exfile]: 943 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 944 fd.write('\n') 945 946 # Now write the args: 947 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 948 949 fd.close() 950 return 951 952def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 953 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 954 testdir=os.path.normpath(testdir) 955 if petsc_arch: 956 petsc_arch=petsc_arch.rstrip(os.path.sep) 957 if len(petsc_arch.split(os.path.sep))>1: 958 petsc_dir,petsc_arch=os.path.split(petsc_arch) 959 output = os.path.join(testdir, 'testfiles') 960 961 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 962 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 963 testdir=testdir) 964 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 965 pEx.write_gnumake(dataDict, output) 966 967if __name__ == '__main__': 968 import optparse 969 parser = optparse.OptionParser() 970 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 971 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 972 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 973 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 974 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 975 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 976 977 opts, extra_args = parser.parse_args() 978 if extra_args: 979 import sys 980 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 981 exit(1) 982 if opts.testdir is None: 983 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 984 985 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 986 verbose=opts.verbose, 987 single_ex=opts.single_executable, srcdir=opts.srcdir, 988 testdir=opts.testdir) 989