1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose) 85 86 self.single_ex=single_ex 87 self.srcdir=srcdir 88 89 # Set locations to handle movement 90 self.inInstallDir=self.getInInstallDir(thisscriptdir) 91 92 if self.inInstallDir: 93 # Case 2 discussed above 94 # set PETSC_ARCH to install directory to allow script to work in both 95 dirlist=thisscriptdir.split(os.path.sep) 96 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 97 self.arch_dir=installdir 98 if self.srcdir is None: 99 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 100 else: 101 if petsc_arch == '': 102 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 103 # Case 1 discussed above 104 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 105 if self.srcdir is None: 106 self.srcdir=os.path.join(self.petsc_dir,'src') 107 108 self.testroot_dir=os.path.abspath(testdir) 109 110 self.ptNaming=True 111 self.verbose=verbose 112 # Whether to write out a useful debugging 113 self.summarize=True if verbose else False 114 115 # For help in setting the requirements 116 self.precision_types="single double __float128 int32".split() 117 self.integer_types="int32 int64 long32 long64".split() 118 self.languages="fortran cuda cxx".split() # Always requires C so do not list 119 120 # Things that are not test 121 self.buildkeys=testparse.buildkeys 122 123 # Adding a dictionary for storing sources, objects, and tests 124 # to make building the dependency tree easier 125 self.sources={} 126 self.objects={} 127 self.tests={} 128 for pkg in PKGS: 129 self.sources[pkg]={} 130 self.objects[pkg]=[] 131 self.tests[pkg]={} 132 for lang in LANGS: 133 self.sources[pkg][lang]={} 134 self.sources[pkg][lang]['srcs']=[] 135 self.tests[pkg][lang]={} 136 137 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 138 139 self.indent=" " 140 if self.verbose: print('Finishing the constructor') 141 return 142 143 def srcrelpath(self,rdir): 144 """ 145 Get relative path to source directory 146 """ 147 return os.path.relpath(rdir,self.srcdir) 148 149 def getInInstallDir(self,thisscriptdir): 150 """ 151 When petsc is installed then this file in installed in: 152 <PREFIX>/share/petsc/examples/config/gmakegentest.py 153 otherwise the path is: 154 <PETSC_DIR>/config/gmakegentest.py 155 We use this difference to determine if we are in installdir 156 """ 157 dirlist=thisscriptdir.split(os.path.sep) 158 if len(dirlist)>4: 159 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 160 if lastfour==os.path.join('share','petsc','examples','config'): 161 return True 162 else: 163 return False 164 else: 165 return False 166 167 def nameSpace(self,srcfile,srcdir): 168 """ 169 Because the scripts have a non-unique naming, the pretty-printing 170 needs to convey the srcdir and srcfile. There are two ways of doing this. 171 """ 172 if self.ptNaming: 173 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 174 cdir=srcdir 175 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 176 nameString=prefix+srcfile 177 else: 178 #nameString=srcdir+": "+srcfile 179 nameString=srcfile 180 return nameString 181 182 def getLanguage(self,srcfile): 183 """ 184 Based on the source, determine associated language as found in gmakegen.LANGS 185 Can we just return srcext[1:\] now? 186 """ 187 langReq=None 188 srcext=os.path.splitext(srcfile)[-1] 189 if srcext in ".F90".split(): langReq="F90" 190 if srcext in ".F".split(): langReq="F" 191 if srcext in ".cxx".split(): langReq="cxx" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getLoopVars(self,inDict,testname, isSubtest=False): 198 """ 199 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 200 Return: 201 inDict['args']: -ksp_monitor 202 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 203 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 204 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 205 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 206 subst should be passed in instead of inDict 207 """ 208 loopVars={}; newargs=[] 209 lsuffix='_' 210 argregex = re.compile(' (?=-[a-zA-Z])') 211 from testparse import parseLoopArgs 212 for key in inDict: 213 if key in ('SKIP', 'regexes'): 214 continue 215 akey=('subargs' if key=='args' else key) # what to assign 216 if akey not in inDict: inDict[akey]='' 217 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 218 # Always generate a loop over nsize, even if there is only one value 219 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 220 keystr = str(inDict[key]) 221 varlist = [] 222 for varset in argregex.split(keystr): 223 if not varset.strip(): continue 224 if '{{' in varset: 225 keyvar,lvars,ftype=parseLoopArgs(varset) 226 if akey not in loopVars: loopVars[akey]={} 227 varlist.append(keyvar) 228 loopVars[akey][keyvar]=[keyvar,lvars] 229 if akey=='nsize': 230 if len(lvars.split()) > 1: 231 lsuffix += akey +'-${' + keyvar + '}' 232 else: 233 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 234 lsuffix+=keyvar+'-${' + keyvar + '}_' 235 else: 236 if key=='args': 237 newargs.append(varset.strip()) 238 if varlist: 239 loopVars[akey]['varlist']=varlist 240 241 # For subtests, args are always substituted in (not top level) 242 if isSubtest: 243 inDict['subargs'] += " ".join(newargs) 244 inDict['args']='' 245 if 'label_suffix' in inDict: 246 inDict['label_suffix']+=lsuffix.rstrip('_') 247 else: 248 inDict['label_suffix']=lsuffix.rstrip('_') 249 else: 250 if loopVars: 251 inDict['args'] = ' '.join(newargs) 252 inDict['label_suffix']=lsuffix.rstrip('_') 253 return loopVars 254 255 def getArgLabel(self,testDict): 256 """ 257 In all of the arguments in the test dictionary, create a simple 258 string for searching within the makefile system. For simplicity in 259 search, remove "-", for strings, etc. 260 Also, concatenate the arg commands 261 For now, ignore nsize -- seems hard to search for anyway 262 """ 263 # Collect all of the args associated with a test 264 argStr=("" if 'args' not in testDict else testDict['args']) 265 if 'subtests' in testDict: 266 for stest in testDict["subtests"]: 267 sd=testDict[stest] 268 argStr=argStr+("" if 'args' not in sd else sd['args']) 269 270 # Now go through and cleanup 271 argStr=re.sub('{{(.*?)}}',"",argStr) 272 argStr=re.sub('-'," ",argStr) 273 for digit in string.digits: argStr=re.sub(digit," ",argStr) 274 argStr=re.sub("\.","",argStr) 275 argStr=re.sub(",","",argStr) 276 argStr=re.sub('\+',' ',argStr) 277 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 278 return argStr.strip() 279 280 def addToSources(self,exfile,rpath,srcDict): 281 """ 282 Put into data structure that allows easy generation of makefile 283 """ 284 pkg=rpath.split(os.path.sep)[0] 285 relpfile=os.path.join(rpath,exfile) 286 lang=self.getLanguage(exfile) 287 if not lang: return 288 self.sources[pkg][lang]['srcs'].append(relpfile) 289 self.sources[pkg][lang][relpfile] = [] 290 if 'depends' in srcDict: 291 depSrcList=srcDict['depends'].split() 292 for depSrc in depSrcList: 293 depObj=os.path.splitext(depSrc)[0]+".o" 294 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 295 296 # In gmakefile, ${TESTDIR} var specifies the object compilation 297 testsdir=rpath+"/" 298 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 299 self.objects[pkg].append(objfile) 300 return 301 302 def addToTests(self,test,rpath,exfile,execname,testDict): 303 """ 304 Put into data structure that allows easy generation of makefile 305 Organized by languages to allow testing of languages 306 """ 307 pkg=rpath.split("/")[0] 308 nmtest=os.path.join(rpath,test) 309 lang=self.getLanguage(exfile) 310 if not lang: return 311 self.tests[pkg][lang][nmtest]={} 312 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 313 self.tests[pkg][lang][nmtest]['exec']=execname 314 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 315 return 316 317 def getExecname(self,exfile,rpath): 318 """ 319 Generate bash script using template found next to this file. 320 This file is read in at constructor time to avoid file I/O 321 """ 322 if self.single_ex: 323 execname=rpath.split("/")[1]+"-ex" 324 else: 325 execname=os.path.splitext(exfile)[0] 326 return execname 327 328 def getSubstVars(self,testDict,rpath,testname): 329 """ 330 Create a dictionary with all of the variables that get substituted 331 into the template commands found in example_template.py 332 """ 333 subst={} 334 335 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 336 if 'nsize' not in testDict: testDict['nsize'] = '1' 337 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 338 for ak in testparse.acceptedkeys: 339 if ak=='test': continue 340 subst[ak]=(testDict[ak] if ak in testDict else '') 341 342 # Now do other variables 343 subst['execname']=testDict['execname'] 344 if 'filter' in testDict: 345 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 346 347 # Others 348 subst['subargs']='' # Default. For variables override 349 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 350 subst['label_suffix']='' 351 subst['comments']="\n#".join(subst['comments'].split("\n")) 352 if subst['comments']: subst['comments']="#"+subst['comments'] 353 subst['exec']="../"+subst['execname'] 354 subst['testroot']=self.testroot_dir 355 subst['testname']=testname 356 dp = self.conf.get('DATAFILESPATH','') 357 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 358 359 # This is used to label some matrices 360 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 361 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 362 363 #Conf vars 364 if self.petsc_arch.find('valgrind')>=0: 365 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 366 else: 367 subst['mpiexec']=self.conf['MPIEXEC'] 368 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 369 subst['petsc_arch']=self.petsc_arch 370 if self.inInstallDir: 371 # Case 2 372 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 373 else: 374 # Case 1 375 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 376 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 377 subst['diff']=self.conf['DIFF'] 378 subst['rm']=self.conf['RM'] 379 subst['grep']=self.conf['GREP'] 380 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 381 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 382 383 # Output file is special because of subtests override 384 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 385 if not "_" in defroot: defroot=defroot+"_1" 386 subst['defroot']=defroot 387 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 388 subst['redirect_file']=defroot+".tmp" 389 if 'output_file' not in testDict: 390 subst['output_file']="output/"+defroot+".out" 391 # Add in the full path here. 392 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 393 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 394 if not subst['TODO']: 395 print("Warning: "+subst['output_file']+" not found.") 396 # Worry about alt files here -- see 397 # src/snes/examples/tutorials/output/ex22*.out 398 altlist=[subst['output_file']] 399 basefile,ext = os.path.splitext(subst['output_file']) 400 for i in range(1,9): 401 altroot=basefile+"_alt" 402 if i > 1: altroot=altroot+"_"+str(i) 403 af=altroot+".out" 404 srcaf=os.path.join(subst['srcdir'],af) 405 fullaf=os.path.join(self.petsc_dir,srcaf) 406 if os.path.isfile(fullaf): altlist.append(srcaf) 407 if len(altlist)>1: subst['altfiles']=altlist 408 #if len(altlist)>1: print("Found alt files: ",altlist) 409 410 subst['regexes']={} 411 for subkey in subst: 412 if subkey=='regexes': continue 413 if not isinstance(subst[subkey],str): continue 414 patt="@"+subkey.upper()+"@" 415 subst['regexes'][subkey]=re.compile(patt) 416 417 return subst 418 419 def _substVars(self,subst,origStr): 420 """ 421 Substitute variables 422 """ 423 Str=origStr 424 for subkey in subst: 425 if subkey=='regexes': continue 426 if not isinstance(subst[subkey],str): continue 427 if subkey.upper() not in Str: continue 428 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 429 return Str 430 431 def getCmds(self,subst,i): 432 """ 433 Generate bash script using template found next to this file. 434 This file is read in at constructor time to avoid file I/O 435 """ 436 nindnt=i # the start and has to be consistent with below 437 cmdindnt=self.indent*nindnt 438 cmdLines="" 439 440 # MPI is the default -- but we have a few odd commands 441 if not subst['command']: 442 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 443 else: 444 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 445 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 446 447 cmdLines+=cmdindnt+'if test $res = 0; then\n' 448 diffindnt=self.indent*(nindnt+1) 449 if not subst['filter_output']: 450 if 'altfiles' not in subst: 451 cmd=diffindnt+self._substVars(subst,example_template.difftest) 452 else: 453 # Have to do it by hand a bit because of variable number of alt files 454 rf=subst['redirect_file'] 455 cmd=diffindnt+example_template.difftest.split('@')[0] 456 for i in range(len(subst['altfiles'])): 457 af=subst['altfiles'][i] 458 cmd+=af+' '+rf 459 if i!=len(subst['altfiles'])-1: 460 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 461 cmd+=' || ${diff_exe} ' 462 else: 463 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 464 cmd+=subst['label_suffix']+' ""' # Quotes are painful 465 else: 466 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 467 cmdLines+=cmd+"\n" 468 cmdLines+=cmdindnt+'else\n' 469 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 470 cmdLines+=cmdindnt+'fi\n' 471 return cmdLines 472 473 def _writeTodoSkip(self,fh,tors,reasons,footer): 474 """ 475 Write out the TODO and SKIP lines in the file 476 The TODO or SKIP variable, tors, should be lower case 477 """ 478 TORS=tors.upper() 479 template=eval("example_template."+tors+"line") 480 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 481 tab = '' 482 if reasons: 483 fh.write('if ! $force; then\n') 484 tab = tab + ' ' 485 if reasons == ["Requires DATAFILESPATH"]: 486 # The only reason not to run is DATAFILESPATH, which we check at run-time 487 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 488 tab = tab + ' ' 489 if reasons: 490 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 491 fh.write(tab+footer+"\n") 492 fh.write(tab+"exit\n") 493 if reasons == ["Requires DATAFILESPATH"]: 494 fh.write(' fi\n') 495 if reasons: 496 fh.write('fi\n') 497 fh.write('\n\n') 498 return 499 500 def getLoopVarsHead(self,loopVars,i,usedVars={}): 501 """ 502 Generate a nicely indented string with the format loops 503 Here is what the data structure looks like 504 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 505 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 506 loopVars['subargs']['pc_type']=["j","cholesky sor"] 507 """ 508 outstr=''; indnt=self.indent 509 510 for key in loopVars: 511 if key in usedVars: continue # Do not duplicate setting vars 512 for var in loopVars[key]['varlist']: 513 varval=loopVars[key][var] 514 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 515 outstr += "\n\n" 516 517 for key in loopVars: 518 for var in loopVars[key]['varlist']: 519 varval=loopVars[key][var] 520 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 521 i = i + 1 522 return (outstr,i) 523 524 def getLoopVarsFoot(self,loopVars,i): 525 outstr=''; indnt=self.indent 526 for key in loopVars: 527 for var in loopVars[key]['varlist']: 528 i = i - 1 529 outstr += indnt * i + "done\n" 530 return (outstr,i) 531 532 def genRunScript(self,testname,root,isRun,srcDict): 533 """ 534 Generate bash script using template found next to this file. 535 This file is read in at constructor time to avoid file I/O 536 """ 537 # runscript_dir directory has to be consistent with gmakefile 538 testDict=srcDict[testname] 539 rpath=self.srcrelpath(root) 540 runscript_dir=os.path.join(self.testroot_dir,rpath) 541 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 542 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 543 544 # Get variables to go into shell scripts. last time testDict used 545 subst=self.getSubstVars(testDict,rpath,testname) 546 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 547 if 'subtests' in testDict: 548 # The subtests inherit inDict, so we don't need top-level loops. 549 loopVars = {} 550 551 #Handle runfiles 552 for lfile in subst.get('localrunfiles','').split(): 553 install_files(os.path.join(root, lfile), 554 os.path.join(runscript_dir, os.path.dirname(lfile))) 555 # Check subtests for local runfiles 556 for stest in subst.get("subtests",[]): 557 for lfile in testDict[stest].get('localrunfiles','').split(): 558 install_files(os.path.join(root, lfile), 559 os.path.join(runscript_dir, os.path.dirname(lfile))) 560 561 # Now substitute the key variables into the header and footer 562 header=self._substVars(subst,example_template.header) 563 # The header is done twice to enable @...@ in header 564 header=self._substVars(subst,header) 565 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 566 567 # Start writing the file 568 fh.write(header+"\n") 569 570 # If there is a TODO or a SKIP then we do it before writing out the 571 # rest of the command (which is useful for working on the test) 572 # SKIP and TODO can be for the source file or for the runs 573 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 574 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 575 576 j=0 # for indentation 577 578 if loopVars: 579 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 580 if (loopHead): fh.write(loopHead+"\n") 581 582 # Subtests are special 583 allLoopVars=list(loopVars.keys()) 584 if 'subtests' in testDict: 585 substP=subst # Subtests can inherit args but be careful 586 k=0 # for label suffixes 587 for stest in testDict["subtests"]: 588 subst=substP.copy() 589 subst.update(testDict[stest]) 590 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 591 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 592 if sLoopVars: 593 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 594 allLoopVars+=list(sLoopVars.keys()) 595 fh.write(sLoopHead+"\n") 596 fh.write(self.getCmds(subst,j)+"\n") 597 if sLoopVars: 598 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 599 fh.write(sLoopFoot+"\n") 600 else: 601 fh.write(self.getCmds(subst,j)+"\n") 602 603 if loopVars: 604 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 605 fh.write(loopFoot+"\n") 606 607 fh.write(footer+"\n") 608 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 609 #if '10_9' in testname: sys.exit() 610 return 611 612 def genScriptsAndInfo(self,exfile,root,srcDict): 613 """ 614 Generate scripts from the source file, determine if built, etc. 615 For every test in the exfile with info in the srcDict: 616 1. Determine if it needs to be run for this arch 617 2. Generate the script 618 3. Generate the data needed to write out the makefile in a 619 convenient way 620 All tests are *always* run, but some may be SKIP'd per the TAP standard 621 """ 622 debug=False 623 rpath=self.srcrelpath(root) 624 execname=self.getExecname(exfile,rpath) 625 isBuilt=self._isBuilt(exfile,srcDict) 626 for test in srcDict: 627 if test in self.buildkeys: continue 628 if debug: print(self.nameSpace(exfile,root), test) 629 srcDict[test]['execname']=execname # Convenience in generating scripts 630 isRun=self._isRun(srcDict[test]) 631 self.genRunScript(test,root,isRun,srcDict) 632 srcDict[test]['isrun']=isRun 633 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 634 635 # This adds to datastructure for building deps 636 if isBuilt: self.addToSources(exfile,rpath,srcDict) 637 return 638 639 def _isBuilt(self,exfile,srcDict): 640 """ 641 Determine if this file should be built. 642 """ 643 # Get the language based on file extension 644 srcDict['SKIP'] = [] 645 lang=self.getLanguage(exfile) 646 if (lang=="F" or lang=="F90"): 647 if not self.have_fortran: 648 srcDict["SKIP"].append("Fortran required for this test") 649 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 650 srcDict["SKIP"].append("Fortran f90freeform required for this test") 651 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 652 srcDict["SKIP"].append("CUDA required for this test") 653 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 654 srcDict["SKIP"].append("C++ required for this test") 655 656 # Deprecated source files 657 if srcDict.get("TODO"): 658 return False 659 660 # isRun can work with srcDict to handle the requires 661 if "requires" in srcDict: 662 if srcDict["requires"]: 663 return self._isRun(srcDict) 664 665 return srcDict['SKIP'] == [] 666 667 668 def _isRun(self,testDict, debug=False): 669 """ 670 Based on the requirements listed in the src file and the petscconf.h 671 info, determine whether this test should be run or not. 672 """ 673 indent=" " 674 675 if 'SKIP' not in testDict: 676 testDict['SKIP'] = [] 677 # MPI requirements 678 if 'MPI_IS_MPIUNI' in self.conf: 679 if testDict.get('nsize', '1') != '1': 680 testDict['SKIP'].append("Parallel test with serial build") 681 682 # The requirements for the test are the sum of all the run subtests 683 if 'subtests' in testDict: 684 if 'requires' not in testDict: testDict['requires']="" 685 for stest in testDict['subtests']: 686 if 'requires' in testDict[stest]: 687 testDict['requires']+=" "+testDict[stest]['requires'] 688 if testDict.get('nsize', '1') != '1': 689 testDict['SKIP'].append("Parallel test with serial build") 690 break 691 692 # Now go through all requirements 693 if 'requires' in testDict: 694 for requirement in testDict['requires'].split(): 695 requirement=requirement.strip() 696 if not requirement: continue 697 if debug: print(indent+"Requirement: ", requirement) 698 isNull=False 699 if requirement.startswith("!"): 700 requirement=requirement[1:]; isNull=True 701 # Precision requirement for reals 702 if requirement in self.precision_types: 703 if self.conf['PETSC_PRECISION']==requirement: 704 if isNull: 705 testDict['SKIP'].append("not "+requirement+" required") 706 continue 707 continue # Success 708 elif not isNull: 709 testDict['SKIP'].append(requirement+" required") 710 continue 711 # Precision requirement for ints 712 if requirement in self.integer_types: 713 if requirement=="int32": 714 if self.conf['PETSC_SIZEOF_INT']==4: 715 if isNull: 716 testDict['SKIP'].append("not int32 required") 717 continue 718 continue # Success 719 elif not isNull: 720 testDict['SKIP'].append("int32 required") 721 continue 722 if requirement=="int64": 723 if self.conf['PETSC_SIZEOF_INT']==8: 724 if isNull: 725 testDict['SKIP'].append("NOT int64 required") 726 continue 727 continue # Success 728 elif not isNull: 729 testDict['SKIP'].append("int64 required") 730 continue 731 if requirement.startswith("long"): 732 reqsize = int(requirement[4:])//8 733 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 734 if longsize==reqsize: 735 if isNull: 736 testDict['SKIP'].append("not %s required" % requirement) 737 continue 738 continue # Success 739 elif not isNull: 740 testDict['SKIP'].append("%s required" % requirement) 741 continue 742 # Datafilespath 743 if requirement=="datafilespath" and not isNull: 744 testDict['SKIP'].append("Requires DATAFILESPATH") 745 continue 746 # Defines -- not sure I have comments matching 747 if "define(" in requirement.lower(): 748 reqdef=requirement.split("(")[1].split(")")[0] 749 if reqdef in self.conf: 750 if isNull: 751 testDict['SKIP'].append("Null requirement not met: "+requirement) 752 continue 753 continue # Success 754 elif not isNull: 755 testDict['SKIP'].append("Required: "+requirement) 756 continue 757 758 # Rest should be packages that we can just get from conf 759 if requirement == "complex": 760 petscconfvar="PETSC_USE_COMPLEX" 761 else: 762 petscconfvar="PETSC_HAVE_"+requirement.upper() 763 if self.conf.get(petscconfvar): 764 if isNull: 765 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 766 continue 767 continue # Success 768 elif not isNull: 769 if debug: print("requirement not found: ", requirement) 770 testDict['SKIP'].append(petscconfvar+" requirement not met") 771 continue 772 773 return testDict['SKIP'] == [] 774 775 def genPetscTests_summarize(self,dataDict): 776 """ 777 Required method to state what happened 778 """ 779 if not self.summarize: return 780 indent=" " 781 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 782 fh=open(fhname,"w") 783 for root in dataDict: 784 relroot=self.srcrelpath(root) 785 pkg=relroot.split("/")[1] 786 fh.write(relroot+"\n") 787 allSrcs=[] 788 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 789 for exfile in dataDict[root]: 790 # Basic information 791 rfile=os.path.join(relroot,exfile) 792 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 793 fh.write(indent+exfile+indent*4+builtStatus+"\n") 794 795 for test in dataDict[root][exfile]: 796 if test in self.buildkeys: continue 797 line=indent*2+test 798 fh.write(line+"\n") 799 # Looks nice to have the keys in order 800 #for key in dataDict[root][exfile][test]: 801 for key in "isrun abstracted nsize args requires script".split(): 802 if key not in dataDict[root][exfile][test]: continue 803 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 804 fh.write(line+"\n") 805 fh.write("\n") 806 fh.write("\n") 807 fh.write("\n") 808 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 809 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 810 fh.close() 811 return 812 813 def genPetscTests(self,root,dirs,files,dataDict): 814 """ 815 Go through and parse the source files in the directory to generate 816 the examples based on the metadata contained in the source files 817 """ 818 debug=False 819 # Use examplesAnalyze to get what the makefles think are sources 820 #self.examplesAnalyze(root,dirs,files,anlzDict) 821 822 dataDict[root]={} 823 824 for exfile in files: 825 #TST: Until we replace files, still leaving the orginals as is 826 #if not exfile.startswith("new_"+"ex"): continue 827 #if not exfile.startswith("ex"): continue 828 829 # Ignore emacs and other temporary files 830 if exfile.startswith("."): continue 831 if exfile.startswith("#"): continue 832 if exfile.endswith("~"): continue 833 # Only parse source files 834 ext=os.path.splitext(exfile)[-1].lstrip('.') 835 if ext not in LANGS: continue 836 837 # Convenience 838 fullex=os.path.join(root,exfile) 839 if self.verbose: print(' --> '+fullex) 840 dataDict[root].update(testparse.parseTestFile(fullex,0)) 841 if exfile in dataDict[root]: 842 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 843 844 return 845 846 def walktree(self,top): 847 """ 848 Walk a directory tree, starting from 'top' 849 """ 850 # Goal of action is to fill this dictionary 851 dataDict={} 852 for root, dirs, files in os.walk(top, topdown=True): 853 dirs.sort() 854 files.sort() 855 if not "examples" in root: continue 856 if "dSYM" in root: continue 857 if os.path.basename(root.rstrip("/")) == 'output': continue 858 if self.verbose: print(root) 859 self.genPetscTests(root,dirs,files,dataDict) 860 # Now summarize this dictionary 861 if self.verbose: self.genPetscTests_summarize(dataDict) 862 return dataDict 863 864 def gen_gnumake(self, fd): 865 """ 866 Overwrite of the method in the base PETSc class 867 """ 868 def write(stem, srcs): 869 for lang in LANGS: 870 if srcs[lang]['srcs']: 871 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 872 for pkg in PKGS: 873 srcs = self.gen_pkg(pkg) 874 write('testsrcs-' + pkg, srcs) 875 # Handle dependencies 876 for lang in LANGS: 877 for exfile in srcs[lang]['srcs']: 878 if exfile in srcs[lang]: 879 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 880 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 881 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 882 if deps: 883 # The executable literally depends on the object file because it is linked 884 fd.write(ex +": " + " ".join(deps) +'\n') 885 # The object file containing 'main' does not normally depend on other object 886 # files, but it does when it includes their modules. This dependency is 887 # overly blunt and could be reduced to only depend on object files for 888 # modules that are used, like "*f90aux.o". 889 fd.write(exfo +": " + " ".join(deps) +'\n') 890 891 return self.gendeps 892 893 def gen_pkg(self, pkg): 894 """ 895 Overwrite of the method in the base PETSc class 896 """ 897 return self.sources[pkg] 898 899 def write_gnumake(self, dataDict, output=None): 900 """ 901 Write out something similar to files from gmakegen.py 902 903 Test depends on script which also depends on source 904 file, but since I don't have a good way generating 905 acting on a single file (oops) just depend on 906 executable which in turn will depend on src file 907 """ 908 # Different options for how to set up the targets 909 compileExecsFirst=False 910 911 # Open file 912 fd = open(output, 'w') 913 914 # Write out the sources 915 gendeps = self.gen_gnumake(fd) 916 917 # Write out the tests and execname targets 918 fd.write("\n#Tests and executables\n") # Delimiter 919 920 for pkg in PKGS: 921 # These grab the ones that are built 922 for lang in LANGS: 923 testdeps=[] 924 for ftest in self.tests[pkg][lang]: 925 test=os.path.basename(ftest) 926 basedir=os.path.dirname(ftest) 927 testdeps.append(self.nameSpace(test,basedir)) 928 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 929 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 930 931 # test targets 932 for ftest in self.tests[pkg][lang]: 933 test=os.path.basename(ftest) 934 basedir=os.path.dirname(ftest) 935 testdir="${TESTDIR}/"+basedir+"/" 936 nmtest=self.nameSpace(test,basedir) 937 rundir=os.path.join(testdir,test) 938 script=test+".sh" 939 940 # Deps 941 exfile=self.tests[pkg][lang][ftest]['exfile'] 942 fullex=os.path.join(self.srcdir,exfile) 943 localexec=self.tests[pkg][lang][ftest]['exec'] 944 execname=os.path.join(testdir,localexec) 945 fullscript=os.path.join(testdir,script) 946 tmpfile=os.path.join(testdir,test,test+".tmp") 947 948 # *.counts depends on the script and either executable (will 949 # be run) or the example source file (SKIP or TODO) 950 fd.write('%s.counts : %s %s' 951 % (os.path.join('$(TESTDIR)/counts', nmtest), 952 fullscript, 953 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 954 ) 955 if exfile in self.sources[pkg][lang]: 956 for dep in self.sources[pkg][lang][exfile]: 957 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 958 fd.write('\n') 959 960 # Now write the args: 961 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 962 963 fd.close() 964 return 965 966def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 967 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 968 testdir=os.path.normpath(testdir) 969 if petsc_arch: 970 petsc_arch=petsc_arch.rstrip(os.path.sep) 971 if len(petsc_arch.split(os.path.sep))>1: 972 petsc_dir,petsc_arch=os.path.split(petsc_arch) 973 output = os.path.join(testdir, 'testfiles') 974 975 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 976 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 977 testdir=testdir) 978 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 979 pEx.write_gnumake(dataDict, output) 980 981if __name__ == '__main__': 982 import optparse 983 parser = optparse.OptionParser() 984 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 985 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 986 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 987 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 988 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 989 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 990 991 opts, extra_args = parser.parse_args() 992 if extra_args: 993 import sys 994 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 995 exit(1) 996 if opts.testdir is None: 997 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 998 999 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1000 verbose=opts.verbose, 1001 single_ex=opts.single_executable, srcdir=opts.srcdir, 1002 testdir=opts.testdir) 1003