1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 85 86 self.single_ex=single_ex 87 self.srcdir=srcdir 88 89 # Set locations to handle movement 90 self.inInstallDir=self.getInInstallDir(thisscriptdir) 91 92 if self.inInstallDir: 93 # Case 2 discussed above 94 # set PETSC_ARCH to install directory to allow script to work in both 95 dirlist=thisscriptdir.split(os.path.sep) 96 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 97 self.arch_dir=installdir 98 if self.srcdir is None: 99 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 100 else: 101 if petsc_arch == '': 102 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 103 # Case 1 discussed above 104 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 105 if self.srcdir is None: 106 self.srcdir=os.path.join(self.petsc_dir,'src') 107 108 self.testroot_dir=os.path.abspath(testdir) 109 110 self.ptNaming=True 111 self.verbose=verbose 112 # Whether to write out a useful debugging 113 self.summarize=True if verbose else False 114 115 # For help in setting the requirements 116 self.precision_types="single double __float128 int32".split() 117 self.integer_types="int32 int64".split() 118 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 119 120 # Things that are not test 121 self.buildkeys=testparse.buildkeys 122 123 # Adding a dictionary for storing sources, objects, and tests 124 # to make building the dependency tree easier 125 self.sources={} 126 self.objects={} 127 self.tests={} 128 for pkg in self.pkg_pkgs: 129 self.sources[pkg]={} 130 self.objects[pkg]=[] 131 self.tests[pkg]={} 132 for lang in LANGS: 133 self.sources[pkg][lang]={} 134 self.sources[pkg][lang]['srcs']=[] 135 self.tests[pkg][lang]={} 136 137 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 138 139 self.indent=" " 140 if self.verbose: print('Finishing the constructor') 141 return 142 143 def srcrelpath(self,rdir): 144 """ 145 Get relative path to source directory 146 """ 147 return os.path.relpath(rdir,self.srcdir) 148 149 def getInInstallDir(self,thisscriptdir): 150 """ 151 When petsc is installed then this file in installed in: 152 <PREFIX>/share/petsc/examples/config/gmakegentest.py 153 otherwise the path is: 154 <PETSC_DIR>/config/gmakegentest.py 155 We use this difference to determine if we are in installdir 156 """ 157 dirlist=thisscriptdir.split(os.path.sep) 158 if len(dirlist)>4: 159 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 160 if lastfour==os.path.join('share','petsc','examples','config'): 161 return True 162 else: 163 return False 164 else: 165 return False 166 167 def nameSpace(self,srcfile,srcdir): 168 """ 169 Because the scripts have a non-unique naming, the pretty-printing 170 needs to convey the srcdir and srcfile. There are two ways of doing this. 171 """ 172 if self.ptNaming: 173 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 174 cdir=srcdir 175 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 176 nameString=prefix+srcfile 177 else: 178 #nameString=srcdir+": "+srcfile 179 nameString=srcfile 180 return nameString 181 182 def getLanguage(self,srcfile): 183 """ 184 Based on the source, determine associated language as found in gmakegen.LANGS 185 Can we just return srcext[1:\] now? 186 """ 187 langReq=None 188 srcext=os.path.splitext(srcfile)[-1] 189 if srcext in ".F90".split(): langReq="F90" 190 if srcext in ".F".split(): langReq="F" 191 if srcext in ".cxx".split(): langReq="cxx" 192 if srcext in ".cpp".split(): langReq="cpp" 193 if srcext == ".cu": langReq="cu" 194 if srcext == ".c": langReq="c" 195 #if not langReq: print("ERROR: ", srcext, srcfile) 196 return langReq 197 198 def _getLoopVars(self,inDict,testname, isSubtest=False): 199 """ 200 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 201 Return: 202 inDict['args']: -ksp_monitor 203 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 204 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 205 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 206 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 207 subst should be passed in instead of inDict 208 """ 209 loopVars={}; newargs=[] 210 lsuffix='_' 211 argregex = re.compile(' (?=-[a-zA-Z])') 212 from testparse import parseLoopArgs 213 for key in inDict: 214 if key in ('SKIP', 'regexes'): 215 continue 216 akey=('subargs' if key=='args' else key) # what to assign 217 if akey not in inDict: inDict[akey]='' 218 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 219 # Always generate a loop over nsize, even if there is only one value 220 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 221 keystr = str(inDict[key]) 222 varlist = [] 223 for varset in argregex.split(keystr): 224 if not varset.strip(): continue 225 if '{{' in varset: 226 keyvar,lvars,ftype=parseLoopArgs(varset) 227 if akey not in loopVars: loopVars[akey]={} 228 varlist.append(keyvar) 229 loopVars[akey][keyvar]=[keyvar,lvars] 230 if akey=='nsize': 231 if len(lvars.split()) > 1: 232 lsuffix += akey +'-${' + keyvar + '}' 233 else: 234 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 235 lsuffix+=keyvar+'-${' + keyvar + '}_' 236 else: 237 if key=='args': 238 newargs.append(varset.strip()) 239 if varlist: 240 loopVars[akey]['varlist']=varlist 241 242 # For subtests, args are always substituted in (not top level) 243 if isSubtest: 244 inDict['subargs'] += " ".join(newargs) 245 inDict['args']='' 246 if 'label_suffix' in inDict: 247 inDict['label_suffix']+=lsuffix.rstrip('_') 248 else: 249 inDict['label_suffix']=lsuffix.rstrip('_') 250 else: 251 if loopVars: 252 inDict['args'] = ' '.join(newargs) 253 inDict['label_suffix']=lsuffix.rstrip('_') 254 return loopVars 255 256 def getArgLabel(self,testDict): 257 """ 258 In all of the arguments in the test dictionary, create a simple 259 string for searching within the makefile system. For simplicity in 260 search, remove "-", for strings, etc. 261 Also, concatenate the arg commands 262 For now, ignore nsize -- seems hard to search for anyway 263 """ 264 # Collect all of the args associated with a test 265 argStr=("" if 'args' not in testDict else testDict['args']) 266 if 'subtests' in testDict: 267 for stest in testDict["subtests"]: 268 sd=testDict[stest] 269 argStr=argStr+("" if 'args' not in sd else sd['args']) 270 271 # Now go through and cleanup 272 argStr=re.sub('{{(.*?)}}',"",argStr) 273 argStr=re.sub('-'," ",argStr) 274 for digit in string.digits: argStr=re.sub(digit," ",argStr) 275 argStr=re.sub("\.","",argStr) 276 argStr=re.sub(",","",argStr) 277 argStr=re.sub('\+',' ',argStr) 278 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 279 return argStr.strip() 280 281 def addToSources(self,exfile,rpath,srcDict): 282 """ 283 Put into data structure that allows easy generation of makefile 284 """ 285 pkg=rpath.split(os.path.sep)[0] 286 relpfile=os.path.join(rpath,exfile) 287 lang=self.getLanguage(exfile) 288 if not lang: return 289 if pkg not in self.sources: return 290 self.sources[pkg][lang]['srcs'].append(relpfile) 291 self.sources[pkg][lang][relpfile] = [] 292 if 'depends' in srcDict: 293 depSrcList=srcDict['depends'].split() 294 for depSrc in depSrcList: 295 depObj=os.path.splitext(depSrc)[0]+".o" 296 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 297 298 # In gmakefile, ${TESTDIR} var specifies the object compilation 299 testsdir=rpath+"/" 300 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 301 self.objects[pkg].append(objfile) 302 return 303 304 def addToTests(self,test,rpath,exfile,execname,testDict): 305 """ 306 Put into data structure that allows easy generation of makefile 307 Organized by languages to allow testing of languages 308 """ 309 pkg=rpath.split("/")[0] 310 nmtest=os.path.join(rpath,test) 311 lang=self.getLanguage(exfile) 312 if not lang: return 313 if pkg not in self.tests: return 314 self.tests[pkg][lang][nmtest]={} 315 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 316 self.tests[pkg][lang][nmtest]['exec']=execname 317 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 318 return 319 320 def getExecname(self,exfile,rpath): 321 """ 322 Generate bash script using template found next to this file. 323 This file is read in at constructor time to avoid file I/O 324 """ 325 if self.single_ex: 326 execname=rpath.split("/")[1]+"-ex" 327 else: 328 execname=os.path.splitext(exfile)[0] 329 return execname 330 331 def getSubstVars(self,testDict,rpath,testname): 332 """ 333 Create a dictionary with all of the variables that get substituted 334 into the template commands found in example_template.py 335 """ 336 subst={} 337 338 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 339 if 'nsize' not in testDict: testDict['nsize'] = '1' 340 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 341 for ak in testparse.acceptedkeys: 342 if ak=='test': continue 343 subst[ak]=(testDict[ak] if ak in testDict else '') 344 345 # Now do other variables 346 subst['execname']=testDict['execname'] 347 if 'filter' in testDict: 348 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 349 350 # Others 351 subst['subargs']='' # Default. For variables override 352 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 353 subst['label_suffix']='' 354 subst['comments']="\n#".join(subst['comments'].split("\n")) 355 if subst['comments']: subst['comments']="#"+subst['comments'] 356 subst['exec']="../"+subst['execname'] 357 subst['testroot']=self.testroot_dir 358 subst['testname']=testname 359 dp = self.conf.get('DATAFILESPATH','') 360 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 361 362 # This is used to label some matrices 363 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 364 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 365 366 #Conf vars 367 if self.petsc_arch.find('valgrind')>=0: 368 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 369 else: 370 subst['mpiexec']=self.conf['MPIEXEC'] 371 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 372 subst['petsc_arch']=self.petsc_arch 373 if self.inInstallDir: 374 # Case 2 375 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 376 else: 377 # Case 1 378 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 379 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 380 subst['diff']=self.conf['DIFF'] 381 subst['rm']=self.conf['RM'] 382 subst['grep']=self.conf['GREP'] 383 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 384 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 385 386 # Output file is special because of subtests override 387 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 388 if not "_" in defroot: defroot=defroot+"_1" 389 subst['defroot']=defroot 390 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 391 subst['redirect_file']=defroot+".tmp" 392 if 'output_file' not in testDict: 393 subst['output_file']="output/"+defroot+".out" 394 # Add in the full path here. 395 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 396 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 397 if not subst['TODO']: 398 print("Warning: "+subst['output_file']+" not found.") 399 # Worry about alt files here -- see 400 # src/snes/examples/tutorials/output/ex22*.out 401 altlist=[subst['output_file']] 402 basefile,ext = os.path.splitext(subst['output_file']) 403 for i in range(1,9): 404 altroot=basefile+"_alt" 405 if i > 1: altroot=altroot+"_"+str(i) 406 af=altroot+".out" 407 srcaf=os.path.join(subst['srcdir'],af) 408 fullaf=os.path.join(self.petsc_dir,srcaf) 409 if os.path.isfile(fullaf): altlist.append(srcaf) 410 if len(altlist)>1: subst['altfiles']=altlist 411 #if len(altlist)>1: print("Found alt files: ",altlist) 412 413 subst['regexes']={} 414 for subkey in subst: 415 if subkey=='regexes': continue 416 if not isinstance(subst[subkey],str): continue 417 patt="@"+subkey.upper()+"@" 418 subst['regexes'][subkey]=re.compile(patt) 419 420 return subst 421 422 def _substVars(self,subst,origStr): 423 """ 424 Substitute variables 425 """ 426 Str=origStr 427 for subkey in subst: 428 if subkey=='regexes': continue 429 if not isinstance(subst[subkey],str): continue 430 if subkey.upper() not in Str: continue 431 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 432 return Str 433 434 def getCmds(self,subst,i): 435 """ 436 Generate bash script using template found next to this file. 437 This file is read in at constructor time to avoid file I/O 438 """ 439 nindnt=i # the start and has to be consistent with below 440 cmdindnt=self.indent*nindnt 441 cmdLines="" 442 443 # MPI is the default -- but we have a few odd commands 444 if not subst['command']: 445 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 446 else: 447 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 448 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 449 450 cmdLines+=cmdindnt+'if test $res = 0; then\n' 451 diffindnt=self.indent*(nindnt+1) 452 if not subst['filter_output']: 453 if 'altfiles' not in subst: 454 cmd=diffindnt+self._substVars(subst,example_template.difftest) 455 else: 456 # Have to do it by hand a bit because of variable number of alt files 457 rf=subst['redirect_file'] 458 cmd=diffindnt+example_template.difftest.split('@')[0] 459 for i in range(len(subst['altfiles'])): 460 af=subst['altfiles'][i] 461 cmd+=af+' '+rf 462 if i!=len(subst['altfiles'])-1: 463 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 464 cmd+=' || ${diff_exe} ' 465 else: 466 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 467 cmd+=subst['label_suffix']+' ""' # Quotes are painful 468 else: 469 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 470 cmdLines+=cmd+"\n" 471 cmdLines+=cmdindnt+'else\n' 472 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 473 cmdLines+=cmdindnt+'fi\n' 474 return cmdLines 475 476 def _writeTodoSkip(self,fh,tors,reasons,footer): 477 """ 478 Write out the TODO and SKIP lines in the file 479 The TODO or SKIP variable, tors, should be lower case 480 """ 481 TORS=tors.upper() 482 template=eval("example_template."+tors+"line") 483 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 484 tab = '' 485 if reasons: 486 fh.write('if ! $force; then\n') 487 tab = tab + ' ' 488 if reasons == ["Requires DATAFILESPATH"]: 489 # The only reason not to run is DATAFILESPATH, which we check at run-time 490 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 491 tab = tab + ' ' 492 if reasons: 493 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 494 fh.write(tab+footer+"\n") 495 fh.write(tab+"exit\n") 496 if reasons == ["Requires DATAFILESPATH"]: 497 fh.write(' fi\n') 498 if reasons: 499 fh.write('fi\n') 500 fh.write('\n\n') 501 return 502 503 def getLoopVarsHead(self,loopVars,i,usedVars={}): 504 """ 505 Generate a nicely indented string with the format loops 506 Here is what the data structure looks like 507 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 508 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 509 loopVars['subargs']['pc_type']=["j","cholesky sor"] 510 """ 511 outstr=''; indnt=self.indent 512 513 for key in loopVars: 514 if key in usedVars: continue # Do not duplicate setting vars 515 for var in loopVars[key]['varlist']: 516 varval=loopVars[key][var] 517 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 518 outstr += "\n\n" 519 520 for key in loopVars: 521 for var in loopVars[key]['varlist']: 522 varval=loopVars[key][var] 523 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 524 i = i + 1 525 return (outstr,i) 526 527 def getLoopVarsFoot(self,loopVars,i): 528 outstr=''; indnt=self.indent 529 for key in loopVars: 530 for var in loopVars[key]['varlist']: 531 i = i - 1 532 outstr += indnt * i + "done\n" 533 return (outstr,i) 534 535 def genRunScript(self,testname,root,isRun,srcDict): 536 """ 537 Generate bash script using template found next to this file. 538 This file is read in at constructor time to avoid file I/O 539 """ 540 # runscript_dir directory has to be consistent with gmakefile 541 testDict=srcDict[testname] 542 rpath=self.srcrelpath(root) 543 runscript_dir=os.path.join(self.testroot_dir,rpath) 544 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 545 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 546 547 # Get variables to go into shell scripts. last time testDict used 548 subst=self.getSubstVars(testDict,rpath,testname) 549 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 550 if 'subtests' in testDict: 551 # The subtests inherit inDict, so we don't need top-level loops. 552 loopVars = {} 553 554 #Handle runfiles 555 for lfile in subst.get('localrunfiles','').split(): 556 install_files(os.path.join(root, lfile), 557 os.path.join(runscript_dir, os.path.dirname(lfile))) 558 # Check subtests for local runfiles 559 for stest in subst.get("subtests",[]): 560 for lfile in testDict[stest].get('localrunfiles','').split(): 561 install_files(os.path.join(root, lfile), 562 os.path.join(runscript_dir, os.path.dirname(lfile))) 563 564 # Now substitute the key variables into the header and footer 565 header=self._substVars(subst,example_template.header) 566 # The header is done twice to enable @...@ in header 567 header=self._substVars(subst,header) 568 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 569 570 # Start writing the file 571 fh.write(header+"\n") 572 573 # If there is a TODO or a SKIP then we do it before writing out the 574 # rest of the command (which is useful for working on the test) 575 # SKIP and TODO can be for the source file or for the runs 576 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 577 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 578 579 j=0 # for indentation 580 581 if loopVars: 582 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 583 if (loopHead): fh.write(loopHead+"\n") 584 585 # Subtests are special 586 allLoopVars=list(loopVars.keys()) 587 if 'subtests' in testDict: 588 substP=subst # Subtests can inherit args but be careful 589 k=0 # for label suffixes 590 for stest in testDict["subtests"]: 591 subst=substP.copy() 592 subst.update(testDict[stest]) 593 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 594 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 595 if sLoopVars: 596 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 597 allLoopVars+=list(sLoopVars.keys()) 598 fh.write(sLoopHead+"\n") 599 fh.write(self.getCmds(subst,j)+"\n") 600 if sLoopVars: 601 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 602 fh.write(sLoopFoot+"\n") 603 else: 604 fh.write(self.getCmds(subst,j)+"\n") 605 606 if loopVars: 607 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 608 fh.write(loopFoot+"\n") 609 610 fh.write(footer+"\n") 611 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 612 #if '10_9' in testname: sys.exit() 613 return 614 615 def genScriptsAndInfo(self,exfile,root,srcDict): 616 """ 617 Generate scripts from the source file, determine if built, etc. 618 For every test in the exfile with info in the srcDict: 619 1. Determine if it needs to be run for this arch 620 2. Generate the script 621 3. Generate the data needed to write out the makefile in a 622 convenient way 623 All tests are *always* run, but some may be SKIP'd per the TAP standard 624 """ 625 debug=False 626 rpath=self.srcrelpath(root) 627 execname=self.getExecname(exfile,rpath) 628 isBuilt=self._isBuilt(exfile,srcDict) 629 for test in srcDict: 630 if test in self.buildkeys: continue 631 if debug: print(self.nameSpace(exfile,root), test) 632 srcDict[test]['execname']=execname # Convenience in generating scripts 633 isRun=self._isRun(srcDict[test]) 634 self.genRunScript(test,root,isRun,srcDict) 635 srcDict[test]['isrun']=isRun 636 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 637 638 # This adds to datastructure for building deps 639 if isBuilt: self.addToSources(exfile,rpath,srcDict) 640 return 641 642 def _isBuilt(self,exfile,srcDict): 643 """ 644 Determine if this file should be built. 645 """ 646 # Get the language based on file extension 647 srcDict['SKIP'] = [] 648 lang=self.getLanguage(exfile) 649 if (lang=="F" or lang=="F90"): 650 if not self.have_fortran: 651 srcDict["SKIP"].append("Fortran required for this test") 652 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 653 srcDict["SKIP"].append("Fortran f90freeform required for this test") 654 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 655 srcDict["SKIP"].append("CUDA required for this test") 656 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 657 srcDict["SKIP"].append("C++ required for this test") 658 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 659 srcDict["SKIP"].append("C++ required for this test") 660 661 # Deprecated source files 662 if srcDict.get("TODO"): 663 return False 664 665 # isRun can work with srcDict to handle the requires 666 if "requires" in srcDict: 667 if srcDict["requires"]: 668 return self._isRun(srcDict) 669 670 return srcDict['SKIP'] == [] 671 672 673 def _isRun(self,testDict, debug=False): 674 """ 675 Based on the requirements listed in the src file and the petscconf.h 676 info, determine whether this test should be run or not. 677 """ 678 indent=" " 679 680 if 'SKIP' not in testDict: 681 testDict['SKIP'] = [] 682 # MPI requirements 683 if 'MPI_IS_MPIUNI' in self.conf: 684 if testDict.get('nsize', '1') != '1': 685 testDict['SKIP'].append("Parallel test with serial build") 686 687 # The requirements for the test are the sum of all the run subtests 688 if 'subtests' in testDict: 689 if 'requires' not in testDict: testDict['requires']="" 690 for stest in testDict['subtests']: 691 if 'requires' in testDict[stest]: 692 testDict['requires']+=" "+testDict[stest]['requires'] 693 if testDict.get('nsize', '1') != '1': 694 testDict['SKIP'].append("Parallel test with serial build") 695 break 696 697 # Now go through all requirements 698 if 'requires' in testDict: 699 for requirement in testDict['requires'].split(): 700 requirement=requirement.strip() 701 if not requirement: continue 702 if debug: print(indent+"Requirement: ", requirement) 703 isNull=False 704 if requirement.startswith("!"): 705 requirement=requirement[1:]; isNull=True 706 # Precision requirement for reals 707 if requirement in self.precision_types: 708 if self.conf['PETSC_PRECISION']==requirement: 709 if isNull: 710 testDict['SKIP'].append("not "+requirement+" required") 711 continue 712 continue # Success 713 elif not isNull: 714 testDict['SKIP'].append(requirement+" required") 715 continue 716 # Precision requirement for ints 717 if requirement in self.integer_types: 718 if requirement=="int32": 719 if self.conf['PETSC_SIZEOF_INT']==4: 720 if isNull: 721 testDict['SKIP'].append("not int32 required") 722 continue 723 continue # Success 724 elif not isNull: 725 testDict['SKIP'].append("int32 required") 726 continue 727 if requirement=="int64": 728 if self.conf['PETSC_SIZEOF_INT']==8: 729 if isNull: 730 testDict['SKIP'].append("NOT int64 required") 731 continue 732 continue # Success 733 elif not isNull: 734 testDict['SKIP'].append("int64 required") 735 continue 736 # Datafilespath 737 if requirement=="datafilespath" and not isNull: 738 testDict['SKIP'].append("Requires DATAFILESPATH") 739 continue 740 # Defines -- not sure I have comments matching 741 if "define(" in requirement.lower(): 742 reqdef=requirement.split("(")[1].split(")")[0] 743 if reqdef in self.conf: 744 if isNull: 745 testDict['SKIP'].append("Null requirement not met: "+requirement) 746 continue 747 continue # Success 748 elif not isNull: 749 testDict['SKIP'].append("Required: "+requirement) 750 continue 751 752 # Rest should be packages that we can just get from conf 753 if requirement == "complex": 754 petscconfvar="PETSC_USE_COMPLEX" 755 pkgconfvar="PETSC_USE_COMPLEX" 756 else: 757 petscconfvar="PETSC_HAVE_"+requirement.upper() 758 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 759 petsccv = self.conf.get(petscconfvar) 760 pkgcv = self.conf.get(pkgconfvar) 761 762 if petsccv or pkgcv: 763 if isNull: 764 if petsccv: 765 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 766 continue 767 else: 768 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 769 continue 770 continue # Success 771 elif not isNull: 772 if not petsccv and not pkgcv: 773 if debug: print("requirement not found: ", requirement) 774 if self.pkg_name == 'petsc': 775 testDict['SKIP'].append(petscconfvar+" requirement not met") 776 else: 777 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 778 continue 779 return testDict['SKIP'] == [] 780 781 def genPetscTests_summarize(self,dataDict): 782 """ 783 Required method to state what happened 784 """ 785 if not self.summarize: return 786 indent=" " 787 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 788 fh=open(fhname,"w") 789 for root in dataDict: 790 relroot=self.srcrelpath(root) 791 pkg=relroot.split("/")[1] 792 fh.write(relroot+"\n") 793 allSrcs=[] 794 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 795 for exfile in dataDict[root]: 796 # Basic information 797 rfile=os.path.join(relroot,exfile) 798 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 799 fh.write(indent+exfile+indent*4+builtStatus+"\n") 800 801 for test in dataDict[root][exfile]: 802 if test in self.buildkeys: continue 803 line=indent*2+test 804 fh.write(line+"\n") 805 # Looks nice to have the keys in order 806 #for key in dataDict[root][exfile][test]: 807 for key in "isrun abstracted nsize args requires script".split(): 808 if key not in dataDict[root][exfile][test]: continue 809 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 810 fh.write(line+"\n") 811 fh.write("\n") 812 fh.write("\n") 813 fh.write("\n") 814 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 815 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 816 fh.close() 817 return 818 819 def genPetscTests(self,root,dirs,files,dataDict): 820 """ 821 Go through and parse the source files in the directory to generate 822 the examples based on the metadata contained in the source files 823 """ 824 debug=False 825 # Use examplesAnalyze to get what the makefles think are sources 826 #self.examplesAnalyze(root,dirs,files,anlzDict) 827 828 dataDict[root]={} 829 830 for exfile in files: 831 #TST: Until we replace files, still leaving the orginals as is 832 #if not exfile.startswith("new_"+"ex"): continue 833 #if not exfile.startswith("ex"): continue 834 835 # Ignore emacs and other temporary files 836 if exfile.startswith("."): continue 837 if exfile.startswith("#"): continue 838 if exfile.endswith("~"): continue 839 # Only parse source files 840 ext=os.path.splitext(exfile)[-1].lstrip('.') 841 if ext not in LANGS: continue 842 843 # Convenience 844 fullex=os.path.join(root,exfile) 845 if self.verbose: print(' --> '+fullex) 846 dataDict[root].update(testparse.parseTestFile(fullex,0)) 847 if exfile in dataDict[root]: 848 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 849 850 return 851 852 def walktree(self,top): 853 """ 854 Walk a directory tree, starting from 'top' 855 """ 856 # Goal of action is to fill this dictionary 857 dataDict={} 858 for root, dirs, files in os.walk(top, topdown=True): 859 dirs.sort() 860 files.sort() 861 if not "examples" in root: continue 862 if "dSYM" in root: continue 863 if os.path.basename(root.rstrip("/")) == 'output': continue 864 if self.verbose: print(root) 865 self.genPetscTests(root,dirs,files,dataDict) 866 # Now summarize this dictionary 867 if self.verbose: self.genPetscTests_summarize(dataDict) 868 return dataDict 869 870 def gen_gnumake(self, fd): 871 """ 872 Overwrite of the method in the base PETSc class 873 """ 874 def write(stem, srcs): 875 for lang in LANGS: 876 if srcs[lang]['srcs']: 877 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 878 for pkg in self.pkg_pkgs: 879 srcs = self.gen_pkg(pkg) 880 write('testsrcs-' + pkg, srcs) 881 # Handle dependencies 882 for lang in LANGS: 883 for exfile in srcs[lang]['srcs']: 884 if exfile in srcs[lang]: 885 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 886 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 887 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 888 if deps: 889 # The executable literally depends on the object file because it is linked 890 fd.write(ex +": " + " ".join(deps) +'\n') 891 # The object file containing 'main' does not normally depend on other object 892 # files, but it does when it includes their modules. This dependency is 893 # overly blunt and could be reduced to only depend on object files for 894 # modules that are used, like "*f90aux.o". 895 fd.write(exfo +": " + " ".join(deps) +'\n') 896 897 return self.gendeps 898 899 def gen_pkg(self, pkg): 900 """ 901 Overwrite of the method in the base PETSc class 902 """ 903 return self.sources[pkg] 904 905 def write_gnumake(self, dataDict, output=None): 906 """ 907 Write out something similar to files from gmakegen.py 908 909 Test depends on script which also depends on source 910 file, but since I don't have a good way generating 911 acting on a single file (oops) just depend on 912 executable which in turn will depend on src file 913 """ 914 # Different options for how to set up the targets 915 compileExecsFirst=False 916 917 # Open file 918 fd = open(output, 'w') 919 920 # Write out the sources 921 gendeps = self.gen_gnumake(fd) 922 923 # Write out the tests and execname targets 924 fd.write("\n#Tests and executables\n") # Delimiter 925 926 for pkg in self.pkg_pkgs: 927 # These grab the ones that are built 928 for lang in LANGS: 929 testdeps=[] 930 for ftest in self.tests[pkg][lang]: 931 test=os.path.basename(ftest) 932 basedir=os.path.dirname(ftest) 933 testdeps.append(self.nameSpace(test,basedir)) 934 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 935 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 936 937 # test targets 938 for ftest in self.tests[pkg][lang]: 939 test=os.path.basename(ftest) 940 basedir=os.path.dirname(ftest) 941 testdir="${TESTDIR}/"+basedir+"/" 942 nmtest=self.nameSpace(test,basedir) 943 rundir=os.path.join(testdir,test) 944 script=test+".sh" 945 946 # Deps 947 exfile=self.tests[pkg][lang][ftest]['exfile'] 948 fullex=os.path.join(self.srcdir,exfile) 949 localexec=self.tests[pkg][lang][ftest]['exec'] 950 execname=os.path.join(testdir,localexec) 951 fullscript=os.path.join(testdir,script) 952 tmpfile=os.path.join(testdir,test,test+".tmp") 953 954 # *.counts depends on the script and either executable (will 955 # be run) or the example source file (SKIP or TODO) 956 fd.write('%s.counts : %s %s' 957 % (os.path.join('$(TESTDIR)/counts', nmtest), 958 fullscript, 959 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 960 ) 961 if exfile in self.sources[pkg][lang]: 962 for dep in self.sources[pkg][lang][exfile]: 963 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 964 fd.write('\n') 965 966 # Now write the args: 967 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 968 969 fd.close() 970 return 971 972def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 973 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 974 testdir=os.path.normpath(testdir) 975 if petsc_arch: 976 petsc_arch=petsc_arch.rstrip(os.path.sep) 977 if len(petsc_arch.split(os.path.sep))>1: 978 petsc_dir,petsc_arch=os.path.split(petsc_arch) 979 output = os.path.join(testdir, 'testfiles') 980 981 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 982 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 983 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 984 testdir=testdir) 985 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 986 pEx.write_gnumake(dataDict, output) 987 988if __name__ == '__main__': 989 import optparse 990 parser = optparse.OptionParser() 991 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 992 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 993 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 994 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 995 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 996 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 997 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 998 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 999 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1000 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1001 1002 opts, extra_args = parser.parse_args() 1003 if extra_args: 1004 import sys 1005 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1006 exit(1) 1007 if opts.testdir is None: 1008 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1009 1010 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1011 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1012 verbose=opts.verbose, 1013 single_ex=opts.single_executable, srcdir=opts.srcdir, 1014 testdir=opts.testdir) 1015