1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_pkgs=pkg_pkgs, verbose=verbose) 85 86 self.single_ex=single_ex 87 self.srcdir=srcdir 88 89 # Set locations to handle movement 90 self.inInstallDir=self.getInInstallDir(thisscriptdir) 91 92 if self.inInstallDir: 93 # Case 2 discussed above 94 # set PETSC_ARCH to install directory to allow script to work in both 95 dirlist=thisscriptdir.split(os.path.sep) 96 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 97 self.arch_dir=installdir 98 if self.srcdir is None: 99 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 100 else: 101 if petsc_arch == '': 102 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 103 # Case 1 discussed above 104 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 105 if self.srcdir is None: 106 self.srcdir=os.path.join(self.petsc_dir,'src') 107 108 self.testroot_dir=os.path.abspath(testdir) 109 110 self.ptNaming=True 111 self.verbose=verbose 112 # Whether to write out a useful debugging 113 self.summarize=True if verbose else False 114 115 # For help in setting the requirements 116 self.precision_types="single double __float128 int32".split() 117 self.integer_types="int32 int64".split() 118 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 119 120 # Things that are not test 121 self.buildkeys=testparse.buildkeys 122 123 # Adding a dictionary for storing sources, objects, and tests 124 # to make building the dependency tree easier 125 self.sources={} 126 self.objects={} 127 self.tests={} 128 for pkg in self.pkg_pkgs: 129 self.sources[pkg]={} 130 self.objects[pkg]=[] 131 self.tests[pkg]={} 132 for lang in LANGS: 133 self.sources[pkg][lang]={} 134 self.sources[pkg][lang]['srcs']=[] 135 self.tests[pkg][lang]={} 136 137 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 138 139 self.indent=" " 140 if self.verbose: print('Finishing the constructor') 141 return 142 143 def srcrelpath(self,rdir): 144 """ 145 Get relative path to source directory 146 """ 147 return os.path.relpath(rdir,self.srcdir) 148 149 def getInInstallDir(self,thisscriptdir): 150 """ 151 When petsc is installed then this file in installed in: 152 <PREFIX>/share/petsc/examples/config/gmakegentest.py 153 otherwise the path is: 154 <PETSC_DIR>/config/gmakegentest.py 155 We use this difference to determine if we are in installdir 156 """ 157 dirlist=thisscriptdir.split(os.path.sep) 158 if len(dirlist)>4: 159 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 160 if lastfour==os.path.join('share','petsc','examples','config'): 161 return True 162 else: 163 return False 164 else: 165 return False 166 167 def nameSpace(self,srcfile,srcdir): 168 """ 169 Because the scripts have a non-unique naming, the pretty-printing 170 needs to convey the srcdir and srcfile. There are two ways of doing this. 171 """ 172 if self.ptNaming: 173 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 174 cdir=srcdir 175 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 176 nameString=prefix+srcfile 177 else: 178 #nameString=srcdir+": "+srcfile 179 nameString=srcfile 180 return nameString 181 182 def getLanguage(self,srcfile): 183 """ 184 Based on the source, determine associated language as found in gmakegen.LANGS 185 Can we just return srcext[1:\] now? 186 """ 187 langReq=None 188 srcext=os.path.splitext(srcfile)[-1] 189 if srcext in ".F90".split(): langReq="F90" 190 if srcext in ".F".split(): langReq="F" 191 if srcext in ".cxx".split(): langReq="cxx" 192 if srcext in ".cpp".split(): langReq="cpp" 193 if srcext == ".cu": langReq="cu" 194 if srcext == ".c": langReq="c" 195 #if not langReq: print("ERROR: ", srcext, srcfile) 196 return langReq 197 198 def _getLoopVars(self,inDict,testname, isSubtest=False): 199 """ 200 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 201 Return: 202 inDict['args']: -ksp_monitor 203 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 204 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 205 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 206 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 207 subst should be passed in instead of inDict 208 """ 209 loopVars={}; newargs=[] 210 lsuffix='_' 211 argregex = re.compile(' (?=-[a-zA-Z])') 212 from testparse import parseLoopArgs 213 for key in inDict: 214 if key in ('SKIP', 'regexes'): 215 continue 216 akey=('subargs' if key=='args' else key) # what to assign 217 if akey not in inDict: inDict[akey]='' 218 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 219 # Always generate a loop over nsize, even if there is only one value 220 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 221 keystr = str(inDict[key]) 222 varlist = [] 223 for varset in argregex.split(keystr): 224 if not varset.strip(): continue 225 if '{{' in varset: 226 keyvar,lvars,ftype=parseLoopArgs(varset) 227 if akey not in loopVars: loopVars[akey]={} 228 varlist.append(keyvar) 229 loopVars[akey][keyvar]=[keyvar,lvars] 230 if akey=='nsize': 231 if len(lvars.split()) > 1: 232 lsuffix += akey +'-${' + keyvar + '}' 233 else: 234 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 235 lsuffix+=keyvar+'-${' + keyvar + '}_' 236 else: 237 if key=='args': 238 newargs.append(varset.strip()) 239 if varlist: 240 loopVars[akey]['varlist']=varlist 241 242 # For subtests, args are always substituted in (not top level) 243 if isSubtest: 244 inDict['subargs'] += " ".join(newargs) 245 inDict['args']='' 246 if 'label_suffix' in inDict: 247 inDict['label_suffix']+=lsuffix.rstrip('_') 248 else: 249 inDict['label_suffix']=lsuffix.rstrip('_') 250 else: 251 if loopVars: 252 inDict['args'] = ' '.join(newargs) 253 inDict['label_suffix']=lsuffix.rstrip('_') 254 return loopVars 255 256 def getArgLabel(self,testDict): 257 """ 258 In all of the arguments in the test dictionary, create a simple 259 string for searching within the makefile system. For simplicity in 260 search, remove "-", for strings, etc. 261 Also, concatenate the arg commands 262 For now, ignore nsize -- seems hard to search for anyway 263 """ 264 # Collect all of the args associated with a test 265 argStr=("" if 'args' not in testDict else testDict['args']) 266 if 'subtests' in testDict: 267 for stest in testDict["subtests"]: 268 sd=testDict[stest] 269 argStr=argStr+("" if 'args' not in sd else sd['args']) 270 271 # Now go through and cleanup 272 argStr=re.sub('{{(.*?)}}',"",argStr) 273 argStr=re.sub('-'," ",argStr) 274 for digit in string.digits: argStr=re.sub(digit," ",argStr) 275 argStr=re.sub("\.","",argStr) 276 argStr=re.sub(",","",argStr) 277 argStr=re.sub('\+',' ',argStr) 278 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 279 return argStr.strip() 280 281 def addToSources(self,exfile,rpath,srcDict): 282 """ 283 Put into data structure that allows easy generation of makefile 284 """ 285 pkg=rpath.split(os.path.sep)[0] 286 relpfile=os.path.join(rpath,exfile) 287 lang=self.getLanguage(exfile) 288 if not lang: return 289 if pkg not in self.sources: return 290 self.sources[pkg][lang]['srcs'].append(relpfile) 291 self.sources[pkg][lang][relpfile] = [] 292 if 'depends' in srcDict: 293 depSrcList=srcDict['depends'].split() 294 for depSrc in depSrcList: 295 depObj=os.path.splitext(depSrc)[0]+".o" 296 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 297 298 # In gmakefile, ${TESTDIR} var specifies the object compilation 299 testsdir=rpath+"/" 300 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 301 self.objects[pkg].append(objfile) 302 return 303 304 def addToTests(self,test,rpath,exfile,execname,testDict): 305 """ 306 Put into data structure that allows easy generation of makefile 307 Organized by languages to allow testing of languages 308 """ 309 pkg=rpath.split("/")[0] 310 nmtest=os.path.join(rpath,test) 311 lang=self.getLanguage(exfile) 312 if not lang: return 313 if pkg not in self.tests: return 314 self.tests[pkg][lang][nmtest]={} 315 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 316 self.tests[pkg][lang][nmtest]['exec']=execname 317 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 318 return 319 320 def getExecname(self,exfile,rpath): 321 """ 322 Generate bash script using template found next to this file. 323 This file is read in at constructor time to avoid file I/O 324 """ 325 if self.single_ex: 326 execname=rpath.split("/")[1]+"-ex" 327 else: 328 execname=os.path.splitext(exfile)[0] 329 return execname 330 331 def getSubstVars(self,testDict,rpath,testname): 332 """ 333 Create a dictionary with all of the variables that get substituted 334 into the template commands found in example_template.py 335 """ 336 subst={} 337 338 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 339 if 'nsize' not in testDict: testDict['nsize'] = '1' 340 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 341 for ak in testparse.acceptedkeys: 342 if ak=='test': continue 343 subst[ak]=(testDict[ak] if ak in testDict else '') 344 345 # Now do other variables 346 subst['execname']=testDict['execname'] 347 if 'filter' in testDict: 348 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 349 350 # Others 351 subst['subargs']='' # Default. For variables override 352 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 353 subst['label_suffix']='' 354 subst['comments']="\n#".join(subst['comments'].split("\n")) 355 if subst['comments']: subst['comments']="#"+subst['comments'] 356 subst['exec']="../"+subst['execname'] 357 subst['testroot']=self.testroot_dir 358 subst['testname']=testname 359 dp = self.conf.get('DATAFILESPATH','') 360 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 361 362 # This is used to label some matrices 363 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 364 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 365 366 #Conf vars 367 if self.petsc_arch.find('valgrind')>=0: 368 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 369 else: 370 subst['mpiexec']=self.conf['MPIEXEC'] 371 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 372 subst['petsc_arch']=self.petsc_arch 373 if self.inInstallDir: 374 # Case 2 375 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 376 else: 377 # Case 1 378 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 379 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 380 subst['diff']=self.conf['DIFF'] 381 subst['rm']=self.conf['RM'] 382 subst['grep']=self.conf['GREP'] 383 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 384 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 385 386 # Output file is special because of subtests override 387 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 388 if not "_" in defroot: defroot=defroot+"_1" 389 subst['defroot']=defroot 390 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 391 subst['redirect_file']=defroot+".tmp" 392 if 'output_file' not in testDict: 393 subst['output_file']="output/"+defroot+".out" 394 # Add in the full path here. 395 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 396 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 397 if not subst['TODO']: 398 print("Warning: "+subst['output_file']+" not found.") 399 # Worry about alt files here -- see 400 # src/snes/examples/tutorials/output/ex22*.out 401 altlist=[subst['output_file']] 402 basefile,ext = os.path.splitext(subst['output_file']) 403 for i in range(1,9): 404 altroot=basefile+"_alt" 405 if i > 1: altroot=altroot+"_"+str(i) 406 af=altroot+".out" 407 srcaf=os.path.join(subst['srcdir'],af) 408 fullaf=os.path.join(self.petsc_dir,srcaf) 409 if os.path.isfile(fullaf): altlist.append(srcaf) 410 if len(altlist)>1: subst['altfiles']=altlist 411 #if len(altlist)>1: print("Found alt files: ",altlist) 412 413 subst['regexes']={} 414 for subkey in subst: 415 if subkey=='regexes': continue 416 if not isinstance(subst[subkey],str): continue 417 patt="@"+subkey.upper()+"@" 418 subst['regexes'][subkey]=re.compile(patt) 419 420 return subst 421 422 def _substVars(self,subst,origStr): 423 """ 424 Substitute variables 425 """ 426 Str=origStr 427 for subkey in subst: 428 if subkey=='regexes': continue 429 if not isinstance(subst[subkey],str): continue 430 if subkey.upper() not in Str: continue 431 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 432 return Str 433 434 def getCmds(self,subst,i): 435 """ 436 Generate bash script using template found next to this file. 437 This file is read in at constructor time to avoid file I/O 438 """ 439 nindnt=i # the start and has to be consistent with below 440 cmdindnt=self.indent*nindnt 441 cmdLines="" 442 443 # MPI is the default -- but we have a few odd commands 444 if not subst['command']: 445 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 446 else: 447 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 448 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 449 450 cmdLines+=cmdindnt+'if test $res = 0; then\n' 451 diffindnt=self.indent*(nindnt+1) 452 if not subst['filter_output']: 453 if 'altfiles' not in subst: 454 cmd=diffindnt+self._substVars(subst,example_template.difftest) 455 else: 456 # Have to do it by hand a bit because of variable number of alt files 457 rf=subst['redirect_file'] 458 cmd=diffindnt+example_template.difftest.split('@')[0] 459 for i in range(len(subst['altfiles'])): 460 af=subst['altfiles'][i] 461 cmd+=af+' '+rf 462 if i!=len(subst['altfiles'])-1: 463 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 464 cmd+=' || ${diff_exe} ' 465 else: 466 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 467 cmd+=subst['label_suffix']+' ""' # Quotes are painful 468 else: 469 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 470 cmdLines+=cmd+"\n" 471 cmdLines+=cmdindnt+'else\n' 472 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 473 cmdLines+=cmdindnt+'fi\n' 474 return cmdLines 475 476 def _writeTodoSkip(self,fh,tors,reasons,footer): 477 """ 478 Write out the TODO and SKIP lines in the file 479 The TODO or SKIP variable, tors, should be lower case 480 """ 481 TORS=tors.upper() 482 template=eval("example_template."+tors+"line") 483 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 484 tab = '' 485 if reasons: 486 fh.write('if ! $force; then\n') 487 tab = tab + ' ' 488 if reasons == ["Requires DATAFILESPATH"]: 489 # The only reason not to run is DATAFILESPATH, which we check at run-time 490 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 491 tab = tab + ' ' 492 if reasons: 493 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 494 fh.write(tab+footer+"\n") 495 fh.write(tab+"exit\n") 496 if reasons == ["Requires DATAFILESPATH"]: 497 fh.write(' fi\n') 498 if reasons: 499 fh.write('fi\n') 500 fh.write('\n\n') 501 return 502 503 def getLoopVarsHead(self,loopVars,i,usedVars={}): 504 """ 505 Generate a nicely indented string with the format loops 506 Here is what the data structure looks like 507 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 508 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 509 loopVars['subargs']['pc_type']=["j","cholesky sor"] 510 """ 511 outstr=''; indnt=self.indent 512 513 for key in loopVars: 514 if key in usedVars: continue # Do not duplicate setting vars 515 for var in loopVars[key]['varlist']: 516 varval=loopVars[key][var] 517 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 518 outstr += "\n\n" 519 520 for key in loopVars: 521 for var in loopVars[key]['varlist']: 522 varval=loopVars[key][var] 523 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 524 i = i + 1 525 return (outstr,i) 526 527 def getLoopVarsFoot(self,loopVars,i): 528 outstr=''; indnt=self.indent 529 for key in loopVars: 530 for var in loopVars[key]['varlist']: 531 i = i - 1 532 outstr += indnt * i + "done\n" 533 return (outstr,i) 534 535 def genRunScript(self,testname,root,isRun,srcDict): 536 """ 537 Generate bash script using template found next to this file. 538 This file is read in at constructor time to avoid file I/O 539 """ 540 # runscript_dir directory has to be consistent with gmakefile 541 testDict=srcDict[testname] 542 rpath=self.srcrelpath(root) 543 runscript_dir=os.path.join(self.testroot_dir,rpath) 544 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 545 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 546 547 # Get variables to go into shell scripts. last time testDict used 548 subst=self.getSubstVars(testDict,rpath,testname) 549 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 550 if 'subtests' in testDict: 551 # The subtests inherit inDict, so we don't need top-level loops. 552 loopVars = {} 553 554 #Handle runfiles 555 for lfile in subst.get('localrunfiles','').split(): 556 install_files(os.path.join(root, lfile), 557 os.path.join(runscript_dir, os.path.dirname(lfile))) 558 # Check subtests for local runfiles 559 for stest in subst.get("subtests",[]): 560 for lfile in testDict[stest].get('localrunfiles','').split(): 561 install_files(os.path.join(root, lfile), 562 os.path.join(runscript_dir, os.path.dirname(lfile))) 563 564 # Now substitute the key variables into the header and footer 565 header=self._substVars(subst,example_template.header) 566 # The header is done twice to enable @...@ in header 567 header=self._substVars(subst,header) 568 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 569 570 # Start writing the file 571 fh.write(header+"\n") 572 573 # If there is a TODO or a SKIP then we do it before writing out the 574 # rest of the command (which is useful for working on the test) 575 # SKIP and TODO can be for the source file or for the runs 576 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 577 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 578 579 j=0 # for indentation 580 581 if loopVars: 582 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 583 if (loopHead): fh.write(loopHead+"\n") 584 585 # Subtests are special 586 allLoopVars=list(loopVars.keys()) 587 if 'subtests' in testDict: 588 substP=subst # Subtests can inherit args but be careful 589 k=0 # for label suffixes 590 for stest in testDict["subtests"]: 591 subst=substP.copy() 592 subst.update(testDict[stest]) 593 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 594 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 595 if sLoopVars: 596 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 597 allLoopVars+=list(sLoopVars.keys()) 598 fh.write(sLoopHead+"\n") 599 fh.write(self.getCmds(subst,j)+"\n") 600 if sLoopVars: 601 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 602 fh.write(sLoopFoot+"\n") 603 else: 604 fh.write(self.getCmds(subst,j)+"\n") 605 606 if loopVars: 607 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 608 fh.write(loopFoot+"\n") 609 610 fh.write(footer+"\n") 611 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 612 #if '10_9' in testname: sys.exit() 613 return 614 615 def genScriptsAndInfo(self,exfile,root,srcDict): 616 """ 617 Generate scripts from the source file, determine if built, etc. 618 For every test in the exfile with info in the srcDict: 619 1. Determine if it needs to be run for this arch 620 2. Generate the script 621 3. Generate the data needed to write out the makefile in a 622 convenient way 623 All tests are *always* run, but some may be SKIP'd per the TAP standard 624 """ 625 debug=False 626 rpath=self.srcrelpath(root) 627 execname=self.getExecname(exfile,rpath) 628 isBuilt=self._isBuilt(exfile,srcDict) 629 for test in srcDict: 630 if test in self.buildkeys: continue 631 if debug: print(self.nameSpace(exfile,root), test) 632 srcDict[test]['execname']=execname # Convenience in generating scripts 633 isRun=self._isRun(srcDict[test]) 634 self.genRunScript(test,root,isRun,srcDict) 635 srcDict[test]['isrun']=isRun 636 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 637 638 # This adds to datastructure for building deps 639 if isBuilt: self.addToSources(exfile,rpath,srcDict) 640 return 641 642 def _isBuilt(self,exfile,srcDict): 643 """ 644 Determine if this file should be built. 645 """ 646 # Get the language based on file extension 647 srcDict['SKIP'] = [] 648 lang=self.getLanguage(exfile) 649 if (lang=="F" or lang=="F90"): 650 if not self.have_fortran: 651 srcDict["SKIP"].append("Fortran required for this test") 652 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 653 srcDict["SKIP"].append("Fortran f90freeform required for this test") 654 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 655 srcDict["SKIP"].append("CUDA required for this test") 656 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 657 srcDict["SKIP"].append("C++ required for this test") 658 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 659 srcDict["SKIP"].append("C++ required for this test") 660 661 # Deprecated source files 662 if srcDict.get("TODO"): 663 return False 664 665 # isRun can work with srcDict to handle the requires 666 if "requires" in srcDict: 667 if srcDict["requires"]: 668 return self._isRun(srcDict) 669 670 return srcDict['SKIP'] == [] 671 672 673 def _isRun(self,testDict, debug=False): 674 """ 675 Based on the requirements listed in the src file and the petscconf.h 676 info, determine whether this test should be run or not. 677 """ 678 indent=" " 679 680 if 'SKIP' not in testDict: 681 testDict['SKIP'] = [] 682 # MPI requirements 683 if 'MPI_IS_MPIUNI' in self.conf: 684 if testDict.get('nsize', '1') != '1': 685 testDict['SKIP'].append("Parallel test with serial build") 686 687 # The requirements for the test are the sum of all the run subtests 688 if 'subtests' in testDict: 689 if 'requires' not in testDict: testDict['requires']="" 690 for stest in testDict['subtests']: 691 if 'requires' in testDict[stest]: 692 testDict['requires']+=" "+testDict[stest]['requires'] 693 if testDict.get('nsize', '1') != '1': 694 testDict['SKIP'].append("Parallel test with serial build") 695 break 696 697 # Now go through all requirements 698 if 'requires' in testDict: 699 for requirement in testDict['requires'].split(): 700 requirement=requirement.strip() 701 if not requirement: continue 702 if debug: print(indent+"Requirement: ", requirement) 703 isNull=False 704 if requirement.startswith("!"): 705 requirement=requirement[1:]; isNull=True 706 # Precision requirement for reals 707 if requirement in self.precision_types: 708 if self.conf['PETSC_PRECISION']==requirement: 709 if isNull: 710 testDict['SKIP'].append("not "+requirement+" required") 711 continue 712 continue # Success 713 elif not isNull: 714 testDict['SKIP'].append(requirement+" required") 715 continue 716 # Precision requirement for ints 717 if requirement in self.integer_types: 718 if requirement=="int32": 719 if self.conf['PETSC_SIZEOF_INT']==4: 720 if isNull: 721 testDict['SKIP'].append("not int32 required") 722 continue 723 continue # Success 724 elif not isNull: 725 testDict['SKIP'].append("int32 required") 726 continue 727 if requirement=="int64": 728 if self.conf['PETSC_SIZEOF_INT']==8: 729 if isNull: 730 testDict['SKIP'].append("NOT int64 required") 731 continue 732 continue # Success 733 elif not isNull: 734 testDict['SKIP'].append("int64 required") 735 continue 736 # Datafilespath 737 if requirement=="datafilespath" and not isNull: 738 testDict['SKIP'].append("Requires DATAFILESPATH") 739 continue 740 # Defines -- not sure I have comments matching 741 if "define(" in requirement.lower(): 742 reqdef=requirement.split("(")[1].split(")")[0] 743 if reqdef in self.conf: 744 if isNull: 745 testDict['SKIP'].append("Null requirement not met: "+requirement) 746 continue 747 continue # Success 748 elif not isNull: 749 testDict['SKIP'].append("Required: "+requirement) 750 continue 751 752 # Rest should be packages that we can just get from conf 753 if requirement == "complex": 754 petscconfvar="PETSC_USE_COMPLEX" 755 else: 756 petscconfvar="PETSC_HAVE_"+requirement.upper() 757 if self.conf.get(petscconfvar): 758 if isNull: 759 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 760 continue 761 continue # Success 762 elif not isNull: 763 if debug: print("requirement not found: ", requirement) 764 testDict['SKIP'].append(petscconfvar+" requirement not met") 765 continue 766 767 return testDict['SKIP'] == [] 768 769 def genPetscTests_summarize(self,dataDict): 770 """ 771 Required method to state what happened 772 """ 773 if not self.summarize: return 774 indent=" " 775 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 776 fh=open(fhname,"w") 777 for root in dataDict: 778 relroot=self.srcrelpath(root) 779 pkg=relroot.split("/")[1] 780 fh.write(relroot+"\n") 781 allSrcs=[] 782 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 783 for exfile in dataDict[root]: 784 # Basic information 785 rfile=os.path.join(relroot,exfile) 786 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 787 fh.write(indent+exfile+indent*4+builtStatus+"\n") 788 789 for test in dataDict[root][exfile]: 790 if test in self.buildkeys: continue 791 line=indent*2+test 792 fh.write(line+"\n") 793 # Looks nice to have the keys in order 794 #for key in dataDict[root][exfile][test]: 795 for key in "isrun abstracted nsize args requires script".split(): 796 if key not in dataDict[root][exfile][test]: continue 797 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 798 fh.write(line+"\n") 799 fh.write("\n") 800 fh.write("\n") 801 fh.write("\n") 802 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 803 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 804 fh.close() 805 return 806 807 def genPetscTests(self,root,dirs,files,dataDict): 808 """ 809 Go through and parse the source files in the directory to generate 810 the examples based on the metadata contained in the source files 811 """ 812 debug=False 813 # Use examplesAnalyze to get what the makefles think are sources 814 #self.examplesAnalyze(root,dirs,files,anlzDict) 815 816 dataDict[root]={} 817 818 for exfile in files: 819 #TST: Until we replace files, still leaving the orginals as is 820 #if not exfile.startswith("new_"+"ex"): continue 821 #if not exfile.startswith("ex"): continue 822 823 # Ignore emacs and other temporary files 824 if exfile.startswith("."): continue 825 if exfile.startswith("#"): continue 826 if exfile.endswith("~"): continue 827 # Only parse source files 828 ext=os.path.splitext(exfile)[-1].lstrip('.') 829 if ext not in LANGS: continue 830 831 # Convenience 832 fullex=os.path.join(root,exfile) 833 if self.verbose: print(' --> '+fullex) 834 dataDict[root].update(testparse.parseTestFile(fullex,0)) 835 if exfile in dataDict[root]: 836 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 837 838 return 839 840 def walktree(self,top): 841 """ 842 Walk a directory tree, starting from 'top' 843 """ 844 # Goal of action is to fill this dictionary 845 dataDict={} 846 for root, dirs, files in os.walk(top, topdown=True): 847 dirs.sort() 848 files.sort() 849 if not "examples" in root: continue 850 if "dSYM" in root: continue 851 if os.path.basename(root.rstrip("/")) == 'output': continue 852 if self.verbose: print(root) 853 self.genPetscTests(root,dirs,files,dataDict) 854 # Now summarize this dictionary 855 if self.verbose: self.genPetscTests_summarize(dataDict) 856 return dataDict 857 858 def gen_gnumake(self, fd): 859 """ 860 Overwrite of the method in the base PETSc class 861 """ 862 def write(stem, srcs): 863 for lang in LANGS: 864 if srcs[lang]['srcs']: 865 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 866 for pkg in self.pkg_pkgs: 867 srcs = self.gen_pkg(pkg) 868 write('testsrcs-' + pkg, srcs) 869 # Handle dependencies 870 for lang in LANGS: 871 for exfile in srcs[lang]['srcs']: 872 if exfile in srcs[lang]: 873 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 874 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 875 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 876 if deps: 877 # The executable literally depends on the object file because it is linked 878 fd.write(ex +": " + " ".join(deps) +'\n') 879 # The object file containing 'main' does not normally depend on other object 880 # files, but it does when it includes their modules. This dependency is 881 # overly blunt and could be reduced to only depend on object files for 882 # modules that are used, like "*f90aux.o". 883 fd.write(exfo +": " + " ".join(deps) +'\n') 884 885 return self.gendeps 886 887 def gen_pkg(self, pkg): 888 """ 889 Overwrite of the method in the base PETSc class 890 """ 891 return self.sources[pkg] 892 893 def write_gnumake(self, dataDict, output=None): 894 """ 895 Write out something similar to files from gmakegen.py 896 897 Test depends on script which also depends on source 898 file, but since I don't have a good way generating 899 acting on a single file (oops) just depend on 900 executable which in turn will depend on src file 901 """ 902 # Different options for how to set up the targets 903 compileExecsFirst=False 904 905 # Open file 906 fd = open(output, 'w') 907 908 # Write out the sources 909 gendeps = self.gen_gnumake(fd) 910 911 # Write out the tests and execname targets 912 fd.write("\n#Tests and executables\n") # Delimiter 913 914 for pkg in self.pkg_pkgs: 915 # These grab the ones that are built 916 for lang in LANGS: 917 testdeps=[] 918 for ftest in self.tests[pkg][lang]: 919 test=os.path.basename(ftest) 920 basedir=os.path.dirname(ftest) 921 testdeps.append(self.nameSpace(test,basedir)) 922 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 923 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 924 925 # test targets 926 for ftest in self.tests[pkg][lang]: 927 test=os.path.basename(ftest) 928 basedir=os.path.dirname(ftest) 929 testdir="${TESTDIR}/"+basedir+"/" 930 nmtest=self.nameSpace(test,basedir) 931 rundir=os.path.join(testdir,test) 932 script=test+".sh" 933 934 # Deps 935 exfile=self.tests[pkg][lang][ftest]['exfile'] 936 fullex=os.path.join(self.srcdir,exfile) 937 localexec=self.tests[pkg][lang][ftest]['exec'] 938 execname=os.path.join(testdir,localexec) 939 fullscript=os.path.join(testdir,script) 940 tmpfile=os.path.join(testdir,test,test+".tmp") 941 942 # *.counts depends on the script and either executable (will 943 # be run) or the example source file (SKIP or TODO) 944 fd.write('%s.counts : %s %s' 945 % (os.path.join('$(TESTDIR)/counts', nmtest), 946 fullscript, 947 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 948 ) 949 if exfile in self.sources[pkg][lang]: 950 for dep in self.sources[pkg][lang][exfile]: 951 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 952 fd.write('\n') 953 954 # Now write the args: 955 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 956 957 fd.close() 958 return 959 960def main(petsc_dir=None, petsc_arch=None, pkg_pkgs=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 961 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 962 testdir=os.path.normpath(testdir) 963 if petsc_arch: 964 petsc_arch=petsc_arch.rstrip(os.path.sep) 965 if len(petsc_arch.split(os.path.sep))>1: 966 petsc_dir,petsc_arch=os.path.split(petsc_arch) 967 output = os.path.join(testdir, 'testfiles') 968 969 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 970 pkg_pkgs=pkg_pkgs, 971 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 972 testdir=testdir) 973 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 974 pEx.write_gnumake(dataDict, output) 975 976if __name__ == '__main__': 977 import optparse 978 parser = optparse.OptionParser() 979 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 980 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 981 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 982 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 983 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 984 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 985 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 986 987 opts, extra_args = parser.parse_args() 988 if extra_args: 989 import sys 990 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 991 exit(1) 992 if opts.testdir is None: 993 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 994 995 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 996 pkg_pkgs=opts.pkg_pkgs, 997 verbose=opts.verbose, 998 single_ex=opts.single_executable, srcdir=opts.srcdir, 999 testdir=opts.testdir) 1000