1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77class generateExamples(Petsc): 78 """ 79 gmakegen.py has basic structure for finding the files, writing out 80 the dependencies, etc. 81 """ 82 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 83 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 84 85 self.single_ex=single_ex 86 self.srcdir=srcdir 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 if self.srcdir is None: 98 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 99 else: 100 if petsc_arch == '': 101 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 102 # Case 1 discussed above 103 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 104 if self.srcdir is None: 105 self.srcdir=os.path.join(self.petsc_dir,'src') 106 107 self.testroot_dir=os.path.abspath(testdir) 108 109 self.ptNaming=True 110 self.verbose=verbose 111 # Whether to write out a useful debugging 112 self.summarize=True if verbose else False 113 114 # For help in setting the requirements 115 self.precision_types="single double __float128 int32".split() 116 self.integer_types="int32 int64 long32 long64".split() 117 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 118 119 # Things that are not test 120 self.buildkeys=testparse.buildkeys 121 122 # Adding a dictionary for storing sources, objects, and tests 123 # to make building the dependency tree easier 124 self.sources={} 125 self.objects={} 126 self.tests={} 127 for pkg in self.pkg_pkgs: 128 self.sources[pkg]={} 129 self.objects[pkg]=[] 130 self.tests[pkg]={} 131 for lang in LANGS: 132 self.sources[pkg][lang]={} 133 self.sources[pkg][lang]['srcs']=[] 134 self.tests[pkg][lang]={} 135 136 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 137 138 self.indent=" " 139 if self.verbose: print('Finishing the constructor') 140 return 141 142 def srcrelpath(self,rdir): 143 """ 144 Get relative path to source directory 145 """ 146 return os.path.relpath(rdir,self.srcdir) 147 148 def getInInstallDir(self,thisscriptdir): 149 """ 150 When petsc is installed then this file in installed in: 151 <PREFIX>/share/petsc/examples/config/gmakegentest.py 152 otherwise the path is: 153 <PETSC_DIR>/config/gmakegentest.py 154 We use this difference to determine if we are in installdir 155 """ 156 dirlist=thisscriptdir.split(os.path.sep) 157 if len(dirlist)>4: 158 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 159 if lastfour==os.path.join('share','petsc','examples','config'): 160 return True 161 else: 162 return False 163 else: 164 return False 165 166 def nameSpace(self,srcfile,srcdir): 167 """ 168 Because the scripts have a non-unique naming, the pretty-printing 169 needs to convey the srcdir and srcfile. There are two ways of doing this. 170 """ 171 if self.ptNaming: 172 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 173 cdir=srcdir 174 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 175 nameString=prefix+srcfile 176 else: 177 #nameString=srcdir+": "+srcfile 178 nameString=srcfile 179 return nameString 180 181 def getLanguage(self,srcfile): 182 """ 183 Based on the source, determine associated language as found in gmakegen.LANGS 184 Can we just return srcext[1:\] now? 185 """ 186 langReq=None 187 srcext=os.path.splitext(srcfile)[-1] 188 if srcext in ".F90".split(): langReq="F90" 189 if srcext in ".F".split(): langReq="F" 190 if srcext in ".cxx".split(): langReq="cxx" 191 if srcext in ".cpp".split(): langReq="cpp" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getLoopVars(self,inDict,testname, isSubtest=False): 198 """ 199 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 200 Return: 201 inDict['args']: -ksp_monitor 202 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 203 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 204 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 205 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 206 subst should be passed in instead of inDict 207 """ 208 loopVars={}; newargs=[] 209 lsuffix='_' 210 argregex = re.compile(' (?=-[a-zA-Z])') 211 from testparse import parseLoopArgs 212 for key in inDict: 213 if key in ('SKIP', 'regexes'): 214 continue 215 akey=('subargs' if key=='args' else key) # what to assign 216 if akey not in inDict: inDict[akey]='' 217 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 218 # Always generate a loop over nsize, even if there is only one value 219 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 220 keystr = str(inDict[key]) 221 varlist = [] 222 for varset in argregex.split(keystr): 223 if not varset.strip(): continue 224 if '{{' in varset: 225 keyvar,lvars,ftype=parseLoopArgs(varset) 226 if akey not in loopVars: loopVars[akey]={} 227 varlist.append(keyvar) 228 loopVars[akey][keyvar]=[keyvar,lvars] 229 if akey=='nsize': 230 if len(lvars.split()) > 1: 231 lsuffix += akey +'-${' + keyvar + '}' 232 else: 233 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 234 lsuffix+=keyvar+'-${' + keyvar + '}_' 235 else: 236 if key=='args': 237 newargs.append(varset.strip()) 238 if varlist: 239 loopVars[akey]['varlist']=varlist 240 241 # For subtests, args are always substituted in (not top level) 242 if isSubtest: 243 inDict['subargs'] += " ".join(newargs) 244 inDict['args']='' 245 if 'label_suffix' in inDict: 246 inDict['label_suffix']+=lsuffix.rstrip('_') 247 else: 248 inDict['label_suffix']=lsuffix.rstrip('_') 249 else: 250 if loopVars: 251 inDict['args'] = ' '.join(newargs) 252 inDict['label_suffix']=lsuffix.rstrip('_') 253 return loopVars 254 255 def getArgLabel(self,testDict): 256 """ 257 In all of the arguments in the test dictionary, create a simple 258 string for searching within the makefile system. For simplicity in 259 search, remove "-", for strings, etc. 260 Also, concatenate the arg commands 261 For now, ignore nsize -- seems hard to search for anyway 262 """ 263 # Collect all of the args associated with a test 264 argStr=("" if 'args' not in testDict else testDict['args']) 265 if 'subtests' in testDict: 266 for stest in testDict["subtests"]: 267 sd=testDict[stest] 268 argStr=argStr+("" if 'args' not in sd else sd['args']) 269 270 # Now go through and cleanup 271 argStr=re.sub('{{(.*?)}}',"",argStr) 272 argStr=re.sub('-'," ",argStr) 273 for digit in string.digits: argStr=re.sub(digit," ",argStr) 274 argStr=re.sub("\.","",argStr) 275 argStr=re.sub(",","",argStr) 276 argStr=re.sub('\+',' ',argStr) 277 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 278 return argStr.strip() 279 280 def addToSources(self,exfile,rpath,srcDict): 281 """ 282 Put into data structure that allows easy generation of makefile 283 """ 284 pkg=rpath.split(os.path.sep)[0] 285 relpfile=os.path.join(rpath,exfile) 286 lang=self.getLanguage(exfile) 287 if not lang: return 288 if pkg not in self.sources: return 289 self.sources[pkg][lang]['srcs'].append(relpfile) 290 self.sources[pkg][lang][relpfile] = [] 291 if 'depends' in srcDict: 292 depSrcList=srcDict['depends'].split() 293 for depSrc in depSrcList: 294 depObj=os.path.splitext(depSrc)[0]+".o" 295 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 296 297 # In gmakefile, ${TESTDIR} var specifies the object compilation 298 testsdir=rpath+"/" 299 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 300 self.objects[pkg].append(objfile) 301 return 302 303 def addToTests(self,test,rpath,exfile,execname,testDict): 304 """ 305 Put into data structure that allows easy generation of makefile 306 Organized by languages to allow testing of languages 307 """ 308 pkg=rpath.split("/")[0] 309 nmtest=os.path.join(rpath,test) 310 lang=self.getLanguage(exfile) 311 if not lang: return 312 if pkg not in self.tests: return 313 self.tests[pkg][lang][nmtest]={} 314 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 315 self.tests[pkg][lang][nmtest]['exec']=execname 316 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 317 return 318 319 def getExecname(self,exfile,rpath): 320 """ 321 Generate bash script using template found next to this file. 322 This file is read in at constructor time to avoid file I/O 323 """ 324 if self.single_ex: 325 execname=rpath.split("/")[1]+"-ex" 326 else: 327 execname=os.path.splitext(exfile)[0] 328 return execname 329 330 def getSubstVars(self,testDict,rpath,testname): 331 """ 332 Create a dictionary with all of the variables that get substituted 333 into the template commands found in example_template.py 334 """ 335 subst={} 336 337 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 338 if 'nsize' not in testDict: testDict['nsize'] = '1' 339 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 340 for ak in testparse.acceptedkeys: 341 if ak=='test': continue 342 subst[ak]=(testDict[ak] if ak in testDict else '') 343 344 # Now do other variables 345 subst['execname']=testDict['execname'] 346 if 'filter' in testDict: 347 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 348 349 # Others 350 subst['subargs']='' # Default. For variables override 351 subst['srcdir']=os.path.join(self.srcdir, rpath) 352 subst['label_suffix']='' 353 subst['comments']="\n#".join(subst['comments'].split("\n")) 354 if subst['comments']: subst['comments']="#"+subst['comments'] 355 subst['exec']="../"+subst['execname'] 356 subst['testroot']=self.testroot_dir 357 subst['testname']=testname 358 dp = self.conf.get('DATAFILESPATH','') 359 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 360 361 # This is used to label some matrices 362 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 363 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 364 365 #Conf vars 366 if self.petsc_arch.find('valgrind')>=0: 367 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 368 else: 369 subst['mpiexec']=self.conf['MPIEXEC'] 370 subst['pkg_name']=self.pkg_name 371 subst['pkg_dir']=self.pkg_dir 372 subst['pkg_arch']=self.petsc_arch 373 374 if self.inInstallDir: 375 # Case 2 376 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 377 else: 378 # Case 1 379 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 380 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 381 subst['diff']=self.conf['DIFF'] 382 subst['rm']=self.conf['RM'] 383 subst['grep']=self.conf['GREP'] 384 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 385 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 386 387 # Output file is special because of subtests override 388 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 389 if not "_" in defroot: defroot=defroot+"_1" 390 subst['defroot']=defroot 391 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 392 subst['redirect_file']=defroot+".tmp" 393 if 'output_file' not in testDict: 394 subst['output_file']="output/"+defroot+".out" 395 # Add in the full path here. 396 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 397 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 398 if not subst['TODO']: 399 print("Warning: "+subst['output_file']+" not found.") 400 # Worry about alt files here -- see 401 # src/snes/examples/tutorials/output/ex22*.out 402 altlist=[subst['output_file']] 403 basefile,ext = os.path.splitext(subst['output_file']) 404 for i in range(1,9): 405 altroot=basefile+"_alt" 406 if i > 1: altroot=altroot+"_"+str(i) 407 af=altroot+".out" 408 srcaf=os.path.join(subst['srcdir'],af) 409 fullaf=os.path.join(self.petsc_dir,srcaf) 410 if os.path.isfile(fullaf): altlist.append(srcaf) 411 if len(altlist)>1: subst['altfiles']=altlist 412 #if len(altlist)>1: print("Found alt files: ",altlist) 413 414 subst['regexes']={} 415 for subkey in subst: 416 if subkey=='regexes': continue 417 if not isinstance(subst[subkey],str): continue 418 patt="@"+subkey.upper()+"@" 419 subst['regexes'][subkey]=re.compile(patt) 420 421 return subst 422 423 def _substVars(self,subst,origStr): 424 """ 425 Substitute variables 426 """ 427 Str=origStr 428 for subkey in subst: 429 if subkey=='regexes': continue 430 if not isinstance(subst[subkey],str): continue 431 if subkey.upper() not in Str: continue 432 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 433 return Str 434 435 def getCmds(self,subst,i): 436 """ 437 Generate bash script using template found next to this file. 438 This file is read in at constructor time to avoid file I/O 439 """ 440 nindnt=i # the start and has to be consistent with below 441 cmdindnt=self.indent*nindnt 442 cmdLines="" 443 444 # MPI is the default -- but we have a few odd commands 445 if not subst['command']: 446 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 447 else: 448 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 449 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 450 451 cmdLines+=cmdindnt+'if test $res = 0; then\n' 452 diffindnt=self.indent*(nindnt+1) 453 if not subst['filter_output']: 454 if 'altfiles' not in subst: 455 cmd=diffindnt+self._substVars(subst,example_template.difftest) 456 else: 457 # Have to do it by hand a bit because of variable number of alt files 458 rf=subst['redirect_file'] 459 cmd=diffindnt+example_template.difftest.split('@')[0] 460 for i in range(len(subst['altfiles'])): 461 af=subst['altfiles'][i] 462 cmd+=af+' '+rf 463 if i!=len(subst['altfiles'])-1: 464 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 465 cmd+=' || ${diff_exe} ' 466 else: 467 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 468 cmd+=subst['label_suffix']+' ""' # Quotes are painful 469 else: 470 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 471 cmdLines+=cmd+"\n" 472 cmdLines+=cmdindnt+'else\n' 473 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 474 cmdLines+=cmdindnt+'fi\n' 475 return cmdLines 476 477 def _writeTodoSkip(self,fh,tors,reasons,footer): 478 """ 479 Write out the TODO and SKIP lines in the file 480 The TODO or SKIP variable, tors, should be lower case 481 """ 482 TORS=tors.upper() 483 template=eval("example_template."+tors+"line") 484 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 485 tab = '' 486 if reasons: 487 fh.write('if ! $force; then\n') 488 tab = tab + ' ' 489 if reasons == ["Requires DATAFILESPATH"]: 490 # The only reason not to run is DATAFILESPATH, which we check at run-time 491 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 492 tab = tab + ' ' 493 if reasons: 494 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 495 fh.write(tab+footer+"\n") 496 fh.write(tab+"exit\n") 497 if reasons == ["Requires DATAFILESPATH"]: 498 fh.write(' fi\n') 499 if reasons: 500 fh.write('fi\n') 501 fh.write('\n\n') 502 return 503 504 def getLoopVarsHead(self,loopVars,i,usedVars={}): 505 """ 506 Generate a nicely indented string with the format loops 507 Here is what the data structure looks like 508 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 509 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 510 loopVars['subargs']['pc_type']=["j","cholesky sor"] 511 """ 512 outstr=''; indnt=self.indent 513 514 for key in loopVars: 515 if key in usedVars: continue # Do not duplicate setting vars 516 for var in loopVars[key]['varlist']: 517 varval=loopVars[key][var] 518 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 519 outstr += "\n\n" 520 521 for key in loopVars: 522 for var in loopVars[key]['varlist']: 523 varval=loopVars[key][var] 524 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 525 i = i + 1 526 return (outstr,i) 527 528 def getLoopVarsFoot(self,loopVars,i): 529 outstr=''; indnt=self.indent 530 for key in loopVars: 531 for var in loopVars[key]['varlist']: 532 i = i - 1 533 outstr += indnt * i + "done\n" 534 return (outstr,i) 535 536 def genRunScript(self,testname,root,isRun,srcDict): 537 """ 538 Generate bash script using template found next to this file. 539 This file is read in at constructor time to avoid file I/O 540 """ 541 # runscript_dir directory has to be consistent with gmakefile 542 testDict=srcDict[testname] 543 rpath=self.srcrelpath(root) 544 runscript_dir=os.path.join(self.testroot_dir,rpath) 545 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 546 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 547 548 # Get variables to go into shell scripts. last time testDict used 549 subst=self.getSubstVars(testDict,rpath,testname) 550 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 551 if 'subtests' in testDict: 552 # The subtests inherit inDict, so we don't need top-level loops. 553 loopVars = {} 554 555 #Handle runfiles 556 for lfile in subst.get('localrunfiles','').split(): 557 install_files(os.path.join(root, lfile), 558 os.path.join(runscript_dir, os.path.dirname(lfile))) 559 # Check subtests for local runfiles 560 for stest in subst.get("subtests",[]): 561 for lfile in testDict[stest].get('localrunfiles','').split(): 562 install_files(os.path.join(root, lfile), 563 os.path.join(runscript_dir, os.path.dirname(lfile))) 564 565 # Now substitute the key variables into the header and footer 566 header=self._substVars(subst,example_template.header) 567 # The header is done twice to enable @...@ in header 568 header=self._substVars(subst,header) 569 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 570 571 # Start writing the file 572 fh.write(header+"\n") 573 574 # If there is a TODO or a SKIP then we do it before writing out the 575 # rest of the command (which is useful for working on the test) 576 # SKIP and TODO can be for the source file or for the runs 577 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 578 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 579 580 j=0 # for indentation 581 582 if loopVars: 583 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 584 if (loopHead): fh.write(loopHead+"\n") 585 586 # Subtests are special 587 allLoopVars=list(loopVars.keys()) 588 if 'subtests' in testDict: 589 substP=subst # Subtests can inherit args but be careful 590 k=0 # for label suffixes 591 for stest in testDict["subtests"]: 592 subst=substP.copy() 593 subst.update(testDict[stest]) 594 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 595 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 596 if sLoopVars: 597 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 598 allLoopVars+=list(sLoopVars.keys()) 599 fh.write(sLoopHead+"\n") 600 fh.write(self.getCmds(subst,j)+"\n") 601 if sLoopVars: 602 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 603 fh.write(sLoopFoot+"\n") 604 else: 605 fh.write(self.getCmds(subst,j)+"\n") 606 607 if loopVars: 608 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 609 fh.write(loopFoot+"\n") 610 611 fh.write(footer+"\n") 612 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 613 #if '10_9' in testname: sys.exit() 614 return 615 616 def genScriptsAndInfo(self,exfile,root,srcDict): 617 """ 618 Generate scripts from the source file, determine if built, etc. 619 For every test in the exfile with info in the srcDict: 620 1. Determine if it needs to be run for this arch 621 2. Generate the script 622 3. Generate the data needed to write out the makefile in a 623 convenient way 624 All tests are *always* run, but some may be SKIP'd per the TAP standard 625 """ 626 debug=False 627 rpath=self.srcrelpath(root) 628 execname=self.getExecname(exfile,rpath) 629 isBuilt=self._isBuilt(exfile,srcDict) 630 for test in srcDict: 631 if test in self.buildkeys: continue 632 if debug: print(self.nameSpace(exfile,root), test) 633 srcDict[test]['execname']=execname # Convenience in generating scripts 634 isRun=self._isRun(srcDict[test]) 635 self.genRunScript(test,root,isRun,srcDict) 636 srcDict[test]['isrun']=isRun 637 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 638 639 # This adds to datastructure for building deps 640 if isBuilt: self.addToSources(exfile,rpath,srcDict) 641 return 642 643 def _isBuilt(self,exfile,srcDict): 644 """ 645 Determine if this file should be built. 646 """ 647 # Get the language based on file extension 648 srcDict['SKIP'] = [] 649 lang=self.getLanguage(exfile) 650 if (lang=="F" or lang=="F90"): 651 if not self.have_fortran: 652 srcDict["SKIP"].append("Fortran required for this test") 653 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 654 srcDict["SKIP"].append("Fortran f90freeform required for this test") 655 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 656 srcDict["SKIP"].append("CUDA required for this test") 657 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 658 srcDict["SKIP"].append("C++ required for this test") 659 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 660 srcDict["SKIP"].append("C++ required for this test") 661 662 # Deprecated source files 663 if srcDict.get("TODO"): 664 return False 665 666 # isRun can work with srcDict to handle the requires 667 if "requires" in srcDict: 668 if srcDict["requires"]: 669 return self._isRun(srcDict) 670 671 return srcDict['SKIP'] == [] 672 673 674 def _isRun(self,testDict, debug=False): 675 """ 676 Based on the requirements listed in the src file and the petscconf.h 677 info, determine whether this test should be run or not. 678 """ 679 indent=" " 680 681 if 'SKIP' not in testDict: 682 testDict['SKIP'] = [] 683 # MPI requirements 684 if 'MPI_IS_MPIUNI' in self.conf: 685 if testDict.get('nsize', '1') != '1': 686 testDict['SKIP'].append("Parallel test with serial build") 687 688 # The requirements for the test are the sum of all the run subtests 689 if 'subtests' in testDict: 690 if 'requires' not in testDict: testDict['requires']="" 691 for stest in testDict['subtests']: 692 if 'requires' in testDict[stest]: 693 testDict['requires']+=" "+testDict[stest]['requires'] 694 if testDict.get('nsize', '1') != '1': 695 testDict['SKIP'].append("Parallel test with serial build") 696 break 697 698 # Now go through all requirements 699 if 'requires' in testDict: 700 for requirement in testDict['requires'].split(): 701 requirement=requirement.strip() 702 if not requirement: continue 703 if debug: print(indent+"Requirement: ", requirement) 704 isNull=False 705 if requirement.startswith("!"): 706 requirement=requirement[1:]; isNull=True 707 # Precision requirement for reals 708 if requirement in self.precision_types: 709 if self.conf['PETSC_PRECISION']==requirement: 710 if isNull: 711 testDict['SKIP'].append("not "+requirement+" required") 712 continue 713 continue # Success 714 elif not isNull: 715 testDict['SKIP'].append(requirement+" required") 716 continue 717 # Precision requirement for ints 718 if requirement in self.integer_types: 719 if requirement=="int32": 720 if self.conf['PETSC_SIZEOF_INT']==4: 721 if isNull: 722 testDict['SKIP'].append("not int32 required") 723 continue 724 continue # Success 725 elif not isNull: 726 testDict['SKIP'].append("int32 required") 727 continue 728 if requirement=="int64": 729 if self.conf['PETSC_SIZEOF_INT']==8: 730 if isNull: 731 testDict['SKIP'].append("NOT int64 required") 732 continue 733 continue # Success 734 elif not isNull: 735 testDict['SKIP'].append("int64 required") 736 continue 737 if requirement.startswith("long"): 738 reqsize = int(requirement[4:])//8 739 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 740 if longsize==reqsize: 741 if isNull: 742 testDict['SKIP'].append("not %s required" % requirement) 743 continue 744 continue # Success 745 elif not isNull: 746 testDict['SKIP'].append("%s required" % requirement) 747 continue 748 # Datafilespath 749 if requirement=="datafilespath" and not isNull: 750 testDict['SKIP'].append("Requires DATAFILESPATH") 751 continue 752 # Defines -- not sure I have comments matching 753 if "define(" in requirement.lower(): 754 reqdef=requirement.split("(")[1].split(")")[0] 755 if reqdef in self.conf: 756 if isNull: 757 testDict['SKIP'].append("Null requirement not met: "+requirement) 758 continue 759 continue # Success 760 elif not isNull: 761 testDict['SKIP'].append("Required: "+requirement) 762 continue 763 764 # Rest should be packages that we can just get from conf 765 if requirement == "complex": 766 petscconfvar="PETSC_USE_COMPLEX" 767 pkgconfvar="PETSC_USE_COMPLEX" 768 else: 769 petscconfvar="PETSC_HAVE_"+requirement.upper() 770 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 771 petsccv = self.conf.get(petscconfvar) 772 pkgcv = self.conf.get(pkgconfvar) 773 774 if petsccv or pkgcv: 775 if isNull: 776 if petsccv: 777 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 778 continue 779 else: 780 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 781 continue 782 continue # Success 783 elif not isNull: 784 if not petsccv and not pkgcv: 785 if debug: print("requirement not found: ", requirement) 786 if self.pkg_name == 'petsc': 787 testDict['SKIP'].append(petscconfvar+" requirement not met") 788 else: 789 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 790 continue 791 return testDict['SKIP'] == [] 792 793 def genPetscTests_summarize(self,dataDict): 794 """ 795 Required method to state what happened 796 """ 797 if not self.summarize: return 798 indent=" " 799 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 800 fh=open(fhname,"w") 801 for root in dataDict: 802 relroot=self.srcrelpath(root) 803 pkg=relroot.split("/")[1] 804 fh.write(relroot+"\n") 805 allSrcs=[] 806 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 807 for exfile in dataDict[root]: 808 # Basic information 809 rfile=os.path.join(relroot,exfile) 810 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 811 fh.write(indent+exfile+indent*4+builtStatus+"\n") 812 813 for test in dataDict[root][exfile]: 814 if test in self.buildkeys: continue 815 line=indent*2+test 816 fh.write(line+"\n") 817 # Looks nice to have the keys in order 818 #for key in dataDict[root][exfile][test]: 819 for key in "isrun abstracted nsize args requires script".split(): 820 if key not in dataDict[root][exfile][test]: continue 821 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 822 fh.write(line+"\n") 823 fh.write("\n") 824 fh.write("\n") 825 fh.write("\n") 826 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 827 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 828 fh.close() 829 return 830 831 def genPetscTests(self,root,dirs,files,dataDict): 832 """ 833 Go through and parse the source files in the directory to generate 834 the examples based on the metadata contained in the source files 835 """ 836 debug=False 837 # Use examplesAnalyze to get what the makefles think are sources 838 #self.examplesAnalyze(root,dirs,files,anlzDict) 839 840 dataDict[root]={} 841 842 for exfile in files: 843 #TST: Until we replace files, still leaving the orginals as is 844 #if not exfile.startswith("new_"+"ex"): continue 845 #if not exfile.startswith("ex"): continue 846 847 # Ignore emacs and other temporary files 848 if exfile.startswith("."): continue 849 if exfile.startswith("#"): continue 850 if exfile.endswith("~"): continue 851 # Only parse source files 852 ext=os.path.splitext(exfile)[-1].lstrip('.') 853 if ext not in LANGS: continue 854 855 # Convenience 856 fullex=os.path.join(root,exfile) 857 if self.verbose: print(' --> '+fullex) 858 dataDict[root].update(testparse.parseTestFile(fullex,0)) 859 if exfile in dataDict[root]: 860 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 861 862 return 863 864 def walktree(self,top): 865 """ 866 Walk a directory tree, starting from 'top' 867 """ 868 # Goal of action is to fill this dictionary 869 dataDict={} 870 for root, dirs, files in os.walk(top, topdown=True): 871 dirs.sort() 872 files.sort() 873 if not "examples" in root: continue 874 if "dSYM" in root: continue 875 if os.path.basename(root.rstrip("/")) == 'output': continue 876 if self.verbose: print(root) 877 self.genPetscTests(root,dirs,files,dataDict) 878 # Now summarize this dictionary 879 if self.verbose: self.genPetscTests_summarize(dataDict) 880 return dataDict 881 882 def gen_gnumake(self, fd): 883 """ 884 Overwrite of the method in the base PETSc class 885 """ 886 def write(stem, srcs): 887 for lang in LANGS: 888 if srcs[lang]['srcs']: 889 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 890 for pkg in self.pkg_pkgs: 891 srcs = self.gen_pkg(pkg) 892 write('testsrcs-' + pkg, srcs) 893 # Handle dependencies 894 for lang in LANGS: 895 for exfile in srcs[lang]['srcs']: 896 if exfile in srcs[lang]: 897 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 898 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 899 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 900 if deps: 901 # The executable literally depends on the object file because it is linked 902 fd.write(ex +": " + " ".join(deps) +'\n') 903 # The object file containing 'main' does not normally depend on other object 904 # files, but it does when it includes their modules. This dependency is 905 # overly blunt and could be reduced to only depend on object files for 906 # modules that are used, like "*f90aux.o". 907 fd.write(exfo +": " + " ".join(deps) +'\n') 908 909 return self.gendeps 910 911 def gen_pkg(self, pkg): 912 """ 913 Overwrite of the method in the base PETSc class 914 """ 915 return self.sources[pkg] 916 917 def write_gnumake(self, dataDict, output=None): 918 """ 919 Write out something similar to files from gmakegen.py 920 921 Test depends on script which also depends on source 922 file, but since I don't have a good way generating 923 acting on a single file (oops) just depend on 924 executable which in turn will depend on src file 925 """ 926 # Different options for how to set up the targets 927 compileExecsFirst=False 928 929 # Open file 930 fd = open(output, 'w') 931 932 # Write out the sources 933 gendeps = self.gen_gnumake(fd) 934 935 # Write out the tests and execname targets 936 fd.write("\n#Tests and executables\n") # Delimiter 937 938 for pkg in self.pkg_pkgs: 939 # These grab the ones that are built 940 for lang in LANGS: 941 testdeps=[] 942 for ftest in self.tests[pkg][lang]: 943 test=os.path.basename(ftest) 944 basedir=os.path.dirname(ftest) 945 testdeps.append(self.nameSpace(test,basedir)) 946 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 947 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 948 949 # test targets 950 for ftest in self.tests[pkg][lang]: 951 test=os.path.basename(ftest) 952 basedir=os.path.dirname(ftest) 953 testdir="${TESTDIR}/"+basedir+"/" 954 nmtest=self.nameSpace(test,basedir) 955 rundir=os.path.join(testdir,test) 956 script=test+".sh" 957 958 # Deps 959 exfile=self.tests[pkg][lang][ftest]['exfile'] 960 fullex=os.path.join(self.srcdir,exfile) 961 localexec=self.tests[pkg][lang][ftest]['exec'] 962 execname=os.path.join(testdir,localexec) 963 fullscript=os.path.join(testdir,script) 964 tmpfile=os.path.join(testdir,test,test+".tmp") 965 966 # *.counts depends on the script and either executable (will 967 # be run) or the example source file (SKIP or TODO) 968 fd.write('%s.counts : %s %s' 969 % (os.path.join('$(TESTDIR)/counts', nmtest), 970 fullscript, 971 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 972 ) 973 if exfile in self.sources[pkg][lang]: 974 for dep in self.sources[pkg][lang][exfile]: 975 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 976 fd.write('\n') 977 978 # Now write the args: 979 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 980 981 fd.close() 982 return 983 984def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 985 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 986 testdir=os.path.normpath(testdir) 987 if petsc_arch: 988 petsc_arch=petsc_arch.rstrip(os.path.sep) 989 if len(petsc_arch.split(os.path.sep))>1: 990 petsc_dir,petsc_arch=os.path.split(petsc_arch) 991 output = os.path.join(testdir, 'testfiles') 992 993 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 994 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 995 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 996 testdir=testdir) 997 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 998 pEx.write_gnumake(dataDict, output) 999 1000if __name__ == '__main__': 1001 import optparse 1002 parser = optparse.OptionParser() 1003 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1004 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1005 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1006 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1007 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1008 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1009 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1010 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1011 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1012 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1013 1014 opts, extra_args = parser.parse_args() 1015 if extra_args: 1016 import sys 1017 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1018 exit(1) 1019 if opts.testdir is None: 1020 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1021 1022 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1023 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1024 verbose=opts.verbose, 1025 single_ex=opts.single_executable, srcdir=opts.srcdir, 1026 testdir=opts.testdir) 1027