1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace('/examples/','_').replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 if self.inInstallDir: 103 # Case 2 discussed above 104 # set PETSC_ARCH to install directory to allow script to work in both 105 dirlist=thisscriptdir.split(os.path.sep) 106 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 107 self.arch_dir=installdir 108 if self.srcdir is None: 109 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 110 else: 111 if petsc_arch == '': 112 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 113 # Case 1 discussed above 114 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 115 if self.srcdir is None: 116 self.srcdir=os.path.join(self.petsc_dir,'src') 117 118 self.testroot_dir=os.path.abspath(testdir) 119 120 self.verbose=verbose 121 # Whether to write out a useful debugging 122 self.summarize=True if verbose else False 123 124 # For help in setting the requirements 125 self.precision_types="single double __float128 int32".split() 126 self.integer_types="int32 int64 long32 long64".split() 127 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 128 129 # Things that are not test 130 self.buildkeys=testparse.buildkeys 131 132 # Adding a dictionary for storing sources, objects, and tests 133 # to make building the dependency tree easier 134 self.sources={} 135 self.objects={} 136 self.tests={} 137 for pkg in self.pkg_pkgs: 138 self.sources[pkg]={} 139 self.objects[pkg]=[] 140 self.tests[pkg]={} 141 for lang in LANGS: 142 self.sources[pkg][lang]={} 143 self.sources[pkg][lang]['srcs']=[] 144 self.tests[pkg][lang]={} 145 146 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 147 148 self.indent=" " 149 if self.verbose: print('Finishing the constructor') 150 return 151 152 def srcrelpath(self,rdir): 153 """ 154 Get relative path to source directory 155 """ 156 return os.path.relpath(rdir,self.srcdir) 157 158 def getInInstallDir(self,thisscriptdir): 159 """ 160 When petsc is installed then this file in installed in: 161 <PREFIX>/share/petsc/examples/config/gmakegentest.py 162 otherwise the path is: 163 <PETSC_DIR>/config/gmakegentest.py 164 We use this difference to determine if we are in installdir 165 """ 166 dirlist=thisscriptdir.split(os.path.sep) 167 if len(dirlist)>4: 168 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 169 if lastfour==os.path.join('share','petsc','examples','config'): 170 return True 171 else: 172 return False 173 else: 174 return False 175 176 def getLanguage(self,srcfile): 177 """ 178 Based on the source, determine associated language as found in gmakegen.LANGS 179 Can we just return srcext[1:\] now? 180 """ 181 langReq=None 182 srcext=os.path.splitext(srcfile)[-1] 183 if srcext in ".F90".split(): langReq="F90" 184 if srcext in ".F".split(): langReq="F" 185 if srcext in ".cxx".split(): langReq="cxx" 186 if srcext in ".cpp".split(): langReq="cpp" 187 if srcext == ".cu": langReq="cu" 188 if srcext == ".c": langReq="c" 189 #if not langReq: print("ERROR: ", srcext, srcfile) 190 return langReq 191 192 def _getLoopVars(self,inDict,testname, isSubtest=False): 193 """ 194 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 195 Return: 196 inDict['args']: -ksp_monitor 197 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 198 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 199 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 200 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 201 subst should be passed in instead of inDict 202 """ 203 loopVars={}; newargs=[] 204 lsuffix='+' 205 argregex = re.compile(' (?=-[a-zA-Z])') 206 from testparse import parseLoopArgs 207 for key in inDict: 208 if key in ('SKIP', 'regexes'): 209 continue 210 akey=('subargs' if key=='args' else key) # what to assign 211 if akey not in inDict: inDict[akey]='' 212 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 213 # Always generate a loop over nsize, even if there is only one value 214 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 215 keystr = str(inDict[key]) 216 varlist = [] 217 for varset in argregex.split(keystr): 218 if not varset.strip(): continue 219 if '{{' in varset: 220 keyvar,lvars,ftype=parseLoopArgs(varset) 221 if akey not in loopVars: loopVars[akey]={} 222 varlist.append(keyvar) 223 loopVars[akey][keyvar]=[keyvar,lvars] 224 if akey=='nsize': 225 if len(lvars.split()) > 1: 226 lsuffix += akey +'-${' + keyvar + '}' 227 else: 228 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 229 lsuffix+=keyvar+'-${' + keyvar + '}_' 230 else: 231 if key=='args': 232 newargs.append(varset.strip()) 233 if varlist: 234 loopVars[akey]['varlist']=varlist 235 236 # For subtests, args are always substituted in (not top level) 237 if isSubtest: 238 inDict['subargs'] += " "+" ".join(newargs) 239 inDict['args']='' 240 if 'label_suffix' in inDict: 241 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 242 else: 243 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 244 else: 245 if loopVars: 246 inDict['args'] = ' '.join(newargs) 247 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 248 return loopVars 249 250 def getArgLabel(self,testDict): 251 """ 252 In all of the arguments in the test dictionary, create a simple 253 string for searching within the makefile system. For simplicity in 254 search, remove "-", for strings, etc. 255 Also, concatenate the arg commands 256 For now, ignore nsize -- seems hard to search for anyway 257 """ 258 # Collect all of the args associated with a test 259 argStr=("" if 'args' not in testDict else testDict['args']) 260 if 'subtests' in testDict: 261 for stest in testDict["subtests"]: 262 sd=testDict[stest] 263 argStr=argStr+("" if 'args' not in sd else sd['args']) 264 265 # Now go through and cleanup 266 argStr=re.sub('{{(.*?)}}',"",argStr) 267 argStr=re.sub('-'," ",argStr) 268 for digit in string.digits: argStr=re.sub(digit," ",argStr) 269 argStr=re.sub("\.","",argStr) 270 argStr=re.sub(",","",argStr) 271 argStr=re.sub('\+',' ',argStr) 272 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 273 return argStr.strip() 274 275 def addToSources(self,exfile,rpath,srcDict): 276 """ 277 Put into data structure that allows easy generation of makefile 278 """ 279 pkg=rpath.split(os.path.sep)[0] 280 relpfile=os.path.join(rpath,exfile) 281 lang=self.getLanguage(exfile) 282 if not lang: return 283 if pkg not in self.sources: return 284 self.sources[pkg][lang]['srcs'].append(relpfile) 285 self.sources[pkg][lang][relpfile] = [] 286 if 'depends' in srcDict: 287 depSrcList=srcDict['depends'].split() 288 for depSrc in depSrcList: 289 depObj=os.path.splitext(depSrc)[0]+".o" 290 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 291 292 # In gmakefile, ${TESTDIR} var specifies the object compilation 293 testsdir=rpath+"/" 294 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 295 self.objects[pkg].append(objfile) 296 return 297 298 def addToTests(self,test,rpath,exfile,execname,testDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 Organized by languages to allow testing of languages 302 """ 303 pkg=rpath.split("/")[0] 304 nmtest=os.path.join(rpath,test) 305 lang=self.getLanguage(exfile) 306 if not lang: return 307 if pkg not in self.tests: return 308 self.tests[pkg][lang][nmtest]={} 309 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 310 self.tests[pkg][lang][nmtest]['exec']=execname 311 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 312 return 313 314 def getExecname(self,exfile,rpath): 315 """ 316 Generate bash script using template found next to this file. 317 This file is read in at constructor time to avoid file I/O 318 """ 319 if self.single_ex: 320 execname=rpath.split("/")[1]+"-ex" 321 else: 322 execname=os.path.splitext(exfile)[0] 323 return execname 324 325 def getSubstVars(self,testDict,rpath,testname): 326 """ 327 Create a dictionary with all of the variables that get substituted 328 into the template commands found in example_template.py 329 """ 330 subst={} 331 332 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 333 if 'nsize' not in testDict: testDict['nsize'] = '1' 334 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 335 for ak in testparse.acceptedkeys: 336 if ak=='test': continue 337 subst[ak]=(testDict[ak] if ak in testDict else '') 338 339 # Now do other variables 340 subst['execname']=testDict['execname'] 341 if 'filter' in testDict: 342 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 343 344 # Others 345 subst['subargs']='' # Default. For variables override 346 subst['srcdir']=os.path.join(self.srcdir, rpath) 347 subst['label_suffix']='' 348 subst['comments']="\n#".join(subst['comments'].split("\n")) 349 if subst['comments']: subst['comments']="#"+subst['comments'] 350 subst['exec']="../"+subst['execname'] 351 subst['testroot']=self.testroot_dir 352 subst['testname']=testname 353 dp = self.conf.get('DATAFILESPATH','') 354 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 355 356 # This is used to label some matrices 357 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 358 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 359 360 #Conf vars 361 if self.petsc_arch.find('valgrind')>=0: 362 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 363 else: 364 subst['mpiexec']=self.conf['MPIEXEC'] 365 subst['pkg_name']=self.pkg_name 366 subst['pkg_dir']=self.pkg_dir 367 subst['pkg_arch']=self.petsc_arch 368 subst['CONFIG_DIR']=thisscriptdir 369 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 370 subst['diff']=self.conf['DIFF'] 371 subst['rm']=self.conf['RM'] 372 subst['grep']=self.conf['GREP'] 373 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 374 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 375 376 # Output file is special because of subtests override 377 defroot = testparse.getDefaultOutputFileRoot(testname) 378 if 'output_file' not in testDict: 379 subst['output_file']="output/"+defroot+".out" 380 subst['redirect_file']=defroot+".tmp" 381 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 382 383 # Add in the full path here. 384 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 385 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 386 if not subst['TODO']: 387 print("Warning: "+subst['output_file']+" not found.") 388 # Worry about alt files here -- see 389 # src/snes/examples/tutorials/output/ex22*.out 390 altlist=[subst['output_file']] 391 basefile,ext = os.path.splitext(subst['output_file']) 392 for i in range(1,9): 393 altroot=basefile+"_alt" 394 if i > 1: altroot=altroot+"_"+str(i) 395 af=altroot+".out" 396 srcaf=os.path.join(subst['srcdir'],af) 397 fullaf=os.path.join(self.petsc_dir,srcaf) 398 if os.path.isfile(fullaf): altlist.append(srcaf) 399 if len(altlist)>1: subst['altfiles']=altlist 400 #if len(altlist)>1: print("Found alt files: ",altlist) 401 402 subst['regexes']={} 403 for subkey in subst: 404 if subkey=='regexes': continue 405 if not isinstance(subst[subkey],str): continue 406 patt="@"+subkey.upper()+"@" 407 subst['regexes'][subkey]=re.compile(patt) 408 409 return subst 410 411 def _substVars(self,subst,origStr): 412 """ 413 Substitute variables 414 """ 415 Str=origStr 416 for subkey in subst: 417 if subkey=='regexes': continue 418 if not isinstance(subst[subkey],str): continue 419 if subkey.upper() not in Str: continue 420 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 421 return Str 422 423 def getCmds(self,subst,i): 424 """ 425 Generate bash script using template found next to this file. 426 This file is read in at constructor time to avoid file I/O 427 """ 428 nindnt=i # the start and has to be consistent with below 429 cmdindnt=self.indent*nindnt 430 cmdLines="" 431 432 # MPI is the default -- but we have a few odd commands 433 if not subst['command']: 434 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 435 else: 436 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 437 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 438 439 cmdLines+=cmdindnt+'if test $res = 0; then\n' 440 diffindnt=self.indent*(nindnt+1) 441 if not subst['filter_output']: 442 if 'altfiles' not in subst: 443 cmd=diffindnt+self._substVars(subst,example_template.difftest) 444 else: 445 # Have to do it by hand a bit because of variable number of alt files 446 rf=subst['redirect_file'] 447 cmd=diffindnt+example_template.difftest.split('@')[0] 448 for i in range(len(subst['altfiles'])): 449 af=subst['altfiles'][i] 450 cmd+=af+' '+rf 451 if i!=len(subst['altfiles'])-1: 452 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 453 cmd+=' || ${diff_exe} ' 454 else: 455 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 456 cmd+=subst['label_suffix']+' ""' # Quotes are painful 457 else: 458 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 459 cmdLines+=cmd+"\n" 460 cmdLines+=cmdindnt+'else\n' 461 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 462 cmdLines+=cmdindnt+'fi\n' 463 return cmdLines 464 465 def _writeTodoSkip(self,fh,tors,reasons,footer): 466 """ 467 Write out the TODO and SKIP lines in the file 468 The TODO or SKIP variable, tors, should be lower case 469 """ 470 TORS=tors.upper() 471 template=eval("example_template."+tors+"line") 472 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 473 tab = '' 474 if reasons: 475 fh.write('if ! $force; then\n') 476 tab = tab + ' ' 477 if reasons == ["Requires DATAFILESPATH"]: 478 # The only reason not to run is DATAFILESPATH, which we check at run-time 479 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 480 tab = tab + ' ' 481 if reasons: 482 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 483 fh.write(tab+footer+"\n") 484 fh.write(tab+"exit\n") 485 if reasons == ["Requires DATAFILESPATH"]: 486 fh.write(' fi\n') 487 if reasons: 488 fh.write('fi\n') 489 fh.write('\n\n') 490 return 491 492 def getLoopVarsHead(self,loopVars,i,usedVars={}): 493 """ 494 Generate a nicely indented string with the format loops 495 Here is what the data structure looks like 496 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 497 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 498 loopVars['subargs']['pc_type']=["j","cholesky sor"] 499 """ 500 outstr=''; indnt=self.indent 501 502 for key in loopVars: 503 if key in usedVars: continue # Do not duplicate setting vars 504 for var in loopVars[key]['varlist']: 505 varval=loopVars[key][var] 506 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 507 outstr += "\n\n" 508 509 for key in loopVars: 510 for var in loopVars[key]['varlist']: 511 varval=loopVars[key][var] 512 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 513 i = i + 1 514 return (outstr,i) 515 516 def getLoopVarsFoot(self,loopVars,i): 517 outstr=''; indnt=self.indent 518 for key in loopVars: 519 for var in loopVars[key]['varlist']: 520 i = i - 1 521 outstr += indnt * i + "done\n" 522 return (outstr,i) 523 524 def genRunScript(self,testname,root,isRun,srcDict): 525 """ 526 Generate bash script using template found next to this file. 527 This file is read in at constructor time to avoid file I/O 528 """ 529 # runscript_dir directory has to be consistent with gmakefile 530 testDict=srcDict[testname] 531 rpath=self.srcrelpath(root) 532 runscript_dir=os.path.join(self.testroot_dir,rpath) 533 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 534 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 535 536 # Get variables to go into shell scripts. last time testDict used 537 subst=self.getSubstVars(testDict,rpath,testname) 538 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 539 if 'subtests' in testDict: 540 # The subtests inherit inDict, so we don't need top-level loops. 541 loopVars = {} 542 543 #Handle runfiles 544 for lfile in subst.get('localrunfiles','').split(): 545 install_files(os.path.join(root, lfile), 546 os.path.join(runscript_dir, os.path.dirname(lfile))) 547 # Check subtests for local runfiles 548 for stest in subst.get("subtests",[]): 549 for lfile in testDict[stest].get('localrunfiles','').split(): 550 install_files(os.path.join(root, lfile), 551 os.path.join(runscript_dir, os.path.dirname(lfile))) 552 553 # Now substitute the key variables into the header and footer 554 header=self._substVars(subst,example_template.header) 555 # The header is done twice to enable @...@ in header 556 header=self._substVars(subst,header) 557 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 558 559 # Start writing the file 560 fh.write(header+"\n") 561 562 # If there is a TODO or a SKIP then we do it before writing out the 563 # rest of the command (which is useful for working on the test) 564 # SKIP and TODO can be for the source file or for the runs 565 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 566 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 567 568 j=0 # for indentation 569 570 if loopVars: 571 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 572 if (loopHead): fh.write(loopHead+"\n") 573 574 # Subtests are special 575 allLoopVars=list(loopVars.keys()) 576 if 'subtests' in testDict: 577 substP=subst # Subtests can inherit args but be careful 578 k=0 # for label suffixes 579 for stest in testDict["subtests"]: 580 subst=substP.copy() 581 subst.update(testDict[stest]) 582 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 583 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 584 if sLoopVars: 585 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 586 allLoopVars+=list(sLoopVars.keys()) 587 fh.write(sLoopHead+"\n") 588 fh.write(self.getCmds(subst,j)+"\n") 589 if sLoopVars: 590 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 591 fh.write(sLoopFoot+"\n") 592 else: 593 fh.write(self.getCmds(subst,j)+"\n") 594 595 if loopVars: 596 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 597 fh.write(loopFoot+"\n") 598 599 fh.write(footer+"\n") 600 fh.close() 601 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 602 #if '10_9' in testname: sys.exit() 603 return 604 605 def genScriptsAndInfo(self,exfile,root,srcDict): 606 """ 607 Generate scripts from the source file, determine if built, etc. 608 For every test in the exfile with info in the srcDict: 609 1. Determine if it needs to be run for this arch 610 2. Generate the script 611 3. Generate the data needed to write out the makefile in a 612 convenient way 613 All tests are *always* run, but some may be SKIP'd per the TAP standard 614 """ 615 debug=False 616 rpath=self.srcrelpath(root) 617 execname=self.getExecname(exfile,rpath) 618 isBuilt=self._isBuilt(exfile,srcDict) 619 for test in srcDict: 620 if test in self.buildkeys: continue 621 if debug: print(nameSpace(exfile,root), test) 622 srcDict[test]['execname']=execname # Convenience in generating scripts 623 isRun=self._isRun(srcDict[test]) 624 self.genRunScript(test,root,isRun,srcDict) 625 srcDict[test]['isrun']=isRun 626 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 627 628 # This adds to datastructure for building deps 629 if isBuilt: self.addToSources(exfile,rpath,srcDict) 630 return 631 632 def _isBuilt(self,exfile,srcDict): 633 """ 634 Determine if this file should be built. 635 """ 636 # Get the language based on file extension 637 srcDict['SKIP'] = [] 638 lang=self.getLanguage(exfile) 639 if (lang=="F" or lang=="F90"): 640 if not self.have_fortran: 641 srcDict["SKIP"].append("Fortran required for this test") 642 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 643 srcDict["SKIP"].append("Fortran f90freeform required for this test") 644 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 645 srcDict["SKIP"].append("CUDA required for this test") 646 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 647 srcDict["SKIP"].append("C++ required for this test") 648 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 649 srcDict["SKIP"].append("C++ required for this test") 650 651 # Deprecated source files 652 if srcDict.get("TODO"): 653 return False 654 655 # isRun can work with srcDict to handle the requires 656 if "requires" in srcDict: 657 if srcDict["requires"]: 658 return self._isRun(srcDict) 659 660 return srcDict['SKIP'] == [] 661 662 663 def _isRun(self,testDict, debug=False): 664 """ 665 Based on the requirements listed in the src file and the petscconf.h 666 info, determine whether this test should be run or not. 667 """ 668 indent=" " 669 670 if 'SKIP' not in testDict: 671 testDict['SKIP'] = [] 672 # MPI requirements 673 if 'MPI_IS_MPIUNI' in self.conf: 674 if testDict.get('nsize', '1') != '1': 675 testDict['SKIP'].append("Parallel test with serial build") 676 677 # The requirements for the test are the sum of all the run subtests 678 if 'subtests' in testDict: 679 if 'requires' not in testDict: testDict['requires']="" 680 for stest in testDict['subtests']: 681 if 'requires' in testDict[stest]: 682 testDict['requires']+=" "+testDict[stest]['requires'] 683 if testDict.get('nsize', '1') != '1': 684 testDict['SKIP'].append("Parallel test with serial build") 685 break 686 687 # Now go through all requirements 688 if 'requires' in testDict: 689 for requirement in testDict['requires'].split(): 690 requirement=requirement.strip() 691 if not requirement: continue 692 if debug: print(indent+"Requirement: ", requirement) 693 isNull=False 694 if requirement.startswith("!"): 695 requirement=requirement[1:]; isNull=True 696 # Precision requirement for reals 697 if requirement in self.precision_types: 698 if self.conf['PETSC_PRECISION']==requirement: 699 if isNull: 700 testDict['SKIP'].append("not "+requirement+" required") 701 continue 702 continue # Success 703 elif not isNull: 704 testDict['SKIP'].append(requirement+" required") 705 continue 706 # Precision requirement for ints 707 if requirement in self.integer_types: 708 if requirement=="int32": 709 if self.conf['PETSC_SIZEOF_INT']==4: 710 if isNull: 711 testDict['SKIP'].append("not int32 required") 712 continue 713 continue # Success 714 elif not isNull: 715 testDict['SKIP'].append("int32 required") 716 continue 717 if requirement=="int64": 718 if self.conf['PETSC_SIZEOF_INT']==8: 719 if isNull: 720 testDict['SKIP'].append("NOT int64 required") 721 continue 722 continue # Success 723 elif not isNull: 724 testDict['SKIP'].append("int64 required") 725 continue 726 if requirement.startswith("long"): 727 reqsize = int(requirement[4:])//8 728 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 729 if longsize==reqsize: 730 if isNull: 731 testDict['SKIP'].append("not %s required" % requirement) 732 continue 733 continue # Success 734 elif not isNull: 735 testDict['SKIP'].append("%s required" % requirement) 736 continue 737 # Datafilespath 738 if requirement=="datafilespath" and not isNull: 739 testDict['SKIP'].append("Requires DATAFILESPATH") 740 continue 741 # Defines -- not sure I have comments matching 742 if "define(" in requirement.lower(): 743 reqdef=requirement.split("(")[1].split(")")[0] 744 if reqdef in self.conf: 745 if isNull: 746 testDict['SKIP'].append("Null requirement not met: "+requirement) 747 continue 748 continue # Success 749 elif not isNull: 750 testDict['SKIP'].append("Required: "+requirement) 751 continue 752 753 # Rest should be packages that we can just get from conf 754 if requirement in ["complex","debug"]: 755 petscconfvar="PETSC_USE_"+requirement.upper() 756 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 757 else: 758 petscconfvar="PETSC_HAVE_"+requirement.upper() 759 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 760 petsccv = self.conf.get(petscconfvar) 761 pkgcv = self.conf.get(pkgconfvar) 762 763 if petsccv or pkgcv: 764 if isNull: 765 if petsccv: 766 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 767 continue 768 else: 769 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 770 continue 771 continue # Success 772 elif not isNull: 773 if not petsccv and not pkgcv: 774 if debug: print("requirement not found: ", requirement) 775 if self.pkg_name == 'petsc': 776 testDict['SKIP'].append(petscconfvar+" requirement not met") 777 else: 778 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 779 continue 780 return testDict['SKIP'] == [] 781 782 def checkOutput(self,exfile,root,srcDict): 783 """ 784 Check and make sure the output files are in the output director 785 """ 786 debug=False 787 rpath=self.srcrelpath(root) 788 for test in srcDict: 789 if test in self.buildkeys: continue 790 if debug: print(rpath, exfile, test) 791 if 'output_file' in srcDict[test]: 792 output_file=srcDict[test]['output_file'] 793 else: 794 defroot = testparse.getDefaultOutputFileRoot(test) 795 if 'TODO' in srcDict[test]: continue 796 output_file="output/"+defroot+".out" 797 798 fullout=os.path.join(root,output_file) 799 if debug: print("---> ",fullout) 800 if not os.path.exists(fullout): 801 self.missing_files.append(fullout) 802 803 return 804 805 def genPetscTests_summarize(self,dataDict): 806 """ 807 Required method to state what happened 808 """ 809 if not self.summarize: return 810 indent=" " 811 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 812 fh=open(fhname,"w") 813 for root in dataDict: 814 relroot=self.srcrelpath(root) 815 pkg=relroot.split("/")[1] 816 fh.write(relroot+"\n") 817 allSrcs=[] 818 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 819 for exfile in dataDict[root]: 820 # Basic information 821 rfile=os.path.join(relroot,exfile) 822 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 823 fh.write(indent+exfile+indent*4+builtStatus+"\n") 824 825 for test in dataDict[root][exfile]: 826 if test in self.buildkeys: continue 827 line=indent*2+test 828 fh.write(line+"\n") 829 # Looks nice to have the keys in order 830 #for key in dataDict[root][exfile][test]: 831 for key in "isrun abstracted nsize args requires script".split(): 832 if key not in dataDict[root][exfile][test]: continue 833 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 834 fh.write(line+"\n") 835 fh.write("\n") 836 fh.write("\n") 837 fh.write("\n") 838 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 839 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 840 fh.close() 841 return 842 843 def genPetscTests(self,root,dirs,files,dataDict): 844 """ 845 Go through and parse the source files in the directory to generate 846 the examples based on the metadata contained in the source files 847 """ 848 debug=False 849 # Use examplesAnalyze to get what the makefles think are sources 850 #self.examplesAnalyze(root,dirs,files,anlzDict) 851 852 dataDict[root]={} 853 854 for exfile in files: 855 #TST: Until we replace files, still leaving the orginals as is 856 #if not exfile.startswith("new_"+"ex"): continue 857 #if not exfile.startswith("ex"): continue 858 859 # Ignore emacs and other temporary files 860 if exfile.startswith("."): continue 861 if exfile.startswith("#"): continue 862 if exfile.endswith("~"): continue 863 # Only parse source files 864 ext=os.path.splitext(exfile)[-1].lstrip('.') 865 if ext not in LANGS: continue 866 867 # Convenience 868 fullex=os.path.join(root,exfile) 869 if self.verbose: print(' --> '+fullex) 870 dataDict[root].update(testparse.parseTestFile(fullex,0)) 871 if exfile in dataDict[root]: 872 if not self.check_output: 873 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 874 else: 875 self.checkOutput(exfile,root,dataDict[root][exfile]) 876 877 return 878 879 def walktree(self,top): 880 """ 881 Walk a directory tree, starting from 'top' 882 """ 883 if self.check_output: 884 print("Checking for missing output files") 885 self.missing_files=[] 886 887 # Goal of action is to fill this dictionary 888 dataDict={} 889 for root, dirs, files in os.walk(top, topdown=True): 890 dirs.sort() 891 files.sort() 892 if not "examples" in root: continue 893 if "dSYM" in root: continue 894 if os.path.basename(root.rstrip("/")) == 'output': continue 895 if self.verbose: print(root) 896 self.genPetscTests(root,dirs,files,dataDict) 897 898 # If checking output, report results 899 if self.check_output: 900 if self.missing_files: 901 for file in set(self.missing_files): # set uniqifies 902 print(file) 903 sys.exit(1) 904 905 # Now summarize this dictionary 906 if self.verbose: self.genPetscTests_summarize(dataDict) 907 return dataDict 908 909 def gen_gnumake(self, fd): 910 """ 911 Overwrite of the method in the base PETSc class 912 """ 913 def write(stem, srcs): 914 for lang in LANGS: 915 if srcs[lang]['srcs']: 916 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 917 for pkg in self.pkg_pkgs: 918 srcs = self.gen_pkg(pkg) 919 write('testsrcs-' + pkg, srcs) 920 # Handle dependencies 921 for lang in LANGS: 922 for exfile in srcs[lang]['srcs']: 923 if exfile in srcs[lang]: 924 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 925 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 926 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 927 if deps: 928 # The executable literally depends on the object file because it is linked 929 fd.write(ex +": " + " ".join(deps) +'\n') 930 # The object file containing 'main' does not normally depend on other object 931 # files, but it does when it includes their modules. This dependency is 932 # overly blunt and could be reduced to only depend on object files for 933 # modules that are used, like "*f90aux.o". 934 fd.write(exfo +": " + " ".join(deps) +'\n') 935 936 return self.gendeps 937 938 def gen_pkg(self, pkg): 939 """ 940 Overwrite of the method in the base PETSc class 941 """ 942 return self.sources[pkg] 943 944 def write_gnumake(self, dataDict, output=None): 945 """ 946 Write out something similar to files from gmakegen.py 947 948 Test depends on script which also depends on source 949 file, but since I don't have a good way generating 950 acting on a single file (oops) just depend on 951 executable which in turn will depend on src file 952 """ 953 # Different options for how to set up the targets 954 compileExecsFirst=False 955 956 # Open file 957 fd = open(output, 'w') 958 959 # Write out the sources 960 gendeps = self.gen_gnumake(fd) 961 962 # Write out the tests and execname targets 963 fd.write("\n#Tests and executables\n") # Delimiter 964 965 for pkg in self.pkg_pkgs: 966 # These grab the ones that are built 967 for lang in LANGS: 968 testdeps=[] 969 for ftest in self.tests[pkg][lang]: 970 test=os.path.basename(ftest) 971 basedir=os.path.dirname(ftest) 972 testdeps.append(nameSpace(test,basedir)) 973 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 974 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 975 976 # test targets 977 for ftest in self.tests[pkg][lang]: 978 test=os.path.basename(ftest) 979 basedir=os.path.dirname(ftest) 980 testdir="${TESTDIR}/"+basedir+"/" 981 nmtest=nameSpace(test,basedir) 982 rundir=os.path.join(testdir,test) 983 script=test+".sh" 984 985 # Deps 986 exfile=self.tests[pkg][lang][ftest]['exfile'] 987 fullex=os.path.join(self.srcdir,exfile) 988 localexec=self.tests[pkg][lang][ftest]['exec'] 989 execname=os.path.join(testdir,localexec) 990 fullscript=os.path.join(testdir,script) 991 tmpfile=os.path.join(testdir,test,test+".tmp") 992 993 # *.counts depends on the script and either executable (will 994 # be run) or the example source file (SKIP or TODO) 995 fd.write('%s.counts : %s %s' 996 % (os.path.join('$(TESTDIR)/counts', nmtest), 997 fullscript, 998 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 999 ) 1000 if exfile in self.sources[pkg][lang]: 1001 for dep in self.sources[pkg][lang][exfile]: 1002 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1003 fd.write('\n') 1004 1005 # Now write the args: 1006 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1007 1008 fd.close() 1009 return 1010 1011 def write_db(self, dataDict, testdir): 1012 """ 1013 Write out the dataDict into a pickle file 1014 """ 1015 fd = open(os.path.join(testdir,'datatest.pkl'), 'wb') 1016 pickle.dump(dataDict,fd) 1017 fd.close() 1018 return 1019 1020def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1021 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1022 srcdir=None, testdir=None, check=False): 1023 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1024 testdir=os.path.normpath(testdir) 1025 if petsc_arch: 1026 petsc_arch=petsc_arch.rstrip(os.path.sep) 1027 if len(petsc_arch.split(os.path.sep))>1: 1028 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1029 output = os.path.join(testdir, 'testfiles') 1030 1031 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1032 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1033 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1034 testdir=testdir,check=check) 1035 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1036 if not pEx.check_output: 1037 pEx.write_gnumake(dataDict, output) 1038 pEx.write_db(dataDict, testdir) 1039 1040if __name__ == '__main__': 1041 import optparse 1042 parser = optparse.OptionParser() 1043 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1044 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1045 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1046 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1047 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1048 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1049 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1050 help='Check whether output files are in output director') 1051 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1052 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1053 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1054 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1055 1056 opts, extra_args = parser.parse_args() 1057 if extra_args: 1058 import sys 1059 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1060 exit(1) 1061 if opts.testdir is None: 1062 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1063 1064 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1065 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1066 verbose=opts.verbose, 1067 single_ex=opts.single_executable, srcdir=opts.srcdir, 1068 testdir=opts.testdir, check=opts.check_output) 1069