1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 85 86 self.single_ex=single_ex 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 self.srcdir=os.path.join(self.petsc_dir,'src') 104 105 self.testroot_dir=os.path.abspath(testdir) 106 107 self.ptNaming=True 108 self.verbose=verbose 109 # Whether to write out a useful debugging 110 self.summarize=True if verbose else False 111 112 # For help in setting the requirements 113 self.precision_types="single double __float128 int32".split() 114 self.integer_types="int32 int64".split() 115 self.languages="fortran cuda cxx".split() # Always requires C so do not list 116 117 # Things that are not test 118 self.buildkeys=testparse.buildkeys 119 120 # Adding a dictionary for storing sources, objects, and tests 121 # to make building the dependency tree easier 122 self.sources={} 123 self.objects={} 124 self.tests={} 125 for pkg in PKGS: 126 self.sources[pkg]={} 127 self.objects[pkg]=[] 128 self.tests[pkg]={} 129 for lang in LANGS: 130 self.sources[pkg][lang]={} 131 self.sources[pkg][lang]['srcs']=[] 132 self.tests[pkg][lang]={} 133 134 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 135 136 self.indent=" " 137 if self.verbose: print('Finishing the constructor') 138 return 139 140 def srcrelpath(self,rdir): 141 """ 142 Get relative path to source directory 143 """ 144 return os.path.relpath(rdir,self.srcdir) 145 146 def getInInstallDir(self,thisscriptdir): 147 """ 148 When petsc is installed then this file in installed in: 149 <PREFIX>/share/petsc/examples/config/gmakegentest.py 150 otherwise the path is: 151 <PETSC_DIR>/config/gmakegentest.py 152 We use this difference to determine if we are in installdir 153 """ 154 dirlist=thisscriptdir.split(os.path.sep) 155 if len(dirlist)>4: 156 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 157 if lastfour==os.path.join('share','petsc','examples','config'): 158 return True 159 else: 160 return False 161 else: 162 return False 163 164 def nameSpace(self,srcfile,srcdir): 165 """ 166 Because the scripts have a non-unique naming, the pretty-printing 167 needs to convey the srcdir and srcfile. There are two ways of doing this. 168 """ 169 if self.ptNaming: 170 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 171 cdir=srcdir 172 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 173 nameString=prefix+srcfile 174 else: 175 #nameString=srcdir+": "+srcfile 176 nameString=srcfile 177 return nameString 178 179 def getLanguage(self,srcfile): 180 """ 181 Based on the source, determine associated language as found in gmakegen.LANGS 182 Can we just return srcext[1:\] now? 183 """ 184 langReq=None 185 srcext=os.path.splitext(srcfile)[-1] 186 if srcext in ".F90".split(): langReq="F90" 187 if srcext in ".F".split(): langReq="F" 188 if srcext in ".cxx".split(): langReq="cxx" 189 if srcext == ".cu": langReq="cu" 190 if srcext == ".c": langReq="c" 191 #if not langReq: print("ERROR: ", srcext, srcfile) 192 return langReq 193 194 def _getLoopVars(self,inDict,testname, isSubtest=False): 195 """ 196 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 197 Return: 198 inDict['args']: -ksp_monitor 199 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 200 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 201 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 202 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 203 subst should be passed in instead of inDict 204 """ 205 loopVars={}; newargs="" 206 lsuffix='_' 207 argregex=re.compile('(?<![a-zA-Z])-(?=[a-zA-Z])') 208 from testparse import parseLoopArgs 209 for key in inDict.keys(): 210 if type(inDict[key])!=bytes: continue 211 keystr = str(inDict[key]) 212 akey=('subargs' if key=='args' else key) # what to assign 213 if akey not in inDict: inDict[akey]='' 214 varlist=[] 215 for varset in argregex.split(keystr): 216 if not varset.strip(): continue 217 if '{{' in varset: 218 keyvar,lvars,ftype=parseLoopArgs(varset) 219 if akey not in loopVars: loopVars[akey]={} 220 varlist.append(keyvar) 221 loopVars[akey][keyvar]=[keyvar,lvars] 222 if akey=='nsize': 223 inDict[akey] = '${' + keyvar + '}' 224 lsuffix+=akey+'-'+inDict[akey]+'_' 225 else: 226 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 227 lsuffix+=keyvar+'-${' + keyvar + '}_' 228 else: 229 if key=='args': newargs+=" -"+varset.strip() 230 if varlist: loopVars[akey]['varlist']=varlist 231 232 233 # For subtests, args are always substituted in (not top level) 234 if isSubtest: 235 inDict['subargs']+=" "+newargs.strip() 236 inDict['args']='' 237 if 'label_suffix' in inDict: 238 inDict['label_suffix']+=lsuffix.rstrip('_') 239 else: 240 inDict['label_suffix']=lsuffix.rstrip('_') 241 else: 242 if loopVars: 243 inDict['args']=newargs.strip() 244 inDict['label_suffix']=lsuffix.rstrip('_') 245 if loopVars: 246 return loopVars 247 else: 248 return None 249 250 def getArgLabel(self,testDict): 251 """ 252 In all of the arguments in the test dictionary, create a simple 253 string for searching within the makefile system. For simplicity in 254 search, remove "-", for strings, etc. 255 Also, concatenate the arg commands 256 For now, ignore nsize -- seems hard to search for anyway 257 """ 258 # Collect all of the args associated with a test 259 argStr=("" if 'args' not in testDict else testDict['args']) 260 if 'subtests' in testDict: 261 for stest in testDict["subtests"]: 262 sd=testDict[stest] 263 argStr=argStr+("" if 'args' not in sd else sd['args']) 264 265 # Now go through and cleanup 266 argStr=re.sub('{{(.*?)}}',"",argStr) 267 argStr=re.sub('-'," ",argStr) 268 for digit in string.digits: argStr=re.sub(digit," ",argStr) 269 argStr=re.sub("\.","",argStr) 270 argStr=re.sub(",","",argStr) 271 argStr=re.sub('\+',' ',argStr) 272 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 273 return argStr.strip() 274 275 def addToSources(self,exfile,rpath,srcDict): 276 """ 277 Put into data structure that allows easy generation of makefile 278 """ 279 pkg=rpath.split(os.path.sep)[0] 280 relpfile=os.path.join(rpath,exfile) 281 lang=self.getLanguage(exfile) 282 if not lang: return 283 self.sources[pkg][lang]['srcs'].append(relpfile) 284 self.sources[pkg][lang][relpfile] = [] 285 if 'depends' in srcDict: 286 depSrcList=srcDict['depends'].split() 287 for depSrc in depSrcList: 288 depObj=os.path.splitext(depSrc)[0]+".o" 289 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 290 291 # In gmakefile, ${TESTDIR} var specifies the object compilation 292 testsdir=rpath+"/" 293 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 294 self.objects[pkg].append(objfile) 295 return 296 297 def addToTests(self,test,rpath,exfile,execname,testDict): 298 """ 299 Put into data structure that allows easy generation of makefile 300 Organized by languages to allow testing of languages 301 """ 302 pkg=rpath.split("/")[0] 303 nmtest=os.path.join(rpath,test) 304 lang=self.getLanguage(exfile) 305 if not lang: return 306 self.tests[pkg][lang][nmtest]={} 307 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 308 self.tests[pkg][lang][nmtest]['exec']=execname 309 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 310 return 311 312 def getExecname(self,exfile,rpath): 313 """ 314 Generate bash script using template found next to this file. 315 This file is read in at constructor time to avoid file I/O 316 """ 317 if self.single_ex: 318 execname=rpath.split("/")[1]+"-ex" 319 else: 320 execname=os.path.splitext(exfile)[0] 321 return execname 322 323 def getSubstVars(self,testDict,rpath,testname): 324 """ 325 Create a dictionary with all of the variables that get substituted 326 into the template commands found in example_template.py 327 """ 328 subst={} 329 330 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 331 if 'nsize' not in testDict: testDict['nsize']=1 332 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 333 for ak in testparse.acceptedkeys: 334 if ak=='test': continue 335 subst[ak]=(testDict[ak] if ak in testDict else '') 336 337 # Now do other variables 338 subst['execname']=testDict['execname'] 339 if 'filter' in testDict: 340 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 341 342 # Others 343 subst['subargs']='' # Default. For variables override 344 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 345 subst['label_suffix']='' 346 subst['comments']="\n#".join(subst['comments'].split("\n")) 347 if subst['comments']: subst['comments']="#"+subst['comments'] 348 subst['exec']="../"+subst['execname'] 349 subst['testroot']=self.testroot_dir 350 subst['testname']=testname 351 dp = self.conf.get('DATAFILESPATH','') 352 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 353 354 # This is used to label some matrices 355 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 356 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 357 358 # These can have for loops and are treated separately later 359 subst['nsize']=str(subst['nsize']) 360 361 #Conf vars 362 if self.petsc_arch.find('valgrind')>=0: 363 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 364 else: 365 subst['mpiexec']=self.conf['MPIEXEC'] 366 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 367 subst['petsc_arch']=self.petsc_arch 368 if self.inInstallDir: 369 # Case 2 370 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 371 else: 372 # Case 1 373 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 374 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 375 subst['diff']=self.conf['DIFF'] 376 subst['rm']=self.conf['RM'] 377 subst['grep']=self.conf['GREP'] 378 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 379 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 380 381 # Output file is special because of subtests override 382 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 383 if not "_" in defroot: defroot=defroot+"_1" 384 subst['defroot']=defroot 385 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 386 subst['redirect_file']=defroot+".tmp" 387 if 'output_file' not in testDict: 388 subst['output_file']="output/"+defroot+".out" 389 # Add in the full path here. 390 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 391 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 392 if not subst['TODO']: 393 print("Warning: "+subst['output_file']+" not found.") 394 # Worry about alt files here -- see 395 # src/snes/examples/tutorials/output/ex22*.out 396 altlist=[subst['output_file']] 397 basefile,ext = os.path.splitext(subst['output_file']) 398 for i in range(1,9): 399 altroot=basefile+"_alt" 400 if i > 1: altroot=altroot+"_"+str(i) 401 af=altroot+".out" 402 srcaf=os.path.join(subst['srcdir'],af) 403 fullaf=os.path.join(self.petsc_dir,srcaf) 404 if os.path.isfile(fullaf): altlist.append(srcaf) 405 if len(altlist)>1: subst['altfiles']=altlist 406 #if len(altlist)>1: print("Found alt files: ",altlist) 407 408 subst['regexes']={} 409 for subkey in subst: 410 if subkey=='regexes': continue 411 if not isinstance(subst[subkey],str): continue 412 patt="@"+subkey.upper()+"@" 413 subst['regexes'][subkey]=re.compile(patt) 414 415 return subst 416 417 def _substVars(self,subst,origStr): 418 """ 419 Substitute variables 420 """ 421 Str=origStr 422 for subkey in subst: 423 if subkey=='regexes': continue 424 if not isinstance(subst[subkey],str): continue 425 if subkey.upper() not in Str: continue 426 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 427 return Str 428 429 def getCmds(self,subst,i): 430 """ 431 Generate bash script using template found next to this file. 432 This file is read in at constructor time to avoid file I/O 433 """ 434 nindnt=i # the start and has to be consistent with below 435 cmdindnt=self.indent*nindnt 436 cmdLines="" 437 438 # MPI is the default -- but we have a few odd commands 439 if not subst['command']: 440 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 441 else: 442 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 443 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 444 445 cmdLines+=cmdindnt+'if test $res = 0; then\n' 446 diffindnt=self.indent*(nindnt+1) 447 if not subst['filter_output']: 448 if 'altfiles' not in subst: 449 cmd=diffindnt+self._substVars(subst,example_template.difftest) 450 else: 451 # Have to do it by hand a bit because of variable number of alt files 452 rf=subst['redirect_file'] 453 cmd=diffindnt+example_template.difftest.split('@')[0] 454 for i in range(len(subst['altfiles'])): 455 af=subst['altfiles'][i] 456 cmd+=af+' '+rf 457 if i!=len(subst['altfiles'])-1: 458 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 459 cmd+=' || ${diff_exe} ' 460 else: 461 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 462 cmd+=subst['label_suffix']+' ""' # Quotes are painful 463 else: 464 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 465 cmdLines+=cmd+"\n" 466 cmdLines+=cmdindnt+'else\n' 467 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 468 cmdLines+=cmdindnt+'fi\n' 469 return cmdLines 470 471 def _writeTodoSkip(self,fh,tors,reasons,footer): 472 """ 473 Write out the TODO and SKIP lines in the file 474 The TODO or SKIP variable, tors, should be lower case 475 """ 476 TORS=tors.upper() 477 template=eval("example_template."+tors+"line") 478 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 479 tab = '' 480 if reasons: 481 fh.write('if ! $force; then\n') 482 tab = tab + ' ' 483 if reasons == ["Requires DATAFILESPATH"]: 484 # The only reason not to run is DATAFILESPATH, which we check at run-time 485 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 486 tab = tab + ' ' 487 if reasons: 488 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 489 fh.write(tab+footer+"\n") 490 fh.write(tab+"exit\n") 491 if reasons == ["Requires DATAFILESPATH"]: 492 fh.write(' fi\n') 493 if reasons: 494 fh.write('fi\n') 495 fh.write('\n\n') 496 return 497 498 def getLoopVarsHead(self,loopVars,i): 499 """ 500 Generate a nicely indented string with the format loops 501 Here is what the data structure looks like 502 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 503 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 504 loopVars['subargs']['pc_type']=["j","cholesky sor"] 505 """ 506 outstr=''; indnt=self.indent 507 for key in loopVars: 508 for var in loopVars[key]['varlist']: 509 varval=loopVars[key][var] 510 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 511 i = i + 1 512 return (outstr,i) 513 514 def getLoopVarsFoot(self,loopVars,i): 515 outstr=''; indnt=self.indent 516 for key in loopVars: 517 for var in loopVars[key]['varlist']: 518 i = i - 1 519 outstr += indnt * i + "done\n" 520 return (outstr,i) 521 522 def genRunScript(self,testname,root,isRun,srcDict): 523 """ 524 Generate bash script using template found next to this file. 525 This file is read in at constructor time to avoid file I/O 526 """ 527 # runscript_dir directory has to be consistent with gmakefile 528 testDict=srcDict[testname] 529 rpath=self.srcrelpath(root) 530 runscript_dir=os.path.join(self.testroot_dir,rpath) 531 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 532 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 533 534 # Get variables to go into shell scripts. last time testDict used 535 subst=self.getSubstVars(testDict,rpath,testname) 536 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 537 538 #Handle runfiles 539 for lfile in subst.get('localrunfiles','').split(): 540 install_files(os.path.join(root, lfile), 541 os.path.join(runscript_dir, os.path.dirname(lfile))) 542 # Check subtests for local runfiles 543 for stest in subst.get("subtests",[]): 544 for lfile in testDict[stest].get('localrunfiles','').split(): 545 install_files(os.path.join(root, lfile), 546 os.path.join(runscript_dir, os.path.dirname(lfile))) 547 548 # Now substitute the key variables into the header and footer 549 header=self._substVars(subst,example_template.header) 550 # The header is done twice to enable @...@ in header 551 header=self._substVars(subst,header) 552 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 553 554 # Start writing the file 555 fh.write(header+"\n") 556 557 # If there is a TODO or a SKIP then we do it before writing out the 558 # rest of the command (which is useful for working on the test) 559 # SKIP and TODO can be for the source file or for the runs 560 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 561 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 562 563 j=0 # for indentation 564 565 if loopVars: 566 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 567 if (loopHead): fh.write(loopHead+"\n") 568 569 # Subtests are special 570 if 'subtests' in testDict: 571 substP=subst # Subtests can inherit args but be careful 572 k=0 # for label suffixes 573 for stest in testDict["subtests"]: 574 subst=substP.copy() 575 subst.update(testDict[stest]) 576 # nsize is special because it is usually overwritten 577 if 'nsize' in testDict[stest]: 578 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 579 else: 580 fh.write("nsize=1\n") 581 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 582 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 583 if sLoopVars: 584 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 585 fh.write(sLoopHead+"\n") 586 fh.write(self.getCmds(subst,j)+"\n") 587 if sLoopVars: 588 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 589 fh.write(sLoopFoot+"\n") 590 else: 591 fh.write(self.getCmds(subst,j)+"\n") 592 593 if loopVars: 594 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 595 fh.write(loopFoot+"\n") 596 597 fh.write(footer+"\n") 598 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 599 #if '10_9' in testname: sys.exit() 600 return 601 602 def genScriptsAndInfo(self,exfile,root,srcDict): 603 """ 604 Generate scripts from the source file, determine if built, etc. 605 For every test in the exfile with info in the srcDict: 606 1. Determine if it needs to be run for this arch 607 2. Generate the script 608 3. Generate the data needed to write out the makefile in a 609 convenient way 610 All tests are *always* run, but some may be SKIP'd per the TAP standard 611 """ 612 debug=False 613 rpath=self.srcrelpath(root) 614 execname=self.getExecname(exfile,rpath) 615 isBuilt=self._isBuilt(exfile,srcDict) 616 for test in srcDict: 617 if test in self.buildkeys: continue 618 if debug: print(self.nameSpace(exfile,root), test) 619 srcDict[test]['execname']=execname # Convenience in generating scripts 620 isRun=self._isRun(srcDict[test]) 621 self.genRunScript(test,root,isRun,srcDict) 622 srcDict[test]['isrun']=isRun 623 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 624 625 # This adds to datastructure for building deps 626 if isBuilt: self.addToSources(exfile,rpath,srcDict) 627 return 628 629 def _isBuilt(self,exfile,srcDict): 630 """ 631 Determine if this file should be built. 632 """ 633 # Get the language based on file extension 634 srcDict['SKIP'] = [] 635 lang=self.getLanguage(exfile) 636 if (lang=="F" or lang=="F90"): 637 if not self.have_fortran: 638 srcDict["SKIP"].append("Fortran required for this test") 639 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 640 srcDict["SKIP"].append("Fortran f90freeform required for this test") 641 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 642 srcDict["SKIP"].append("CUDA required for this test") 643 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 644 srcDict["SKIP"].append("C++ required for this test") 645 646 # Deprecated source files 647 if srcDict.get("TODO"): 648 return False 649 650 # isRun can work with srcDict to handle the requires 651 if "requires" in srcDict: 652 if srcDict["requires"]: 653 return self._isRun(srcDict) 654 655 return srcDict['SKIP'] == [] 656 657 658 def _isRun(self,testDict, debug=False): 659 """ 660 Based on the requirements listed in the src file and the petscconf.h 661 info, determine whether this test should be run or not. 662 """ 663 indent=" " 664 665 if 'SKIP' not in testDict: 666 testDict['SKIP'] = [] 667 # MPI requirements 668 if 'MPI_IS_MPIUNI' in self.conf: 669 nsize=testDict.get('nsize','1') 670 if str(nsize) != '1': 671 testDict['SKIP'].append("Parallel test with serial build") 672 673 # The requirements for the test are the sum of all the run subtests 674 if 'subtests' in testDict: 675 if 'requires' not in testDict: testDict['requires']="" 676 for stest in testDict['subtests']: 677 if 'requires' in testDict[stest]: 678 testDict['requires']+=" "+testDict[stest]['requires'] 679 nsize=testDict[stest].get('nsize','1') 680 if str(nsize) != '1': 681 testDict['SKIP'].append("Parallel test with serial build") 682 break 683 684 # Now go through all requirements 685 if 'requires' in testDict: 686 for requirement in testDict['requires'].split(): 687 requirement=requirement.strip() 688 if not requirement: continue 689 if debug: print(indent+"Requirement: ", requirement) 690 isNull=False 691 if requirement.startswith("!"): 692 requirement=requirement[1:]; isNull=True 693 # Precision requirement for reals 694 if requirement in self.precision_types: 695 if self.conf['PETSC_PRECISION']==requirement: 696 if isNull: 697 testDict['SKIP'].append("not "+requirement+" required") 698 continue 699 continue # Success 700 elif not isNull: 701 testDict['SKIP'].append(requirement+" required") 702 continue 703 # Precision requirement for ints 704 if requirement in self.integer_types: 705 if requirement=="int32": 706 if self.conf['PETSC_SIZEOF_INT']==4: 707 if isNull: 708 testDict['SKIP'].append("not int32 required") 709 continue 710 continue # Success 711 elif not isNull: 712 testDict['SKIP'].append("int32 required") 713 continue 714 if requirement=="int64": 715 if self.conf['PETSC_SIZEOF_INT']==8: 716 if isNull: 717 testDict['SKIP'].append("NOT int64 required") 718 continue 719 continue # Success 720 elif not isNull: 721 testDict['SKIP'].append("int64 required") 722 continue 723 # Datafilespath 724 if requirement=="datafilespath" and not isNull: 725 testDict['SKIP'].append("Requires DATAFILESPATH") 726 continue 727 # Defines -- not sure I have comments matching 728 if "define(" in requirement.lower(): 729 reqdef=requirement.split("(")[1].split(")")[0] 730 if reqdef in self.conf: 731 if isNull: 732 testDict['SKIP'].append("Null requirement not met: "+requirement) 733 continue 734 continue # Success 735 elif not isNull: 736 testDict['SKIP'].append("Required: "+requirement) 737 continue 738 739 # Rest should be packages that we can just get from conf 740 if requirement == "complex": 741 petscconfvar="PETSC_USE_COMPLEX" 742 else: 743 petscconfvar="PETSC_HAVE_"+requirement.upper() 744 if self.conf.get(petscconfvar): 745 if isNull: 746 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 747 continue 748 continue # Success 749 elif not isNull: 750 if debug: print("requirement not found: ", requirement) 751 testDict['SKIP'].append(petscconfvar+" requirement not met") 752 continue 753 754 return testDict['SKIP'] == [] 755 756 def genPetscTests_summarize(self,dataDict): 757 """ 758 Required method to state what happened 759 """ 760 if not self.summarize: return 761 indent=" " 762 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 763 fh=open(fhname,"w") 764 for root in dataDict: 765 relroot=self.srcrelpath(root) 766 pkg=relroot.split("/")[1] 767 fh.write(relroot+"\n") 768 allSrcs=[] 769 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 770 for exfile in dataDict[root]: 771 # Basic information 772 rfile=os.path.join(relroot,exfile) 773 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 774 fh.write(indent+exfile+indent*4+builtStatus+"\n") 775 776 for test in dataDict[root][exfile]: 777 if test in self.buildkeys: continue 778 line=indent*2+test 779 fh.write(line+"\n") 780 # Looks nice to have the keys in order 781 #for key in dataDict[root][exfile][test]: 782 for key in "isrun abstracted nsize args requires script".split(): 783 if key not in dataDict[root][exfile][test]: continue 784 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 785 fh.write(line+"\n") 786 fh.write("\n") 787 fh.write("\n") 788 fh.write("\n") 789 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 790 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 791 fh.close() 792 return 793 794 def genPetscTests(self,root,dirs,files,dataDict): 795 """ 796 Go through and parse the source files in the directory to generate 797 the examples based on the metadata contained in the source files 798 """ 799 debug=False 800 # Use examplesAnalyze to get what the makefles think are sources 801 #self.examplesAnalyze(root,dirs,files,anlzDict) 802 803 dataDict[root]={} 804 805 for exfile in files: 806 #TST: Until we replace files, still leaving the orginals as is 807 #if not exfile.startswith("new_"+"ex"): continue 808 #if not exfile.startswith("ex"): continue 809 810 # Ignore emacs and other temporary files 811 if exfile.startswith("."): continue 812 if exfile.startswith("#"): continue 813 if exfile.endswith("~"): continue 814 # Only parse source files 815 ext=os.path.splitext(exfile)[-1].lstrip('.') 816 if ext not in LANGS: continue 817 818 # Convenience 819 fullex=os.path.join(root,exfile) 820 if self.verbose: print(' --> '+fullex) 821 dataDict[root].update(testparse.parseTestFile(fullex,0)) 822 if exfile in dataDict[root]: 823 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 824 825 return 826 827 def walktree(self,top): 828 """ 829 Walk a directory tree, starting from 'top' 830 """ 831 # Goal of action is to fill this dictionary 832 dataDict={} 833 for root, dirs, files in os.walk(top, topdown=True): 834 dirs.sort() 835 files.sort() 836 if not "examples" in root: continue 837 if "dSYM" in root: continue 838 if os.path.basename(root.rstrip("/")) == 'output': continue 839 if self.verbose: print(root) 840 self.genPetscTests(root,dirs,files,dataDict) 841 # Now summarize this dictionary 842 if self.verbose: self.genPetscTests_summarize(dataDict) 843 return dataDict 844 845 def gen_gnumake(self, fd): 846 """ 847 Overwrite of the method in the base PETSc class 848 """ 849 def write(stem, srcs): 850 for lang in LANGS: 851 if srcs[lang]['srcs']: 852 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 853 for pkg in PKGS: 854 srcs = self.gen_pkg(pkg) 855 write('testsrcs-' + pkg, srcs) 856 # Handle dependencies 857 for lang in LANGS: 858 for exfile in srcs[lang]['srcs']: 859 if exfile in srcs[lang]: 860 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 861 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 862 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 863 if deps: 864 # The executable literally depends on the object file because it is linked 865 fd.write(ex +": " + " ".join(deps) +'\n') 866 # The object file containing 'main' does not normally depend on other object 867 # files, but it does when it includes their modules. This dependency is 868 # overly blunt and could be reduced to only depend on object files for 869 # modules that are used, like "*f90aux.o". 870 fd.write(exfo +": " + " ".join(deps) +'\n') 871 872 return self.gendeps 873 874 def gen_pkg(self, pkg): 875 """ 876 Overwrite of the method in the base PETSc class 877 """ 878 return self.sources[pkg] 879 880 def write_gnumake(self, dataDict, output=None): 881 """ 882 Write out something similar to files from gmakegen.py 883 884 Test depends on script which also depends on source 885 file, but since I don't have a good way generating 886 acting on a single file (oops) just depend on 887 executable which in turn will depend on src file 888 """ 889 # Different options for how to set up the targets 890 compileExecsFirst=False 891 892 # Open file 893 fd = open(output, 'w') 894 895 # Write out the sources 896 gendeps = self.gen_gnumake(fd) 897 898 # Write out the tests and execname targets 899 fd.write("\n#Tests and executables\n") # Delimiter 900 901 for pkg in PKGS: 902 # These grab the ones that are built 903 for lang in LANGS: 904 testdeps=[] 905 for ftest in self.tests[pkg][lang]: 906 test=os.path.basename(ftest) 907 basedir=os.path.dirname(ftest) 908 testdeps.append(self.nameSpace(test,basedir)) 909 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 910 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 911 912 # test targets 913 for ftest in self.tests[pkg][lang]: 914 test=os.path.basename(ftest) 915 basedir=os.path.dirname(ftest) 916 testdir="${TESTDIR}/"+basedir+"/" 917 nmtest=self.nameSpace(test,basedir) 918 rundir=os.path.join(testdir,test) 919 script=test+".sh" 920 921 # Deps 922 exfile=self.tests[pkg][lang][ftest]['exfile'] 923 fullex=os.path.join(self.srcdir,exfile) 924 localexec=self.tests[pkg][lang][ftest]['exec'] 925 execname=os.path.join(testdir,localexec) 926 fullscript=os.path.join(testdir,script) 927 tmpfile=os.path.join(testdir,test,test+".tmp") 928 929 # *.counts depends on the script and either executable (will 930 # be run) or the example source file (SKIP or TODO) 931 fd.write('%s.counts : %s %s' 932 % (os.path.join('$(TESTDIR)/counts', nmtest), 933 fullscript, 934 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 935 ) 936 if exfile in self.sources[pkg][lang]: 937 for dep in self.sources[pkg][lang][exfile]: 938 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 939 fd.write('\n') 940 941 # Now write the args: 942 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 943 944 fd.close() 945 return 946 947def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 948 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 949 if petsc_arch: 950 if len(petsc_arch.split(os.path.sep))>1: 951 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 952 output = os.path.join(testdir, 'testfiles') 953 954 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 955 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 956 testdir=testdir) 957 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 958 pEx.write_gnumake(dataDict, output) 959 960if __name__ == '__main__': 961 import optparse 962 parser = optparse.OptionParser() 963 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 964 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 965 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 966 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 967 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 968 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 969 970 opts, extra_args = parser.parse_args() 971 if extra_args: 972 import sys 973 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 974 exit(1) 975 if opts.testdir is None: 976 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 977 978 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 979 verbose=opts.verbose, 980 single_ex=opts.single_executable, srcdir=opts.srcdir, 981 testdir=opts.testdir) 982