1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 85 86 self.single_ex=single_ex 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 self.srcdir=os.path.join(self.petsc_dir,'src') 104 105 self.testroot_dir=os.path.abspath(testdir) 106 107 self.ptNaming=True 108 self.verbose=verbose 109 # Whether to write out a useful debugging 110 self.summarize=True if verbose else False 111 112 # For help in setting the requirements 113 self.precision_types="single double __float128 int32".split() 114 self.integer_types="int32 int64".split() 115 self.languages="fortran cuda cxx".split() # Always requires C so do not list 116 117 # Things that are not test 118 self.buildkeys=testparse.buildkeys 119 120 # Adding a dictionary for storing sources, objects, and tests 121 # to make building the dependency tree easier 122 self.sources={} 123 self.objects={} 124 self.tests={} 125 for pkg in PKGS: 126 self.sources[pkg]={} 127 self.objects[pkg]=[] 128 self.tests[pkg]={} 129 for lang in LANGS: 130 self.sources[pkg][lang]={} 131 self.sources[pkg][lang]['srcs']=[] 132 self.tests[pkg][lang]={} 133 134 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 135 136 self.indent=" " 137 if self.verbose: print('Finishing the constructor') 138 return 139 140 def srcrelpath(self,rdir): 141 """ 142 Get relative path to source directory 143 """ 144 return os.path.relpath(rdir,self.srcdir) 145 146 def getInInstallDir(self,thisscriptdir): 147 """ 148 When petsc is installed then this file in installed in: 149 <PREFIX>/share/petsc/examples/config/gmakegentest.py 150 otherwise the path is: 151 <PETSC_DIR>/config/gmakegentest.py 152 We use this difference to determine if we are in installdir 153 """ 154 dirlist=thisscriptdir.split(os.path.sep) 155 if len(dirlist)>4: 156 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 157 if lastfour==os.path.join('share','petsc','examples','config'): 158 return True 159 else: 160 return False 161 else: 162 return False 163 164 def nameSpace(self,srcfile,srcdir): 165 """ 166 Because the scripts have a non-unique naming, the pretty-printing 167 needs to convey the srcdir and srcfile. There are two ways of doing this. 168 """ 169 if self.ptNaming: 170 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 171 cdir=srcdir 172 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 173 nameString=prefix+srcfile 174 else: 175 #nameString=srcdir+": "+srcfile 176 nameString=srcfile 177 return nameString 178 179 def getLanguage(self,srcfile): 180 """ 181 Based on the source, determine associated language as found in gmakegen.LANGS 182 Can we just return srcext[1:\] now? 183 """ 184 langReq=None 185 srcext=os.path.splitext(srcfile)[-1] 186 if srcext in ".F90".split(): langReq="F90" 187 if srcext in ".F".split(): langReq="F" 188 if srcext in ".cxx".split(): langReq="cxx" 189 if srcext == ".cu": langReq="cu" 190 if srcext == ".c": langReq="c" 191 #if not langReq: print("ERROR: ", srcext, srcfile) 192 return langReq 193 194 def _getLoopVars(self,inDict,testname, isSubtest=False): 195 """ 196 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 197 Return: 198 inDict['args']: -ksp_monitor 199 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 200 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 201 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 202 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 203 subst should be passed in instead of inDict 204 """ 205 loopVars={}; newargs="" 206 lsuffix='_' 207 argregex=re.compile('(?<![a-zA-Z])-(?=[a-zA-Z])') 208 from testparse import parseLoopArgs 209 for key in inDict: 210 if key in ('SKIP', 'regexes'): 211 continue 212 akey=('subargs' if key=='args' else key) # what to assign 213 if akey not in inDict: inDict[akey]='' 214 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 215 # Always generate a loop over nsize, even if there is only one value 216 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 217 keystr = str(inDict[key]) 218 varlist = [] 219 for varset in argregex.split(keystr): 220 if not varset.strip(): continue 221 if '{{' in varset: 222 keyvar,lvars,ftype=parseLoopArgs(varset) 223 if akey not in loopVars: loopVars[akey]={} 224 varlist.append(keyvar) 225 loopVars[akey][keyvar]=[keyvar,lvars] 226 if akey=='nsize': 227 if len(lvars.split()) > 1: 228 inDict[akey] = '${' + keyvar + '}' 229 lsuffix+=akey+'-'+inDict[akey]+'_' 230 else: 231 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 232 lsuffix+=keyvar+'-${' + keyvar + '}_' 233 else: 234 if key=='args': newargs+=" -"+varset.strip() 235 if varlist: loopVars[akey]['varlist']=varlist 236 237 # For subtests, args are always substituted in (not top level) 238 if isSubtest: 239 inDict['subargs']+=" "+newargs.strip() 240 inDict['args']='' 241 if 'label_suffix' in inDict: 242 inDict['label_suffix']+=lsuffix.rstrip('_') 243 else: 244 inDict['label_suffix']=lsuffix.rstrip('_') 245 else: 246 if loopVars: 247 inDict['args']=newargs.strip() 248 inDict['label_suffix']=lsuffix.rstrip('_') 249 return loopVars 250 251 def getArgLabel(self,testDict): 252 """ 253 In all of the arguments in the test dictionary, create a simple 254 string for searching within the makefile system. For simplicity in 255 search, remove "-", for strings, etc. 256 Also, concatenate the arg commands 257 For now, ignore nsize -- seems hard to search for anyway 258 """ 259 # Collect all of the args associated with a test 260 argStr=("" if 'args' not in testDict else testDict['args']) 261 if 'subtests' in testDict: 262 for stest in testDict["subtests"]: 263 sd=testDict[stest] 264 argStr=argStr+("" if 'args' not in sd else sd['args']) 265 266 # Now go through and cleanup 267 argStr=re.sub('{{(.*?)}}',"",argStr) 268 argStr=re.sub('-'," ",argStr) 269 for digit in string.digits: argStr=re.sub(digit," ",argStr) 270 argStr=re.sub("\.","",argStr) 271 argStr=re.sub(",","",argStr) 272 argStr=re.sub('\+',' ',argStr) 273 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 274 return argStr.strip() 275 276 def addToSources(self,exfile,rpath,srcDict): 277 """ 278 Put into data structure that allows easy generation of makefile 279 """ 280 pkg=rpath.split(os.path.sep)[0] 281 relpfile=os.path.join(rpath,exfile) 282 lang=self.getLanguage(exfile) 283 if not lang: return 284 self.sources[pkg][lang]['srcs'].append(relpfile) 285 self.sources[pkg][lang][relpfile] = [] 286 if 'depends' in srcDict: 287 depSrcList=srcDict['depends'].split() 288 for depSrc in depSrcList: 289 depObj=os.path.splitext(depSrc)[0]+".o" 290 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 291 292 # In gmakefile, ${TESTDIR} var specifies the object compilation 293 testsdir=rpath+"/" 294 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 295 self.objects[pkg].append(objfile) 296 return 297 298 def addToTests(self,test,rpath,exfile,execname,testDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 Organized by languages to allow testing of languages 302 """ 303 pkg=rpath.split("/")[0] 304 nmtest=os.path.join(rpath,test) 305 lang=self.getLanguage(exfile) 306 if not lang: return 307 self.tests[pkg][lang][nmtest]={} 308 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 309 self.tests[pkg][lang][nmtest]['exec']=execname 310 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 311 return 312 313 def getExecname(self,exfile,rpath): 314 """ 315 Generate bash script using template found next to this file. 316 This file is read in at constructor time to avoid file I/O 317 """ 318 if self.single_ex: 319 execname=rpath.split("/")[1]+"-ex" 320 else: 321 execname=os.path.splitext(exfile)[0] 322 return execname 323 324 def getSubstVars(self,testDict,rpath,testname): 325 """ 326 Create a dictionary with all of the variables that get substituted 327 into the template commands found in example_template.py 328 """ 329 subst={} 330 331 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 332 if 'nsize' not in testDict: testDict['nsize'] = '1' 333 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 334 for ak in testparse.acceptedkeys: 335 if ak=='test': continue 336 subst[ak]=(testDict[ak] if ak in testDict else '') 337 338 # Now do other variables 339 subst['execname']=testDict['execname'] 340 if 'filter' in testDict: 341 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 342 343 # Others 344 subst['subargs']='' # Default. For variables override 345 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 346 subst['label_suffix']='' 347 subst['comments']="\n#".join(subst['comments'].split("\n")) 348 if subst['comments']: subst['comments']="#"+subst['comments'] 349 subst['exec']="../"+subst['execname'] 350 subst['testroot']=self.testroot_dir 351 subst['testname']=testname 352 dp = self.conf.get('DATAFILESPATH','') 353 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 354 355 # This is used to label some matrices 356 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 357 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 358 359 #Conf vars 360 if self.petsc_arch.find('valgrind')>=0: 361 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 362 else: 363 subst['mpiexec']=self.conf['MPIEXEC'] 364 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 365 subst['petsc_arch']=self.petsc_arch 366 if self.inInstallDir: 367 # Case 2 368 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 369 else: 370 # Case 1 371 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 372 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 373 subst['diff']=self.conf['DIFF'] 374 subst['rm']=self.conf['RM'] 375 subst['grep']=self.conf['GREP'] 376 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 377 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 378 379 # Output file is special because of subtests override 380 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 381 if not "_" in defroot: defroot=defroot+"_1" 382 subst['defroot']=defroot 383 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 384 subst['redirect_file']=defroot+".tmp" 385 if 'output_file' not in testDict: 386 subst['output_file']="output/"+defroot+".out" 387 # Add in the full path here. 388 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 389 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 390 if not subst['TODO']: 391 print("Warning: "+subst['output_file']+" not found.") 392 # Worry about alt files here -- see 393 # src/snes/examples/tutorials/output/ex22*.out 394 altlist=[subst['output_file']] 395 basefile,ext = os.path.splitext(subst['output_file']) 396 for i in range(1,9): 397 altroot=basefile+"_alt" 398 if i > 1: altroot=altroot+"_"+str(i) 399 af=altroot+".out" 400 srcaf=os.path.join(subst['srcdir'],af) 401 fullaf=os.path.join(self.petsc_dir,srcaf) 402 if os.path.isfile(fullaf): altlist.append(srcaf) 403 if len(altlist)>1: subst['altfiles']=altlist 404 #if len(altlist)>1: print("Found alt files: ",altlist) 405 406 subst['regexes']={} 407 for subkey in subst: 408 if subkey=='regexes': continue 409 if not isinstance(subst[subkey],str): continue 410 patt="@"+subkey.upper()+"@" 411 subst['regexes'][subkey]=re.compile(patt) 412 413 return subst 414 415 def _substVars(self,subst,origStr): 416 """ 417 Substitute variables 418 """ 419 Str=origStr 420 for subkey in subst: 421 if subkey=='regexes': continue 422 if not isinstance(subst[subkey],str): continue 423 if subkey.upper() not in Str: continue 424 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 425 return Str 426 427 def getCmds(self,subst,i): 428 """ 429 Generate bash script using template found next to this file. 430 This file is read in at constructor time to avoid file I/O 431 """ 432 nindnt=i # the start and has to be consistent with below 433 cmdindnt=self.indent*nindnt 434 cmdLines="" 435 436 # MPI is the default -- but we have a few odd commands 437 if not subst['command']: 438 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 439 else: 440 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 441 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 442 443 cmdLines+=cmdindnt+'if test $res = 0; then\n' 444 diffindnt=self.indent*(nindnt+1) 445 if not subst['filter_output']: 446 if 'altfiles' not in subst: 447 cmd=diffindnt+self._substVars(subst,example_template.difftest) 448 else: 449 # Have to do it by hand a bit because of variable number of alt files 450 rf=subst['redirect_file'] 451 cmd=diffindnt+example_template.difftest.split('@')[0] 452 for i in range(len(subst['altfiles'])): 453 af=subst['altfiles'][i] 454 cmd+=af+' '+rf 455 if i!=len(subst['altfiles'])-1: 456 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 457 cmd+=' || ${diff_exe} ' 458 else: 459 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 460 cmd+=subst['label_suffix']+' ""' # Quotes are painful 461 else: 462 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 463 cmdLines+=cmd+"\n" 464 cmdLines+=cmdindnt+'else\n' 465 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 466 cmdLines+=cmdindnt+'fi\n' 467 return cmdLines 468 469 def _writeTodoSkip(self,fh,tors,reasons,footer): 470 """ 471 Write out the TODO and SKIP lines in the file 472 The TODO or SKIP variable, tors, should be lower case 473 """ 474 TORS=tors.upper() 475 template=eval("example_template."+tors+"line") 476 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 477 tab = '' 478 if reasons: 479 fh.write('if ! $force; then\n') 480 tab = tab + ' ' 481 if reasons == ["Requires DATAFILESPATH"]: 482 # The only reason not to run is DATAFILESPATH, which we check at run-time 483 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 484 tab = tab + ' ' 485 if reasons: 486 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 487 fh.write(tab+footer+"\n") 488 fh.write(tab+"exit\n") 489 if reasons == ["Requires DATAFILESPATH"]: 490 fh.write(' fi\n') 491 if reasons: 492 fh.write('fi\n') 493 fh.write('\n\n') 494 return 495 496 def getLoopVarsHead(self,loopVars,i): 497 """ 498 Generate a nicely indented string with the format loops 499 Here is what the data structure looks like 500 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 501 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 502 loopVars['subargs']['pc_type']=["j","cholesky sor"] 503 """ 504 outstr=''; indnt=self.indent 505 for key in loopVars: 506 for var in loopVars[key]['varlist']: 507 varval=loopVars[key][var] 508 outstr += indnt * i + "for {0} in ${{{0}:-{1}}}; do\n".format(*varval) 509 i = i + 1 510 return (outstr,i) 511 512 def getLoopVarsFoot(self,loopVars,i): 513 outstr=''; indnt=self.indent 514 for key in loopVars: 515 for var in loopVars[key]['varlist']: 516 i = i - 1 517 outstr += indnt * i + "done\n" 518 return (outstr,i) 519 520 def genRunScript(self,testname,root,isRun,srcDict): 521 """ 522 Generate bash script using template found next to this file. 523 This file is read in at constructor time to avoid file I/O 524 """ 525 # runscript_dir directory has to be consistent with gmakefile 526 testDict=srcDict[testname] 527 rpath=self.srcrelpath(root) 528 runscript_dir=os.path.join(self.testroot_dir,rpath) 529 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 530 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 531 532 # Get variables to go into shell scripts. last time testDict used 533 subst=self.getSubstVars(testDict,rpath,testname) 534 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 535 536 #Handle runfiles 537 for lfile in subst.get('localrunfiles','').split(): 538 install_files(os.path.join(root, lfile), 539 os.path.join(runscript_dir, os.path.dirname(lfile))) 540 # Check subtests for local runfiles 541 for stest in subst.get("subtests",[]): 542 for lfile in testDict[stest].get('localrunfiles','').split(): 543 install_files(os.path.join(root, lfile), 544 os.path.join(runscript_dir, os.path.dirname(lfile))) 545 546 # Now substitute the key variables into the header and footer 547 header=self._substVars(subst,example_template.header) 548 # The header is done twice to enable @...@ in header 549 header=self._substVars(subst,header) 550 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 551 552 # Start writing the file 553 fh.write(header+"\n") 554 555 # If there is a TODO or a SKIP then we do it before writing out the 556 # rest of the command (which is useful for working on the test) 557 # SKIP and TODO can be for the source file or for the runs 558 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 559 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 560 561 j=0 # for indentation 562 563 if loopVars: 564 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 565 if (loopHead): fh.write(loopHead+"\n") 566 567 # Subtests are special 568 if 'subtests' in testDict: 569 substP=subst # Subtests can inherit args but be careful 570 k=0 # for label suffixes 571 for stest in testDict["subtests"]: 572 subst=substP.copy() 573 subst.update(testDict[stest]) 574 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 575 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 576 if sLoopVars: 577 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 578 fh.write(sLoopHead+"\n") 579 fh.write(self.getCmds(subst,j)+"\n") 580 if sLoopVars: 581 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 582 fh.write(sLoopFoot+"\n") 583 else: 584 fh.write(self.getCmds(subst,j)+"\n") 585 586 if loopVars: 587 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 588 fh.write(loopFoot+"\n") 589 590 fh.write(footer+"\n") 591 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 592 #if '10_9' in testname: sys.exit() 593 return 594 595 def genScriptsAndInfo(self,exfile,root,srcDict): 596 """ 597 Generate scripts from the source file, determine if built, etc. 598 For every test in the exfile with info in the srcDict: 599 1. Determine if it needs to be run for this arch 600 2. Generate the script 601 3. Generate the data needed to write out the makefile in a 602 convenient way 603 All tests are *always* run, but some may be SKIP'd per the TAP standard 604 """ 605 debug=False 606 rpath=self.srcrelpath(root) 607 execname=self.getExecname(exfile,rpath) 608 isBuilt=self._isBuilt(exfile,srcDict) 609 for test in srcDict: 610 if test in self.buildkeys: continue 611 if debug: print(self.nameSpace(exfile,root), test) 612 srcDict[test]['execname']=execname # Convenience in generating scripts 613 isRun=self._isRun(srcDict[test]) 614 self.genRunScript(test,root,isRun,srcDict) 615 srcDict[test]['isrun']=isRun 616 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 617 618 # This adds to datastructure for building deps 619 if isBuilt: self.addToSources(exfile,rpath,srcDict) 620 return 621 622 def _isBuilt(self,exfile,srcDict): 623 """ 624 Determine if this file should be built. 625 """ 626 # Get the language based on file extension 627 srcDict['SKIP'] = [] 628 lang=self.getLanguage(exfile) 629 if (lang=="F" or lang=="F90"): 630 if not self.have_fortran: 631 srcDict["SKIP"].append("Fortran required for this test") 632 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 633 srcDict["SKIP"].append("Fortran f90freeform required for this test") 634 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 635 srcDict["SKIP"].append("CUDA required for this test") 636 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 637 srcDict["SKIP"].append("C++ required for this test") 638 639 # Deprecated source files 640 if srcDict.get("TODO"): 641 return False 642 643 # isRun can work with srcDict to handle the requires 644 if "requires" in srcDict: 645 if srcDict["requires"]: 646 return self._isRun(srcDict) 647 648 return srcDict['SKIP'] == [] 649 650 651 def _isRun(self,testDict, debug=False): 652 """ 653 Based on the requirements listed in the src file and the petscconf.h 654 info, determine whether this test should be run or not. 655 """ 656 indent=" " 657 658 if 'SKIP' not in testDict: 659 testDict['SKIP'] = [] 660 # MPI requirements 661 if 'MPI_IS_MPIUNI' in self.conf: 662 if testDict.get('nsize', '1') != '1': 663 testDict['SKIP'].append("Parallel test with serial build") 664 665 # The requirements for the test are the sum of all the run subtests 666 if 'subtests' in testDict: 667 if 'requires' not in testDict: testDict['requires']="" 668 for stest in testDict['subtests']: 669 if 'requires' in testDict[stest]: 670 testDict['requires']+=" "+testDict[stest]['requires'] 671 if testDict.get('nsize', '1') != '1': 672 testDict['SKIP'].append("Parallel test with serial build") 673 break 674 675 # Now go through all requirements 676 if 'requires' in testDict: 677 for requirement in testDict['requires'].split(): 678 requirement=requirement.strip() 679 if not requirement: continue 680 if debug: print(indent+"Requirement: ", requirement) 681 isNull=False 682 if requirement.startswith("!"): 683 requirement=requirement[1:]; isNull=True 684 # Precision requirement for reals 685 if requirement in self.precision_types: 686 if self.conf['PETSC_PRECISION']==requirement: 687 if isNull: 688 testDict['SKIP'].append("not "+requirement+" required") 689 continue 690 continue # Success 691 elif not isNull: 692 testDict['SKIP'].append(requirement+" required") 693 continue 694 # Precision requirement for ints 695 if requirement in self.integer_types: 696 if requirement=="int32": 697 if self.conf['PETSC_SIZEOF_INT']==4: 698 if isNull: 699 testDict['SKIP'].append("not int32 required") 700 continue 701 continue # Success 702 elif not isNull: 703 testDict['SKIP'].append("int32 required") 704 continue 705 if requirement=="int64": 706 if self.conf['PETSC_SIZEOF_INT']==8: 707 if isNull: 708 testDict['SKIP'].append("NOT int64 required") 709 continue 710 continue # Success 711 elif not isNull: 712 testDict['SKIP'].append("int64 required") 713 continue 714 # Datafilespath 715 if requirement=="datafilespath" and not isNull: 716 testDict['SKIP'].append("Requires DATAFILESPATH") 717 continue 718 # Defines -- not sure I have comments matching 719 if "define(" in requirement.lower(): 720 reqdef=requirement.split("(")[1].split(")")[0] 721 if reqdef in self.conf: 722 if isNull: 723 testDict['SKIP'].append("Null requirement not met: "+requirement) 724 continue 725 continue # Success 726 elif not isNull: 727 testDict['SKIP'].append("Required: "+requirement) 728 continue 729 730 # Rest should be packages that we can just get from conf 731 if requirement == "complex": 732 petscconfvar="PETSC_USE_COMPLEX" 733 else: 734 petscconfvar="PETSC_HAVE_"+requirement.upper() 735 if self.conf.get(petscconfvar): 736 if isNull: 737 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 738 continue 739 continue # Success 740 elif not isNull: 741 if debug: print("requirement not found: ", requirement) 742 testDict['SKIP'].append(petscconfvar+" requirement not met") 743 continue 744 745 return testDict['SKIP'] == [] 746 747 def genPetscTests_summarize(self,dataDict): 748 """ 749 Required method to state what happened 750 """ 751 if not self.summarize: return 752 indent=" " 753 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 754 fh=open(fhname,"w") 755 for root in dataDict: 756 relroot=self.srcrelpath(root) 757 pkg=relroot.split("/")[1] 758 fh.write(relroot+"\n") 759 allSrcs=[] 760 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 761 for exfile in dataDict[root]: 762 # Basic information 763 rfile=os.path.join(relroot,exfile) 764 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 765 fh.write(indent+exfile+indent*4+builtStatus+"\n") 766 767 for test in dataDict[root][exfile]: 768 if test in self.buildkeys: continue 769 line=indent*2+test 770 fh.write(line+"\n") 771 # Looks nice to have the keys in order 772 #for key in dataDict[root][exfile][test]: 773 for key in "isrun abstracted nsize args requires script".split(): 774 if key not in dataDict[root][exfile][test]: continue 775 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 776 fh.write(line+"\n") 777 fh.write("\n") 778 fh.write("\n") 779 fh.write("\n") 780 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 781 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 782 fh.close() 783 return 784 785 def genPetscTests(self,root,dirs,files,dataDict): 786 """ 787 Go through and parse the source files in the directory to generate 788 the examples based on the metadata contained in the source files 789 """ 790 debug=False 791 # Use examplesAnalyze to get what the makefles think are sources 792 #self.examplesAnalyze(root,dirs,files,anlzDict) 793 794 dataDict[root]={} 795 796 for exfile in files: 797 #TST: Until we replace files, still leaving the orginals as is 798 #if not exfile.startswith("new_"+"ex"): continue 799 #if not exfile.startswith("ex"): continue 800 801 # Ignore emacs and other temporary files 802 if exfile.startswith("."): continue 803 if exfile.startswith("#"): continue 804 if exfile.endswith("~"): continue 805 # Only parse source files 806 ext=os.path.splitext(exfile)[-1].lstrip('.') 807 if ext not in LANGS: continue 808 809 # Convenience 810 fullex=os.path.join(root,exfile) 811 if self.verbose: print(' --> '+fullex) 812 dataDict[root].update(testparse.parseTestFile(fullex,0)) 813 if exfile in dataDict[root]: 814 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 815 816 return 817 818 def walktree(self,top): 819 """ 820 Walk a directory tree, starting from 'top' 821 """ 822 # Goal of action is to fill this dictionary 823 dataDict={} 824 for root, dirs, files in os.walk(top, topdown=True): 825 dirs.sort() 826 files.sort() 827 if not "examples" in root: continue 828 if "dSYM" in root: continue 829 if os.path.basename(root.rstrip("/")) == 'output': continue 830 if self.verbose: print(root) 831 self.genPetscTests(root,dirs,files,dataDict) 832 # Now summarize this dictionary 833 if self.verbose: self.genPetscTests_summarize(dataDict) 834 return dataDict 835 836 def gen_gnumake(self, fd): 837 """ 838 Overwrite of the method in the base PETSc class 839 """ 840 def write(stem, srcs): 841 for lang in LANGS: 842 if srcs[lang]['srcs']: 843 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 844 for pkg in PKGS: 845 srcs = self.gen_pkg(pkg) 846 write('testsrcs-' + pkg, srcs) 847 # Handle dependencies 848 for lang in LANGS: 849 for exfile in srcs[lang]['srcs']: 850 if exfile in srcs[lang]: 851 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 852 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 853 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 854 if deps: 855 # The executable literally depends on the object file because it is linked 856 fd.write(ex +": " + " ".join(deps) +'\n') 857 # The object file containing 'main' does not normally depend on other object 858 # files, but it does when it includes their modules. This dependency is 859 # overly blunt and could be reduced to only depend on object files for 860 # modules that are used, like "*f90aux.o". 861 fd.write(exfo +": " + " ".join(deps) +'\n') 862 863 return self.gendeps 864 865 def gen_pkg(self, pkg): 866 """ 867 Overwrite of the method in the base PETSc class 868 """ 869 return self.sources[pkg] 870 871 def write_gnumake(self, dataDict, output=None): 872 """ 873 Write out something similar to files from gmakegen.py 874 875 Test depends on script which also depends on source 876 file, but since I don't have a good way generating 877 acting on a single file (oops) just depend on 878 executable which in turn will depend on src file 879 """ 880 # Different options for how to set up the targets 881 compileExecsFirst=False 882 883 # Open file 884 fd = open(output, 'w') 885 886 # Write out the sources 887 gendeps = self.gen_gnumake(fd) 888 889 # Write out the tests and execname targets 890 fd.write("\n#Tests and executables\n") # Delimiter 891 892 for pkg in PKGS: 893 # These grab the ones that are built 894 for lang in LANGS: 895 testdeps=[] 896 for ftest in self.tests[pkg][lang]: 897 test=os.path.basename(ftest) 898 basedir=os.path.dirname(ftest) 899 testdeps.append(self.nameSpace(test,basedir)) 900 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 901 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 902 903 # test targets 904 for ftest in self.tests[pkg][lang]: 905 test=os.path.basename(ftest) 906 basedir=os.path.dirname(ftest) 907 testdir="${TESTDIR}/"+basedir+"/" 908 nmtest=self.nameSpace(test,basedir) 909 rundir=os.path.join(testdir,test) 910 script=test+".sh" 911 912 # Deps 913 exfile=self.tests[pkg][lang][ftest]['exfile'] 914 fullex=os.path.join(self.srcdir,exfile) 915 localexec=self.tests[pkg][lang][ftest]['exec'] 916 execname=os.path.join(testdir,localexec) 917 fullscript=os.path.join(testdir,script) 918 tmpfile=os.path.join(testdir,test,test+".tmp") 919 920 # *.counts depends on the script and either executable (will 921 # be run) or the example source file (SKIP or TODO) 922 fd.write('%s.counts : %s %s' 923 % (os.path.join('$(TESTDIR)/counts', nmtest), 924 fullscript, 925 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 926 ) 927 if exfile in self.sources[pkg][lang]: 928 for dep in self.sources[pkg][lang][exfile]: 929 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 930 fd.write('\n') 931 932 # Now write the args: 933 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 934 935 fd.close() 936 return 937 938def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 939 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 940 if petsc_arch: 941 if len(petsc_arch.split(os.path.sep))>1: 942 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 943 output = os.path.join(testdir, 'testfiles') 944 945 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 946 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 947 testdir=testdir) 948 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 949 pEx.write_gnumake(dataDict, output) 950 951if __name__ == '__main__': 952 import optparse 953 parser = optparse.OptionParser() 954 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 955 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 956 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 957 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 958 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 959 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 960 961 opts, extra_args = parser.parse_args() 962 if extra_args: 963 import sys 964 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 965 exit(1) 966 if opts.testdir is None: 967 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 968 969 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 970 verbose=opts.verbose, 971 single_ex=opts.single_executable, srcdir=opts.srcdir, 972 testdir=opts.testdir) 973