1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 85 86 self.single_ex=single_ex 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 self.srcdir=os.path.join(self.petsc_dir,'src') 104 105 self.testroot_dir=os.path.abspath(testdir) 106 107 self.ptNaming=True 108 self.verbose=verbose 109 # Whether to write out a useful debugging 110 self.summarize=True if verbose else False 111 112 # For help in setting the requirements 113 self.precision_types="single double __float128 int32".split() 114 self.integer_types="int32 int64".split() 115 self.languages="fortran cuda cxx".split() # Always requires C so do not list 116 117 # Things that are not test 118 self.buildkeys=testparse.buildkeys 119 120 # Adding a dictionary for storing sources, objects, and tests 121 # to make building the dependency tree easier 122 self.sources={} 123 self.objects={} 124 self.tests={} 125 for pkg in PKGS: 126 self.sources[pkg]={} 127 self.objects[pkg]=[] 128 self.tests[pkg]={} 129 for lang in LANGS: 130 self.sources[pkg][lang]={} 131 self.sources[pkg][lang]['srcs']=[] 132 self.tests[pkg][lang]={} 133 134 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 135 136 self.indent=" " 137 if self.verbose: print('Finishing the constructor') 138 return 139 140 def srcrelpath(self,rdir): 141 """ 142 Get relative path to source directory 143 """ 144 return os.path.relpath(rdir,self.srcdir) 145 146 def getInInstallDir(self,thisscriptdir): 147 """ 148 When petsc is installed then this file in installed in: 149 <PREFIX>/share/petsc/examples/config/gmakegentest.py 150 otherwise the path is: 151 <PETSC_DIR>/config/gmakegentest.py 152 We use this difference to determine if we are in installdir 153 """ 154 dirlist=thisscriptdir.split(os.path.sep) 155 if len(dirlist)>4: 156 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 157 if lastfour==os.path.join('share','petsc','examples','config'): 158 return True 159 else: 160 return False 161 else: 162 return False 163 164 def nameSpace(self,srcfile,srcdir): 165 """ 166 Because the scripts have a non-unique naming, the pretty-printing 167 needs to convey the srcdir and srcfile. There are two ways of doing this. 168 """ 169 if self.ptNaming: 170 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 171 cdir=srcdir 172 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 173 nameString=prefix+srcfile 174 else: 175 #nameString=srcdir+": "+srcfile 176 nameString=srcfile 177 return nameString 178 179 def getLanguage(self,srcfile): 180 """ 181 Based on the source, determine associated language as found in gmakegen.LANGS 182 Can we just return srcext[1:\] now? 183 """ 184 langReq=None 185 srcext=os.path.splitext(srcfile)[-1] 186 if srcext in ".F90".split(): langReq="F90" 187 if srcext in ".F".split(): langReq="F" 188 if srcext in ".cxx".split(): langReq="cxx" 189 if srcext == ".cu": langReq="cu" 190 if srcext == ".c": langReq="c" 191 #if not langReq: print("ERROR: ", srcext, srcfile) 192 return langReq 193 194 def _getLoopVars(self,inDict,testname, isSubtest=False): 195 """ 196 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 197 Return: 198 inDict['args']: -ksp_monitor 199 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 200 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 201 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 202 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 203 subst should be passed in instead of inDict 204 """ 205 loopVars={}; newargs="" 206 lkeys=inDict.keys() 207 lsuffix='_' 208 argregex=re.compile('(?<![a-zA-Z])-(?=[a-zA-Z])') 209 from testparse import parseLoopArgs 210 for key in lkeys: 211 if type(inDict[key])!=bytes: continue 212 keystr = str(inDict[key]) 213 akey=('subargs' if key=='args' else key) # what to assign 214 if akey not in inDict: inDict[akey]='' 215 varlist=[] 216 for varset in argregex.split(keystr): 217 if not varset.strip(): continue 218 if '{{' in varset: 219 keyvar,lvars,ftype=parseLoopArgs(varset) 220 if akey not in loopVars: loopVars[akey]={} 221 varlist.append(keyvar) 222 loopVars[akey][keyvar]=[keyvar,lvars] 223 if akey=='nsize': 224 inDict[akey] = '${' + keyvar + '}' 225 lsuffix+=akey+'-'+inDict[akey]+'_' 226 else: 227 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 228 lsuffix+=keyvar+'-${' + keyvar + '}_' 229 else: 230 if key=='args': newargs+=" -"+varset.strip() 231 if varlist: loopVars[akey]['varlist']=varlist 232 233 234 # For subtests, args are always substituted in (not top level) 235 if isSubtest: 236 inDict['subargs']+=" "+newargs.strip() 237 inDict['args']='' 238 if 'label_suffix' in inDict: 239 inDict['label_suffix']+=lsuffix.rstrip('_') 240 else: 241 inDict['label_suffix']=lsuffix.rstrip('_') 242 else: 243 if loopVars.keys(): 244 inDict['args']=newargs.strip() 245 inDict['label_suffix']=lsuffix.rstrip('_') 246 if loopVars.keys(): 247 return loopVars 248 else: 249 return None 250 251 def getArgLabel(self,testDict): 252 """ 253 In all of the arguments in the test dictionary, create a simple 254 string for searching within the makefile system. For simplicity in 255 search, remove "-", for strings, etc. 256 Also, concatenate the arg commands 257 For now, ignore nsize -- seems hard to search for anyway 258 """ 259 # Collect all of the args associated with a test 260 argStr=("" if 'args' not in testDict else testDict['args']) 261 if 'subtests' in testDict: 262 for stest in testDict["subtests"]: 263 sd=testDict[stest] 264 argStr=argStr+("" if 'args' not in sd else sd['args']) 265 266 # Now go through and cleanup 267 argStr=re.sub('{{(.*?)}}',"",argStr) 268 argStr=re.sub('-'," ",argStr) 269 for digit in string.digits: argStr=re.sub(digit," ",argStr) 270 argStr=re.sub("\.","",argStr) 271 argStr=re.sub(",","",argStr) 272 argStr=re.sub('\+',' ',argStr) 273 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 274 return argStr.strip() 275 276 def addToSources(self,exfile,rpath,srcDict): 277 """ 278 Put into data structure that allows easy generation of makefile 279 """ 280 pkg=rpath.split(os.path.sep)[0] 281 relpfile=os.path.join(rpath,exfile) 282 lang=self.getLanguage(exfile) 283 if not lang: return 284 self.sources[pkg][lang]['srcs'].append(relpfile) 285 self.sources[pkg][lang][relpfile] = [] 286 if 'depends' in srcDict: 287 depSrcList=srcDict['depends'].split() 288 for depSrc in depSrcList: 289 depObj=os.path.splitext(depSrc)[0]+".o" 290 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 291 292 # In gmakefile, ${TESTDIR} var specifies the object compilation 293 testsdir=rpath+"/" 294 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 295 self.objects[pkg].append(objfile) 296 return 297 298 def addToTests(self,test,rpath,exfile,execname,testDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 Organized by languages to allow testing of languages 302 """ 303 pkg=rpath.split("/")[0] 304 nmtest=os.path.join(rpath,test) 305 lang=self.getLanguage(exfile) 306 if not lang: return 307 self.tests[pkg][lang][nmtest]={} 308 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 309 self.tests[pkg][lang][nmtest]['exec']=execname 310 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 311 return 312 313 def getExecname(self,exfile,rpath): 314 """ 315 Generate bash script using template found next to this file. 316 This file is read in at constructor time to avoid file I/O 317 """ 318 if self.single_ex: 319 execname=rpath.split("/")[1]+"-ex" 320 else: 321 execname=os.path.splitext(exfile)[0] 322 return execname 323 324 def getSubstVars(self,testDict,rpath,testname): 325 """ 326 Create a dictionary with all of the variables that get substituted 327 into the template commands found in example_template.py 328 """ 329 subst={} 330 331 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 332 if 'nsize' not in testDict: testDict['nsize']=1 333 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 334 for ak in testparse.acceptedkeys: 335 if ak=='test': continue 336 subst[ak]=(testDict[ak] if ak in testDict else '') 337 338 # Now do other variables 339 subst['execname']=testDict['execname'] 340 if 'filter' in testDict: 341 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 342 343 # Others 344 subst['subargs']='' # Default. For variables override 345 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 346 subst['label_suffix']='' 347 subst['comments']="\n#".join(subst['comments'].split("\n")) 348 if subst['comments']: subst['comments']="#"+subst['comments'] 349 subst['exec']="../"+subst['execname'] 350 subst['testroot']=self.testroot_dir 351 subst['testname']=testname 352 dp = self.conf.get('DATAFILESPATH','') 353 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 354 355 # This is used to label some matrices 356 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 357 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 358 359 # These can have for loops and are treated separately later 360 subst['nsize']=str(subst['nsize']) 361 362 #Conf vars 363 if self.petsc_arch.find('valgrind')>=0: 364 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 365 else: 366 subst['mpiexec']=self.conf['MPIEXEC'] 367 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 368 subst['petsc_arch']=self.petsc_arch 369 if self.inInstallDir: 370 # Case 2 371 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 372 else: 373 # Case 1 374 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 375 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 376 subst['diff']=self.conf['DIFF'] 377 subst['rm']=self.conf['RM'] 378 subst['grep']=self.conf['GREP'] 379 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 380 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 381 382 # Output file is special because of subtests override 383 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 384 if not "_" in defroot: defroot=defroot+"_1" 385 subst['defroot']=defroot 386 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 387 subst['redirect_file']=defroot+".tmp" 388 if 'output_file' not in testDict: 389 subst['output_file']="output/"+defroot+".out" 390 # Add in the full path here. 391 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 392 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 393 if not subst['TODO']: 394 print("Warning: "+subst['output_file']+" not found.") 395 # Worry about alt files here -- see 396 # src/snes/examples/tutorials/output/ex22*.out 397 altlist=[subst['output_file']] 398 basefile,ext = os.path.splitext(subst['output_file']) 399 for i in range(1,9): 400 altroot=basefile+"_alt" 401 if i > 1: altroot=altroot+"_"+str(i) 402 af=altroot+".out" 403 srcaf=os.path.join(subst['srcdir'],af) 404 fullaf=os.path.join(self.petsc_dir,srcaf) 405 if os.path.isfile(fullaf): altlist.append(srcaf) 406 if len(altlist)>1: subst['altfiles']=altlist 407 #if len(altlist)>1: print("Found alt files: ",altlist) 408 409 subst['regexes']={} 410 for subkey in subst: 411 if subkey=='regexes': continue 412 if not isinstance(subst[subkey],str): continue 413 patt="@"+subkey.upper()+"@" 414 subst['regexes'][subkey]=re.compile(patt) 415 416 return subst 417 418 def _substVars(self,subst,origStr): 419 """ 420 Substitute variables 421 """ 422 Str=origStr 423 for subkey in subst: 424 if subkey=='regexes': continue 425 if not isinstance(subst[subkey],str): continue 426 if subkey.upper() not in Str: continue 427 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 428 return Str 429 430 def getCmds(self,subst,i): 431 """ 432 Generate bash script using template found next to this file. 433 This file is read in at constructor time to avoid file I/O 434 """ 435 nindnt=i # the start and has to be consistent with below 436 cmdindnt=self.indent*nindnt 437 cmdLines="" 438 439 # MPI is the default -- but we have a few odd commands 440 if not subst['command']: 441 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 442 else: 443 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 444 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 445 446 cmdLines+=cmdindnt+'if test $res = 0; then\n' 447 diffindnt=self.indent*(nindnt+1) 448 if not subst['filter_output']: 449 if 'altfiles' not in subst: 450 cmd=diffindnt+self._substVars(subst,example_template.difftest) 451 else: 452 # Have to do it by hand a bit because of variable number of alt files 453 rf=subst['redirect_file'] 454 cmd=diffindnt+example_template.difftest.split('@')[0] 455 for i in range(len(subst['altfiles'])): 456 af=subst['altfiles'][i] 457 cmd+=af+' '+rf 458 if i!=len(subst['altfiles'])-1: 459 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 460 cmd+=' || ${diff_exe} ' 461 else: 462 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 463 cmd+=subst['label_suffix']+' ""' # Quotes are painful 464 else: 465 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 466 cmdLines+=cmd+"\n" 467 cmdLines+=cmdindnt+'else\n' 468 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 469 cmdLines+=cmdindnt+'fi\n' 470 return cmdLines 471 472 def _writeTodoSkip(self,fh,tors,reasons,footer): 473 """ 474 Write out the TODO and SKIP lines in the file 475 The TODO or SKIP variable, tors, should be lower case 476 """ 477 TORS=tors.upper() 478 template=eval("example_template."+tors+"line") 479 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 480 tab = '' 481 if reasons: 482 fh.write('if ! $force; then\n') 483 tab = tab + ' ' 484 if reasons == ["Requires DATAFILESPATH"]: 485 # The only reason not to run is DATAFILESPATH, which we check at run-time 486 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 487 tab = tab + ' ' 488 if reasons: 489 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 490 fh.write(tab+footer+"\n") 491 fh.write(tab+"exit\n") 492 if reasons == ["Requires DATAFILESPATH"]: 493 fh.write(' fi\n') 494 if reasons: 495 fh.write('fi\n') 496 fh.write('\n\n') 497 return 498 499 def getLoopVarsHead(self,loopVars,i): 500 """ 501 Generate a nicely indented string with the format loops 502 Here is what the data structure looks like 503 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 504 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 505 loopVars['subargs']['pc_type']=["j","cholesky sor"] 506 """ 507 outstr=''; indnt=self.indent 508 for key in loopVars: 509 for var in loopVars[key]['varlist']: 510 varval=loopVars[key][var] 511 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 512 i = i + 1 513 return (outstr,i) 514 515 def getLoopVarsFoot(self,loopVars,i): 516 outstr=''; indnt=self.indent 517 for key in loopVars: 518 for var in loopVars[key]['varlist']: 519 i = i - 1 520 outstr += indnt * i + "done\n" 521 return (outstr,i) 522 523 def genRunScript(self,testname,root,isRun,srcDict): 524 """ 525 Generate bash script using template found next to this file. 526 This file is read in at constructor time to avoid file I/O 527 """ 528 # runscript_dir directory has to be consistent with gmakefile 529 testDict=srcDict[testname] 530 rpath=self.srcrelpath(root) 531 runscript_dir=os.path.join(self.testroot_dir,rpath) 532 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 533 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 534 535 # Get variables to go into shell scripts. last time testDict used 536 subst=self.getSubstVars(testDict,rpath,testname) 537 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 538 539 #Handle runfiles 540 for lfile in subst.get('localrunfiles','').split(): 541 install_files(os.path.join(root, lfile), 542 os.path.join(runscript_dir, os.path.dirname(lfile))) 543 # Check subtests for local runfiles 544 for stest in subst.get("subtests",[]): 545 for lfile in testDict[stest].get('localrunfiles','').split(): 546 install_files(os.path.join(root, lfile), 547 os.path.join(runscript_dir, os.path.dirname(lfile))) 548 549 # Now substitute the key variables into the header and footer 550 header=self._substVars(subst,example_template.header) 551 # The header is done twice to enable @...@ in header 552 header=self._substVars(subst,header) 553 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 554 555 # Start writing the file 556 fh.write(header+"\n") 557 558 # If there is a TODO or a SKIP then we do it before writing out the 559 # rest of the command (which is useful for working on the test) 560 # SKIP and TODO can be for the source file or for the runs 561 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 562 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 563 564 j=0 # for indentation 565 566 if loopVars: 567 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 568 if (loopHead): fh.write(loopHead+"\n") 569 570 # Subtests are special 571 if 'subtests' in testDict: 572 substP=subst # Subtests can inherit args but be careful 573 k=0 # for label suffixes 574 for stest in testDict["subtests"]: 575 subst=substP.copy() 576 subst.update(testDict[stest]) 577 # nsize is special because it is usually overwritten 578 if 'nsize' in testDict[stest]: 579 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 580 else: 581 fh.write("nsize=1\n") 582 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 583 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 584 if sLoopVars: 585 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 586 fh.write(sLoopHead+"\n") 587 fh.write(self.getCmds(subst,j)+"\n") 588 if sLoopVars: 589 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 590 fh.write(sLoopFoot+"\n") 591 else: 592 fh.write(self.getCmds(subst,j)+"\n") 593 594 if loopVars: 595 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 596 fh.write(loopFoot+"\n") 597 598 fh.write(footer+"\n") 599 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 600 #if '10_9' in testname: sys.exit() 601 return 602 603 def genScriptsAndInfo(self,exfile,root,srcDict): 604 """ 605 Generate scripts from the source file, determine if built, etc. 606 For every test in the exfile with info in the srcDict: 607 1. Determine if it needs to be run for this arch 608 2. Generate the script 609 3. Generate the data needed to write out the makefile in a 610 convenient way 611 All tests are *always* run, but some may be SKIP'd per the TAP standard 612 """ 613 debug=False 614 rpath=self.srcrelpath(root) 615 execname=self.getExecname(exfile,rpath) 616 isBuilt=self._isBuilt(exfile,srcDict) 617 for test in srcDict: 618 if test in self.buildkeys: continue 619 if debug: print(self.nameSpace(exfile,root), test) 620 srcDict[test]['execname']=execname # Convenience in generating scripts 621 isRun=self._isRun(srcDict[test]) 622 self.genRunScript(test,root,isRun,srcDict) 623 srcDict[test]['isrun']=isRun 624 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 625 626 # This adds to datastructure for building deps 627 if isBuilt: self.addToSources(exfile,rpath,srcDict) 628 return 629 630 def _isBuilt(self,exfile,srcDict): 631 """ 632 Determine if this file should be built. 633 """ 634 # Get the language based on file extension 635 srcDict['SKIP'] = [] 636 lang=self.getLanguage(exfile) 637 if (lang=="F" or lang=="F90"): 638 if not self.have_fortran: 639 srcDict["SKIP"].append("Fortran required for this test") 640 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 641 srcDict["SKIP"].append("Fortran f90freeform required for this test") 642 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 643 srcDict["SKIP"].append("CUDA required for this test") 644 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 645 srcDict["SKIP"].append("C++ required for this test") 646 647 # Deprecated source files 648 if srcDict.get("TODO"): 649 return False 650 651 # isRun can work with srcDict to handle the requires 652 if "requires" in srcDict: 653 if srcDict["requires"]: 654 return self._isRun(srcDict) 655 656 return srcDict['SKIP'] == [] 657 658 659 def _isRun(self,testDict, debug=False): 660 """ 661 Based on the requirements listed in the src file and the petscconf.h 662 info, determine whether this test should be run or not. 663 """ 664 indent=" " 665 666 if 'SKIP' not in testDict: 667 testDict['SKIP'] = [] 668 # MPI requirements 669 nsize=testDict.get('nsize',1) 670 # nsize could have for loops at this point 671 if not isinstance(nsize,int): 672 try: 673 nsize=int(nsize) 674 except: 675 pass 676 if isinstance(nsize, int): 677 if nsize>1 and 'MPI_IS_MPIUNI' in self.conf: 678 if debug: print(indent+"Cannot run parallel tests") 679 testDict['SKIP'].append("Parallel test with serial build") 680 681 # The requirements for the test are the sum of all the run subtests 682 if 'subtests' in testDict: 683 if 'requires' not in testDict: testDict['requires']="" 684 for stest in testDict['subtests']: 685 if 'requires' in testDict[stest]: 686 testDict['requires']+=" "+testDict[stest]['requires'] 687 nsize=testDict[stest].get('nsize',1) 688 if not isinstance(nsize,int): 689 try: 690 nsize=int(nsize) 691 except: 692 pass 693 if isinstance(nsize, int): 694 if nsize>1 and 'MPI_IS_MPIUNI' in self.conf: 695 testDict['SKIP'].append("Parallel test with serial build") 696 697 698 # Now go through all requirements 699 if 'requires' in testDict: 700 for requirement in testDict['requires'].split(): 701 requirement=requirement.strip() 702 if not requirement: continue 703 if debug: print(indent+"Requirement: ", requirement) 704 isNull=False 705 if requirement.startswith("!"): 706 requirement=requirement[1:]; isNull=True 707 # Precision requirement for reals 708 if requirement in self.precision_types: 709 if self.conf['PETSC_PRECISION']==requirement: 710 if isNull: 711 testDict['SKIP'].append("not "+requirement+" required") 712 continue 713 continue # Success 714 elif not isNull: 715 testDict['SKIP'].append(requirement+" required") 716 continue 717 # Precision requirement for ints 718 if requirement in self.integer_types: 719 if requirement=="int32": 720 if self.conf['PETSC_SIZEOF_INT']==4: 721 if isNull: 722 testDict['SKIP'].append("not int32 required") 723 continue 724 continue # Success 725 elif not isNull: 726 testDict['SKIP'].append("int32 required") 727 continue 728 if requirement=="int64": 729 if self.conf['PETSC_SIZEOF_INT']==8: 730 if isNull: 731 testDict['SKIP'].append("NOT int64 required") 732 continue 733 continue # Success 734 elif not isNull: 735 testDict['SKIP'].append("int64 required") 736 continue 737 # Datafilespath 738 if requirement=="datafilespath" and not isNull: 739 testDict['SKIP'].append("Requires DATAFILESPATH") 740 continue 741 # Defines -- not sure I have comments matching 742 if "define(" in requirement.lower(): 743 reqdef=requirement.split("(")[1].split(")")[0] 744 if reqdef in self.conf: 745 if isNull: 746 testDict['SKIP'].append("Null requirement not met: "+requirement) 747 continue 748 continue # Success 749 elif not isNull: 750 testDict['SKIP'].append("Required: "+requirement) 751 continue 752 753 # Rest should be packages that we can just get from conf 754 if requirement == "complex": 755 petscconfvar="PETSC_USE_COMPLEX" 756 else: 757 petscconfvar="PETSC_HAVE_"+requirement.upper() 758 if self.conf.get(petscconfvar): 759 if isNull: 760 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 761 continue 762 continue # Success 763 elif not isNull: 764 if debug: print("requirement not found: ", requirement) 765 testDict['SKIP'].append(petscconfvar+" requirement not met") 766 continue 767 768 return testDict['SKIP'] == [] 769 770 def genPetscTests_summarize(self,dataDict): 771 """ 772 Required method to state what happened 773 """ 774 if not self.summarize: return 775 indent=" " 776 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 777 fh=open(fhname,"w") 778 #print("See ", fhname) 779 for root in dataDict: 780 relroot=self.srcrelpath(root) 781 pkg=relroot.split("/")[1] 782 fh.write(relroot+"\n") 783 allSrcs=[] 784 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 785 for exfile in dataDict[root]: 786 # Basic information 787 rfile=os.path.join(relroot,exfile) 788 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 789 fh.write(indent+exfile+indent*4+builtStatus+"\n") 790 791 for test in dataDict[root][exfile]: 792 if test in self.buildkeys: continue 793 line=indent*2+test 794 fh.write(line+"\n") 795 # Looks nice to have the keys in order 796 #for key in dataDict[root][exfile][test]: 797 for key in "isrun abstracted nsize args requires script".split(): 798 if key not in dataDict[root][exfile][test]: continue 799 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 800 fh.write(line+"\n") 801 fh.write("\n") 802 fh.write("\n") 803 fh.write("\n") 804 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 805 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 806 fh.close() 807 return 808 809 def genPetscTests(self,root,dirs,files,dataDict): 810 """ 811 Go through and parse the source files in the directory to generate 812 the examples based on the metadata contained in the source files 813 """ 814 debug=False 815 # Use examplesAnalyze to get what the makefles think are sources 816 #self.examplesAnalyze(root,dirs,files,anlzDict) 817 818 dataDict[root]={} 819 820 for exfile in files: 821 #TST: Until we replace files, still leaving the orginals as is 822 #if not exfile.startswith("new_"+"ex"): continue 823 #if not exfile.startswith("ex"): continue 824 825 # Ignore emacs and other temporary files 826 if exfile.startswith("."): continue 827 if exfile.startswith("#"): continue 828 if exfile.endswith("~"): continue 829 # Only parse source files 830 ext=os.path.splitext(exfile)[-1].lstrip('.') 831 if ext not in LANGS: continue 832 833 # Convenience 834 fullex=os.path.join(root,exfile) 835 if self.verbose: print(' --> '+fullex) 836 dataDict[root].update(testparse.parseTestFile(fullex,0)) 837 if exfile in dataDict[root]: 838 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 839 840 return 841 842 def walktree(self,top): 843 """ 844 Walk a directory tree, starting from 'top' 845 """ 846 #print("action", action) 847 # Goal of action is to fill this dictionary 848 dataDict={} 849 for root, dirs, files in os.walk(top, topdown=True): 850 dirs.sort() 851 files.sort() 852 if not "examples" in root: continue 853 if "dSYM" in root: continue 854 if os.path.basename(root.rstrip("/")) == 'output': continue 855 if self.verbose: print(root) 856 self.genPetscTests(root,dirs,files,dataDict) 857 # Now summarize this dictionary 858 if self.verbose: self.genPetscTests_summarize(dataDict) 859 return dataDict 860 861 def gen_gnumake(self, fd): 862 """ 863 Overwrite of the method in the base PETSc class 864 """ 865 def write(stem, srcs): 866 for lang in LANGS: 867 if srcs[lang]['srcs']: 868 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 869 for pkg in PKGS: 870 srcs = self.gen_pkg(pkg) 871 write('testsrcs-' + pkg, srcs) 872 # Handle dependencies 873 for lang in LANGS: 874 for exfile in srcs[lang]['srcs']: 875 if exfile in srcs[lang]: 876 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 877 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 878 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 879 if deps: 880 # The executable literally depends on the object file because it is linked 881 fd.write(ex +": " + " ".join(deps) +'\n') 882 # The object file containing 'main' does not normally depend on other object 883 # files, but it does when it includes their modules. This dependency is 884 # overly blunt and could be reduced to only depend on object files for 885 # modules that are used, like "*f90aux.o". 886 fd.write(exfo +": " + " ".join(deps) +'\n') 887 888 return self.gendeps 889 890 def gen_pkg(self, pkg): 891 """ 892 Overwrite of the method in the base PETSc class 893 """ 894 return self.sources[pkg] 895 896 def write_gnumake(self, dataDict, output=None): 897 """ 898 Write out something similar to files from gmakegen.py 899 900 Test depends on script which also depends on source 901 file, but since I don't have a good way generating 902 acting on a single file (oops) just depend on 903 executable which in turn will depend on src file 904 """ 905 # Different options for how to set up the targets 906 compileExecsFirst=False 907 908 # Open file 909 fd = open(output, 'w') 910 911 # Write out the sources 912 gendeps = self.gen_gnumake(fd) 913 914 # Write out the tests and execname targets 915 fd.write("\n#Tests and executables\n") # Delimiter 916 917 for pkg in PKGS: 918 # These grab the ones that are built 919 for lang in LANGS: 920 testdeps=[] 921 for ftest in self.tests[pkg][lang]: 922 test=os.path.basename(ftest) 923 basedir=os.path.dirname(ftest) 924 testdeps.append(self.nameSpace(test,basedir)) 925 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 926 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 927 928 # test targets 929 for ftest in self.tests[pkg][lang]: 930 test=os.path.basename(ftest) 931 basedir=os.path.dirname(ftest) 932 testdir="${TESTDIR}/"+basedir+"/" 933 nmtest=self.nameSpace(test,basedir) 934 rundir=os.path.join(testdir,test) 935 script=test+".sh" 936 937 # Deps 938 exfile=self.tests[pkg][lang][ftest]['exfile'] 939 fullex=os.path.join(self.srcdir,exfile) 940 localexec=self.tests[pkg][lang][ftest]['exec'] 941 execname=os.path.join(testdir,localexec) 942 fullscript=os.path.join(testdir,script) 943 tmpfile=os.path.join(testdir,test,test+".tmp") 944 945 # *.counts depends on the script and either executable (will 946 # be run) or the example source file (SKIP or TODO) 947 fd.write('%s.counts : %s %s' 948 % (os.path.join('$(TESTDIR)/counts', nmtest), 949 fullscript, 950 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 951 ) 952 if exfile in self.sources[pkg][lang]: 953 for dep in self.sources[pkg][lang][exfile]: 954 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 955 fd.write('\n') 956 957 # Now write the args: 958 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 959 960 fd.close() 961 return 962 963def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 964 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 965 if petsc_arch: 966 if len(petsc_arch.split(os.path.sep))>1: 967 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 968 output = os.path.join(testdir, 'testfiles') 969 970 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 971 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 972 testdir=testdir) 973 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 974 pEx.write_gnumake(dataDict, output) 975 976if __name__ == '__main__': 977 import optparse 978 parser = optparse.OptionParser() 979 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 980 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 981 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 982 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 983 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 984 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 985 986 opts, extra_args = parser.parse_args() 987 if extra_args: 988 import sys 989 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 990 exit(1) 991 if opts.testdir is None: 992 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 993 994 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 995 verbose=opts.verbose, 996 single_ex=opts.single_executable, srcdir=opts.srcdir, 997 testdir=opts.testdir) 998