1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 85 86 self.single_ex=single_ex 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 self.srcdir=os.path.join(self.petsc_dir,'src') 104 105 self.testroot_dir=os.path.abspath(testdir) 106 107 self.ptNaming=True 108 self.verbose=verbose 109 # Whether to write out a useful debugging 110 self.summarize=True if verbose else False 111 112 # For help in setting the requirements 113 self.precision_types="single double __float128 int32".split() 114 self.integer_types="int32 int64".split() 115 self.languages="fortran cuda cxx".split() # Always requires C so do not list 116 117 # Things that are not test 118 self.buildkeys=testparse.buildkeys 119 120 # Adding a dictionary for storing sources, objects, and tests 121 # to make building the dependency tree easier 122 self.sources={} 123 self.objects={} 124 self.tests={} 125 for pkg in PKGS: 126 self.sources[pkg]={} 127 self.objects[pkg]=[] 128 self.tests[pkg]={} 129 for lang in LANGS: 130 self.sources[pkg][lang]={} 131 self.sources[pkg][lang]['srcs']=[] 132 self.tests[pkg][lang]={} 133 134 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 135 136 self.indent=" " 137 if self.verbose: print('Finishing the constructor') 138 return 139 140 def srcrelpath(self,rdir): 141 """ 142 Get relative path to source directory 143 """ 144 return os.path.relpath(rdir,self.srcdir) 145 146 def getInInstallDir(self,thisscriptdir): 147 """ 148 When petsc is installed then this file in installed in: 149 <PREFIX>/share/petsc/examples/config/gmakegentest.py 150 otherwise the path is: 151 <PETSC_DIR>/config/gmakegentest.py 152 We use this difference to determine if we are in installdir 153 """ 154 dirlist=thisscriptdir.split(os.path.sep) 155 if len(dirlist)>4: 156 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 157 if lastfour==os.path.join('share','petsc','examples','config'): 158 return True 159 else: 160 return False 161 else: 162 return False 163 164 def nameSpace(self,srcfile,srcdir): 165 """ 166 Because the scripts have a non-unique naming, the pretty-printing 167 needs to convey the srcdir and srcfile. There are two ways of doing this. 168 """ 169 if self.ptNaming: 170 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 171 cdir=srcdir 172 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 173 nameString=prefix+srcfile 174 else: 175 #nameString=srcdir+": "+srcfile 176 nameString=srcfile 177 return nameString 178 179 def getLanguage(self,srcfile): 180 """ 181 Based on the source, determine associated language as found in gmakegen.LANGS 182 Can we just return srcext[1:\] now? 183 """ 184 langReq=None 185 srcext=os.path.splitext(srcfile)[-1] 186 if srcext in ".F90".split(): langReq="F90" 187 if srcext in ".F".split(): langReq="F" 188 if srcext in ".cxx".split(): langReq="cxx" 189 if srcext == ".cu": langReq="cu" 190 if srcext == ".c": langReq="c" 191 #if not langReq: print("ERROR: ", srcext, srcfile) 192 return langReq 193 194 def _getLoopVars(self,inDict,testname, isSubtest=False): 195 """ 196 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 197 Return: 198 inDict['args']: -ksp_monitor 199 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 200 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 201 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 202 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 203 subst should be passed in instead of inDict 204 """ 205 loopVars={}; newargs=[] 206 lsuffix='_' 207 argregex=re.compile('(?<![\w])(?=-[a-zA-Z])') 208 from testparse import parseLoopArgs 209 for key in inDict: 210 if key in ('SKIP', 'regexes'): 211 continue 212 akey=('subargs' if key=='args' else key) # what to assign 213 if akey not in inDict: inDict[akey]='' 214 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 215 # Always generate a loop over nsize, even if there is only one value 216 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 217 keystr = str(inDict[key]) 218 varlist = [] 219 for varset in argregex.split(keystr): 220 if not varset.strip(): continue 221 if '{{' in varset: 222 keyvar,lvars,ftype=parseLoopArgs(varset) 223 if akey not in loopVars: loopVars[akey]={} 224 varlist.append(keyvar) 225 loopVars[akey][keyvar]=[keyvar,lvars] 226 if akey=='nsize': 227 if len(lvars.split()) > 1: 228 inDict[akey] = '${' + keyvar + '}' 229 lsuffix+=akey+'-'+inDict[akey]+'_' 230 else: 231 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 232 lsuffix+=keyvar+'-${' + keyvar + '}_' 233 else: 234 if key=='args': 235 newargs.append(varset.strip()) 236 if varlist: 237 loopVars[akey]['varlist']=varlist 238 239 # For subtests, args are always substituted in (not top level) 240 if isSubtest: 241 inDict['subargs'] += " ".join(newargs) 242 inDict['args']='' 243 if 'label_suffix' in inDict: 244 inDict['label_suffix']+=lsuffix.rstrip('_') 245 else: 246 inDict['label_suffix']=lsuffix.rstrip('_') 247 else: 248 if loopVars: 249 inDict['args'] = ' '.join(newargs) 250 inDict['label_suffix']=lsuffix.rstrip('_') 251 return loopVars 252 253 def getArgLabel(self,testDict): 254 """ 255 In all of the arguments in the test dictionary, create a simple 256 string for searching within the makefile system. For simplicity in 257 search, remove "-", for strings, etc. 258 Also, concatenate the arg commands 259 For now, ignore nsize -- seems hard to search for anyway 260 """ 261 # Collect all of the args associated with a test 262 argStr=("" if 'args' not in testDict else testDict['args']) 263 if 'subtests' in testDict: 264 for stest in testDict["subtests"]: 265 sd=testDict[stest] 266 argStr=argStr+("" if 'args' not in sd else sd['args']) 267 268 # Now go through and cleanup 269 argStr=re.sub('{{(.*?)}}',"",argStr) 270 argStr=re.sub('-'," ",argStr) 271 for digit in string.digits: argStr=re.sub(digit," ",argStr) 272 argStr=re.sub("\.","",argStr) 273 argStr=re.sub(",","",argStr) 274 argStr=re.sub('\+',' ',argStr) 275 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 276 return argStr.strip() 277 278 def addToSources(self,exfile,rpath,srcDict): 279 """ 280 Put into data structure that allows easy generation of makefile 281 """ 282 pkg=rpath.split(os.path.sep)[0] 283 relpfile=os.path.join(rpath,exfile) 284 lang=self.getLanguage(exfile) 285 if not lang: return 286 self.sources[pkg][lang]['srcs'].append(relpfile) 287 self.sources[pkg][lang][relpfile] = [] 288 if 'depends' in srcDict: 289 depSrcList=srcDict['depends'].split() 290 for depSrc in depSrcList: 291 depObj=os.path.splitext(depSrc)[0]+".o" 292 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 293 294 # In gmakefile, ${TESTDIR} var specifies the object compilation 295 testsdir=rpath+"/" 296 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 297 self.objects[pkg].append(objfile) 298 return 299 300 def addToTests(self,test,rpath,exfile,execname,testDict): 301 """ 302 Put into data structure that allows easy generation of makefile 303 Organized by languages to allow testing of languages 304 """ 305 pkg=rpath.split("/")[0] 306 nmtest=os.path.join(rpath,test) 307 lang=self.getLanguage(exfile) 308 if not lang: return 309 self.tests[pkg][lang][nmtest]={} 310 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 311 self.tests[pkg][lang][nmtest]['exec']=execname 312 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 313 return 314 315 def getExecname(self,exfile,rpath): 316 """ 317 Generate bash script using template found next to this file. 318 This file is read in at constructor time to avoid file I/O 319 """ 320 if self.single_ex: 321 execname=rpath.split("/")[1]+"-ex" 322 else: 323 execname=os.path.splitext(exfile)[0] 324 return execname 325 326 def getSubstVars(self,testDict,rpath,testname): 327 """ 328 Create a dictionary with all of the variables that get substituted 329 into the template commands found in example_template.py 330 """ 331 subst={} 332 333 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 334 if 'nsize' not in testDict: testDict['nsize'] = '1' 335 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 336 for ak in testparse.acceptedkeys: 337 if ak=='test': continue 338 subst[ak]=(testDict[ak] if ak in testDict else '') 339 340 # Now do other variables 341 subst['execname']=testDict['execname'] 342 if 'filter' in testDict: 343 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 344 345 # Others 346 subst['subargs']='' # Default. For variables override 347 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 348 subst['label_suffix']='' 349 subst['comments']="\n#".join(subst['comments'].split("\n")) 350 if subst['comments']: subst['comments']="#"+subst['comments'] 351 subst['exec']="../"+subst['execname'] 352 subst['testroot']=self.testroot_dir 353 subst['testname']=testname 354 dp = self.conf.get('DATAFILESPATH','') 355 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 356 357 # This is used to label some matrices 358 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 359 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 360 361 #Conf vars 362 if self.petsc_arch.find('valgrind')>=0: 363 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 364 else: 365 subst['mpiexec']=self.conf['MPIEXEC'] 366 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 367 subst['petsc_arch']=self.petsc_arch 368 if self.inInstallDir: 369 # Case 2 370 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 371 else: 372 # Case 1 373 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 374 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 375 subst['diff']=self.conf['DIFF'] 376 subst['rm']=self.conf['RM'] 377 subst['grep']=self.conf['GREP'] 378 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 379 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 380 381 # Output file is special because of subtests override 382 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 383 if not "_" in defroot: defroot=defroot+"_1" 384 subst['defroot']=defroot 385 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 386 subst['redirect_file']=defroot+".tmp" 387 if 'output_file' not in testDict: 388 subst['output_file']="output/"+defroot+".out" 389 # Add in the full path here. 390 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 391 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 392 if not subst['TODO']: 393 print("Warning: "+subst['output_file']+" not found.") 394 # Worry about alt files here -- see 395 # src/snes/examples/tutorials/output/ex22*.out 396 altlist=[subst['output_file']] 397 basefile,ext = os.path.splitext(subst['output_file']) 398 for i in range(1,9): 399 altroot=basefile+"_alt" 400 if i > 1: altroot=altroot+"_"+str(i) 401 af=altroot+".out" 402 srcaf=os.path.join(subst['srcdir'],af) 403 fullaf=os.path.join(self.petsc_dir,srcaf) 404 if os.path.isfile(fullaf): altlist.append(srcaf) 405 if len(altlist)>1: subst['altfiles']=altlist 406 #if len(altlist)>1: print("Found alt files: ",altlist) 407 408 subst['regexes']={} 409 for subkey in subst: 410 if subkey=='regexes': continue 411 if not isinstance(subst[subkey],str): continue 412 patt="@"+subkey.upper()+"@" 413 subst['regexes'][subkey]=re.compile(patt) 414 415 return subst 416 417 def _substVars(self,subst,origStr): 418 """ 419 Substitute variables 420 """ 421 Str=origStr 422 for subkey in subst: 423 if subkey=='regexes': continue 424 if not isinstance(subst[subkey],str): continue 425 if subkey.upper() not in Str: continue 426 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 427 return Str 428 429 def getCmds(self,subst,i): 430 """ 431 Generate bash script using template found next to this file. 432 This file is read in at constructor time to avoid file I/O 433 """ 434 nindnt=i # the start and has to be consistent with below 435 cmdindnt=self.indent*nindnt 436 cmdLines="" 437 438 # MPI is the default -- but we have a few odd commands 439 if not subst['command']: 440 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 441 else: 442 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 443 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 444 445 cmdLines+=cmdindnt+'if test $res = 0; then\n' 446 diffindnt=self.indent*(nindnt+1) 447 if not subst['filter_output']: 448 if 'altfiles' not in subst: 449 cmd=diffindnt+self._substVars(subst,example_template.difftest) 450 else: 451 # Have to do it by hand a bit because of variable number of alt files 452 rf=subst['redirect_file'] 453 cmd=diffindnt+example_template.difftest.split('@')[0] 454 for i in range(len(subst['altfiles'])): 455 af=subst['altfiles'][i] 456 cmd+=af+' '+rf 457 if i!=len(subst['altfiles'])-1: 458 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 459 cmd+=' || ${diff_exe} ' 460 else: 461 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 462 cmd+=subst['label_suffix']+' ""' # Quotes are painful 463 else: 464 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 465 cmdLines+=cmd+"\n" 466 cmdLines+=cmdindnt+'else\n' 467 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 468 cmdLines+=cmdindnt+'fi\n' 469 return cmdLines 470 471 def _writeTodoSkip(self,fh,tors,reasons,footer): 472 """ 473 Write out the TODO and SKIP lines in the file 474 The TODO or SKIP variable, tors, should be lower case 475 """ 476 TORS=tors.upper() 477 template=eval("example_template."+tors+"line") 478 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 479 tab = '' 480 if reasons: 481 fh.write('if ! $force; then\n') 482 tab = tab + ' ' 483 if reasons == ["Requires DATAFILESPATH"]: 484 # The only reason not to run is DATAFILESPATH, which we check at run-time 485 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 486 tab = tab + ' ' 487 if reasons: 488 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 489 fh.write(tab+footer+"\n") 490 fh.write(tab+"exit\n") 491 if reasons == ["Requires DATAFILESPATH"]: 492 fh.write(' fi\n') 493 if reasons: 494 fh.write('fi\n') 495 fh.write('\n\n') 496 return 497 498 def getLoopVarsHead(self,loopVars,i): 499 """ 500 Generate a nicely indented string with the format loops 501 Here is what the data structure looks like 502 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 503 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 504 loopVars['subargs']['pc_type']=["j","cholesky sor"] 505 """ 506 outstr=''; indnt=self.indent 507 for key in loopVars: 508 for var in loopVars[key]['varlist']: 509 varval=loopVars[key][var] 510 outstr += indnt * i + "for {0} in ${{{0}:-{1}}}; do\n".format(*varval) 511 i = i + 1 512 return (outstr,i) 513 514 def getLoopVarsFoot(self,loopVars,i): 515 outstr=''; indnt=self.indent 516 for key in loopVars: 517 for var in loopVars[key]['varlist']: 518 i = i - 1 519 outstr += indnt * i + "done\n" 520 return (outstr,i) 521 522 def genRunScript(self,testname,root,isRun,srcDict): 523 """ 524 Generate bash script using template found next to this file. 525 This file is read in at constructor time to avoid file I/O 526 """ 527 # runscript_dir directory has to be consistent with gmakefile 528 testDict=srcDict[testname] 529 rpath=self.srcrelpath(root) 530 runscript_dir=os.path.join(self.testroot_dir,rpath) 531 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 532 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 533 534 # Get variables to go into shell scripts. last time testDict used 535 subst=self.getSubstVars(testDict,rpath,testname) 536 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 537 538 #Handle runfiles 539 for lfile in subst.get('localrunfiles','').split(): 540 install_files(os.path.join(root, lfile), 541 os.path.join(runscript_dir, os.path.dirname(lfile))) 542 # Check subtests for local runfiles 543 for stest in subst.get("subtests",[]): 544 for lfile in testDict[stest].get('localrunfiles','').split(): 545 install_files(os.path.join(root, lfile), 546 os.path.join(runscript_dir, os.path.dirname(lfile))) 547 548 # Now substitute the key variables into the header and footer 549 header=self._substVars(subst,example_template.header) 550 # The header is done twice to enable @...@ in header 551 header=self._substVars(subst,header) 552 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 553 554 # Start writing the file 555 fh.write(header+"\n") 556 557 # If there is a TODO or a SKIP then we do it before writing out the 558 # rest of the command (which is useful for working on the test) 559 # SKIP and TODO can be for the source file or for the runs 560 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 561 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 562 563 j=0 # for indentation 564 565 if loopVars: 566 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 567 if (loopHead): fh.write(loopHead+"\n") 568 569 # Subtests are special 570 if 'subtests' in testDict: 571 substP=subst # Subtests can inherit args but be careful 572 k=0 # for label suffixes 573 for stest in testDict["subtests"]: 574 subst=substP.copy() 575 subst.update(testDict[stest]) 576 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 577 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 578 if sLoopVars: 579 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 580 fh.write(sLoopHead+"\n") 581 fh.write(self.getCmds(subst,j)+"\n") 582 if sLoopVars: 583 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 584 fh.write(sLoopFoot+"\n") 585 else: 586 fh.write(self.getCmds(subst,j)+"\n") 587 588 if loopVars: 589 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 590 fh.write(loopFoot+"\n") 591 592 fh.write(footer+"\n") 593 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 594 #if '10_9' in testname: sys.exit() 595 return 596 597 def genScriptsAndInfo(self,exfile,root,srcDict): 598 """ 599 Generate scripts from the source file, determine if built, etc. 600 For every test in the exfile with info in the srcDict: 601 1. Determine if it needs to be run for this arch 602 2. Generate the script 603 3. Generate the data needed to write out the makefile in a 604 convenient way 605 All tests are *always* run, but some may be SKIP'd per the TAP standard 606 """ 607 debug=False 608 rpath=self.srcrelpath(root) 609 execname=self.getExecname(exfile,rpath) 610 isBuilt=self._isBuilt(exfile,srcDict) 611 for test in srcDict: 612 if test in self.buildkeys: continue 613 if debug: print(self.nameSpace(exfile,root), test) 614 srcDict[test]['execname']=execname # Convenience in generating scripts 615 isRun=self._isRun(srcDict[test]) 616 self.genRunScript(test,root,isRun,srcDict) 617 srcDict[test]['isrun']=isRun 618 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 619 620 # This adds to datastructure for building deps 621 if isBuilt: self.addToSources(exfile,rpath,srcDict) 622 return 623 624 def _isBuilt(self,exfile,srcDict): 625 """ 626 Determine if this file should be built. 627 """ 628 # Get the language based on file extension 629 srcDict['SKIP'] = [] 630 lang=self.getLanguage(exfile) 631 if (lang=="F" or lang=="F90"): 632 if not self.have_fortran: 633 srcDict["SKIP"].append("Fortran required for this test") 634 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 635 srcDict["SKIP"].append("Fortran f90freeform required for this test") 636 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 637 srcDict["SKIP"].append("CUDA required for this test") 638 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 639 srcDict["SKIP"].append("C++ required for this test") 640 641 # Deprecated source files 642 if srcDict.get("TODO"): 643 return False 644 645 # isRun can work with srcDict to handle the requires 646 if "requires" in srcDict: 647 if srcDict["requires"]: 648 return self._isRun(srcDict) 649 650 return srcDict['SKIP'] == [] 651 652 653 def _isRun(self,testDict, debug=False): 654 """ 655 Based on the requirements listed in the src file and the petscconf.h 656 info, determine whether this test should be run or not. 657 """ 658 indent=" " 659 660 if 'SKIP' not in testDict: 661 testDict['SKIP'] = [] 662 # MPI requirements 663 if 'MPI_IS_MPIUNI' in self.conf: 664 if testDict.get('nsize', '1') != '1': 665 testDict['SKIP'].append("Parallel test with serial build") 666 667 # The requirements for the test are the sum of all the run subtests 668 if 'subtests' in testDict: 669 if 'requires' not in testDict: testDict['requires']="" 670 for stest in testDict['subtests']: 671 if 'requires' in testDict[stest]: 672 testDict['requires']+=" "+testDict[stest]['requires'] 673 if testDict.get('nsize', '1') != '1': 674 testDict['SKIP'].append("Parallel test with serial build") 675 break 676 677 # Now go through all requirements 678 if 'requires' in testDict: 679 for requirement in testDict['requires'].split(): 680 requirement=requirement.strip() 681 if not requirement: continue 682 if debug: print(indent+"Requirement: ", requirement) 683 isNull=False 684 if requirement.startswith("!"): 685 requirement=requirement[1:]; isNull=True 686 # Precision requirement for reals 687 if requirement in self.precision_types: 688 if self.conf['PETSC_PRECISION']==requirement: 689 if isNull: 690 testDict['SKIP'].append("not "+requirement+" required") 691 continue 692 continue # Success 693 elif not isNull: 694 testDict['SKIP'].append(requirement+" required") 695 continue 696 # Precision requirement for ints 697 if requirement in self.integer_types: 698 if requirement=="int32": 699 if self.conf['PETSC_SIZEOF_INT']==4: 700 if isNull: 701 testDict['SKIP'].append("not int32 required") 702 continue 703 continue # Success 704 elif not isNull: 705 testDict['SKIP'].append("int32 required") 706 continue 707 if requirement=="int64": 708 if self.conf['PETSC_SIZEOF_INT']==8: 709 if isNull: 710 testDict['SKIP'].append("NOT int64 required") 711 continue 712 continue # Success 713 elif not isNull: 714 testDict['SKIP'].append("int64 required") 715 continue 716 # Datafilespath 717 if requirement=="datafilespath" and not isNull: 718 testDict['SKIP'].append("Requires DATAFILESPATH") 719 continue 720 # Defines -- not sure I have comments matching 721 if "define(" in requirement.lower(): 722 reqdef=requirement.split("(")[1].split(")")[0] 723 if reqdef in self.conf: 724 if isNull: 725 testDict['SKIP'].append("Null requirement not met: "+requirement) 726 continue 727 continue # Success 728 elif not isNull: 729 testDict['SKIP'].append("Required: "+requirement) 730 continue 731 732 # Rest should be packages that we can just get from conf 733 if requirement == "complex": 734 petscconfvar="PETSC_USE_COMPLEX" 735 else: 736 petscconfvar="PETSC_HAVE_"+requirement.upper() 737 if self.conf.get(petscconfvar): 738 if isNull: 739 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 740 continue 741 continue # Success 742 elif not isNull: 743 if debug: print("requirement not found: ", requirement) 744 testDict['SKIP'].append(petscconfvar+" requirement not met") 745 continue 746 747 return testDict['SKIP'] == [] 748 749 def genPetscTests_summarize(self,dataDict): 750 """ 751 Required method to state what happened 752 """ 753 if not self.summarize: return 754 indent=" " 755 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 756 fh=open(fhname,"w") 757 for root in dataDict: 758 relroot=self.srcrelpath(root) 759 pkg=relroot.split("/")[1] 760 fh.write(relroot+"\n") 761 allSrcs=[] 762 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 763 for exfile in dataDict[root]: 764 # Basic information 765 rfile=os.path.join(relroot,exfile) 766 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 767 fh.write(indent+exfile+indent*4+builtStatus+"\n") 768 769 for test in dataDict[root][exfile]: 770 if test in self.buildkeys: continue 771 line=indent*2+test 772 fh.write(line+"\n") 773 # Looks nice to have the keys in order 774 #for key in dataDict[root][exfile][test]: 775 for key in "isrun abstracted nsize args requires script".split(): 776 if key not in dataDict[root][exfile][test]: continue 777 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 778 fh.write(line+"\n") 779 fh.write("\n") 780 fh.write("\n") 781 fh.write("\n") 782 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 783 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 784 fh.close() 785 return 786 787 def genPetscTests(self,root,dirs,files,dataDict): 788 """ 789 Go through and parse the source files in the directory to generate 790 the examples based on the metadata contained in the source files 791 """ 792 debug=False 793 # Use examplesAnalyze to get what the makefles think are sources 794 #self.examplesAnalyze(root,dirs,files,anlzDict) 795 796 dataDict[root]={} 797 798 for exfile in files: 799 #TST: Until we replace files, still leaving the orginals as is 800 #if not exfile.startswith("new_"+"ex"): continue 801 #if not exfile.startswith("ex"): continue 802 803 # Ignore emacs and other temporary files 804 if exfile.startswith("."): continue 805 if exfile.startswith("#"): continue 806 if exfile.endswith("~"): continue 807 # Only parse source files 808 ext=os.path.splitext(exfile)[-1].lstrip('.') 809 if ext not in LANGS: continue 810 811 # Convenience 812 fullex=os.path.join(root,exfile) 813 if self.verbose: print(' --> '+fullex) 814 dataDict[root].update(testparse.parseTestFile(fullex,0)) 815 if exfile in dataDict[root]: 816 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 817 818 return 819 820 def walktree(self,top): 821 """ 822 Walk a directory tree, starting from 'top' 823 """ 824 # Goal of action is to fill this dictionary 825 dataDict={} 826 for root, dirs, files in os.walk(top, topdown=True): 827 dirs.sort() 828 files.sort() 829 if not "examples" in root: continue 830 if "dSYM" in root: continue 831 if os.path.basename(root.rstrip("/")) == 'output': continue 832 if self.verbose: print(root) 833 self.genPetscTests(root,dirs,files,dataDict) 834 # Now summarize this dictionary 835 if self.verbose: self.genPetscTests_summarize(dataDict) 836 return dataDict 837 838 def gen_gnumake(self, fd): 839 """ 840 Overwrite of the method in the base PETSc class 841 """ 842 def write(stem, srcs): 843 for lang in LANGS: 844 if srcs[lang]['srcs']: 845 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 846 for pkg in PKGS: 847 srcs = self.gen_pkg(pkg) 848 write('testsrcs-' + pkg, srcs) 849 # Handle dependencies 850 for lang in LANGS: 851 for exfile in srcs[lang]['srcs']: 852 if exfile in srcs[lang]: 853 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 854 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 855 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 856 if deps: 857 # The executable literally depends on the object file because it is linked 858 fd.write(ex +": " + " ".join(deps) +'\n') 859 # The object file containing 'main' does not normally depend on other object 860 # files, but it does when it includes their modules. This dependency is 861 # overly blunt and could be reduced to only depend on object files for 862 # modules that are used, like "*f90aux.o". 863 fd.write(exfo +": " + " ".join(deps) +'\n') 864 865 return self.gendeps 866 867 def gen_pkg(self, pkg): 868 """ 869 Overwrite of the method in the base PETSc class 870 """ 871 return self.sources[pkg] 872 873 def write_gnumake(self, dataDict, output=None): 874 """ 875 Write out something similar to files from gmakegen.py 876 877 Test depends on script which also depends on source 878 file, but since I don't have a good way generating 879 acting on a single file (oops) just depend on 880 executable which in turn will depend on src file 881 """ 882 # Different options for how to set up the targets 883 compileExecsFirst=False 884 885 # Open file 886 fd = open(output, 'w') 887 888 # Write out the sources 889 gendeps = self.gen_gnumake(fd) 890 891 # Write out the tests and execname targets 892 fd.write("\n#Tests and executables\n") # Delimiter 893 894 for pkg in PKGS: 895 # These grab the ones that are built 896 for lang in LANGS: 897 testdeps=[] 898 for ftest in self.tests[pkg][lang]: 899 test=os.path.basename(ftest) 900 basedir=os.path.dirname(ftest) 901 testdeps.append(self.nameSpace(test,basedir)) 902 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 903 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 904 905 # test targets 906 for ftest in self.tests[pkg][lang]: 907 test=os.path.basename(ftest) 908 basedir=os.path.dirname(ftest) 909 testdir="${TESTDIR}/"+basedir+"/" 910 nmtest=self.nameSpace(test,basedir) 911 rundir=os.path.join(testdir,test) 912 script=test+".sh" 913 914 # Deps 915 exfile=self.tests[pkg][lang][ftest]['exfile'] 916 fullex=os.path.join(self.srcdir,exfile) 917 localexec=self.tests[pkg][lang][ftest]['exec'] 918 execname=os.path.join(testdir,localexec) 919 fullscript=os.path.join(testdir,script) 920 tmpfile=os.path.join(testdir,test,test+".tmp") 921 922 # *.counts depends on the script and either executable (will 923 # be run) or the example source file (SKIP or TODO) 924 fd.write('%s.counts : %s %s' 925 % (os.path.join('$(TESTDIR)/counts', nmtest), 926 fullscript, 927 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 928 ) 929 if exfile in self.sources[pkg][lang]: 930 for dep in self.sources[pkg][lang][exfile]: 931 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 932 fd.write('\n') 933 934 # Now write the args: 935 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 936 937 fd.close() 938 return 939 940def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 941 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 942 if petsc_arch: 943 if len(petsc_arch.split(os.path.sep))>1: 944 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 945 output = os.path.join(testdir, 'testfiles') 946 947 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 948 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 949 testdir=testdir) 950 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 951 pEx.write_gnumake(dataDict, output) 952 953if __name__ == '__main__': 954 import optparse 955 parser = optparse.OptionParser() 956 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 957 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 958 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 959 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 960 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 961 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 962 963 opts, extra_args = parser.parse_args() 964 if extra_args: 965 import sys 966 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 967 exit(1) 968 if opts.testdir is None: 969 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 970 971 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 972 verbose=opts.verbose, 973 single_ex=opts.single_executable, srcdir=opts.srcdir, 974 testdir=opts.testdir) 975