xref: /petsc/config/gmakegentest.py (revision cf9fd96efb2244135eb1d215a552a023f1eb8b10)

#!/usr/bin/env python3

from __future__ import print_function
import pickle
import os,shutil, string, re
import sys
import logging, time
import types
import shlex
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
from collections import defaultdict
from gmakegen import *

import inspect
thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
sys.path.insert(0,thisscriptdir)
import testparse
import example_template


"""

There are 2 modes of running tests: Normal builds and run from prefix of
install.  They affect where to find things:


Case 1.  Normal builds:

     +---------------------+----------------------------------+
     | PETSC_DIR           | <git dir>                        |
     +---------------------+----------------------------------+
     | PETSC_ARCH          | arch-foo                         |
     +---------------------+----------------------------------+
     | PETSC_LIBDIR        | PETSC_DIR/PETSC_ARCH/lib         |
     +---------------------+----------------------------------+
     | PETSC_EXAMPLESDIR   | PETSC_DIR/src                    |
     +---------------------+----------------------------------+
     | PETSC_TESTDIR       | PETSC_DIR/PETSC_ARCH/tests       |
     +---------------------+----------------------------------+
     | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test         |
     +---------------------+----------------------------------+
     | PETSC_GMAKEGENTEST  | PETSC_DIR/config/gmakegentest.py |
     +---------------------+----------------------------------+


Case 2.  From install dir:

     +---------------------+-------------------------------------------------------+
     | PETSC_DIR           | <prefix dir>                                          |
     +---------------------+-------------------------------------------------------+
     | PETSC_ARCH          | ''                                                    |
     +---------------------+-------------------------------------------------------+
     | PETSC_LIBDIR        | PETSC_DIR/PETSC_ARCH/lib                              |
     +---------------------+-------------------------------------------------------+
     | PETSC_EXAMPLESDIR   | PETSC_DIR/share/petsc/examples/src                    |
     +---------------------+-------------------------------------------------------+
     | PETSC_TESTDIR       | PETSC_DIR/PETSC_ARCH/tests                            |
     +---------------------+-------------------------------------------------------+
     | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test         |
     +---------------------+-------------------------------------------------------+
     | PETSC_GMAKEGENTEST  | PETSC_DIR/share/petsc/examples/config/gmakegentest.py |
     +---------------------+-------------------------------------------------------+

"""

def install_files(source, destdir):
  """Install file or directory 'source' to 'destdir'.  Does not preserve
  mode (permissions).
  """
  if not os.path.isdir(destdir):
    os.makedirs(destdir)
  if os.path.isdir(source):
    for name in os.listdir(source):
      install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source)))
  else:
    shutil.copyfile(source, os.path.join(destdir, os.path.basename(source)))

def nameSpace(srcfile,srcdir):
  """
  Because the scripts have a non-unique naming, the pretty-printing
  needs to convey the srcdir and srcfile.  There are two ways of doing this.
  """
  if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile)
  prefix=srcdir.replace("/","_")+"-"
  nameString=prefix+srcfile
  return nameString

class generateExamples(Petsc):
  """
    gmakegen.py has basic structure for finding the files, writing out
      the dependencies, etc.
  """
  def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False):
    super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs)

    self.single_ex=single_ex
    self.srcdir=srcdir
    self.check_output=check

    # Set locations to handle movement
    self.inInstallDir=self.getInInstallDir(thisscriptdir)

    # Special configuration for CI testing
    if self.petsc_arch.find('valgrind') >= 0:
      self.conf['PETSCTEST_VALGRIND']=1

    if self.inInstallDir:
      # Case 2 discussed above
      # set PETSC_ARCH to install directory to allow script to work in both
      dirlist=thisscriptdir.split(os.path.sep)
      installdir=os.path.sep.join(dirlist[0:len(dirlist)-4])
      self.arch_dir=installdir
      if self.srcdir is None:
        self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src')
    else:
      if petsc_arch == '':
        raise RuntimeError('PETSC_ARCH must be set when running from build directory')
      # Case 1 discussed above
      self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch)
      if self.srcdir is None:
        self.srcdir=os.path.join(self.petsc_dir,'src')

    self.testroot_dir=os.path.abspath(testdir)

    self.verbose=verbose
    # Whether to write out a useful debugging
    self.summarize=True if verbose else False

    # For help in setting the requirements
    self.precision_types="__fp16 single double __float128".split()
    self.integer_types="int32 int64 long32 long64".split()
    self.languages="fortran cuda hip sycl cxx cpp".split()    # Always requires C so do not list

    # Things that are not test
    self.buildkeys=testparse.buildkeys

    # Adding a dictionary for storing sources, objects, and tests
    # to make building the dependency tree easier
    self.sources={}
    self.objects={}
    self.tests={}
    for pkg in self.pkg_pkgs:
      self.sources[pkg]={}
      self.objects[pkg]=[]
      self.tests[pkg]={}
      for lang in LANGS:
        self.sources[pkg][lang]={}
        self.sources[pkg][lang]['srcs']=[]
        self.tests[pkg][lang]={}

    if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir)

    self.indent="   "
    if self.verbose: print('Finishing the constructor')
    return

  def srcrelpath(self,rdir):
    """
    Get relative path to source directory
    """
    return os.path.relpath(rdir,self.srcdir)

  def getInInstallDir(self,thisscriptdir):
    """
    When PETSc is installed then this file in installed in:
         <PREFIX>/share/petsc/examples/config/gmakegentest.py
    otherwise the path is:
         <PETSC_DIR>/config/gmakegentest.py
    We use this difference to determine if we are in installdir
    """
    dirlist=thisscriptdir.split(os.path.sep)
    if len(dirlist)>4:
      lastfour=os.path.sep.join(dirlist[len(dirlist)-4:])
      if lastfour==os.path.join('share','petsc','examples','config'):
        return True
      else:
        return False
    else:
      return False

  def getLanguage(self,srcfile):
    """
    Based on the source, determine associated language as found in gmakegen.LANGS
    Can we just return srcext[1:] now?
    """
    langReq=None
    srcext = getlangext(srcfile)
    if srcext in ".F90".split(): langReq="F90"
    if srcext in ".F".split(): langReq="F"
    if srcext in ".cxx".split(): langReq="cxx"
    if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx"
    if srcext in ".hip.cpp".split(): langReq="hip_cpp"
    if srcext in ".raja.cxx".split(): langReq="raja_cxx"
    if srcext in ".cpp".split(): langReq="cpp"
    if srcext == ".cu": langReq="cu"
    if srcext == ".c": langReq="c"
    #if not langReq: print("ERROR: ", srcext, srcfile)
    return langReq

  def _getAltList(self,output_file,srcdir):
    ''' Calculate AltList based on output file-- see
       src/snes/tutorials/output/ex22*.out
    '''
    altlist=[output_file]
    basefile = getlangsplit(output_file)
    for i in range(1,9):
      altroot=basefile+"_alt"
      if i > 1: altroot=altroot+"_"+str(i)
      af=altroot+".out"
      srcaf=os.path.join(srcdir,af)
      fullaf=os.path.join(self.petsc_dir,srcaf)
      if os.path.isfile(fullaf): altlist.append(srcaf)

    return altlist


  def _getLoopVars(self,inDict,testname, isSubtest=False):
    """
    Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor'
    Return:
      inDict['args']: -ksp_monitor
      inDict['subargs']: -bs ${bs} -pc_type ${pc_type}
      loopVars['subargs']['varlist']=['bs' 'pc_type']   # Don't worry about OrderedDict
      loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]]
      loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]]
    subst should be passed in instead of inDict
    """
    loopVars={}; newargs=[]
    lsuffix='+'
    argregex = re.compile(' (?=-[a-zA-Z])')
    from testparse import parseLoopArgs
    for key in inDict:
      if key in ('SKIP', 'regexes'):
        continue
      akey=('subargs' if key=='args' else key)  # what to assign
      if akey not in inDict: inDict[akey]=''
      if akey == 'nsize' and not inDict['nsize'].startswith('{{'):
        # Always generate a loop over nsize, even if there is only one value
        inDict['nsize'] = '{{' + inDict['nsize'] + '}}'
      keystr = str(inDict[key])
      varlist = []
      for varset in argregex.split(keystr):
        if not varset.strip(): continue
        if '{{' in varset:
          keyvar,lvars,ftype=parseLoopArgs(varset)
          if akey not in loopVars: loopVars[akey]={}
          varlist.append(keyvar)
          loopVars[akey][keyvar]=[keyvar,lvars]
          if akey=='nsize':
            if len(lvars.split()) > 1:
              lsuffix += akey +'-${i' + keyvar + '}'
          else:
            inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}'
            lsuffix+=keyvar+'-${i' + keyvar + '}_'
        else:
          if key=='args':
            newargs.append(varset.strip())
        if varlist:
          loopVars[akey]['varlist']=varlist

    # For subtests, args are always substituted in (not top level)
    if isSubtest:
      inDict['subargs'] += " "+" ".join(newargs)
      inDict['args']=''
      if 'label_suffix' in inDict:
        inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_')
      else:
        inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_')
    else:
      if loopVars:
        inDict['args'] = ' '.join(newargs)
        inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_')
    return loopVars

  def getArgLabel(self,testDict):
    """
    In all of the arguments in the test dictionary, create a simple
    string for searching within the makefile system.  For simplicity in
    search, remove "-", for strings, etc.
    Also, concatenate the arg commands
    For now, ignore nsize -- seems hard to search for anyway
    """
    # Collect all of the args associated with a test
    argStr=("" if 'args' not in testDict else testDict['args'])
    if 'subtests' in testDict:
      for stest in testDict["subtests"]:
         sd=testDict[stest]
         argStr=argStr+("" if 'args' not in sd else sd['args'])

    # Now go through and cleanup
    argStr=re.sub('{{(.*?)}}',"",argStr)
    argStr=re.sub('-'," ",argStr)
    for digit in string.digits: argStr=re.sub(digit," ",argStr)
    argStr=re.sub(r"\.","",argStr)
    argStr=re.sub(",","",argStr)
    argStr=re.sub(r'\+',' ',argStr)
    argStr=re.sub(' +',' ',argStr)  # Remove repeated white space
    return argStr.strip()

  def addToSources(self,exfile,rpath,srcDict):
    """
      Put into data structure that allows easy generation of makefile
    """
    pkg=rpath.split(os.path.sep)[0]
    relpfile=os.path.join(rpath,exfile)
    lang=self.getLanguage(exfile)
    if not lang: return
    if pkg not in self.sources: return
    self.sources[pkg][lang]['srcs'].append(relpfile)
    self.sources[pkg][lang][relpfile] = []
    if 'depends' in srcDict:
      depSrcList=srcDict['depends'].split()
      for depSrc in depSrcList:
        depObj = getlangsplit(depSrc)+'.o'
        self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj))

    # In gmakefile, ${TESTDIR} var specifies the object compilation
    testsdir=rpath+"/"
    objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o'
    self.objects[pkg].append(objfile)
    return

  def addToTests(self,test,rpath,exfile,execname,testDict):
    """
      Put into data structure that allows easy generation of makefile
      Organized by languages to allow testing of languages
    """
    pkg=rpath.split("/")[0]
    nmtest=os.path.join(rpath,test)
    lang=self.getLanguage(exfile)
    if not lang: return
    if pkg not in self.tests: return
    self.tests[pkg][lang][nmtest]={}
    self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile)
    self.tests[pkg][lang][nmtest]['exec']=execname
    self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict)
    return

  def getExecname(self,exfile,rpath):
    """
      Generate bash script using template found next to this file.
      This file is read in at constructor time to avoid file I/O
    """
    if self.single_ex:
      execname=rpath.split("/")[1]+"-ex"
    else:
      execname=getlangsplit(exfile)
    return execname

  def getSubstVars(self,testDict,rpath,testname):
    """
      Create a dictionary with all of the variables that get substituted
      into the template commands found in example_template.py
    """
    # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests)
    if 'nsize' not in testDict: testDict['nsize'] = '1'
    if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1"
    subst = {key : testDict.get(key, '') for key in testparse.acceptedkeys if key != 'test'}

    # Now do other variables
    subst['env'] = '\n'.join('export '+cmd for cmd in shlex.split(subst['env']))
    subst['execname']=testDict['execname']
    subst['error']=''
    if 'filter' in testDict:
      if testDict['filter'].startswith("Error:"):
        subst['error']="Error"
        subst['filter']=testDict['filter'].lstrip("Error:")
      else:
        subst['filter']=testDict['filter']

    # Others
    subst['subargs']=''  # Default.  For variables override
    subst['srcdir']=os.path.join(self.srcdir, rpath)
    subst['label_suffix']=''
    subst['comments']="\n#".join(subst['comments'].split("\n"))
    if subst['comments']: subst['comments']="#"+subst['comments']
    subst['executable']="../"+subst['execname']
    subst['testroot']=self.testroot_dir
    subst['testname']=testname
    dp = self.conf.get('DATAFILESPATH','')
    subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}'

    # This is used to label some matrices
    subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE'])
    subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE'])

    subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS']

    #Conf vars
    if self.petsc_arch.find('valgrind')>=0:
      subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC']
    else:
      subst['mpiexec']=self.conf['MPIEXEC']
    subst['mpiexec_tail']=self.conf['MPIEXEC_TAIL']
    subst['pkg_name']=self.pkg_name
    subst['pkg_dir']=self.pkg_dir
    subst['pkg_arch']=self.petsc_arch
    subst['CONFIG_DIR']=thisscriptdir
    subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin')
    subst['diff']=self.conf['DIFF']
    subst['rm']=self.conf['RM']
    subst['grep']=self.conf['GREP']
    subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR']
    subst['wpetsc_dir']=self.conf['wPETSC_DIR']

    # Output file is special because of subtests override
    defroot = testparse.getDefaultOutputFileRoot(testname)
    if 'output_file' not in testDict:
      subst['output_file']="output/"+defroot+".out"
    subst['redirect_file']=defroot+".tmp"
    subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir']))

    # Add in the full path here.
    subst['output_file']=os.path.join(subst['srcdir'],subst['output_file'])

    subst['regexes']={}
    for subkey in subst:
      if subkey=='regexes': continue
      if not isinstance(subst[subkey],str): continue
      patt="@"+subkey.upper()+"@"
      subst['regexes'][subkey]=re.compile(patt)

    return subst

  def _substVars(self,subst,origStr):
    """
      Substitute variables
    """
    Str=origStr
    for subkey, subvalue in subst.items():
      if subkey=='regexes': continue
      if not isinstance(subvalue,str): continue
      if subkey.upper() not in Str: continue
      Str=subst['regexes'][subkey].sub(lambda x: subvalue,Str)
    return Str

  def getCmds(self,subst,i, debug=False):
    """
      Generate bash script using template found next to this file.
      This file is read in at constructor time to avoid file I/O
    """
    nindnt=i # the start and has to be consistent with below
    cmdindnt=self.indent*nindnt
    cmdLines=""

    # MPI is the default -- but we have a few odd commands
    if subst['temporaries']:
      if '*' in subst['temporaries']:
        raise RuntimeError('{}/{}: list of temporary files to remove may not include wildcards'.format(subst['srcdir'], subst['execname']))
      cmd=cmdindnt+self._substVars(subst,example_template.preclean)
      cmdLines+=cmd+"\n"
    if not subst['command']:
      cmd=cmdindnt+self._substVars(subst,example_template.mpitest)
    else:
      cmd=cmdindnt+self._substVars(subst,example_template.commandtest)
    cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n"

    cmdLines+=cmdindnt+'if test $res = 0; then\n'
    diffindnt=self.indent*(nindnt+1)

    # Do some checks on existence of output_file and alt files
    if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])):
      if not subst['TODO']:
        print("Warning: "+subst['output_file']+" not found.")
    altlist=self._getAltList(subst['output_file'], subst['srcdir'])

    # altlist always has output_file
    if len(altlist)==1:
      cmd=diffindnt+self._substVars(subst,example_template.difftest)
    else:
      if debug: print("Found alt files: ",altlist)
      # Have to do it by hand a bit because of variable number of alt files
      rf=subst['redirect_file']
      cmd=diffindnt+example_template.difftest.split('@')[0]
      for i in range(len(altlist)):
        af=altlist[i]
        cmd+=af+' '+rf
        if i!=len(altlist)-1:
          cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out'
          cmd+=' || ${diff_exe} '
        else:
          cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}'
          cmd+=subst['label_suffix']+' ""'  # Quotes are painful
    cmdLines+=cmd+"\n"
    cmdLines+=cmdindnt+'else\n'
    cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n'
    cmdLines+=cmdindnt+'fi\n'
    return cmdLines

  def _writeTodoSkip(self,fh,tors,reasons,footer):
    """
    Write out the TODO and SKIP lines in the file
    The TODO or SKIP variable, tors, should be lower case
    """
    TORS=tors.upper()
    template=eval("example_template."+tors+"line")
    tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template)
    tab = ''
    if reasons:
      fh.write('if ! $force; then\n')
      tab = tab + '    '
    if reasons == ["Requires DATAFILESPATH"]:
      # The only reason not to run is DATAFILESPATH, which we check at run-time
      fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n')
      tab = tab + '    '
    if reasons:
      fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n")
      fh.write(tab+footer+"\n")
      fh.write(tab+"exit\n")
    if reasons == ["Requires DATAFILESPATH"]:
      fh.write('    fi\n')
    if reasons:
      fh.write('fi\n')
    fh.write('\n\n')
    return

  def getLoopVarsHead(self,loopVars,i,usedVars={}):
    """
    Generate a nicely indented string with the format loops
    Here is what the data structure looks like
      loopVars['subargs']['varlist']=['bs' 'pc_type']   # Don't worry about OrderedDict
      loopVars['subargs']['bs']=["i","1 2 3 4 5"]
      loopVars['subargs']['pc_type']=["j","cholesky sor"]
    """
    outstr=''; indnt=self.indent

    for key in loopVars:
      for var in loopVars[key]['varlist']:
        varval=loopVars[key][var]
        outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval)
    outstr += "\n\n"

    for key in loopVars:
      for var in loopVars[key]['varlist']:
        varval=loopVars[key][var]
        outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval)
        i = i + 1
    return (outstr,i)

  def getLoopVarsFoot(self,loopVars,i):
    outstr=''; indnt=self.indent
    for key in loopVars:
      for var in loopVars[key]['varlist']:
        i = i - 1
        outstr += indnt * i + "done\n"
    return (outstr,i)

  def genRunScript(self,testname,root,isRun,srcDict):
    """
      Generate bash script using template found next to this file.
      This file is read in at constructor time to avoid file I/O
    """
    def opener(path,flags,*args,**kwargs):
      kwargs.setdefault('mode',0o755)
      return os.open(path,flags,*args,**kwargs)

    # runscript_dir directory has to be consistent with gmakefile
    testDict=srcDict[testname]
    rpath=self.srcrelpath(root)
    runscript_dir=os.path.join(self.testroot_dir,rpath)
    if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir)
    with open(os.path.join(runscript_dir,testname+".sh"),"w",opener=opener) as fh:

      # Get variables to go into shell scripts.  last time testDict used
      subst=self.getSubstVars(testDict,rpath,testname)
      loopVars = self._getLoopVars(subst,testname)  # Alters subst as well
      if 'subtests' in testDict:
        # The subtests inherit inDict, so we don't need top-level loops.
        loopVars = {}

      #Handle runfiles
      for lfile in subst.get('localrunfiles','').split():
        install_files(os.path.join(root, lfile),
                      os.path.join(runscript_dir, os.path.dirname(lfile)))
      # Check subtests for local runfiles
      for stest in subst.get("subtests",[]):
        for lfile in testDict[stest].get('localrunfiles','').split():
          install_files(os.path.join(root, lfile),
                        os.path.join(runscript_dir, os.path.dirname(lfile)))

      # Now substitute the key variables into the header and footer
      header=self._substVars(subst,example_template.header)
      # The header is done twice to enable @...@ in header
      header=self._substVars(subst,header)
      footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer)

      # Start writing the file
      fh.write(header+"\n")

      # If there is a TODO or a SKIP then we do it before writing out the
      # rest of the command (which is useful for working on the test)
      # SKIP and TODO can be for the source file or for the runs
      self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer)
      self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer)

      j=0  # for indentation

      if loopVars:
        (loopHead,j) = self.getLoopVarsHead(loopVars,j)
        if (loopHead): fh.write(loopHead+"\n")

      # Subtests are special
      allLoopVars=list(loopVars.keys())
      if 'subtests' in testDict:
        substP=subst   # Subtests can inherit args but be careful
        k=0  # for label suffixes
        for stest in testDict["subtests"]:
          subst=substP.copy()
          subst.update(testDict[stest])
          subst['label_suffix']='+'+string.ascii_letters[k]; k+=1
          sLoopVars = self._getLoopVars(subst,testname,isSubtest=True)
          if sLoopVars:
            (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars)
            allLoopVars+=list(sLoopVars.keys())
            fh.write(sLoopHead+"\n")
          fh.write(self.getCmds(subst,j)+"\n")
          if sLoopVars:
            (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j)
            fh.write(sLoopFoot+"\n")
      else:
        fh.write(self.getCmds(subst,j)+"\n")

      if loopVars:
        (loopFoot,j) = self.getLoopVarsFoot(loopVars,j)
        fh.write(loopFoot+"\n")

      fh.write(footer+"\n")
    return

  def  genScriptsAndInfo(self,exfile,root,srcDict):
    """
    Generate scripts from the source file, determine if built, etc.
     For every test in the exfile with info in the srcDict:
      1. Determine if it needs to be run for this arch
      2. Generate the script
      3. Generate the data needed to write out the makefile in a
         convenient way
     All tests are *always* run, but some may be SKIP'd per the TAP standard
    """
    debug=False
    rpath=self.srcrelpath(root)
    execname=self.getExecname(exfile,rpath)
    isBuilt=self._isBuilt(exfile,srcDict)
    for test in srcDict.copy():
      if test in self.buildkeys: continue
      isRun=self._isRun(srcDict[test])
      # if the next two lines are dropped all scripts are generating included the unneeded
      # if the unneeded are generated when run they will skip their tests automatically
      # not generating them saves setup time
      allow = False
      if 'SKIP' in srcDict[test]:
        allow = srcDict[test]['SKIP'] in [['Requires DATAFILESPATH'], ['PETSC_HAVE_PYVISTA requirement not met']]
      if not isRun and not allow:
        del srcDict[test]
        continue
      if 'TODO' in srcDict[test]:
        del srcDict[test]
        continue
      srcDict[test]['execname']=execname   # Convenience in generating scripts
      self.genRunScript(test,root,isRun,srcDict)
      srcDict[test]['isrun']=isRun
      self.addToTests(test,rpath,exfile,execname,srcDict[test])

    # This adds to datastructure for building deps
    if isBuilt: self.addToSources(exfile,rpath,srcDict)
    return

  def _isBuilt(self,exfile,srcDict):
    """
    Determine if this file should be built.
    """
    # Get the language based on file extension
    srcDict['SKIP'] = []
    lang=self.getLanguage(exfile)
    if (lang=="F" or lang=="F90"):
      if not self.have_fortran:
        srcDict["SKIP"].append("Fortran required for this test")
      elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf:
        srcDict["SKIP"].append("Fortran f90freeform required for this test")
    if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf:
      srcDict["SKIP"].append("CUDA required for this test")
    if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf:
      srcDict["SKIP"].append("HIP required for this test")
    if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf:
      srcDict["SKIP"].append("SYCL required for this test")
    if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf:
      srcDict["SKIP"].append("KOKKOS required for this test")
    if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf:
      srcDict["SKIP"].append("RAJA required for this test")
    if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf:
      srcDict["SKIP"].append("C++ required for this test")
    if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf:
      srcDict["SKIP"].append("C++ required for this test")

    # Deprecated source files
    if srcDict.get("TODO"):
      return False

    # isRun can work with srcDict to handle the requires
    if "requires" in srcDict:
      if srcDict["requires"]:
        return self._isRun(srcDict)

    return srcDict['SKIP'] == []


  def _isRun(self,testDict, debug=False):
    """
    Based on the requirements listed in the src file and the petscconf.h
    info, determine whether this test should be run or not.
    """
    indent="  "

    if 'SKIP' not in testDict:
      testDict['SKIP'] = []
    # MPI requirements
    if 'MPI_IS_MPIUNI' in self.conf:
      if testDict.get('nsize', '1') != '1':
        testDict['SKIP'].append("Parallel test with serial build")

      # The requirements for the test are the sum of all the run subtests
      if 'subtests' in testDict:
        if 'requires' not in testDict: testDict['requires']=""
        for stest in testDict['subtests']:
          if 'requires' in testDict[stest]:
            testDict['requires']+=" "+testDict[stest]['requires']
          if testDict[stest].get('nsize', '1') != '1':
            testDict['SKIP'].append("Parallel test with serial build")
            break

    # Now go through all requirements
    if 'requires' in testDict:
      for requirement in testDict['requires'].split():
        requirement=requirement.strip()
        if not requirement: continue
        if debug: print(indent+"Requirement: ", requirement)
        isNull=False
        if requirement.startswith("!"):
          requirement=requirement[1:]; isNull=True
        # 32-bit vs 64-bit pointers
        if requirement == "64bitptr":
          if self.conf['PETSC_SIZEOF_VOID_P']==8:
            if isNull:
              testDict['SKIP'].append("not 64bit-ptr required")
              continue
            continue  # Success
          elif not isNull:
            testDict['SKIP'].append("64bit-ptr required")
            continue
        # Precision requirement for reals
        if requirement in self.precision_types:
          if self.conf['PETSC_PRECISION']==requirement:
            if isNull:
              testDict['SKIP'].append("not "+requirement+" required")
              continue
            continue  # Success
          elif not isNull:
            testDict['SKIP'].append(requirement+" required")
            continue
        # Precision requirement for ints
        if requirement in self.integer_types:
          if requirement=="int32":
            if self.conf['PETSC_SIZEOF_INT']==4:
              if isNull:
                testDict['SKIP'].append("not int32 required")
                continue
              continue  # Success
            elif not isNull:
              testDict['SKIP'].append("int32 required")
              continue
          if requirement=="int64":
            if self.conf['PETSC_SIZEOF_INT']==8:
              if isNull:
                testDict['SKIP'].append("NOT int64 required")
                continue
              continue  # Success
            elif not isNull:
              testDict['SKIP'].append("int64 required")
              continue
          if requirement.startswith("long"):
            reqsize = int(requirement[4:])//8
            longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip())
            if longsize==reqsize:
              if isNull:
                testDict['SKIP'].append("not %s required" % requirement)
                continue
              continue  # Success
            elif not isNull:
              testDict['SKIP'].append("%s required" % requirement)
              continue
        # Datafilespath
        if requirement=="datafilespath" and not isNull:
          testDict['SKIP'].append("Requires DATAFILESPATH")
          continue
        # Defines -- not sure I have comments matching
        if "defined(" in requirement.lower():
          reqdef=requirement.split("(")[1].split(")")[0]
          if reqdef in self.conf:
            if isNull:
              testDict['SKIP'].append("Null requirement not met: "+requirement)
              continue
            continue  # Success
          elif not isNull:
            testDict['SKIP'].append("Required: "+requirement)
            continue

        # Rest should be packages that we can just get from conf
        if requirement in ["complex","debug"]:
          petscconfvar="PETSC_USE_"+requirement.upper()
          pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper()
        else:
          petscconfvar="PETSC_HAVE_"+requirement.upper()
          pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper()
        petsccv = self.conf.get(petscconfvar)
        pkgcv = self.conf.get(pkgconfvar)

        if petsccv or pkgcv:
          if isNull:
            if petsccv:
              testDict['SKIP'].append("Not "+petscconfvar+" requirement not met")
              continue
            else:
              testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met")
              continue
          continue  # Success
        elif not isNull:
          if not petsccv and not pkgcv:
            if debug: print("requirement not found: ", requirement)
            if self.pkg_name == 'petsc':
              testDict['SKIP'].append(petscconfvar+" requirement not met")
            else:
              testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met")
            continue
    return testDict['SKIP'] == []

  def  checkOutput(self,exfile,root,srcDict):
    """
     Check and make sure the output files are in the output directory
    """
    debug=False
    rpath=self.srcrelpath(root)
    for test in srcDict:
      if test in self.buildkeys: continue
      if debug: print(rpath, exfile, test)
      if 'output_file' in srcDict[test]:
        output_file=srcDict[test]['output_file']
      else:
        defroot = testparse.getDefaultOutputFileRoot(test)
        if 'TODO' in srcDict[test]: continue
        output_file="output/"+defroot+".out"

      fullout=os.path.join(root,output_file)
      if debug: print("---> ",fullout)
      if not os.path.exists(fullout):
        self.missing_files.append(fullout)

    return

  def genPetscTests_summarize(self,dataDict):
    """
    Required method to state what happened
    """
    if not self.summarize: return
    indent="   "
    fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt')
    with open(fhname, "w") as fh:
      for root in dataDict:
        relroot=self.srcrelpath(root)
        pkg=relroot.split("/")[1]
        if not pkg in self.sources: continue
        fh.write(relroot+"\n")
        allSrcs=[]
        for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs']
        for exfile in dataDict[root]:
          # Basic  information
          rfile=os.path.join(relroot,exfile)
          builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built")
          fh.write(indent+exfile+indent*4+builtStatus+"\n")
          for test in dataDict[root][exfile]:
            if test in self.buildkeys: continue
            line=indent*2+test
            fh.write(line+"\n")
            # Looks nice to have the keys in order
            #for key in dataDict[root][exfile][test]:
            for key in "isrun abstracted nsize args requires script".split():
              if key not in dataDict[root][exfile][test]: continue
              line=indent*3+key+": "+str(dataDict[root][exfile][test][key])
              fh.write(line+"\n")
            fh.write("\n")
          fh.write("\n")
        fh.write("\n")
    return

  def genPetscTests(self,root,dirs,files,dataDict):
    """
     Go through and parse the source files in the directory to generate
     the examples based on the metadata contained in the source files
    """
    debug=False

    data = {}
    for exfile in files:
      #TST: Until we replace files, still leaving the originals as is
      #if not exfile.startswith("new_"+"ex"): continue
      #if not exfile.startswith("ex"): continue

      # Ignore emacs and other temporary files
      if exfile.startswith((".", "#")) or exfile.endswith("~"): continue
      # Only parse source files
      ext=getlangext(exfile).lstrip('.').replace('.','_')
      if ext not in LANGS: continue

      # Convenience
      fullex=os.path.join(root,exfile)
      if self.verbose: print('   --> '+fullex)
      data.update(testparse.parseTestFile(fullex,0))
      if exfile in data:
        if self.check_output:
          self.checkOutput(exfile,root,data[exfile])
        else:
          self.genScriptsAndInfo(exfile,root,data[exfile])

    dataDict[root] = data
    return

  def walktree(self,top):
    """
    Walk a directory tree, starting from 'top'
    """
    if self.check_output:
      print("Checking for missing output files")
      self.missing_files=[]

    # Goal of action is to fill this dictionary
    dataDict={}
    for root, dirs, files in os.walk(top, topdown=True):
      dirs.sort()
      files.sort()
      if "/tests" not in root and "/tutorials" not in root: continue
      if "dSYM" in root: continue
      if "tutorials"+os.sep+"build" in root: continue
      if os.path.basename(root.rstrip("/")) == 'output': continue
      if self.verbose: print(root)
      self.genPetscTests(root,dirs,files,dataDict)

    # If checking output, report results
    if self.check_output:
      if self.missing_files:
        for file in set(self.missing_files):  # set uniqifies
          print(file)
        sys.exit(1)

    # Now summarize this dictionary
    if self.verbose: self.genPetscTests_summarize(dataDict)
    return dataDict

  def gen_gnumake(self, fd):
    """
     Overwrite of the method in the base PETSc class
    """
    def write(stem, srcs):
      for lang in LANGS:
        if srcs[lang]['srcs']:
          fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs'])))
    for pkg in self.pkg_pkgs:
        srcs = self.gen_pkg(pkg)
        write('testsrcs-' + pkg, srcs)
        # Handle dependencies
        for lang in LANGS:
            for exfile in srcs[lang]['srcs']:
                if exfile in srcs[lang]:
                    ex='$(TESTDIR)/'+getlangsplit(exfile)
                    exfo=ex+'.o'
                    deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]]
                    if deps:
                        # The executable literally depends on the object file because it is linked
                        fd.write(ex   +": " + " ".join(deps) +'\n')
                        # The object file containing 'main' does not normally depend on other object
                        # files, but it does when it includes their modules.  This dependency is
                        # overly blunt and could be reduced to only depend on object files for
                        # modules that are used, like "*f90aux.o".
                        fd.write(exfo +": " + " ".join(deps) +'\n')

    return self.gendeps

  def gen_pkg(self, pkg):
    """
     Overwrite of the method in the base PETSc class
    """
    return self.sources[pkg]

  def write_gnumake(self, dataDict, output=None):
    """
     Write out something similar to files from gmakegen.py

     Test depends on script which also depends on source
     file, but since I don't have a good way generating
     acting on a single file (oops) just depend on
     executable which in turn will depend on src file
    """
    # Different options for how to set up the targets
    compileExecsFirst=False

    # Open file
    with open(output, 'w') as fd:
      # Write out the sources
      gendeps = self.gen_gnumake(fd)

      # Write out the tests and execname targets
      fd.write("\n#Tests and executables\n")    # Delimiter

      for pkg in self.pkg_pkgs:
        # These grab the ones that are built
        for lang in LANGS:
          testdeps=[]
          for ftest in self.tests[pkg][lang]:
            test=os.path.basename(ftest)
            basedir=os.path.dirname(ftest)
            testdeps.append(nameSpace(test,basedir))
          fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n")
          fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.')))

          # test targets
          for ftest in self.tests[pkg][lang]:
            test=os.path.basename(ftest)
            basedir=os.path.dirname(ftest)
            testdir="${TESTDIR}/"+basedir+"/"
            nmtest=nameSpace(test,basedir)
            rundir=os.path.join(testdir,test)
            script=test+".sh"

            # Deps
            exfile=self.tests[pkg][lang][ftest]['exfile']
            fullex=os.path.join(self.srcdir,exfile)
            localexec=self.tests[pkg][lang][ftest]['exec']
            execname=os.path.join(testdir,localexec)
            fullscript=os.path.join(testdir,script)
            tmpfile=os.path.join(testdir,test,test+".tmp")

            # *.counts depends on the script and either executable (will
            # be run) or the example source file (SKIP or TODO)
            fd.write('%s.counts : %s %s'
                % (os.path.join('$(TESTDIR)/counts', nmtest),
                   fullscript,
                   execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)
                )
            if exfile in self.sources[pkg][lang]:
              for dep in self.sources[pkg][lang][exfile]:
                fd.write(' %s' % os.path.join('$(TESTDIR)',dep))
            fd.write('\n')

            # Now write the args:
            fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n")

    return

  def write_db(self, dataDict, testdir):
    """
     Write out the dataDict into a pickle file
    """
    with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd:
      pickle.dump(dataDict,fd)
    return

def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None,
         pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False,
         srcdir=None, testdir=None, check=False):
    # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience
    testdir=os.path.normpath(testdir)
    if petsc_arch:
        petsc_arch=petsc_arch.rstrip(os.path.sep)
        if len(petsc_arch.split(os.path.sep))>1:
            petsc_dir,petsc_arch=os.path.split(petsc_arch)
    output = os.path.join(testdir, 'testfiles')

    pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch,
                         pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs,
                         verbose=verbose, single_ex=single_ex, srcdir=srcdir,
                         testdir=testdir,check=check)
    dataDict=pEx.walktree(os.path.join(pEx.srcdir))
    if not pEx.check_output:
        pEx.write_gnumake(dataDict, output)
        pEx.write_db(dataDict, testdir)

if __name__ == '__main__':
    import optparse
    parser = optparse.OptionParser()
    parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False)
    parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR'))
    parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH'))
    parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None)
    parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir.  Default is false')
    parser.add_option('-t', '--testdir', dest='testdir',  help='Test directory [$PETSC_ARCH/tests]')
    parser.add_option('-c', '--check-output', dest='check_output', action="store_true",
                      help='Check whether output files are in output director')
    parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None)
    parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None)
    parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None)
    parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None)

    opts, extra_args = parser.parse_args()
    if extra_args:
        import sys
        sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args))
        exit(1)
    if opts.testdir is None:
      opts.testdir = os.path.join(opts.petsc_arch, 'tests')

    main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch,
         pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs,
         verbose=opts.verbose,
         single_ex=opts.single_executable, srcdir=opts.srcdir,
         testdir=opts.testdir, check=opts.check_output)