xref: /petsc/config/BuildSystem/retrieval.py (revision fbfe4939ef67fee8a73b0c97887690d1107faa89)

from __future__ import absolute_import
import logger

import os
try:
  from urllib import urlretrieve
except ImportError:
  from urllib.request import urlretrieve
try:
  import urlparse as urlparse_local # novermin
except ImportError:
  from urllib import parse as urlparse_local
import config.base
import socket
import shutil

# Fix parsing for nonstandard schemes
urlparse_local.uses_netloc.extend(['bk', 'ssh', 'svn'])

class Retriever(logger.Logger):
  def __init__(self, sourceControl, clArgs = None, argDB = None):
    logger.Logger.__init__(self, clArgs, argDB)
    self.sourceControl = sourceControl
    self.stamp = None
    return

  def isDirectoryGitRepo(self, directory):
    from config.base import Configure
    for loc in ['.git','']:
      cmd = '%s rev-parse --resolve-git-dir  %s'  % (self.sourceControl.git, os.path.join(directory,loc))
      (output, error, ret) = Configure.executeShellCommand(cmd, checkCommand = Configure.passCheckCommand, log = self.log)
      if not ret:
        return True
    return False

  @staticmethod
  def removeTarget(t):
    if os.path.islink(t) or os.path.isfile(t):
      os.unlink(t) # same as os.remove(t)
    elif os.path.isdir(t):
      shutil.rmtree(t)

  @staticmethod
  def getDownloadFailureMessage(package, url, filename=None):
    slashFilename = '/'+filename if filename else ''
    return '''\
Unable to download package %s from: %s
* If URL specified manually - perhaps there is a typo?
* If your network is disconnected - please reconnect and rerun ./configure
* Or perhaps you have a firewall blocking the download
* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually
* or you can download the above URL manually, to /yourselectedlocation%s
  and use the configure option:
  --download-%s=/yourselectedlocation%s
    ''' % (package.upper(), url, slashFilename, package, slashFilename)

  @staticmethod
  def removePrefix(url,prefix):
    '''Replacement for str.removeprefix() supported only since Python 3.9'''
    if url.startswith(prefix):
      return url[len(prefix):]
    return url

  def genericRetrieve(self, url, root, package):
    '''Fetch package from version control repository or tarfile indicated by URL and extract it into root'''

    parsed = urlparse_local.urlparse(url)
    if parsed[0] == 'dir':
      f = self.dirRetrieve
    elif parsed[0] == 'link':
      f = self.linkRetrieve
    elif parsed[0] == 'git':
      f = self.gitRetrieve
    elif parsed[0] == 'ssh'   and parsed[2].endswith('.git'):
      f = self.gitRetrieve
    elif parsed[0] == 'https' and parsed[2].endswith('.git'):
      f = self.gitRetrieve
    elif parsed[0] == 'hg':
      f = self.hgRetrieve
    elif parsed[0] == 'ssh' and parsed[1].startswith('hg@'):
      f = self.hgRetrieve
    elif os.path.isdir(url):
      if self.isDirectoryGitRepo(url):
        f = self.gitRetrieve
      else:
        f = self.dirRetrieve
    else:
      f = self.tarballRetrieve
    return f(url, root, package)

  def dirRetrieve(self, url, root, package):
    self.logPrint('Retrieving %s as directory' % url, 3, 'install')
    d = self.removePrefix(url, 'dir://')
    if not os.path.isdir(d): raise RuntimeError('URL %s is not a directory' % url)

    t = os.path.join(root,os.path.basename(d))
    self.removeTarget(t)
    shutil.copytree(d,t)

  def linkRetrieve(self, url, root, package):
    self.logPrint('Retrieving %s as link' % url, 3, 'install')
    d = self.removePrefix(url, 'link://')
    if not os.path.isdir(d): raise RuntimeError('URL %s is not pointing to a directory' % url)

    t = os.path.join(root,os.path.basename(d))
    self.removeTarget(t)
    os.symlink(os.path.abspath(d),t)

  def gitRetrieve(self, url, root, package):
    self.logPrint('Retrieving %s as git repo' % url, 3, 'install')
    if not hasattr(self.sourceControl, 'git'):
      raise RuntimeError('self.sourceControl.git not set')
    d = self.removePrefix(url, 'git://')
    if os.path.isdir(d) and not self.isDirectoryGitRepo(d):
      raise RuntimeError('URL %s is a directory but not a git repository' % url)

    newgitrepo = os.path.join(root,'git.'+package)
    self.removeTarget(newgitrepo)

    try:
      config.base.Configure.executeShellCommand('%s clone %s %s' % (self.sourceControl.git, d, newgitrepo), log = self.log)
    except  RuntimeError as e:
      self.logPrint('ERROR: '+str(e))
      err = str(e)
      failureMessage = self.getDownloadFailureMessage(package, url)
      raise RuntimeError('Unable to clone '+package+'\n'+err+failureMessage)

  def hgRetrieve(self, url, root, package):
    self.logPrint('Retrieving %s as hg repo' % url, 3, 'install')
    if not hasattr(self.sourceControl, 'hg'):
      raise RuntimeError('self.sourceControl.hg not set')
    d = self.removePrefix(url, 'hg://')

    newgitrepo = os.path.join(root,'hg.'+package)
    self.removeTarget(newgitrepo)
    try:
      config.base.Configure.executeShellCommand('%s clone %s %s' % (self.sourceControl.hg, d, newgitrepo), log = self.log)
    except  RuntimeError as e:
      self.logPrint('ERROR: '+str(e))
      err = str(e)
      failureMessage = self.getDownloadFailureMessage(package, url)
      raise RuntimeError('Unable to clone '+package+'\n'+err+failureMessage)

  def tarballRetrieve(self, url, root, package):
    parsed = urlparse_local.urlparse(url)
    filename = os.path.basename(parsed[2])
    localFile = os.path.join(root,'_d_'+filename)
    self.logPrint('Retrieving %s as tarball to %s' % (url,localFile) , 3, 'install')
    ext =  os.path.splitext(localFile)[1]
    if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']:
      raise RuntimeError('Unknown compression type in URL: '+ url)

    self.removeTarget(localFile)

    if parsed[0] == 'file' and not parsed[1]:
      url = parsed[2]
    if os.path.exists(url):
      if not os.path.isfile(url):
        raise RuntimeError('Local path exists but is not a regular file: '+ url)
      # copy local file
      shutil.copyfile(url, localFile)
    else:
      # fetch remote file
      try:
        sav_timeout = socket.getdefaulttimeout()
        socket.setdefaulttimeout(30)
        urlretrieve(url, localFile)
        socket.setdefaulttimeout(sav_timeout)
      except Exception as e:
        socket.setdefaulttimeout(sav_timeout)
        failureMessage = self.getDownloadFailureMessage(package, url, filename)
        raise RuntimeError(failureMessage)

    self.logPrint('Extracting '+localFile)
    if ext in ['.zip','.ZIP']:
      config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log)
      output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log)
      dirname = os.path.normpath(output[0].strip())
    else:
      failureMessage = '''\
Downloaded package %s from: %s is not a tarball.
[or installed python cannot process compressed files]
* If you are behind a firewall - please fix your proxy and rerun ./configure
  For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to  http://proxyout.lanl.gov
* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually
* or you can download the above URL manually, to /yourselectedlocation/%s
  and use the configure option:
  --download-%s=/yourselectedlocation/%s
''' % (package.upper(), url, filename, package, filename)
      import tarfile
      try:
        tf  = tarfile.open(os.path.join(root, localFile))
      except tarfile.ReadError as e:
        raise RuntimeError(str(e)+'\n'+failureMessage)
      if not tf: raise RuntimeError(failureMessage)
      #git puts 'pax_global_header' as the first entry and some tar utils process this as a file
      firstname = tf.getnames()[0]
      if firstname == 'pax_global_header':
        firstmember = tf.getmembers()[1]
      else:
        firstmember = tf.getmembers()[0]
      # some tarfiles list packagename/ but some list packagename/filename in the first entry
      if firstmember.isdir():
        dirname = firstmember.name
      else:
        dirname = os.path.dirname(firstmember.name)
      tf.extractall(root)
      tf.close()

    # fix file permissions for the untared tarballs.
    try:
      # check if 'dirname' is set'
      if dirname:
        config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find  '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log)
      else:
        self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions')
    except RuntimeError as e:
      raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e))
    os.unlink(localFile)