1import logger 2 3import os 4import urllib 5import urlparse 6import config.base 7# Fix parsing for nonstandard schemes 8urlparse.uses_netloc.extend(['bk', 'ssh', 'svn']) 9 10class Retriever(logger.Logger): 11 def __init__(self, sourceControl, clArgs = None, argDB = None): 12 logger.Logger.__init__(self, clArgs, argDB) 13 self.sourceControl = sourceControl 14 self.stamp = None 15 return 16 17 def getAuthorizedUrl(self, url): 18 '''This returns a tuple of the unauthorized and authorized URLs for the given URL, and a flag indicating which was input''' 19 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 20 if not location: 21 url = urlparse.urlunparse(('', '', path, parameters, query, fragment)) 22 authUrl = None 23 wasAuth = 0 24 else: 25 index = location.find('@') 26 if index >= 0: 27 login = location[0:index] 28 authUrl = url 29 url = urlparse.urlunparse((scheme, location[index+1:], path, parameters, query, fragment)) 30 wasAuth = 1 31 else: 32 login = location.split('.')[0] 33 authUrl = urlparse.urlunparse((scheme, login+'@'+location, path, parameters, query, fragment)) 34 wasAuth = 0 35 return (url, authUrl, wasAuth) 36 37 def testAuthorizedUrl(self, authUrl): 38 '''Raise an exception if the URL cannot receive an SSH login without a password''' 39 if not authUrl: 40 raise RuntimeError('Url is empty') 41 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(authUrl) 42 return self.executeShellCommand('echo "quit" | ssh -oBatchMode=yes '+location) 43 44 def genericRetrieve(self, url, root, name): 45 '''Fetch the gzipped tarfile indicated by url and expand it into root 46 - All the logic for removing old versions, updating etc. must move''' 47 48 archive = '_d_'+name 49 if url.endswith(".bz2") or url.endswith(".tbz"): 50 archive += '.tar' 51 archiveZip = archive+'.bz2' 52 elif url.endswith('.tgz') or url.endswith('.tar.gz'): 53 archive += '.tar' 54 archiveZip = archive+'.gz' 55 elif url.endswith(".zip") or url.endswith('.ZIP'): 56 archiveZip = archive+'.zip' 57 else: 58 raise RuntimeError('Unknown compression type in URL: '+ url) 59 localFile = os.path.join(root, archiveZip) 60 61 self.logPrint('Downloading '+url+' to '+localFile) 62 63 if os.path.exists(localFile): 64 os.remove(localFile) 65 httpfail=-1 66 ftpfail=-1 67 try: 68 urllib.urlretrieve(url, localFile) 69 httpfail=0 70 except Exception, e: 71 httpfail=1 72 if httpfail and (url.find('http://ftp.mcs.anl.gov') >=0): 73 furl = url.replace('http://','ftp://') 74 self.logPrintBox('Warning failed download: '+url+'\nReattempting with: '+furl) 75 try: 76 urllib.urlretrieve(furl, localFile) 77 ftpfail=0 78 except Exception, e: 79 self.logPrintBox('Failed download with alternate: '+furl) 80 ftpfail=1 81 if ((ftpfail == 1) or ((ftpfail == -1) and httpfail)): 82 filename = os.path.basename(urlparse.urlparse(url)[2]) 83 failureMessage = '''\ 84Unable to download package %s from: %s 85* If URL specified manually - perhaps there is a typo? 86* If your network is disconnected - please reconnect and rerun ./configure 87* Alternatively, you can download the above URL manually, to /yourselectedlocation/%s 88 and use the configure option: 89 --download-%s=/yourselectedlocation/%s 90''' % (name, url, filename, name.lower(), filename) 91 raise RuntimeError(failureMessage) 92 self.logPrint('Uncompressing '+localFile) 93 if not archiveZip.endswith(".zip"): 94 localFile = os.path.join(root, archive) 95 # just in case old local .tar file is still hanging around get rid of it 96 if os.path.exists(localFile): 97 os.remove(localFile) 98 try: 99 if archiveZip.endswith(".bz2"): 100 config.base.Configure.executeShellCommand('cd '+root+'; bunzip2 '+archiveZip, log = self.log) 101 elif archiveZip.endswith(".zip"): 102 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+archiveZip, log = self.log) 103 else: 104 config.base.Configure.executeShellCommand('cd '+root+'; gunzip '+archiveZip, log = self.log) 105 except RuntimeError, e: 106 filename = os.path.basename(urlparse.urlparse(url)[2]) 107 if str(e).find("not in gzip format") > -1: 108 failureMessage = '''\ 109Unable to unzip downloaded package %s from: %s 110* If you are behind a firewall - please fix your proxy and rerun ./configure 111* For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 112* Alternatively, you can download the above URL manually, to /yourselectedlocation/%s 113 and use the configure option: 114 --download-%s=/yourselectedlocation/%s 115''' % (name, url, filename, name.lower(), filename) 116 raise RuntimeError(failureMessage) 117 else: 118 raise RuntimeError('Error unzipping '+archiveZip+': '+str(e)) 119 self.logPrint('Expanding '+localFile) 120 try: 121 if not archiveZip.endswith(".zip"): 122 config.base.Configure.executeShellCommand('cd '+root+'; tar -xf '+archive, log = self.log) 123 except RuntimeError, e: 124 raise RuntimeError('Error doing tar -xf '+archive+': '+str(e)) 125 # now find the dirname - and do a chmod 126 try: 127 if archiveZip.endswith(".zip"): 128 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+archive+' | head -n 1', log = self.log) 129 else: 130 output = config.base.Configure.executeShellCommand('cd '+root+'; tar -tf '+archive+' | head -n 1', log = self.log) 131 dirname = os.path.normpath(output[0].strip()) 132 # some tarfiles list packagename/ but some list packagename/filename in the first entry - so handle both cases 133 apath,bpath=os.path.split(dirname) 134 if (apath != ''): dirname = apath 135 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 136 except RuntimeError, e: 137 raise RuntimeError('Error changing permissions for '+archive+': '+str(e)) 138 os.unlink(localFile) 139 return 140 141 def ftpRetrieve(self, url, root, name,force): 142 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via ftp', 3, 'install') 143 return self.genericRetrieve(url, root, name) 144 145 def httpRetrieve(self, url, root, name,force): 146 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via http', 3, 'install') 147 return self.genericRetrieve(url, root, name) 148 149 def fileRetrieve(self, url, root, name,force): 150 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via cp', 3, 'install') 151 return self.genericRetrieve(url, root, name) 152 153 def svnRetrieve(self, url, root, name,force): 154 if not hasattr(self.sourceControl, 'svn'): 155 raise RuntimeError('Cannot retrieve a SVN repository since svn was not found') 156 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via svn', 3, 'install') 157 try: 158 config.base.Configure.executeShellCommand(self.sourceControl.svn+' checkout http'+url[3:]+' '+os.path.join(root, name)) 159 except RuntimeError: 160 pass 161 162 163 # This is the old code for updating a BK repository 164 # Stamp used to be stored with a url 165 def bkUpdate(self): 166 if not self.stamp is None and url in self.stamp: 167 if not self.stamp[url] == self.bkHeadRevision(root): 168 raise RuntimeError('Existing stamp for '+url+' does not match revision of repository in '+root) 169 (url, authUrl, wasAuth) = self.getAuthorizedUrl(self.getBKParentURL(root)) 170 if not wasAuth: 171 self.debugPrint('Changing parent from '+url+' --> '+authUrl, 1, 'install') 172 output = self.executeShellCommand('cd '+root+'; bk parent '+authUrl) 173 try: 174 self.testAuthorizedUrl(authUrl) 175 output = self.executeShellCommand('cd '+root+'; bk pull') 176 except RuntimeError, e: 177 (url, authUrl, wasAuth) = self.getAuthorizedUrl(self.getBKParentURL(root)) 178 if wasAuth: 179 self.debugPrint('Changing parent from '+authUrl+' --> '+url, 1, 'install') 180 output = self.executeShellCommand('cd '+root+'; bk parent '+url) 181 output = self.executeShellCommand('cd '+root+'; bk pull') 182 else: 183 raise e 184 return 185 186 def bkClone(self, url, root, name): 187 '''Clone a Bitkeeper repository located at url into root/name 188 - If self.stamp exists, clone only up to that revision''' 189 failureMessage = '''\ 190Unable to bk clone %s 191You may be off the network. Connect to the internet and run ./configure again 192or from the directory %s try: 193 bk clone %s 194and if that succeeds then rerun ./configure 195''' % (name, root, url, name) 196 try: 197 if not self.stamp is None and url in self.stamp: 198 (output, error, status) = self.executeShellCommand('bk clone -r'+self.stamp[url]+' '+url+' '+os.path.join(root, name)) 199 else: 200 (output, error, status) = self.executeShellCommand('bk clone '+url+' '+os.path.join(root, name)) 201 except RuntimeError, e: 202 status = 1 203 output = str(e) 204 error = '' 205 if status: 206 if output.find('ommand not found') >= 0: 207 failureMessage = 'Unable to locate bk (Bitkeeper) to download repository; make sure bk is in your path' 208 elif output.find('Cannot resolve host') >= 0: 209 failureMessage = output+'\n'+error+'\n'+failureMessage 210 else: 211 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 212 try: 213 self.bkClone(urlparse.urlunparse(('http', location, path, parameters, query, fragment)), root, name) 214 except RuntimeError, e: 215 failureMessage += '\n'+str(e) 216 else: 217 return 218 raise RuntimeError(failureMessage) 219 return 220 221 def bkRetrieve(self, url, root, name): 222 if not hasattr(self.sourceControl, 'bk'): 223 raise RuntimeError('Cannot retrieve a BitKeeper repository since BK was not found') 224 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via bk', 3, 'install') 225 (url, authUrl, wasAuth) = self.getAuthorizedUrl(url) 226 try: 227 self.testAuthorizedUrl(authUrl) 228 self.bkClone(authUrl, root, name) 229 except RuntimeError: 230 pass 231 else: 232 return 233 return self.bkClone(url, root, name) 234 235 def retrieve(self, url, root = None, canExist = 0, force = 0): 236 '''Retrieve the project corresponding to url 237 - If root is None, the local root directory is automatically determined. If the project 238 was already installed, this root is used. Otherwise a guess is made based upon the url. 239 - If canExist is True and the root exists, an update is done instead of a full download. 240 The canExist is automatically true if the project has been installed. The retrievalCanExist 241 flag can also be used to set this. 242 - If force is True, a full download is mandated. 243 Providing the root is an easy way to make a copy, for instance when making tarballs. 244 ''' 245 if root is None: 246 root = self.getInstallRoot(url) 247 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 248 if hasattr(self,scheme+'Retrieve'): 249 getattr(self, scheme+'Retrieve')(url, os.path.abspath(root), canExist, force) 250 else: 251 raise RuntimeError('Invalid transport for retrieval: '+scheme) 252 return 253 254 ############################################## 255 # This is the old shit 256 ############################################## 257 def removeRoot(self, root, canExist, force = 0): 258 '''Returns 1 if removes root''' 259 if os.path.exists(root): 260 if canExist: 261 if force: 262 import shutil 263 shutil.rmtree(root) 264 return 1 265 else: 266 return 0 267 else: 268 raise RuntimeError('Root directory '+root+' already exists') 269 return 1 270 271 def getBKParentURL(self, root): 272 '''Return the parent URL for the BK repository at "root"''' 273 return self.executeShellCommand('cd '+root+'; bk parent')[21:] 274 275 def bkHeadRevision(self, root): 276 '''Return the last change set revision in the repository''' 277 return self.executeShellCommand('cd '+root+'; bk changes -and:REV: | head -1') 278 279 def bkfileRetrieve(self, url, root, canExist = 0, force = 0): 280 self.debugPrint('Retrieving '+url+' --> '+root+' via local bk', 3, 'install') 281 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 282 return self.bkRetrieve(urlparse.urlunparse(('file', location, path, parameters, query, fragment)), root, canExist, force) 283 284 def sshRetrieve(self, url, root, canExist = 0, force = 0): 285 command = 'hg clone '+url+' '+os.path.join(root,os.path.basename(url)) 286 output = config.base.Configure.executeShellCommand(command) 287 return root 288 289 def oldRetrieve(self, url, root = None, canExist = 0, force = 0): 290 '''Retrieve the project corresponding to url 291 - If root is None, the local root directory is automatically determined. If the project 292 was already installed, this root is used. Otherwise a guess is made based upon the url. 293 - If canExist is True and the root exists, an update is done instead of a full download. 294 The canExist is automatically true if the project has been installed. The retrievalCanExist 295 flag can also be used to set this. 296 - If force is True, a full download is mandated. 297 Providing the root is an easy way to make a copy, for instance when making tarballs. 298 ''' 299 origUrl = url 300 url = self.getMappedUrl(origUrl) 301 project = self.getInstalledProject(url) 302 if not project is None and root is None: 303 root = project.getRoot() 304 canExist = 1 305 if root is None: 306 root = self.getInstallRoot(origUrl) 307 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 308 try: 309 if self.argDB['retrievalCanExist']: 310 canExist = 1 311 return getattr(self, scheme+'Retrieve')(url, os.path.abspath(root), canExist, force) 312 except AttributeError: 313 raise RuntimeError('Invalid transport for retrieval: '+scheme) 314