1179860b2SJed Brown#!/usr/bin/env python 2179860b2SJed Brown'''A remote dictionary server 3179860b2SJed Brown 4179860b2SJed Brown RDict is a typed, hierarchical, persistent dictionary intended to manage 5179860b2SJed Brown all arguments or options for a program. The interface remains exactly the 6179860b2SJed Brown same as dict, but the storage is more complicated. 7179860b2SJed Brown 8179860b2SJed Brown Argument typing is handled by wrapping all values stored in the dictionary 9179860b2SJed Brown with nargs.Arg or a subclass. A user can call setType() to set the type of 10179860b2SJed Brown an argument without any value being present. Whenever __getitem__() or 11179860b2SJed Brown __setitem__() is called, values are extracted or replaced in the wrapper. 12179860b2SJed Brown These wrappers can be accessed directly using getType(), setType(), and 13179860b2SJed Brown types(). 14179860b2SJed Brown 15179860b2SJed Brown Hierarchy is allowed using a single "parent" dictionary. All operations 16179860b2SJed Brown cascade to the parent. For instance, the length of the dictionary is the 17179860b2SJed Brown number of local keys plus the number of keys in the parent, and its 18179860b2SJed Brown parent, etc. Also, a dictionary need not have a parent. If a key does not 19179860b2SJed Brown appear in the local dicitonary, the call if passed to the parent. However, 20179860b2SJed Brown in this case we see that local keys can shadow those in a parent. 21179860b2SJed Brown Communication with the parent is handled using sockets, with the parent 22179860b2SJed Brown being a server and the interactive dictionary a client. 23179860b2SJed Brown 24179860b2SJed Brown The default persistence mechanism is a pickle file, RDict.db, written 25179860b2SJed Brown whenever an argument is changed locally. A timer thread is created after 26179860b2SJed Brown an initial change, so that many rapid changes do not cause many writes. 27179860b2SJed Brown Each dictionary only saves its local entries, so all parents also 28179860b2SJed Brown separately save data in different RDict.db files. Each time a dictionary 29179860b2SJed Brown is created, the current directory is searched for an RDict.db file, and 30179860b2SJed Brown if found the contents are loaded into the dictionary. 31179860b2SJed Brown 32179860b2SJed Brown This script also provides some default actions: 33179860b2SJed Brown 34179860b2SJed Brown - server [parent] 35179860b2SJed Brown Starts a server in the current directory with an optional parent. This 36179860b2SJed Brown server will accept socket connections from other dictionaries and act 37179860b2SJed Brown as a parent. 38179860b2SJed Brown 39179860b2SJed Brown - client [parent] 40179860b2SJed Brown Creates a dictionary in the current directory with an optional parent 41179860b2SJed Brown and lists the contents. Notice that the contents may come from either 42179860b2SJed Brown an RDict.db file in the current directory, or from the parent. 43179860b2SJed Brown 44179860b2SJed Brown - clear [parent] 45179860b2SJed Brown Creates a dictionary in the current directory with an optional parent 46179860b2SJed Brown and clears the contents. Notice that this will also clear the parent. 47179860b2SJed Brown 48179860b2SJed Brown - insert <parent> <key> <value> 49179860b2SJed Brown Creates a dictionary in the current directory with a parent, and inserts 50179860b2SJed Brown the key-value pair. If "parent" is "None", no parent is assigned. 51179860b2SJed Brown 52179860b2SJed Brown - remove <parent> <key> 53179860b2SJed Brown Creates a dictionary in the current directory with a parent, and removes 54179860b2SJed Brown the given key. If "parent" is "None", no parent is assigned. 55179860b2SJed Brown''' 56179860b2SJed Browntry: 57179860b2SJed Brown import project # This is necessary for us to create Project objects on load 58179860b2SJed Brown import build.buildGraph # This is necessary for us to create BuildGraph objects on load 59179860b2SJed Brownexcept ImportError: 60179860b2SJed Brown pass 61179860b2SJed Brownimport nargs 62179860b2SJed Brown 63179860b2SJed Brownimport cPickle 64179860b2SJed Brownimport os 65179860b2SJed Brownimport sys 66179860b2SJed BrownuseThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 67179860b2SJed Brownif useThreads is None: 68*31de54b5SMatthew G. Knepley useThreads = 1 69179860b2SJed Brownelse: 70179860b2SJed Brown useThreads = int(useThreads) 71179860b2SJed Brown 72179860b2SJed Brownclass RDict(dict): 73179860b2SJed Brown '''An RDict is a typed dictionary, which may be hierarchically composed. All elements derive from the 74179860b2SJed BrownArg class, which wraps the usual value.''' 75179860b2SJed Brown # The server will self-shutdown after this many seconds 76179860b2SJed Brown shutdownDelay = 60*60*5 77179860b2SJed Brown 78179860b2SJed Brown def __init__(self, parentAddr = None, parentDirectory = None, load = 1, autoShutdown = 1, readonly = False): 79179860b2SJed Brown import atexit 80179860b2SJed Brown import time 81179860b2SJed Brown import xdrlib 82179860b2SJed Brown 83179860b2SJed Brown self.logFile = None 84179860b2SJed Brown self.setupLogFile() 85179860b2SJed Brown self.target = ['default'] 86179860b2SJed Brown self.parent = None 87179860b2SJed Brown self.saveTimer = None 88179860b2SJed Brown self.shutdownTimer = None 89179860b2SJed Brown self.lastAccess = time.time() 90179860b2SJed Brown self.saveFilename = 'RDict.db' 91179860b2SJed Brown self.addrFilename = 'RDict.loc' 92179860b2SJed Brown self.parentAddr = parentAddr 93179860b2SJed Brown self.isServer = 0 94179860b2SJed Brown self.readonly = readonly 95179860b2SJed Brown self.parentDirectory = parentDirectory 96179860b2SJed Brown self.packer = xdrlib.Packer() 97179860b2SJed Brown self.unpacker = xdrlib.Unpacker('') 98179860b2SJed Brown self.stopCmd = cPickle.dumps(('stop',)) 99179860b2SJed Brown self.writeLogLine('Greetings') 100179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 101179860b2SJed Brown if load: self.load() 102179860b2SJed Brown if autoShutdown and useThreads: 103179860b2SJed Brown atexit.register(self.shutdown) 104179860b2SJed Brown self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 105179860b2SJed Brown return 106179860b2SJed Brown 107179860b2SJed Brown def __getstate__(self): 108179860b2SJed Brown '''Remove any parent socket object, the XDR translators, and the log file from the dictionary before pickling''' 109179860b2SJed Brown self.writeLogLine('Pickling RDict') 110179860b2SJed Brown d = self.__dict__.copy() 111179860b2SJed Brown if 'parent' in d: del d['parent'] 112179860b2SJed Brown if 'saveTimer' in d: del d['saveTimer'] 113179860b2SJed Brown if '_setCommandLine' in d: del d['_setCommandLine'] 114179860b2SJed Brown del d['packer'] 115179860b2SJed Brown del d['unpacker'] 116179860b2SJed Brown del d['logFile'] 117179860b2SJed Brown return d 118179860b2SJed Brown 119179860b2SJed Brown def __setstate__(self, d): 120179860b2SJed Brown '''Reconnect the parent socket object, recreate the XDR translators and reopen the log file after unpickling''' 121179860b2SJed Brown self.logFile = file('RDict.log', 'a') 122179860b2SJed Brown self.writeLogLine('Unpickling RDict') 123179860b2SJed Brown self.__dict__.update(d) 124179860b2SJed Brown import xdrlib 125179860b2SJed Brown self.packer = xdrlib.Packer() 126179860b2SJed Brown self.unpacker = xdrlib.Unpacker('') 127179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 128179860b2SJed Brown return 129179860b2SJed Brown 130179860b2SJed Brown def setupLogFile(self, filename = 'RDict.log'): 131179860b2SJed Brown if not self.logFile is None: 132179860b2SJed Brown self.logFile.close() 133179860b2SJed Brown if os.path.isfile(filename) and os.stat(filename).st_size > 10*1024*1024: 134179860b2SJed Brown if os.path.isfile(filename+'.bkp'): 135179860b2SJed Brown os.remove(filename+'.bkp') 136179860b2SJed Brown os.rename(filename, filename+'.bkp') 137179860b2SJed Brown self.logFile = file(filename, 'w') 138179860b2SJed Brown else: 139179860b2SJed Brown self.logFile = file(filename, 'a') 140179860b2SJed Brown return 141179860b2SJed Brown 142179860b2SJed Brown def writeLogLine(self, message): 143179860b2SJed Brown '''Writes the message to the log along with the current time''' 144179860b2SJed Brown import time 145179860b2SJed Brown self.logFile.write('('+str(os.getpid())+')('+str(id(self))+')'+message+' ['+time.asctime(time.localtime())+']\n') 146179860b2SJed Brown self.logFile.flush() 147179860b2SJed Brown return 148179860b2SJed Brown 149179860b2SJed Brown def __len__(self): 150179860b2SJed Brown '''Returns the length of both the local and parent dictionaries''' 151179860b2SJed Brown length = dict.__len__(self) 152179860b2SJed Brown if not self.parent is None: 153179860b2SJed Brown length = length + self.send() 154179860b2SJed Brown return length 155179860b2SJed Brown 156179860b2SJed Brown def getType(self, key): 157179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Returns the Arg object or None if not found.''' 158179860b2SJed Brown if dict.has_key(self, key): 159179860b2SJed Brown self.writeLogLine('getType: Getting local type for '+key+' '+str(dict.__getitem__(self, key))) 160179860b2SJed Brown return dict.__getitem__(self, key) 161179860b2SJed Brown elif not self.parent is None: 162179860b2SJed Brown return self.send(key) 163179860b2SJed Brown return None 164179860b2SJed Brown 165179860b2SJed Brown def __getitem__(self, key): 166179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Returns the value of the Arg. 167179860b2SJed Brown - If the value has not been set, the user will be prompted for input''' 168179860b2SJed Brown if dict.has_key(self, key): 169179860b2SJed Brown self.writeLogLine('__getitem__: '+key+' has local type') 170179860b2SJed Brown pass 171179860b2SJed Brown elif not self.parent is None: 172179860b2SJed Brown self.writeLogLine('__getitem__: Checking parent value') 173179860b2SJed Brown if self.send(key, operation = 'has_key'): 174179860b2SJed Brown self.writeLogLine('__getitem__: Parent has value') 175179860b2SJed Brown return self.send(key) 176179860b2SJed Brown else: 177179860b2SJed Brown self.writeLogLine('__getitem__: Checking parent type') 178179860b2SJed Brown arg = self.send(key, operation = 'getType') 179179860b2SJed Brown if not arg: 180179860b2SJed Brown self.writeLogLine('__getitem__: Parent has no type') 181179860b2SJed Brown arg = nargs.Arg(key) 182179860b2SJed Brown try: 183179860b2SJed Brown value = arg.getValue() 184179860b2SJed Brown except AttributeError, e: 185179860b2SJed Brown self.writeLogLine('__getitem__: Parent had invalid entry: '+str(e)) 186179860b2SJed Brown arg = nargs.Arg(key) 187179860b2SJed Brown value = arg.getValue() 188179860b2SJed Brown self.writeLogLine('__getitem__: Setting parent value '+str(value)) 189179860b2SJed Brown self.send(key, value, operation = '__setitem__') 190179860b2SJed Brown return value 191179860b2SJed Brown else: 192179860b2SJed Brown self.writeLogLine('__getitem__: Setting local type for '+key) 193179860b2SJed Brown dict.__setitem__(self, key, nargs.Arg(key)) 194179860b2SJed Brown self.save() 195179860b2SJed Brown self.writeLogLine('__getitem__: Setting local value for '+key) 196179860b2SJed Brown return dict.__getitem__(self, key).getValue() 197179860b2SJed Brown 198179860b2SJed Brown def setType(self, key, value, forceLocal = 0): 199179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Sets the type for this key. 200179860b2SJed Brown - If a value for the key already exists, it is converted to the new type''' 201179860b2SJed Brown if not isinstance(value, nargs.Arg): 202179860b2SJed Brown raise TypeError('An argument type must be a subclass of Arg') 203179860b2SJed Brown value.setKey(key) 204179860b2SJed Brown if forceLocal or self.parent is None or dict.has_key(self, key): 205179860b2SJed Brown if dict.has_key(self, key): 206179860b2SJed Brown v = dict.__getitem__(self, key) 207179860b2SJed Brown if v.isValueSet(): 208179860b2SJed Brown try: 209179860b2SJed Brown value.setValue(v.getValue()) 210179860b2SJed Brown except TypeError: pass 211179860b2SJed Brown dict.__setitem__(self, key, value) 212179860b2SJed Brown self.save() 213179860b2SJed Brown else: 214179860b2SJed Brown return self.send(key, value) 215179860b2SJed Brown return 216179860b2SJed Brown 217179860b2SJed Brown def __setitem__(self, key, value): 218179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Sets the value of the Arg.''' 219179860b2SJed Brown if not dict.has_key(self, key): 220179860b2SJed Brown if not self.parent is None: 221179860b2SJed Brown return self.send(key, value) 222179860b2SJed Brown else: 223179860b2SJed Brown dict.__setitem__(self, key, nargs.Arg(key)) 224179860b2SJed Brown dict.__getitem__(self, key).setValue(value) 225179860b2SJed Brown self.writeLogLine('__setitem__: Set value for '+key+' to '+str(dict.__getitem__(self, key))) 226179860b2SJed Brown self.save() 227179860b2SJed Brown return 228179860b2SJed Brown 229179860b2SJed Brown def __delitem__(self, key): 230179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Deletes the Arg completely.''' 231179860b2SJed Brown if dict.has_key(self, key): 232179860b2SJed Brown dict.__delitem__(self, key) 233179860b2SJed Brown self.save() 234179860b2SJed Brown elif not self.parent is None: 235179860b2SJed Brown self.send(key) 236179860b2SJed Brown return 237179860b2SJed Brown 238179860b2SJed Brown def clear(self): 239179860b2SJed Brown '''Clears both the local and parent dictionaries''' 240179860b2SJed Brown if dict.__len__(self): 241179860b2SJed Brown dict.clear(self) 242179860b2SJed Brown self.save() 243179860b2SJed Brown if not self.parent is None: 244179860b2SJed Brown self.send() 245179860b2SJed Brown return 246179860b2SJed Brown 247179860b2SJed Brown def __contains__(self, key): 248179860b2SJed Brown '''This method just calls self.has_key(key)''' 249179860b2SJed Brown return self.has_key(key) 250179860b2SJed Brown 251179860b2SJed Brown def has_key(self, key): 252179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Then checks whether the value has been set''' 253179860b2SJed Brown if dict.has_key(self, key): 254179860b2SJed Brown if dict.__getitem__(self, key).isValueSet(): 255179860b2SJed Brown self.writeLogLine('has_key: Have value for '+key) 256179860b2SJed Brown else: 257179860b2SJed Brown self.writeLogLine('has_key: Do not have value for '+key) 258179860b2SJed Brown return dict.__getitem__(self, key).isValueSet() 259179860b2SJed Brown elif not self.parent is None: 260179860b2SJed Brown return self.send(key) 261179860b2SJed Brown return 0 262179860b2SJed Brown 263179860b2SJed Brown def get(self, key, default=None): 264179860b2SJed Brown if self.has_key(key): 265179860b2SJed Brown return self.__getitem__(key) 266179860b2SJed Brown else: 267179860b2SJed Brown return default 268179860b2SJed Brown 269179860b2SJed Brown def hasType(self, key): 270179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Then checks whether the type has been set''' 271179860b2SJed Brown if dict.has_key(self, key): 272179860b2SJed Brown return 1 273179860b2SJed Brown elif not self.parent is None: 274179860b2SJed Brown return self.send(key) 275179860b2SJed Brown return 0 276179860b2SJed Brown 277179860b2SJed Brown def items(self): 278179860b2SJed Brown '''Return a list of all accessible items, as (key, value) pairs.''' 279179860b2SJed Brown l = dict.items(self) 280179860b2SJed Brown if not self.parent is None: 281179860b2SJed Brown l.extend(self.send()) 282179860b2SJed Brown return l 283179860b2SJed Brown 284179860b2SJed Brown def localitems(self): 285179860b2SJed Brown '''Return a list of all the items stored locally, as (key, value) pairs.''' 286179860b2SJed Brown return dict.items(self) 287179860b2SJed Brown 288179860b2SJed Brown def keys(self): 289179860b2SJed Brown '''Returns the list of keys in both the local and parent dictionaries''' 290179860b2SJed Brown keyList = filter(lambda key: dict.__getitem__(self, key).isValueSet(), dict.keys(self)) 291179860b2SJed Brown if not self.parent is None: 292179860b2SJed Brown keyList.extend(self.send()) 293179860b2SJed Brown return keyList 294179860b2SJed Brown 295179860b2SJed Brown def types(self): 296179860b2SJed Brown '''Returns the list of keys for which types are defined in both the local and parent dictionaries''' 297179860b2SJed Brown keyList = dict.keys(self) 298179860b2SJed Brown if not self.parent is None: 299179860b2SJed Brown keyList.extend(self.send()) 300179860b2SJed Brown return keyList 301179860b2SJed Brown 302179860b2SJed Brown def update(self, d): 303179860b2SJed Brown '''Update the dictionary with the contents of d''' 304179860b2SJed Brown for k in d: 305179860b2SJed Brown self[k] = d[k] 306179860b2SJed Brown return 307179860b2SJed Brown 308179860b2SJed Brown def updateTypes(self, d): 309179860b2SJed Brown '''Update types locally, which is equivalent to the dict.update() method''' 310179860b2SJed Brown return dict.update(self, d) 311179860b2SJed Brown 312179860b2SJed Brown def insertArg(self, key, value, arg): 313179860b2SJed Brown '''Insert a (key, value) pair into the dictionary. If key is None, arg is put into the target list.''' 314179860b2SJed Brown if not key is None: 315179860b2SJed Brown self[key] = value 316179860b2SJed Brown else: 317179860b2SJed Brown if not self.target == ['default']: 318179860b2SJed Brown self.target.append(arg) 319179860b2SJed Brown else: 320179860b2SJed Brown self.target = [arg] 321179860b2SJed Brown return 322179860b2SJed Brown 323179860b2SJed Brown def insertArgs(self, args): 324179860b2SJed Brown '''Insert some text arguments into the dictionary (list and dictionaries are recognized)''' 325179860b2SJed Brown import UserDict 326179860b2SJed Brown 327179860b2SJed Brown if isinstance(args, list): 328179860b2SJed Brown for arg in args: 329179860b2SJed Brown (key, value) = nargs.Arg.parseArgument(arg) 330179860b2SJed Brown self.insertArg(key, value, arg) 331179860b2SJed Brown # Necessary since os.environ is a UserDict 332179860b2SJed Brown elif isinstance(args, dict) or isinstance(args, UserDict.UserDict): 333179860b2SJed Brown for key in args.keys(): 334179860b2SJed Brown if isinstance(args[key], str): 335179860b2SJed Brown value = nargs.Arg.parseValue(args[key]) 336179860b2SJed Brown else: 337179860b2SJed Brown value = args[key] 338179860b2SJed Brown self.insertArg(key, value, None) 339179860b2SJed Brown elif isinstance(args, str): 340179860b2SJed Brown (key, value) = nargs.Arg.parseArgument(args) 341179860b2SJed Brown self.insertArg(key, value, args) 342179860b2SJed Brown return 343179860b2SJed Brown 344179860b2SJed Brown def hasParent(self): 345179860b2SJed Brown '''Return True if this RDict has a parent dictionary''' 346179860b2SJed Brown return not self.parent is None 347179860b2SJed Brown 348179860b2SJed Brown def getServerAddr(self, dir): 349179860b2SJed Brown '''Read the server socket address (in pickled form) from a file, usually RDict.loc 350179860b2SJed Brown - If we fail to connect to the server specified in the file, we spawn it using startServer()''' 351179860b2SJed Brown filename = os.path.join(dir, self.addrFilename) 352179860b2SJed Brown if not os.path.exists(filename): 353179860b2SJed Brown self.startServer(filename) 354179860b2SJed Brown if not os.path.exists(filename): 355179860b2SJed Brown raise RuntimeError('Server address file does not exist: '+filename) 356179860b2SJed Brown try: 357179860b2SJed Brown f = open(filename, 'r') 358179860b2SJed Brown addr = cPickle.load(f) 359179860b2SJed Brown f.close() 360179860b2SJed Brown return addr 361179860b2SJed Brown except Exception, e: 362179860b2SJed Brown self.writeLogLine('CLIENT: Exception during server address determination: '+str(e.__class__)+': '+str(e)) 363179860b2SJed Brown raise RuntimeError('Could not get server address in '+filename) 364179860b2SJed Brown 365179860b2SJed Brown def writeServerAddr(self, server): 366179860b2SJed Brown '''Write the server socket address (in pickled form) to a file, usually RDict.loc.''' 367179860b2SJed Brown f = file(self.addrFilename, 'w') 368179860b2SJed Brown cPickle.dump(server.server_address, f) 369179860b2SJed Brown f.close() 370179860b2SJed Brown self.writeLogLine('SERVER: Wrote lock file '+os.path.abspath(self.addrFilename)) 371179860b2SJed Brown return 372179860b2SJed Brown 373179860b2SJed Brown def startServer(self, addrFilename): 374179860b2SJed Brown '''Spawn a new RDict server in the parent directory''' 375179860b2SJed Brown import RDict # Need this to locate server script 376179860b2SJed Brown import sys 377179860b2SJed Brown import time 378179860b2SJed Brown import distutils.sysconfig 379179860b2SJed Brown 380179860b2SJed Brown self.writeLogLine('CLIENT: Spawning a new server with lock file '+os.path.abspath(addrFilename)) 381179860b2SJed Brown if os.path.exists(addrFilename): 382179860b2SJed Brown os.remove(addrFilename) 383179860b2SJed Brown oldDir = os.getcwd() 384179860b2SJed Brown source = os.path.join(os.path.dirname(os.path.abspath(sys.modules['RDict'].__file__)), 'RDict.py') 385179860b2SJed Brown interpreter = os.path.join(distutils.sysconfig.get_config_var('BINDIR'), distutils.sysconfig.get_config_var('PYTHON')) 386179860b2SJed Brown if not os.path.isfile(interpreter): 387179860b2SJed Brown interpreter = 'python' 388179860b2SJed Brown os.chdir(os.path.dirname(addrFilename)) 389179860b2SJed Brown self.writeLogLine('CLIENT: Executing '+interpreter+' '+source+' server"') 390179860b2SJed Brown try: 391179860b2SJed Brown os.spawnvp(os.P_NOWAIT, interpreter, [interpreter, source, 'server']) 392179860b2SJed Brown except: 393179860b2SJed Brown self.writeLogLine('CLIENT: os.spawnvp failed.\n \ 394179860b2SJed Brown This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 395179860b2SJed Brown you can fix this problem by running /bin/rebaseall. If you do not have\n \ 396179860b2SJed Brown this program, you can install it with the CYGWIN installer in the package\n \ 397179860b2SJed Brown Rebase, under the category System. You must run /bin/rebaseall after\n \ 398179860b2SJed Brown turning off all cygwin services -- in particular sshd, if any such services\n \ 399179860b2SJed Brown are running. For more information about rebase, go to http://www.cygwin.com') 400179860b2SJed Brown print '\n \ 401179860b2SJed Brown This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 402179860b2SJed Brown you can fix this problem by running /bin/rebaseall. If you do not have\n \ 403179860b2SJed Brown this program, you can install it with the CYGWIN installer in the package\n \ 404179860b2SJed Brown Rebase, under the category System. You must run /bin/rebaseall after\n \ 405179860b2SJed Brown turning off all cygwin services -- in particular sshd, if any such services\n \ 406179860b2SJed Brown are running. For more information about rebase, go to http://www.cygwin.com\n' 407179860b2SJed Brown raise 408179860b2SJed Brown os.chdir(oldDir) 409179860b2SJed Brown timeout = 1 410179860b2SJed Brown for i in range(10): 411179860b2SJed Brown time.sleep(timeout) 412179860b2SJed Brown timeout *= 2 413179860b2SJed Brown if timeout > 100: timeout = 100 414179860b2SJed Brown if os.path.exists(addrFilename): return 415179860b2SJed Brown self.writeLogLine('CLIENT: Could not start server') 416179860b2SJed Brown return 417179860b2SJed Brown 418179860b2SJed Brown def connectParent(self, addr, dir): 419179860b2SJed Brown '''Try to connect to a parent RDict server 420179860b2SJed Brown - If addr and dir are both None, this operation fails 421179860b2SJed Brown - If addr is None, check for an address file in dir''' 422179860b2SJed Brown if addr is None: 423179860b2SJed Brown if dir is None: return 0 424179860b2SJed Brown addr = self.getServerAddr(dir) 425179860b2SJed Brown 426179860b2SJed Brown import socket 427179860b2SJed Brown import errno 428179860b2SJed Brown connected = 0 429179860b2SJed Brown s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 430179860b2SJed Brown timeout = 1 431179860b2SJed Brown for i in range(10): 432179860b2SJed Brown try: 433179860b2SJed Brown self.writeLogLine('CLIENT: Trying to connect to '+str(addr)) 434179860b2SJed Brown s.connect(addr) 435179860b2SJed Brown connected = 1 436179860b2SJed Brown break 437179860b2SJed Brown except socket.error, e: 438179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect: '+str(e)) 439179860b2SJed Brown if e[0] == errno.ECONNREFUSED: 440179860b2SJed Brown try: 441179860b2SJed Brown import time 442179860b2SJed Brown time.sleep(timeout) 443179860b2SJed Brown timeout *= 2 444179860b2SJed Brown if timeout > 100: timeout = 100 445179860b2SJed Brown except KeyboardInterrupt: 446179860b2SJed Brown break 447179860b2SJed Brown # Try to spawn parent 448179860b2SJed Brown if dir: 449179860b2SJed Brown filename = os.path.join(dir, self.addrFilename) 450179860b2SJed Brown if os.path.isfile(filename): 451179860b2SJed Brown os.remove(filename) 452179860b2SJed Brown self.startServer(filename) 453179860b2SJed Brown except Exception, e: 454179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect: '+str(e.__class__)+': '+str(e)) 455179860b2SJed Brown if not connected: 456179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect to parent') 457179860b2SJed Brown return 0 458179860b2SJed Brown self.parent = s 459179860b2SJed Brown self.writeLogLine('CLIENT: Connected to '+str(self.parent)) 460179860b2SJed Brown return 1 461179860b2SJed Brown 462179860b2SJed Brown def sendPacket(self, s, packet, source = 'Unknown', isPickled = 0): 463179860b2SJed Brown '''Pickle the input packet. Send first the size of the pickled string in 32-bit integer, and then the string itself''' 464179860b2SJed Brown self.writeLogLine(source+': Sending packet '+str(packet)) 465179860b2SJed Brown if isPickled: 466179860b2SJed Brown p = packet 467179860b2SJed Brown else: 468179860b2SJed Brown p = cPickle.dumps(packet) 469179860b2SJed Brown self.packer.reset() 470179860b2SJed Brown self.packer.pack_uint(len(p)) 471179860b2SJed Brown if hasattr(s, 'write'): 472179860b2SJed Brown s.write(self.packer.get_buffer()) 473179860b2SJed Brown s.write(p) 474179860b2SJed Brown else: 475179860b2SJed Brown s.sendall(self.packer.get_buffer()) 476179860b2SJed Brown s.sendall(p) 477179860b2SJed Brown self.writeLogLine(source+': Sent packet') 478179860b2SJed Brown return 479179860b2SJed Brown 480179860b2SJed Brown def recvPacket(self, s, source = 'Unknown'): 481179860b2SJed Brown '''Receive first the size of the pickled string in a 32-bit integer, and then the string itself. Return the unpickled object''' 482179860b2SJed Brown self.writeLogLine(source+': Receiving packet') 483179860b2SJed Brown if hasattr(s, 'read'): 484179860b2SJed Brown s.read(4) 485179860b2SJed Brown value = cPickle.load(s) 486179860b2SJed Brown else: 487179860b2SJed Brown # I probably need to check that it actually read these 4 bytes 488179860b2SJed Brown self.unpacker.reset(s.recv(4)) 489179860b2SJed Brown length = self.unpacker.unpack_uint() 490179860b2SJed Brown objString = '' 491179860b2SJed Brown while len(objString) < length: 492179860b2SJed Brown objString += s.recv(length - len(objString)) 493179860b2SJed Brown value = cPickle.loads(objString) 494179860b2SJed Brown self.writeLogLine(source+': Received packet '+str(value)) 495179860b2SJed Brown return value 496179860b2SJed Brown 497179860b2SJed Brown def send(self, key = None, value = None, operation = None): 498179860b2SJed Brown '''Send a request to the parent''' 499179860b2SJed Brown import inspect 500179860b2SJed Brown 501179860b2SJed Brown objString = '' 502179860b2SJed Brown for i in range(3): 503179860b2SJed Brown try: 504179860b2SJed Brown packet = [] 505179860b2SJed Brown if operation is None: 506179860b2SJed Brown operation = inspect.stack()[1][3] 507179860b2SJed Brown packet.append(operation) 508179860b2SJed Brown if not key is None: 509179860b2SJed Brown packet.append(key) 510179860b2SJed Brown if not value is None: 511179860b2SJed Brown packet.append(value) 512179860b2SJed Brown self.sendPacket(self.parent, tuple(packet), source = 'CLIENT') 513179860b2SJed Brown response = self.recvPacket(self.parent, source = 'CLIENT') 514179860b2SJed Brown break 515179860b2SJed Brown except IOError, e: 516179860b2SJed Brown self.writeLogLine('CLIENT: IOError '+str(e)) 517179860b2SJed Brown if e.errno == 32: 518179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 519179860b2SJed Brown except Exception, e: 520179860b2SJed Brown self.writeLogLine('CLIENT: Exception '+str(e)+' '+str(e.__class__)) 521179860b2SJed Brown try: 522179860b2SJed Brown if isinstance(response, Exception): 523179860b2SJed Brown self.writeLogLine('CLIENT: Got an exception '+str(response)) 524179860b2SJed Brown raise response 525179860b2SJed Brown else: 526179860b2SJed Brown self.writeLogLine('CLIENT: Received value '+str(response)+' '+str(type(response))) 527179860b2SJed Brown except UnboundLocalError: 528179860b2SJed Brown self.writeLogLine('CLIENT: Could not unpickle response') 529179860b2SJed Brown response = None 530179860b2SJed Brown return response 531179860b2SJed Brown 532179860b2SJed Brown def serve(self): 533179860b2SJed Brown '''Start a server''' 534179860b2SJed Brown import socket 535179860b2SJed Brown import SocketServer 536179860b2SJed Brown 537179860b2SJed Brown if not useThreads: 538179860b2SJed Brown raise RuntimeError('Cannot run a server if threads are disabled') 539179860b2SJed Brown 540179860b2SJed Brown class ProcessHandler(SocketServer.StreamRequestHandler): 541179860b2SJed Brown def handle(self): 542179860b2SJed Brown import time 543179860b2SJed Brown 544179860b2SJed Brown self.server.rdict.lastAccess = time.time() 545179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Started new handler') 546179860b2SJed Brown while 1: 547179860b2SJed Brown try: 548179860b2SJed Brown value = self.server.rdict.recvPacket(self.rfile, source = 'SERVER') 549179860b2SJed Brown except EOFError, e: 550179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: EOFError receiving packet '+str(e)+' '+str(e.__class__)) 551179860b2SJed Brown return 552179860b2SJed Brown except Exception, e: 553179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Error receiving packet '+str(e)+' '+str(e.__class__)) 554179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 555179860b2SJed Brown continue 556179860b2SJed Brown if value[0] == 'stop': break 557179860b2SJed Brown try: 558179860b2SJed Brown response = getattr(self.server.rdict, value[0])(*value[1:]) 559179860b2SJed Brown except Exception, e: 560179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Error executing operation '+str(e)+' '+str(e.__class__)) 561179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 562179860b2SJed Brown else: 563179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, response, source = 'SERVER') 564179860b2SJed Brown return 565179860b2SJed Brown 566179860b2SJed Brown # check if server is running 567179860b2SJed Brown if os.path.exists(self.addrFilename): 568179860b2SJed Brown rdict = RDict(parentDirectory = '.') 569179860b2SJed Brown hasParent = rdict.hasParent() 570179860b2SJed Brown del rdict 571179860b2SJed Brown if hasParent: 572179860b2SJed Brown self.writeLogLine('SERVER: Another server is already running') 573179860b2SJed Brown raise RuntimeError('Server already running') 574179860b2SJed Brown 575179860b2SJed Brown # Daemonize server 576179860b2SJed Brown self.writeLogLine('SERVER: Daemonizing server') 577179860b2SJed Brown if os.fork(): # Launch child 578179860b2SJed Brown os._exit(0) # Kill off parent, so we are not a process group leader and get a new PID 579179860b2SJed Brown os.setsid() # Set session ID, so that we have no controlling terminal 580179860b2SJed Brown # We choose to leave cwd at RDict.py: os.chdir('/') # Make sure root directory is not on a mounted drive 581179860b2SJed Brown os.umask(077) # Fix creation mask 582179860b2SJed Brown for i in range(3): # Crappy stopgap for closing descriptors 583179860b2SJed Brown try: 584179860b2SJed Brown os.close(i) 585179860b2SJed Brown except OSError, e: 586179860b2SJed Brown if e.errno != errno.EBADF: 587179860b2SJed Brown raise RuntimeError('Could not close default descriptor '+str(i)) 588179860b2SJed Brown 589179860b2SJed Brown # wish there was a better way to get a usable socket 590179860b2SJed Brown self.writeLogLine('SERVER: Establishing socket server') 591179860b2SJed Brown basePort = 8000 592179860b2SJed Brown flag = 'nosocket' 593179860b2SJed Brown p = 1 594179860b2SJed Brown while p < 1000 and flag == 'nosocket': 595179860b2SJed Brown try: 596179860b2SJed Brown server = SocketServer.ThreadingTCPServer((socket.gethostname(), basePort+p), ProcessHandler) 597179860b2SJed Brown flag = 'socket' 598179860b2SJed Brown except Exception, e: 599179860b2SJed Brown p = p + 1 600179860b2SJed Brown if flag == 'nosocket': 601179860b2SJed Brown p = 1 602179860b2SJed Brown while p < 1000 and flag == 'nosocket': 603179860b2SJed Brown try: 604179860b2SJed Brown server = SocketServer.ThreadingTCPServer(('localhost', basePort+p), ProcessHandler) 605179860b2SJed Brown flag = 'socket' 606179860b2SJed Brown except Exception, e: 607179860b2SJed Brown p = p + 1 608179860b2SJed Brown if flag == 'nosocket': 609179860b2SJed Brown self.writeLogLine('SERVER: Could not established socket server on port '+str(basePort+p)) 610179860b2SJed Brown raise RuntimeError,'Cannot get available socket' 611179860b2SJed Brown self.writeLogLine('SERVER: Established socket server on port '+str(basePort+p)) 612179860b2SJed Brown 613179860b2SJed Brown self.isServer = 1 614179860b2SJed Brown self.writeServerAddr(server) 615179860b2SJed Brown self.serverShutdown(os.getpid()) 616179860b2SJed Brown 617179860b2SJed Brown server.rdict = self 618179860b2SJed Brown self.writeLogLine('SERVER: Started server') 619179860b2SJed Brown server.serve_forever() 620179860b2SJed Brown return 621179860b2SJed Brown 622179860b2SJed Brown def load(self): 623179860b2SJed Brown '''Load the saved dictionary''' 624179860b2SJed Brown if not self.parentDirectory is None and os.path.samefile(os.getcwd(), self.parentDirectory): 625179860b2SJed Brown return 626179860b2SJed Brown self.saveFilename = os.path.abspath(self.saveFilename) 627179860b2SJed Brown if os.path.exists(self.saveFilename): 628179860b2SJed Brown try: 629179860b2SJed Brown dbFile = file(self.saveFilename) 630179860b2SJed Brown data = cPickle.load(dbFile) 631179860b2SJed Brown self.updateTypes(data) 632179860b2SJed Brown dbFile.close() 633179860b2SJed Brown self.writeLogLine('Loaded dictionary from '+self.saveFilename) 634179860b2SJed Brown except Exception, e: 635179860b2SJed Brown self.writeLogLine('Problem loading dictionary from '+self.saveFilename+'\n--> '+str(e)) 636179860b2SJed Brown else: 637179860b2SJed Brown self.writeLogLine('No dictionary to load in this file: '+self.saveFilename) 638179860b2SJed Brown return 639179860b2SJed Brown 640179860b2SJed Brown def save(self, force = 0): 641179860b2SJed Brown '''Save the dictionary after 5 seconds, ignoring all subsequent calls until the save 642179860b2SJed Brown - Giving force = True will cause an immediate save''' 643179860b2SJed Brown if self.readonly: return 64408eb64ffSMatthew G. Knepley if force: 645179860b2SJed Brown self.saveTimer = None 646179860b2SJed Brown # This should be a critical section 647179860b2SJed Brown dbFile = file(self.saveFilename, 'w') 648179860b2SJed Brown data = dict(filter(lambda i: not i[1].getTemporary(), self.localitems())) 649179860b2SJed Brown cPickle.dump(data, dbFile) 650179860b2SJed Brown dbFile.close() 651179860b2SJed Brown self.writeLogLine('Saved local dictionary to '+os.path.abspath(self.saveFilename)) 652179860b2SJed Brown elif not self.saveTimer: 653179860b2SJed Brown import threading 654179860b2SJed Brown self.saveTimer = threading.Timer(5, self.save, [], {'force': 1}) 655179860b2SJed Brown self.saveTimer.setDaemon(1) 656179860b2SJed Brown self.saveTimer.start() 657179860b2SJed Brown return 658179860b2SJed Brown 659179860b2SJed Brown def shutdown(self): 660179860b2SJed Brown '''Shutdown the dictionary, writing out changes and notifying parent''' 661179860b2SJed Brown if self.saveTimer: 662179860b2SJed Brown self.saveTimer.cancel() 663179860b2SJed Brown self.save(force = 1) 664179860b2SJed Brown if self.isServer and os.path.isfile(self.addrFilename): 665179860b2SJed Brown os.remove(self.addrFilename) 666179860b2SJed Brown if not self.parent is None: 667179860b2SJed Brown self.sendPacket(self.parent, self.stopCmd, isPickled = 1) 668179860b2SJed Brown self.parent.close() 669179860b2SJed Brown self.parent = None 670179860b2SJed Brown self.writeLogLine('Shutting down') 671179860b2SJed Brown self.logFile.close() 672179860b2SJed Brown return 673179860b2SJed Brown 674179860b2SJed Brown def serverShutdown(self, pid, delay = shutdownDelay): 675179860b2SJed Brown if self.shutdownTimer is None: 676179860b2SJed Brown import threading 677179860b2SJed Brown 678179860b2SJed Brown self.shutdownTimer = threading.Timer(delay, self.serverShutdown, [pid], {'delay': 0}) 679179860b2SJed Brown self.shutdownTimer.setDaemon(1) 680179860b2SJed Brown self.shutdownTimer.start() 681179860b2SJed Brown self.writeLogLine('SERVER: Set shutdown timer for process '+str(pid)+' at '+str(delay)+' seconds') 682179860b2SJed Brown else: 683179860b2SJed Brown try: 684179860b2SJed Brown import signal 685179860b2SJed Brown import time 686179860b2SJed Brown 687179860b2SJed Brown idleTime = time.time() - self.lastAccess 688179860b2SJed Brown self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 689179860b2SJed Brown self.writeLogLine('SERVER: Idle time '+str(idleTime)) 690179860b2SJed Brown if idleTime < RDict.shutdownDelay: 691179860b2SJed Brown self.writeLogLine('SERVER: Extending shutdown timer for '+str(pid)+' by '+str(RDict.shutdownDelay - idleTime)+' seconds') 692179860b2SJed Brown self.shutdownTimer = None 693179860b2SJed Brown self.serverShutdown(pid, RDict.shutdownDelay - idleTime) 694179860b2SJed Brown else: 695179860b2SJed Brown self.writeLogLine('SERVER: Killing server '+str(pid)) 696179860b2SJed Brown os.kill(pid, signal.SIGTERM) 697179860b2SJed Brown except Exception, e: 698179860b2SJed Brown self.writeLogLine('SERVER: Exception killing server: '+str(e)) 699179860b2SJed Brown return 700179860b2SJed Brown 701179860b2SJed Brownif __name__ == '__main__': 702179860b2SJed Brown import sys 703179860b2SJed Brown try: 704179860b2SJed Brown if len(sys.argv) < 2: 705179860b2SJed Brown print 'RDict.py [server | client | clear | insert | remove] [parent]' 706179860b2SJed Brown else: 707179860b2SJed Brown action = sys.argv[1] 708179860b2SJed Brown parent = None 709179860b2SJed Brown if len(sys.argv) > 2: 710179860b2SJed Brown if not sys.argv[2] == 'None': parent = sys.argv[2] 711179860b2SJed Brown if action == 'server': 712179860b2SJed Brown RDict(parentDirectory = parent).serve() 713179860b2SJed Brown elif action == 'client': 714179860b2SJed Brown print 'Entries in server dictionary' 715179860b2SJed Brown rdict = RDict(parentDirectory = parent) 716179860b2SJed Brown for key in rdict.types(): 717179860b2SJed Brown if not key.startswith('cacheKey') and not key.startswith('stamp-'): 718179860b2SJed Brown print str(key)+' '+str(rdict.getType(key)) 719179860b2SJed Brown elif action == 'cacheClient': 720179860b2SJed Brown print 'Cache entries in server dictionary' 721179860b2SJed Brown rdict = RDict(parentDirectory = parent) 722179860b2SJed Brown for key in rdict.types(): 723179860b2SJed Brown if key.startswith('cacheKey'): 724179860b2SJed Brown print str(key)+' '+str(rdict.getType(key)) 725179860b2SJed Brown elif action == 'stampClient': 726179860b2SJed Brown print 'Stamp entries in server dictionary' 727179860b2SJed Brown rdict = RDict(parentDirectory = parent) 728179860b2SJed Brown for key in rdict.types(): 729179860b2SJed Brown if key.startswith('stamp-'): 730179860b2SJed Brown print str(key)+' '+str(rdict.getType(key)) 731179860b2SJed Brown elif action == 'clear': 732179860b2SJed Brown print 'Clearing all dictionaries' 733179860b2SJed Brown RDict(parentDirectory = parent).clear() 734179860b2SJed Brown elif action == 'insert': 735179860b2SJed Brown rdict = RDict(parentDirectory = parent) 736179860b2SJed Brown rdict[sys.argv[3]] = sys.argv[4] 737179860b2SJed Brown elif action == 'remove': 738179860b2SJed Brown rdict = RDict(parentDirectory = parent) 739179860b2SJed Brown del rdict[sys.argv[3]] 740179860b2SJed Brown else: 741179860b2SJed Brown sys.exit('Unknown action: '+action) 742179860b2SJed Brown except Exception, e: 743179860b2SJed Brown import traceback 744179860b2SJed Brown print traceback.print_tb(sys.exc_info()[2]) 745179860b2SJed Brown sys.exit(str(e)) 746179860b2SJed Brown sys.exit(0) 747