1*179860b2SJed Brown#!/usr/bin/env python 2*179860b2SJed Brown'''A remote dictionary server 3*179860b2SJed Brown 4*179860b2SJed Brown RDict is a typed, hierarchical, persistent dictionary intended to manage 5*179860b2SJed Brown all arguments or options for a program. The interface remains exactly the 6*179860b2SJed Brown same as dict, but the storage is more complicated. 7*179860b2SJed Brown 8*179860b2SJed Brown Argument typing is handled by wrapping all values stored in the dictionary 9*179860b2SJed Brown with nargs.Arg or a subclass. A user can call setType() to set the type of 10*179860b2SJed Brown an argument without any value being present. Whenever __getitem__() or 11*179860b2SJed Brown __setitem__() is called, values are extracted or replaced in the wrapper. 12*179860b2SJed Brown These wrappers can be accessed directly using getType(), setType(), and 13*179860b2SJed Brown types(). 14*179860b2SJed Brown 15*179860b2SJed Brown Hierarchy is allowed using a single "parent" dictionary. All operations 16*179860b2SJed Brown cascade to the parent. For instance, the length of the dictionary is the 17*179860b2SJed Brown number of local keys plus the number of keys in the parent, and its 18*179860b2SJed Brown parent, etc. Also, a dictionary need not have a parent. If a key does not 19*179860b2SJed Brown appear in the local dicitonary, the call if passed to the parent. However, 20*179860b2SJed Brown in this case we see that local keys can shadow those in a parent. 21*179860b2SJed Brown Communication with the parent is handled using sockets, with the parent 22*179860b2SJed Brown being a server and the interactive dictionary a client. 23*179860b2SJed Brown 24*179860b2SJed Brown The default persistence mechanism is a pickle file, RDict.db, written 25*179860b2SJed Brown whenever an argument is changed locally. A timer thread is created after 26*179860b2SJed Brown an initial change, so that many rapid changes do not cause many writes. 27*179860b2SJed Brown Each dictionary only saves its local entries, so all parents also 28*179860b2SJed Brown separately save data in different RDict.db files. Each time a dictionary 29*179860b2SJed Brown is created, the current directory is searched for an RDict.db file, and 30*179860b2SJed Brown if found the contents are loaded into the dictionary. 31*179860b2SJed Brown 32*179860b2SJed Brown This script also provides some default actions: 33*179860b2SJed Brown 34*179860b2SJed Brown - server [parent] 35*179860b2SJed Brown Starts a server in the current directory with an optional parent. This 36*179860b2SJed Brown server will accept socket connections from other dictionaries and act 37*179860b2SJed Brown as a parent. 38*179860b2SJed Brown 39*179860b2SJed Brown - client [parent] 40*179860b2SJed Brown Creates a dictionary in the current directory with an optional parent 41*179860b2SJed Brown and lists the contents. Notice that the contents may come from either 42*179860b2SJed Brown an RDict.db file in the current directory, or from the parent. 43*179860b2SJed Brown 44*179860b2SJed Brown - clear [parent] 45*179860b2SJed Brown Creates a dictionary in the current directory with an optional parent 46*179860b2SJed Brown and clears the contents. Notice that this will also clear the parent. 47*179860b2SJed Brown 48*179860b2SJed Brown - insert <parent> <key> <value> 49*179860b2SJed Brown Creates a dictionary in the current directory with a parent, and inserts 50*179860b2SJed Brown the key-value pair. If "parent" is "None", no parent is assigned. 51*179860b2SJed Brown 52*179860b2SJed Brown - remove <parent> <key> 53*179860b2SJed Brown Creates a dictionary in the current directory with a parent, and removes 54*179860b2SJed Brown the given key. If "parent" is "None", no parent is assigned. 55*179860b2SJed Brown''' 56*179860b2SJed Browntry: 57*179860b2SJed Brown import project # This is necessary for us to create Project objects on load 58*179860b2SJed Brown import build.buildGraph # This is necessary for us to create BuildGraph objects on load 59*179860b2SJed Brownexcept ImportError: 60*179860b2SJed Brown pass 61*179860b2SJed Brownimport nargs 62*179860b2SJed Brown 63*179860b2SJed Brownimport cPickle 64*179860b2SJed Brownimport os 65*179860b2SJed Brownimport sys 66*179860b2SJed BrownuseThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 67*179860b2SJed Brownif useThreads is None: 68*179860b2SJed Brown useThreads = 1 69*179860b2SJed Brownelse: 70*179860b2SJed Brown useThreads = int(useThreads) 71*179860b2SJed Brown 72*179860b2SJed Brownclass RDict(dict): 73*179860b2SJed Brown '''An RDict is a typed dictionary, which may be hierarchically composed. All elements derive from the 74*179860b2SJed BrownArg class, which wraps the usual value.''' 75*179860b2SJed Brown # The server will self-shutdown after this many seconds 76*179860b2SJed Brown shutdownDelay = 60*60*5 77*179860b2SJed Brown 78*179860b2SJed Brown def __init__(self, parentAddr = None, parentDirectory = None, load = 1, autoShutdown = 1, readonly = False): 79*179860b2SJed Brown import atexit 80*179860b2SJed Brown import time 81*179860b2SJed Brown import xdrlib 82*179860b2SJed Brown 83*179860b2SJed Brown self.logFile = None 84*179860b2SJed Brown self.setupLogFile() 85*179860b2SJed Brown self.target = ['default'] 86*179860b2SJed Brown self.parent = None 87*179860b2SJed Brown self.saveTimer = None 88*179860b2SJed Brown self.shutdownTimer = None 89*179860b2SJed Brown self.lastAccess = time.time() 90*179860b2SJed Brown self.saveFilename = 'RDict.db' 91*179860b2SJed Brown self.addrFilename = 'RDict.loc' 92*179860b2SJed Brown self.parentAddr = parentAddr 93*179860b2SJed Brown self.isServer = 0 94*179860b2SJed Brown self.readonly = readonly 95*179860b2SJed Brown self.parentDirectory = parentDirectory 96*179860b2SJed Brown self.packer = xdrlib.Packer() 97*179860b2SJed Brown self.unpacker = xdrlib.Unpacker('') 98*179860b2SJed Brown self.stopCmd = cPickle.dumps(('stop',)) 99*179860b2SJed Brown self.writeLogLine('Greetings') 100*179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 101*179860b2SJed Brown if load: self.load() 102*179860b2SJed Brown if autoShutdown and useThreads: 103*179860b2SJed Brown atexit.register(self.shutdown) 104*179860b2SJed Brown self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 105*179860b2SJed Brown return 106*179860b2SJed Brown 107*179860b2SJed Brown def __getstate__(self): 108*179860b2SJed Brown '''Remove any parent socket object, the XDR translators, and the log file from the dictionary before pickling''' 109*179860b2SJed Brown self.writeLogLine('Pickling RDict') 110*179860b2SJed Brown d = self.__dict__.copy() 111*179860b2SJed Brown if 'parent' in d: del d['parent'] 112*179860b2SJed Brown if 'saveTimer' in d: del d['saveTimer'] 113*179860b2SJed Brown if '_setCommandLine' in d: del d['_setCommandLine'] 114*179860b2SJed Brown del d['packer'] 115*179860b2SJed Brown del d['unpacker'] 116*179860b2SJed Brown del d['logFile'] 117*179860b2SJed Brown return d 118*179860b2SJed Brown 119*179860b2SJed Brown def __setstate__(self, d): 120*179860b2SJed Brown '''Reconnect the parent socket object, recreate the XDR translators and reopen the log file after unpickling''' 121*179860b2SJed Brown self.logFile = file('RDict.log', 'a') 122*179860b2SJed Brown self.writeLogLine('Unpickling RDict') 123*179860b2SJed Brown self.__dict__.update(d) 124*179860b2SJed Brown import xdrlib 125*179860b2SJed Brown self.packer = xdrlib.Packer() 126*179860b2SJed Brown self.unpacker = xdrlib.Unpacker('') 127*179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 128*179860b2SJed Brown return 129*179860b2SJed Brown 130*179860b2SJed Brown def setupLogFile(self, filename = 'RDict.log'): 131*179860b2SJed Brown if not self.logFile is None: 132*179860b2SJed Brown self.logFile.close() 133*179860b2SJed Brown if os.path.isfile(filename) and os.stat(filename).st_size > 10*1024*1024: 134*179860b2SJed Brown if os.path.isfile(filename+'.bkp'): 135*179860b2SJed Brown os.remove(filename+'.bkp') 136*179860b2SJed Brown os.rename(filename, filename+'.bkp') 137*179860b2SJed Brown self.logFile = file(filename, 'w') 138*179860b2SJed Brown else: 139*179860b2SJed Brown self.logFile = file(filename, 'a') 140*179860b2SJed Brown return 141*179860b2SJed Brown 142*179860b2SJed Brown def writeLogLine(self, message): 143*179860b2SJed Brown '''Writes the message to the log along with the current time''' 144*179860b2SJed Brown import time 145*179860b2SJed Brown self.logFile.write('('+str(os.getpid())+')('+str(id(self))+')'+message+' ['+time.asctime(time.localtime())+']\n') 146*179860b2SJed Brown self.logFile.flush() 147*179860b2SJed Brown return 148*179860b2SJed Brown 149*179860b2SJed Brown def __len__(self): 150*179860b2SJed Brown '''Returns the length of both the local and parent dictionaries''' 151*179860b2SJed Brown length = dict.__len__(self) 152*179860b2SJed Brown if not self.parent is None: 153*179860b2SJed Brown length = length + self.send() 154*179860b2SJed Brown return length 155*179860b2SJed Brown 156*179860b2SJed Brown def getType(self, key): 157*179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Returns the Arg object or None if not found.''' 158*179860b2SJed Brown if dict.has_key(self, key): 159*179860b2SJed Brown self.writeLogLine('getType: Getting local type for '+key+' '+str(dict.__getitem__(self, key))) 160*179860b2SJed Brown return dict.__getitem__(self, key) 161*179860b2SJed Brown elif not self.parent is None: 162*179860b2SJed Brown return self.send(key) 163*179860b2SJed Brown return None 164*179860b2SJed Brown 165*179860b2SJed Brown def __getitem__(self, key): 166*179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Returns the value of the Arg. 167*179860b2SJed Brown - If the value has not been set, the user will be prompted for input''' 168*179860b2SJed Brown if dict.has_key(self, key): 169*179860b2SJed Brown self.writeLogLine('__getitem__: '+key+' has local type') 170*179860b2SJed Brown pass 171*179860b2SJed Brown elif not self.parent is None: 172*179860b2SJed Brown self.writeLogLine('__getitem__: Checking parent value') 173*179860b2SJed Brown if self.send(key, operation = 'has_key'): 174*179860b2SJed Brown self.writeLogLine('__getitem__: Parent has value') 175*179860b2SJed Brown return self.send(key) 176*179860b2SJed Brown else: 177*179860b2SJed Brown self.writeLogLine('__getitem__: Checking parent type') 178*179860b2SJed Brown arg = self.send(key, operation = 'getType') 179*179860b2SJed Brown if not arg: 180*179860b2SJed Brown self.writeLogLine('__getitem__: Parent has no type') 181*179860b2SJed Brown arg = nargs.Arg(key) 182*179860b2SJed Brown try: 183*179860b2SJed Brown value = arg.getValue() 184*179860b2SJed Brown except AttributeError, e: 185*179860b2SJed Brown self.writeLogLine('__getitem__: Parent had invalid entry: '+str(e)) 186*179860b2SJed Brown arg = nargs.Arg(key) 187*179860b2SJed Brown value = arg.getValue() 188*179860b2SJed Brown self.writeLogLine('__getitem__: Setting parent value '+str(value)) 189*179860b2SJed Brown self.send(key, value, operation = '__setitem__') 190*179860b2SJed Brown return value 191*179860b2SJed Brown else: 192*179860b2SJed Brown self.writeLogLine('__getitem__: Setting local type for '+key) 193*179860b2SJed Brown dict.__setitem__(self, key, nargs.Arg(key)) 194*179860b2SJed Brown self.save() 195*179860b2SJed Brown self.writeLogLine('__getitem__: Setting local value for '+key) 196*179860b2SJed Brown return dict.__getitem__(self, key).getValue() 197*179860b2SJed Brown 198*179860b2SJed Brown def setType(self, key, value, forceLocal = 0): 199*179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Sets the type for this key. 200*179860b2SJed Brown - If a value for the key already exists, it is converted to the new type''' 201*179860b2SJed Brown if not isinstance(value, nargs.Arg): 202*179860b2SJed Brown raise TypeError('An argument type must be a subclass of Arg') 203*179860b2SJed Brown value.setKey(key) 204*179860b2SJed Brown if forceLocal or self.parent is None or dict.has_key(self, key): 205*179860b2SJed Brown if dict.has_key(self, key): 206*179860b2SJed Brown v = dict.__getitem__(self, key) 207*179860b2SJed Brown if v.isValueSet(): 208*179860b2SJed Brown try: 209*179860b2SJed Brown value.setValue(v.getValue()) 210*179860b2SJed Brown except TypeError: pass 211*179860b2SJed Brown dict.__setitem__(self, key, value) 212*179860b2SJed Brown self.save() 213*179860b2SJed Brown else: 214*179860b2SJed Brown return self.send(key, value) 215*179860b2SJed Brown return 216*179860b2SJed Brown 217*179860b2SJed Brown def __setitem__(self, key, value): 218*179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Sets the value of the Arg.''' 219*179860b2SJed Brown if not dict.has_key(self, key): 220*179860b2SJed Brown if not self.parent is None: 221*179860b2SJed Brown return self.send(key, value) 222*179860b2SJed Brown else: 223*179860b2SJed Brown dict.__setitem__(self, key, nargs.Arg(key)) 224*179860b2SJed Brown dict.__getitem__(self, key).setValue(value) 225*179860b2SJed Brown self.writeLogLine('__setitem__: Set value for '+key+' to '+str(dict.__getitem__(self, key))) 226*179860b2SJed Brown self.save() 227*179860b2SJed Brown return 228*179860b2SJed Brown 229*179860b2SJed Brown def __delitem__(self, key): 230*179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Deletes the Arg completely.''' 231*179860b2SJed Brown if dict.has_key(self, key): 232*179860b2SJed Brown dict.__delitem__(self, key) 233*179860b2SJed Brown self.save() 234*179860b2SJed Brown elif not self.parent is None: 235*179860b2SJed Brown self.send(key) 236*179860b2SJed Brown return 237*179860b2SJed Brown 238*179860b2SJed Brown def clear(self): 239*179860b2SJed Brown '''Clears both the local and parent dictionaries''' 240*179860b2SJed Brown if dict.__len__(self): 241*179860b2SJed Brown dict.clear(self) 242*179860b2SJed Brown self.save() 243*179860b2SJed Brown if not self.parent is None: 244*179860b2SJed Brown self.send() 245*179860b2SJed Brown return 246*179860b2SJed Brown 247*179860b2SJed Brown def __contains__(self, key): 248*179860b2SJed Brown '''This method just calls self.has_key(key)''' 249*179860b2SJed Brown return self.has_key(key) 250*179860b2SJed Brown 251*179860b2SJed Brown def has_key(self, key): 252*179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Then checks whether the value has been set''' 253*179860b2SJed Brown if dict.has_key(self, key): 254*179860b2SJed Brown if dict.__getitem__(self, key).isValueSet(): 255*179860b2SJed Brown self.writeLogLine('has_key: Have value for '+key) 256*179860b2SJed Brown else: 257*179860b2SJed Brown self.writeLogLine('has_key: Do not have value for '+key) 258*179860b2SJed Brown return dict.__getitem__(self, key).isValueSet() 259*179860b2SJed Brown elif not self.parent is None: 260*179860b2SJed Brown return self.send(key) 261*179860b2SJed Brown return 0 262*179860b2SJed Brown 263*179860b2SJed Brown def get(self, key, default=None): 264*179860b2SJed Brown if self.has_key(key): 265*179860b2SJed Brown return self.__getitem__(key) 266*179860b2SJed Brown else: 267*179860b2SJed Brown return default 268*179860b2SJed Brown 269*179860b2SJed Brown def hasType(self, key): 270*179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Then checks whether the type has been set''' 271*179860b2SJed Brown if dict.has_key(self, key): 272*179860b2SJed Brown return 1 273*179860b2SJed Brown elif not self.parent is None: 274*179860b2SJed Brown return self.send(key) 275*179860b2SJed Brown return 0 276*179860b2SJed Brown 277*179860b2SJed Brown def items(self): 278*179860b2SJed Brown '''Return a list of all accessible items, as (key, value) pairs.''' 279*179860b2SJed Brown l = dict.items(self) 280*179860b2SJed Brown if not self.parent is None: 281*179860b2SJed Brown l.extend(self.send()) 282*179860b2SJed Brown return l 283*179860b2SJed Brown 284*179860b2SJed Brown def localitems(self): 285*179860b2SJed Brown '''Return a list of all the items stored locally, as (key, value) pairs.''' 286*179860b2SJed Brown return dict.items(self) 287*179860b2SJed Brown 288*179860b2SJed Brown def keys(self): 289*179860b2SJed Brown '''Returns the list of keys in both the local and parent dictionaries''' 290*179860b2SJed Brown keyList = filter(lambda key: dict.__getitem__(self, key).isValueSet(), dict.keys(self)) 291*179860b2SJed Brown if not self.parent is None: 292*179860b2SJed Brown keyList.extend(self.send()) 293*179860b2SJed Brown return keyList 294*179860b2SJed Brown 295*179860b2SJed Brown def types(self): 296*179860b2SJed Brown '''Returns the list of keys for which types are defined in both the local and parent dictionaries''' 297*179860b2SJed Brown keyList = dict.keys(self) 298*179860b2SJed Brown if not self.parent is None: 299*179860b2SJed Brown keyList.extend(self.send()) 300*179860b2SJed Brown return keyList 301*179860b2SJed Brown 302*179860b2SJed Brown def update(self, d): 303*179860b2SJed Brown '''Update the dictionary with the contents of d''' 304*179860b2SJed Brown for k in d: 305*179860b2SJed Brown self[k] = d[k] 306*179860b2SJed Brown return 307*179860b2SJed Brown 308*179860b2SJed Brown def updateTypes(self, d): 309*179860b2SJed Brown '''Update types locally, which is equivalent to the dict.update() method''' 310*179860b2SJed Brown return dict.update(self, d) 311*179860b2SJed Brown 312*179860b2SJed Brown def insertArg(self, key, value, arg): 313*179860b2SJed Brown '''Insert a (key, value) pair into the dictionary. If key is None, arg is put into the target list.''' 314*179860b2SJed Brown if not key is None: 315*179860b2SJed Brown self[key] = value 316*179860b2SJed Brown else: 317*179860b2SJed Brown if not self.target == ['default']: 318*179860b2SJed Brown self.target.append(arg) 319*179860b2SJed Brown else: 320*179860b2SJed Brown self.target = [arg] 321*179860b2SJed Brown return 322*179860b2SJed Brown 323*179860b2SJed Brown def insertArgs(self, args): 324*179860b2SJed Brown '''Insert some text arguments into the dictionary (list and dictionaries are recognized)''' 325*179860b2SJed Brown import UserDict 326*179860b2SJed Brown 327*179860b2SJed Brown if isinstance(args, list): 328*179860b2SJed Brown for arg in args: 329*179860b2SJed Brown (key, value) = nargs.Arg.parseArgument(arg) 330*179860b2SJed Brown self.insertArg(key, value, arg) 331*179860b2SJed Brown # Necessary since os.environ is a UserDict 332*179860b2SJed Brown elif isinstance(args, dict) or isinstance(args, UserDict.UserDict): 333*179860b2SJed Brown for key in args.keys(): 334*179860b2SJed Brown if isinstance(args[key], str): 335*179860b2SJed Brown value = nargs.Arg.parseValue(args[key]) 336*179860b2SJed Brown else: 337*179860b2SJed Brown value = args[key] 338*179860b2SJed Brown self.insertArg(key, value, None) 339*179860b2SJed Brown elif isinstance(args, str): 340*179860b2SJed Brown (key, value) = nargs.Arg.parseArgument(args) 341*179860b2SJed Brown self.insertArg(key, value, args) 342*179860b2SJed Brown return 343*179860b2SJed Brown 344*179860b2SJed Brown def hasParent(self): 345*179860b2SJed Brown '''Return True if this RDict has a parent dictionary''' 346*179860b2SJed Brown return not self.parent is None 347*179860b2SJed Brown 348*179860b2SJed Brown def getServerAddr(self, dir): 349*179860b2SJed Brown '''Read the server socket address (in pickled form) from a file, usually RDict.loc 350*179860b2SJed Brown - If we fail to connect to the server specified in the file, we spawn it using startServer()''' 351*179860b2SJed Brown filename = os.path.join(dir, self.addrFilename) 352*179860b2SJed Brown if not os.path.exists(filename): 353*179860b2SJed Brown self.startServer(filename) 354*179860b2SJed Brown if not os.path.exists(filename): 355*179860b2SJed Brown raise RuntimeError('Server address file does not exist: '+filename) 356*179860b2SJed Brown try: 357*179860b2SJed Brown f = open(filename, 'r') 358*179860b2SJed Brown addr = cPickle.load(f) 359*179860b2SJed Brown f.close() 360*179860b2SJed Brown return addr 361*179860b2SJed Brown except Exception, e: 362*179860b2SJed Brown self.writeLogLine('CLIENT: Exception during server address determination: '+str(e.__class__)+': '+str(e)) 363*179860b2SJed Brown raise RuntimeError('Could not get server address in '+filename) 364*179860b2SJed Brown 365*179860b2SJed Brown def writeServerAddr(self, server): 366*179860b2SJed Brown '''Write the server socket address (in pickled form) to a file, usually RDict.loc.''' 367*179860b2SJed Brown f = file(self.addrFilename, 'w') 368*179860b2SJed Brown cPickle.dump(server.server_address, f) 369*179860b2SJed Brown f.close() 370*179860b2SJed Brown self.writeLogLine('SERVER: Wrote lock file '+os.path.abspath(self.addrFilename)) 371*179860b2SJed Brown return 372*179860b2SJed Brown 373*179860b2SJed Brown def startServer(self, addrFilename): 374*179860b2SJed Brown '''Spawn a new RDict server in the parent directory''' 375*179860b2SJed Brown import RDict # Need this to locate server script 376*179860b2SJed Brown import sys 377*179860b2SJed Brown import time 378*179860b2SJed Brown import distutils.sysconfig 379*179860b2SJed Brown 380*179860b2SJed Brown self.writeLogLine('CLIENT: Spawning a new server with lock file '+os.path.abspath(addrFilename)) 381*179860b2SJed Brown if os.path.exists(addrFilename): 382*179860b2SJed Brown os.remove(addrFilename) 383*179860b2SJed Brown oldDir = os.getcwd() 384*179860b2SJed Brown source = os.path.join(os.path.dirname(os.path.abspath(sys.modules['RDict'].__file__)), 'RDict.py') 385*179860b2SJed Brown interpreter = os.path.join(distutils.sysconfig.get_config_var('BINDIR'), distutils.sysconfig.get_config_var('PYTHON')) 386*179860b2SJed Brown if not os.path.isfile(interpreter): 387*179860b2SJed Brown interpreter = 'python' 388*179860b2SJed Brown os.chdir(os.path.dirname(addrFilename)) 389*179860b2SJed Brown self.writeLogLine('CLIENT: Executing '+interpreter+' '+source+' server"') 390*179860b2SJed Brown try: 391*179860b2SJed Brown os.spawnvp(os.P_NOWAIT, interpreter, [interpreter, source, 'server']) 392*179860b2SJed Brown except: 393*179860b2SJed Brown self.writeLogLine('CLIENT: os.spawnvp failed.\n \ 394*179860b2SJed Brown This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 395*179860b2SJed Brown you can fix this problem by running /bin/rebaseall. If you do not have\n \ 396*179860b2SJed Brown this program, you can install it with the CYGWIN installer in the package\n \ 397*179860b2SJed Brown Rebase, under the category System. You must run /bin/rebaseall after\n \ 398*179860b2SJed Brown turning off all cygwin services -- in particular sshd, if any such services\n \ 399*179860b2SJed Brown are running. For more information about rebase, go to http://www.cygwin.com') 400*179860b2SJed Brown print '\n \ 401*179860b2SJed Brown This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 402*179860b2SJed Brown you can fix this problem by running /bin/rebaseall. If you do not have\n \ 403*179860b2SJed Brown this program, you can install it with the CYGWIN installer in the package\n \ 404*179860b2SJed Brown Rebase, under the category System. You must run /bin/rebaseall after\n \ 405*179860b2SJed Brown turning off all cygwin services -- in particular sshd, if any such services\n \ 406*179860b2SJed Brown are running. For more information about rebase, go to http://www.cygwin.com\n' 407*179860b2SJed Brown raise 408*179860b2SJed Brown os.chdir(oldDir) 409*179860b2SJed Brown timeout = 1 410*179860b2SJed Brown for i in range(10): 411*179860b2SJed Brown time.sleep(timeout) 412*179860b2SJed Brown timeout *= 2 413*179860b2SJed Brown if timeout > 100: timeout = 100 414*179860b2SJed Brown if os.path.exists(addrFilename): return 415*179860b2SJed Brown self.writeLogLine('CLIENT: Could not start server') 416*179860b2SJed Brown return 417*179860b2SJed Brown 418*179860b2SJed Brown def connectParent(self, addr, dir): 419*179860b2SJed Brown '''Try to connect to a parent RDict server 420*179860b2SJed Brown - If addr and dir are both None, this operation fails 421*179860b2SJed Brown - If addr is None, check for an address file in dir''' 422*179860b2SJed Brown if addr is None: 423*179860b2SJed Brown if dir is None: return 0 424*179860b2SJed Brown addr = self.getServerAddr(dir) 425*179860b2SJed Brown 426*179860b2SJed Brown import socket 427*179860b2SJed Brown import errno 428*179860b2SJed Brown connected = 0 429*179860b2SJed Brown s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 430*179860b2SJed Brown timeout = 1 431*179860b2SJed Brown for i in range(10): 432*179860b2SJed Brown try: 433*179860b2SJed Brown self.writeLogLine('CLIENT: Trying to connect to '+str(addr)) 434*179860b2SJed Brown s.connect(addr) 435*179860b2SJed Brown connected = 1 436*179860b2SJed Brown break 437*179860b2SJed Brown except socket.error, e: 438*179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect: '+str(e)) 439*179860b2SJed Brown if e[0] == errno.ECONNREFUSED: 440*179860b2SJed Brown try: 441*179860b2SJed Brown import time 442*179860b2SJed Brown time.sleep(timeout) 443*179860b2SJed Brown timeout *= 2 444*179860b2SJed Brown if timeout > 100: timeout = 100 445*179860b2SJed Brown except KeyboardInterrupt: 446*179860b2SJed Brown break 447*179860b2SJed Brown # Try to spawn parent 448*179860b2SJed Brown if dir: 449*179860b2SJed Brown filename = os.path.join(dir, self.addrFilename) 450*179860b2SJed Brown if os.path.isfile(filename): 451*179860b2SJed Brown os.remove(filename) 452*179860b2SJed Brown self.startServer(filename) 453*179860b2SJed Brown except Exception, e: 454*179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect: '+str(e.__class__)+': '+str(e)) 455*179860b2SJed Brown if not connected: 456*179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect to parent') 457*179860b2SJed Brown return 0 458*179860b2SJed Brown self.parent = s 459*179860b2SJed Brown self.writeLogLine('CLIENT: Connected to '+str(self.parent)) 460*179860b2SJed Brown return 1 461*179860b2SJed Brown 462*179860b2SJed Brown def sendPacket(self, s, packet, source = 'Unknown', isPickled = 0): 463*179860b2SJed Brown '''Pickle the input packet. Send first the size of the pickled string in 32-bit integer, and then the string itself''' 464*179860b2SJed Brown self.writeLogLine(source+': Sending packet '+str(packet)) 465*179860b2SJed Brown if isPickled: 466*179860b2SJed Brown p = packet 467*179860b2SJed Brown else: 468*179860b2SJed Brown p = cPickle.dumps(packet) 469*179860b2SJed Brown self.packer.reset() 470*179860b2SJed Brown self.packer.pack_uint(len(p)) 471*179860b2SJed Brown if hasattr(s, 'write'): 472*179860b2SJed Brown s.write(self.packer.get_buffer()) 473*179860b2SJed Brown s.write(p) 474*179860b2SJed Brown else: 475*179860b2SJed Brown s.sendall(self.packer.get_buffer()) 476*179860b2SJed Brown s.sendall(p) 477*179860b2SJed Brown self.writeLogLine(source+': Sent packet') 478*179860b2SJed Brown return 479*179860b2SJed Brown 480*179860b2SJed Brown def recvPacket(self, s, source = 'Unknown'): 481*179860b2SJed Brown '''Receive first the size of the pickled string in a 32-bit integer, and then the string itself. Return the unpickled object''' 482*179860b2SJed Brown self.writeLogLine(source+': Receiving packet') 483*179860b2SJed Brown if hasattr(s, 'read'): 484*179860b2SJed Brown s.read(4) 485*179860b2SJed Brown value = cPickle.load(s) 486*179860b2SJed Brown else: 487*179860b2SJed Brown # I probably need to check that it actually read these 4 bytes 488*179860b2SJed Brown self.unpacker.reset(s.recv(4)) 489*179860b2SJed Brown length = self.unpacker.unpack_uint() 490*179860b2SJed Brown objString = '' 491*179860b2SJed Brown while len(objString) < length: 492*179860b2SJed Brown objString += s.recv(length - len(objString)) 493*179860b2SJed Brown value = cPickle.loads(objString) 494*179860b2SJed Brown self.writeLogLine(source+': Received packet '+str(value)) 495*179860b2SJed Brown return value 496*179860b2SJed Brown 497*179860b2SJed Brown def send(self, key = None, value = None, operation = None): 498*179860b2SJed Brown '''Send a request to the parent''' 499*179860b2SJed Brown import inspect 500*179860b2SJed Brown 501*179860b2SJed Brown objString = '' 502*179860b2SJed Brown for i in range(3): 503*179860b2SJed Brown try: 504*179860b2SJed Brown packet = [] 505*179860b2SJed Brown if operation is None: 506*179860b2SJed Brown operation = inspect.stack()[1][3] 507*179860b2SJed Brown packet.append(operation) 508*179860b2SJed Brown if not key is None: 509*179860b2SJed Brown packet.append(key) 510*179860b2SJed Brown if not value is None: 511*179860b2SJed Brown packet.append(value) 512*179860b2SJed Brown self.sendPacket(self.parent, tuple(packet), source = 'CLIENT') 513*179860b2SJed Brown response = self.recvPacket(self.parent, source = 'CLIENT') 514*179860b2SJed Brown break 515*179860b2SJed Brown except IOError, e: 516*179860b2SJed Brown self.writeLogLine('CLIENT: IOError '+str(e)) 517*179860b2SJed Brown if e.errno == 32: 518*179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 519*179860b2SJed Brown except Exception, e: 520*179860b2SJed Brown self.writeLogLine('CLIENT: Exception '+str(e)+' '+str(e.__class__)) 521*179860b2SJed Brown try: 522*179860b2SJed Brown if isinstance(response, Exception): 523*179860b2SJed Brown self.writeLogLine('CLIENT: Got an exception '+str(response)) 524*179860b2SJed Brown raise response 525*179860b2SJed Brown else: 526*179860b2SJed Brown self.writeLogLine('CLIENT: Received value '+str(response)+' '+str(type(response))) 527*179860b2SJed Brown except UnboundLocalError: 528*179860b2SJed Brown self.writeLogLine('CLIENT: Could not unpickle response') 529*179860b2SJed Brown response = None 530*179860b2SJed Brown return response 531*179860b2SJed Brown 532*179860b2SJed Brown def serve(self): 533*179860b2SJed Brown '''Start a server''' 534*179860b2SJed Brown import socket 535*179860b2SJed Brown import SocketServer 536*179860b2SJed Brown 537*179860b2SJed Brown if not useThreads: 538*179860b2SJed Brown raise RuntimeError('Cannot run a server if threads are disabled') 539*179860b2SJed Brown 540*179860b2SJed Brown class ProcessHandler(SocketServer.StreamRequestHandler): 541*179860b2SJed Brown def handle(self): 542*179860b2SJed Brown import time 543*179860b2SJed Brown 544*179860b2SJed Brown self.server.rdict.lastAccess = time.time() 545*179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Started new handler') 546*179860b2SJed Brown while 1: 547*179860b2SJed Brown try: 548*179860b2SJed Brown value = self.server.rdict.recvPacket(self.rfile, source = 'SERVER') 549*179860b2SJed Brown except EOFError, e: 550*179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: EOFError receiving packet '+str(e)+' '+str(e.__class__)) 551*179860b2SJed Brown return 552*179860b2SJed Brown except Exception, e: 553*179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Error receiving packet '+str(e)+' '+str(e.__class__)) 554*179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 555*179860b2SJed Brown continue 556*179860b2SJed Brown if value[0] == 'stop': break 557*179860b2SJed Brown try: 558*179860b2SJed Brown response = getattr(self.server.rdict, value[0])(*value[1:]) 559*179860b2SJed Brown except Exception, e: 560*179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Error executing operation '+str(e)+' '+str(e.__class__)) 561*179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 562*179860b2SJed Brown else: 563*179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, response, source = 'SERVER') 564*179860b2SJed Brown return 565*179860b2SJed Brown 566*179860b2SJed Brown # check if server is running 567*179860b2SJed Brown if os.path.exists(self.addrFilename): 568*179860b2SJed Brown rdict = RDict(parentDirectory = '.') 569*179860b2SJed Brown hasParent = rdict.hasParent() 570*179860b2SJed Brown del rdict 571*179860b2SJed Brown if hasParent: 572*179860b2SJed Brown self.writeLogLine('SERVER: Another server is already running') 573*179860b2SJed Brown raise RuntimeError('Server already running') 574*179860b2SJed Brown 575*179860b2SJed Brown # Daemonize server 576*179860b2SJed Brown self.writeLogLine('SERVER: Daemonizing server') 577*179860b2SJed Brown if os.fork(): # Launch child 578*179860b2SJed Brown os._exit(0) # Kill off parent, so we are not a process group leader and get a new PID 579*179860b2SJed Brown os.setsid() # Set session ID, so that we have no controlling terminal 580*179860b2SJed Brown # We choose to leave cwd at RDict.py: os.chdir('/') # Make sure root directory is not on a mounted drive 581*179860b2SJed Brown os.umask(077) # Fix creation mask 582*179860b2SJed Brown for i in range(3): # Crappy stopgap for closing descriptors 583*179860b2SJed Brown try: 584*179860b2SJed Brown os.close(i) 585*179860b2SJed Brown except OSError, e: 586*179860b2SJed Brown if e.errno != errno.EBADF: 587*179860b2SJed Brown raise RuntimeError('Could not close default descriptor '+str(i)) 588*179860b2SJed Brown 589*179860b2SJed Brown # wish there was a better way to get a usable socket 590*179860b2SJed Brown self.writeLogLine('SERVER: Establishing socket server') 591*179860b2SJed Brown basePort = 8000 592*179860b2SJed Brown flag = 'nosocket' 593*179860b2SJed Brown p = 1 594*179860b2SJed Brown while p < 1000 and flag == 'nosocket': 595*179860b2SJed Brown try: 596*179860b2SJed Brown server = SocketServer.ThreadingTCPServer((socket.gethostname(), basePort+p), ProcessHandler) 597*179860b2SJed Brown flag = 'socket' 598*179860b2SJed Brown except Exception, e: 599*179860b2SJed Brown p = p + 1 600*179860b2SJed Brown if flag == 'nosocket': 601*179860b2SJed Brown p = 1 602*179860b2SJed Brown while p < 1000 and flag == 'nosocket': 603*179860b2SJed Brown try: 604*179860b2SJed Brown server = SocketServer.ThreadingTCPServer(('localhost', basePort+p), ProcessHandler) 605*179860b2SJed Brown flag = 'socket' 606*179860b2SJed Brown except Exception, e: 607*179860b2SJed Brown p = p + 1 608*179860b2SJed Brown if flag == 'nosocket': 609*179860b2SJed Brown self.writeLogLine('SERVER: Could not established socket server on port '+str(basePort+p)) 610*179860b2SJed Brown raise RuntimeError,'Cannot get available socket' 611*179860b2SJed Brown self.writeLogLine('SERVER: Established socket server on port '+str(basePort+p)) 612*179860b2SJed Brown 613*179860b2SJed Brown self.isServer = 1 614*179860b2SJed Brown self.writeServerAddr(server) 615*179860b2SJed Brown self.serverShutdown(os.getpid()) 616*179860b2SJed Brown 617*179860b2SJed Brown server.rdict = self 618*179860b2SJed Brown self.writeLogLine('SERVER: Started server') 619*179860b2SJed Brown server.serve_forever() 620*179860b2SJed Brown return 621*179860b2SJed Brown 622*179860b2SJed Brown def load(self): 623*179860b2SJed Brown '''Load the saved dictionary''' 624*179860b2SJed Brown if not self.parentDirectory is None and os.path.samefile(os.getcwd(), self.parentDirectory): 625*179860b2SJed Brown return 626*179860b2SJed Brown self.saveFilename = os.path.abspath(self.saveFilename) 627*179860b2SJed Brown if os.path.exists(self.saveFilename): 628*179860b2SJed Brown try: 629*179860b2SJed Brown dbFile = file(self.saveFilename) 630*179860b2SJed Brown data = cPickle.load(dbFile) 631*179860b2SJed Brown self.updateTypes(data) 632*179860b2SJed Brown dbFile.close() 633*179860b2SJed Brown self.writeLogLine('Loaded dictionary from '+self.saveFilename) 634*179860b2SJed Brown except Exception, e: 635*179860b2SJed Brown self.writeLogLine('Problem loading dictionary from '+self.saveFilename+'\n--> '+str(e)) 636*179860b2SJed Brown else: 637*179860b2SJed Brown self.writeLogLine('No dictionary to load in this file: '+self.saveFilename) 638*179860b2SJed Brown return 639*179860b2SJed Brown 640*179860b2SJed Brown def save(self, force = 0): 641*179860b2SJed Brown '''Save the dictionary after 5 seconds, ignoring all subsequent calls until the save 642*179860b2SJed Brown - Giving force = True will cause an immediate save''' 643*179860b2SJed Brown if self.readonly: return 644*179860b2SJed Brown if force or not useThreads: 645*179860b2SJed Brown self.saveTimer = None 646*179860b2SJed Brown # This should be a critical section 647*179860b2SJed Brown dbFile = file(self.saveFilename, 'w') 648*179860b2SJed Brown data = dict(filter(lambda i: not i[1].getTemporary(), self.localitems())) 649*179860b2SJed Brown cPickle.dump(data, dbFile) 650*179860b2SJed Brown dbFile.close() 651*179860b2SJed Brown self.writeLogLine('Saved local dictionary to '+os.path.abspath(self.saveFilename)) 652*179860b2SJed Brown elif not self.saveTimer: 653*179860b2SJed Brown import threading 654*179860b2SJed Brown self.saveTimer = threading.Timer(5, self.save, [], {'force': 1}) 655*179860b2SJed Brown self.saveTimer.setDaemon(1) 656*179860b2SJed Brown self.saveTimer.start() 657*179860b2SJed Brown return 658*179860b2SJed Brown 659*179860b2SJed Brown def shutdown(self): 660*179860b2SJed Brown '''Shutdown the dictionary, writing out changes and notifying parent''' 661*179860b2SJed Brown if self.saveTimer: 662*179860b2SJed Brown self.saveTimer.cancel() 663*179860b2SJed Brown self.save(force = 1) 664*179860b2SJed Brown if self.isServer and os.path.isfile(self.addrFilename): 665*179860b2SJed Brown os.remove(self.addrFilename) 666*179860b2SJed Brown if not self.parent is None: 667*179860b2SJed Brown self.sendPacket(self.parent, self.stopCmd, isPickled = 1) 668*179860b2SJed Brown self.parent.close() 669*179860b2SJed Brown self.parent = None 670*179860b2SJed Brown self.writeLogLine('Shutting down') 671*179860b2SJed Brown self.logFile.close() 672*179860b2SJed Brown return 673*179860b2SJed Brown 674*179860b2SJed Brown def serverShutdown(self, pid, delay = shutdownDelay): 675*179860b2SJed Brown if self.shutdownTimer is None: 676*179860b2SJed Brown import threading 677*179860b2SJed Brown 678*179860b2SJed Brown self.shutdownTimer = threading.Timer(delay, self.serverShutdown, [pid], {'delay': 0}) 679*179860b2SJed Brown self.shutdownTimer.setDaemon(1) 680*179860b2SJed Brown self.shutdownTimer.start() 681*179860b2SJed Brown self.writeLogLine('SERVER: Set shutdown timer for process '+str(pid)+' at '+str(delay)+' seconds') 682*179860b2SJed Brown else: 683*179860b2SJed Brown try: 684*179860b2SJed Brown import signal 685*179860b2SJed Brown import time 686*179860b2SJed Brown 687*179860b2SJed Brown idleTime = time.time() - self.lastAccess 688*179860b2SJed Brown self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 689*179860b2SJed Brown self.writeLogLine('SERVER: Idle time '+str(idleTime)) 690*179860b2SJed Brown if idleTime < RDict.shutdownDelay: 691*179860b2SJed Brown self.writeLogLine('SERVER: Extending shutdown timer for '+str(pid)+' by '+str(RDict.shutdownDelay - idleTime)+' seconds') 692*179860b2SJed Brown self.shutdownTimer = None 693*179860b2SJed Brown self.serverShutdown(pid, RDict.shutdownDelay - idleTime) 694*179860b2SJed Brown else: 695*179860b2SJed Brown self.writeLogLine('SERVER: Killing server '+str(pid)) 696*179860b2SJed Brown os.kill(pid, signal.SIGTERM) 697*179860b2SJed Brown except Exception, e: 698*179860b2SJed Brown self.writeLogLine('SERVER: Exception killing server: '+str(e)) 699*179860b2SJed Brown return 700*179860b2SJed Brown 701*179860b2SJed Brownif __name__ == '__main__': 702*179860b2SJed Brown import sys 703*179860b2SJed Brown try: 704*179860b2SJed Brown if len(sys.argv) < 2: 705*179860b2SJed Brown print 'RDict.py [server | client | clear | insert | remove] [parent]' 706*179860b2SJed Brown else: 707*179860b2SJed Brown action = sys.argv[1] 708*179860b2SJed Brown parent = None 709*179860b2SJed Brown if len(sys.argv) > 2: 710*179860b2SJed Brown if not sys.argv[2] == 'None': parent = sys.argv[2] 711*179860b2SJed Brown if action == 'server': 712*179860b2SJed Brown RDict(parentDirectory = parent).serve() 713*179860b2SJed Brown elif action == 'client': 714*179860b2SJed Brown print 'Entries in server dictionary' 715*179860b2SJed Brown rdict = RDict(parentDirectory = parent) 716*179860b2SJed Brown for key in rdict.types(): 717*179860b2SJed Brown if not key.startswith('cacheKey') and not key.startswith('stamp-'): 718*179860b2SJed Brown print str(key)+' '+str(rdict.getType(key)) 719*179860b2SJed Brown elif action == 'cacheClient': 720*179860b2SJed Brown print 'Cache entries in server dictionary' 721*179860b2SJed Brown rdict = RDict(parentDirectory = parent) 722*179860b2SJed Brown for key in rdict.types(): 723*179860b2SJed Brown if key.startswith('cacheKey'): 724*179860b2SJed Brown print str(key)+' '+str(rdict.getType(key)) 725*179860b2SJed Brown elif action == 'stampClient': 726*179860b2SJed Brown print 'Stamp entries in server dictionary' 727*179860b2SJed Brown rdict = RDict(parentDirectory = parent) 728*179860b2SJed Brown for key in rdict.types(): 729*179860b2SJed Brown if key.startswith('stamp-'): 730*179860b2SJed Brown print str(key)+' '+str(rdict.getType(key)) 731*179860b2SJed Brown elif action == 'clear': 732*179860b2SJed Brown print 'Clearing all dictionaries' 733*179860b2SJed Brown RDict(parentDirectory = parent).clear() 734*179860b2SJed Brown elif action == 'insert': 735*179860b2SJed Brown rdict = RDict(parentDirectory = parent) 736*179860b2SJed Brown rdict[sys.argv[3]] = sys.argv[4] 737*179860b2SJed Brown elif action == 'remove': 738*179860b2SJed Brown rdict = RDict(parentDirectory = parent) 739*179860b2SJed Brown del rdict[sys.argv[3]] 740*179860b2SJed Brown else: 741*179860b2SJed Brown sys.exit('Unknown action: '+action) 742*179860b2SJed Brown except Exception, e: 743*179860b2SJed Brown import traceback 744*179860b2SJed Brown print traceback.print_tb(sys.exc_info()[2]) 745*179860b2SJed Brown sys.exit(str(e)) 746*179860b2SJed Brown sys.exit(0) 747