xref: /petsc/config/BuildSystem/RDict.py (revision 179860b23afbef20daed3359c1645679d1efa988)
1*179860b2SJed Brown#!/usr/bin/env python
2*179860b2SJed Brown'''A remote dictionary server
3*179860b2SJed Brown
4*179860b2SJed Brown    RDict is a typed, hierarchical, persistent dictionary intended to manage
5*179860b2SJed Brown    all arguments or options for a program. The interface remains exactly the
6*179860b2SJed Brown    same as dict, but the storage is more complicated.
7*179860b2SJed Brown
8*179860b2SJed Brown    Argument typing is handled by wrapping all values stored in the dictionary
9*179860b2SJed Brown    with nargs.Arg or a subclass. A user can call setType() to set the type of
10*179860b2SJed Brown    an argument without any value being present. Whenever __getitem__() or
11*179860b2SJed Brown    __setitem__() is called, values are extracted or replaced in the wrapper.
12*179860b2SJed Brown    These wrappers can be accessed directly using getType(), setType(), and
13*179860b2SJed Brown    types().
14*179860b2SJed Brown
15*179860b2SJed Brown    Hierarchy is allowed using a single "parent" dictionary. All operations
16*179860b2SJed Brown    cascade to the parent. For instance, the length of the dictionary is the
17*179860b2SJed Brown    number of local keys plus the number of keys in the parent, and its
18*179860b2SJed Brown    parent, etc. Also, a dictionary need not have a parent. If a key does not
19*179860b2SJed Brown    appear in the local dicitonary, the call if passed to the parent. However,
20*179860b2SJed Brown    in this case we see that local keys can shadow those in a parent.
21*179860b2SJed Brown    Communication with the parent is handled using sockets, with the parent
22*179860b2SJed Brown    being a server and the interactive dictionary a client.
23*179860b2SJed Brown
24*179860b2SJed Brown    The default persistence mechanism is a pickle file, RDict.db, written
25*179860b2SJed Brown    whenever an argument is changed locally. A timer thread is created after
26*179860b2SJed Brown    an initial change, so that many rapid changes do not cause many writes.
27*179860b2SJed Brown    Each dictionary only saves its local entries, so all parents also
28*179860b2SJed Brown    separately save data in different RDict.db files. Each time a dictionary
29*179860b2SJed Brown    is created, the current directory is searched for an RDict.db file, and
30*179860b2SJed Brown    if found the contents are loaded into the dictionary.
31*179860b2SJed Brown
32*179860b2SJed Brown    This script also provides some default actions:
33*179860b2SJed Brown
34*179860b2SJed Brown      - server [parent]
35*179860b2SJed Brown        Starts a server in the current directory with an optional parent. This
36*179860b2SJed Brown        server will accept socket connections from other dictionaries and act
37*179860b2SJed Brown        as a parent.
38*179860b2SJed Brown
39*179860b2SJed Brown      - client [parent]
40*179860b2SJed Brown        Creates a dictionary in the current directory with an optional parent
41*179860b2SJed Brown        and lists the contents. Notice that the contents may come from either
42*179860b2SJed Brown        an RDict.db file in the current directory, or from the parent.
43*179860b2SJed Brown
44*179860b2SJed Brown      - clear [parent]
45*179860b2SJed Brown        Creates a dictionary in the current directory with an optional parent
46*179860b2SJed Brown        and clears the contents. Notice that this will also clear the parent.
47*179860b2SJed Brown
48*179860b2SJed Brown      - insert <parent> <key> <value>
49*179860b2SJed Brown        Creates a dictionary in the current directory with a parent, and inserts
50*179860b2SJed Brown        the key-value pair. If "parent" is "None", no parent is assigned.
51*179860b2SJed Brown
52*179860b2SJed Brown      - remove <parent> <key>
53*179860b2SJed Brown        Creates a dictionary in the current directory with a parent, and removes
54*179860b2SJed Brown        the given key. If "parent" is "None", no parent is assigned.
55*179860b2SJed Brown'''
56*179860b2SJed Browntry:
57*179860b2SJed Brown  import project          # This is necessary for us to create Project objects on load
58*179860b2SJed Brown  import build.buildGraph # This is necessary for us to create BuildGraph objects on load
59*179860b2SJed Brownexcept ImportError:
60*179860b2SJed Brown  pass
61*179860b2SJed Brownimport nargs
62*179860b2SJed Brown
63*179860b2SJed Brownimport cPickle
64*179860b2SJed Brownimport os
65*179860b2SJed Brownimport sys
66*179860b2SJed BrownuseThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:])
67*179860b2SJed Brownif useThreads is None:
68*179860b2SJed Brown  useThreads = 1
69*179860b2SJed Brownelse:
70*179860b2SJed Brown  useThreads = int(useThreads)
71*179860b2SJed Brown
72*179860b2SJed Brownclass RDict(dict):
73*179860b2SJed Brown  '''An RDict is a typed dictionary, which may be hierarchically composed. All elements derive from the
74*179860b2SJed BrownArg class, which wraps the usual value.'''
75*179860b2SJed Brown  # The server will self-shutdown after this many seconds
76*179860b2SJed Brown  shutdownDelay = 60*60*5
77*179860b2SJed Brown
78*179860b2SJed Brown  def __init__(self, parentAddr = None, parentDirectory = None, load = 1, autoShutdown = 1, readonly = False):
79*179860b2SJed Brown    import atexit
80*179860b2SJed Brown    import time
81*179860b2SJed Brown    import xdrlib
82*179860b2SJed Brown
83*179860b2SJed Brown    self.logFile         = None
84*179860b2SJed Brown    self.setupLogFile()
85*179860b2SJed Brown    self.target          = ['default']
86*179860b2SJed Brown    self.parent          = None
87*179860b2SJed Brown    self.saveTimer       = None
88*179860b2SJed Brown    self.shutdownTimer   = None
89*179860b2SJed Brown    self.lastAccess      = time.time()
90*179860b2SJed Brown    self.saveFilename    = 'RDict.db'
91*179860b2SJed Brown    self.addrFilename    = 'RDict.loc'
92*179860b2SJed Brown    self.parentAddr      = parentAddr
93*179860b2SJed Brown    self.isServer        = 0
94*179860b2SJed Brown    self.readonly        = readonly
95*179860b2SJed Brown    self.parentDirectory = parentDirectory
96*179860b2SJed Brown    self.packer          = xdrlib.Packer()
97*179860b2SJed Brown    self.unpacker        = xdrlib.Unpacker('')
98*179860b2SJed Brown    self.stopCmd         = cPickle.dumps(('stop',))
99*179860b2SJed Brown    self.writeLogLine('Greetings')
100*179860b2SJed Brown    self.connectParent(self.parentAddr, self.parentDirectory)
101*179860b2SJed Brown    if load: self.load()
102*179860b2SJed Brown    if autoShutdown and useThreads:
103*179860b2SJed Brown      atexit.register(self.shutdown)
104*179860b2SJed Brown    self.writeLogLine('SERVER: Last access '+str(self.lastAccess))
105*179860b2SJed Brown    return
106*179860b2SJed Brown
107*179860b2SJed Brown  def __getstate__(self):
108*179860b2SJed Brown    '''Remove any parent socket object, the XDR translators, and the log file from the dictionary before pickling'''
109*179860b2SJed Brown    self.writeLogLine('Pickling RDict')
110*179860b2SJed Brown    d = self.__dict__.copy()
111*179860b2SJed Brown    if 'parent'    in d: del d['parent']
112*179860b2SJed Brown    if 'saveTimer' in d: del d['saveTimer']
113*179860b2SJed Brown    if '_setCommandLine' in d: del d['_setCommandLine']
114*179860b2SJed Brown    del d['packer']
115*179860b2SJed Brown    del d['unpacker']
116*179860b2SJed Brown    del d['logFile']
117*179860b2SJed Brown    return d
118*179860b2SJed Brown
119*179860b2SJed Brown  def __setstate__(self, d):
120*179860b2SJed Brown    '''Reconnect the parent socket object, recreate the XDR translators and reopen the log file after unpickling'''
121*179860b2SJed Brown    self.logFile  = file('RDict.log', 'a')
122*179860b2SJed Brown    self.writeLogLine('Unpickling RDict')
123*179860b2SJed Brown    self.__dict__.update(d)
124*179860b2SJed Brown    import xdrlib
125*179860b2SJed Brown    self.packer   = xdrlib.Packer()
126*179860b2SJed Brown    self.unpacker = xdrlib.Unpacker('')
127*179860b2SJed Brown    self.connectParent(self.parentAddr, self.parentDirectory)
128*179860b2SJed Brown    return
129*179860b2SJed Brown
130*179860b2SJed Brown  def setupLogFile(self, filename = 'RDict.log'):
131*179860b2SJed Brown    if not self.logFile is None:
132*179860b2SJed Brown      self.logFile.close()
133*179860b2SJed Brown    if os.path.isfile(filename) and os.stat(filename).st_size > 10*1024*1024:
134*179860b2SJed Brown      if os.path.isfile(filename+'.bkp'):
135*179860b2SJed Brown        os.remove(filename+'.bkp')
136*179860b2SJed Brown      os.rename(filename, filename+'.bkp')
137*179860b2SJed Brown      self.logFile = file(filename, 'w')
138*179860b2SJed Brown    else:
139*179860b2SJed Brown      self.logFile = file(filename, 'a')
140*179860b2SJed Brown    return
141*179860b2SJed Brown
142*179860b2SJed Brown  def writeLogLine(self, message):
143*179860b2SJed Brown    '''Writes the message to the log along with the current time'''
144*179860b2SJed Brown    import time
145*179860b2SJed Brown    self.logFile.write('('+str(os.getpid())+')('+str(id(self))+')'+message+' ['+time.asctime(time.localtime())+']\n')
146*179860b2SJed Brown    self.logFile.flush()
147*179860b2SJed Brown    return
148*179860b2SJed Brown
149*179860b2SJed Brown  def __len__(self):
150*179860b2SJed Brown    '''Returns the length of both the local and parent dictionaries'''
151*179860b2SJed Brown    length = dict.__len__(self)
152*179860b2SJed Brown    if not self.parent is None:
153*179860b2SJed Brown      length = length + self.send()
154*179860b2SJed Brown    return length
155*179860b2SJed Brown
156*179860b2SJed Brown  def getType(self, key):
157*179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Returns the Arg object or None if not found.'''
158*179860b2SJed Brown    if dict.has_key(self, key):
159*179860b2SJed Brown      self.writeLogLine('getType: Getting local type for '+key+' '+str(dict.__getitem__(self, key)))
160*179860b2SJed Brown      return dict.__getitem__(self, key)
161*179860b2SJed Brown    elif not self.parent is None:
162*179860b2SJed Brown      return self.send(key)
163*179860b2SJed Brown    return None
164*179860b2SJed Brown
165*179860b2SJed Brown  def __getitem__(self, key):
166*179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Returns the value of the Arg.
167*179860b2SJed Brown       - If the value has not been set, the user will be prompted for input'''
168*179860b2SJed Brown    if dict.has_key(self, key):
169*179860b2SJed Brown      self.writeLogLine('__getitem__: '+key+' has local type')
170*179860b2SJed Brown      pass
171*179860b2SJed Brown    elif not self.parent is None:
172*179860b2SJed Brown      self.writeLogLine('__getitem__: Checking parent value')
173*179860b2SJed Brown      if self.send(key, operation = 'has_key'):
174*179860b2SJed Brown        self.writeLogLine('__getitem__: Parent has value')
175*179860b2SJed Brown        return self.send(key)
176*179860b2SJed Brown      else:
177*179860b2SJed Brown        self.writeLogLine('__getitem__: Checking parent type')
178*179860b2SJed Brown        arg = self.send(key, operation = 'getType')
179*179860b2SJed Brown        if not arg:
180*179860b2SJed Brown          self.writeLogLine('__getitem__: Parent has no type')
181*179860b2SJed Brown          arg = nargs.Arg(key)
182*179860b2SJed Brown        try:
183*179860b2SJed Brown          value = arg.getValue()
184*179860b2SJed Brown        except AttributeError, e:
185*179860b2SJed Brown          self.writeLogLine('__getitem__: Parent had invalid entry: '+str(e))
186*179860b2SJed Brown          arg   = nargs.Arg(key)
187*179860b2SJed Brown          value = arg.getValue()
188*179860b2SJed Brown        self.writeLogLine('__getitem__: Setting parent value '+str(value))
189*179860b2SJed Brown        self.send(key, value, operation = '__setitem__')
190*179860b2SJed Brown        return value
191*179860b2SJed Brown    else:
192*179860b2SJed Brown      self.writeLogLine('__getitem__: Setting local type for '+key)
193*179860b2SJed Brown      dict.__setitem__(self, key, nargs.Arg(key))
194*179860b2SJed Brown      self.save()
195*179860b2SJed Brown    self.writeLogLine('__getitem__: Setting local value for '+key)
196*179860b2SJed Brown    return dict.__getitem__(self, key).getValue()
197*179860b2SJed Brown
198*179860b2SJed Brown  def setType(self, key, value, forceLocal = 0):
199*179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Sets the type for this key.
200*179860b2SJed Brown       - If a value for the key already exists, it is converted to the new type'''
201*179860b2SJed Brown    if not isinstance(value, nargs.Arg):
202*179860b2SJed Brown      raise TypeError('An argument type must be a subclass of Arg')
203*179860b2SJed Brown    value.setKey(key)
204*179860b2SJed Brown    if forceLocal or self.parent is None or dict.has_key(self, key):
205*179860b2SJed Brown      if dict.has_key(self, key):
206*179860b2SJed Brown        v = dict.__getitem__(self, key)
207*179860b2SJed Brown        if v.isValueSet():
208*179860b2SJed Brown          try:
209*179860b2SJed Brown            value.setValue(v.getValue())
210*179860b2SJed Brown          except TypeError: pass
211*179860b2SJed Brown      dict.__setitem__(self, key, value)
212*179860b2SJed Brown      self.save()
213*179860b2SJed Brown    else:
214*179860b2SJed Brown      return self.send(key, value)
215*179860b2SJed Brown    return
216*179860b2SJed Brown
217*179860b2SJed Brown  def __setitem__(self, key, value):
218*179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Sets the value of the Arg.'''
219*179860b2SJed Brown    if not dict.has_key(self, key):
220*179860b2SJed Brown      if not self.parent is None:
221*179860b2SJed Brown        return self.send(key, value)
222*179860b2SJed Brown      else:
223*179860b2SJed Brown        dict.__setitem__(self, key, nargs.Arg(key))
224*179860b2SJed Brown    dict.__getitem__(self, key).setValue(value)
225*179860b2SJed Brown    self.writeLogLine('__setitem__: Set value for '+key+' to '+str(dict.__getitem__(self, key)))
226*179860b2SJed Brown    self.save()
227*179860b2SJed Brown    return
228*179860b2SJed Brown
229*179860b2SJed Brown  def __delitem__(self, key):
230*179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Deletes the Arg completely.'''
231*179860b2SJed Brown    if dict.has_key(self, key):
232*179860b2SJed Brown      dict.__delitem__(self, key)
233*179860b2SJed Brown      self.save()
234*179860b2SJed Brown    elif not self.parent is None:
235*179860b2SJed Brown      self.send(key)
236*179860b2SJed Brown    return
237*179860b2SJed Brown
238*179860b2SJed Brown  def clear(self):
239*179860b2SJed Brown    '''Clears both the local and parent dictionaries'''
240*179860b2SJed Brown    if dict.__len__(self):
241*179860b2SJed Brown      dict.clear(self)
242*179860b2SJed Brown      self.save()
243*179860b2SJed Brown    if not self.parent is None:
244*179860b2SJed Brown      self.send()
245*179860b2SJed Brown    return
246*179860b2SJed Brown
247*179860b2SJed Brown  def __contains__(self, key):
248*179860b2SJed Brown    '''This method just calls self.has_key(key)'''
249*179860b2SJed Brown    return self.has_key(key)
250*179860b2SJed Brown
251*179860b2SJed Brown  def has_key(self, key):
252*179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Then checks whether the value has been set'''
253*179860b2SJed Brown    if dict.has_key(self, key):
254*179860b2SJed Brown      if dict.__getitem__(self, key).isValueSet():
255*179860b2SJed Brown        self.writeLogLine('has_key: Have value for '+key)
256*179860b2SJed Brown      else:
257*179860b2SJed Brown        self.writeLogLine('has_key: Do not have value for '+key)
258*179860b2SJed Brown      return dict.__getitem__(self, key).isValueSet()
259*179860b2SJed Brown    elif not self.parent is None:
260*179860b2SJed Brown      return self.send(key)
261*179860b2SJed Brown    return 0
262*179860b2SJed Brown
263*179860b2SJed Brown  def get(self, key, default=None):
264*179860b2SJed Brown    if self.has_key(key):
265*179860b2SJed Brown      return self.__getitem__(key)
266*179860b2SJed Brown    else:
267*179860b2SJed Brown      return default
268*179860b2SJed Brown
269*179860b2SJed Brown  def hasType(self, key):
270*179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Then checks whether the type has been set'''
271*179860b2SJed Brown    if dict.has_key(self, key):
272*179860b2SJed Brown      return 1
273*179860b2SJed Brown    elif not self.parent is None:
274*179860b2SJed Brown      return self.send(key)
275*179860b2SJed Brown    return 0
276*179860b2SJed Brown
277*179860b2SJed Brown  def items(self):
278*179860b2SJed Brown    '''Return a list of all accessible items, as (key, value) pairs.'''
279*179860b2SJed Brown    l = dict.items(self)
280*179860b2SJed Brown    if not self.parent is None:
281*179860b2SJed Brown      l.extend(self.send())
282*179860b2SJed Brown    return l
283*179860b2SJed Brown
284*179860b2SJed Brown  def localitems(self):
285*179860b2SJed Brown    '''Return a list of all the items stored locally, as (key, value) pairs.'''
286*179860b2SJed Brown    return dict.items(self)
287*179860b2SJed Brown
288*179860b2SJed Brown  def keys(self):
289*179860b2SJed Brown    '''Returns the list of keys in both the local and parent dictionaries'''
290*179860b2SJed Brown    keyList = filter(lambda key: dict.__getitem__(self, key).isValueSet(), dict.keys(self))
291*179860b2SJed Brown    if not self.parent is None:
292*179860b2SJed Brown      keyList.extend(self.send())
293*179860b2SJed Brown    return keyList
294*179860b2SJed Brown
295*179860b2SJed Brown  def types(self):
296*179860b2SJed Brown    '''Returns the list of keys for which types are defined in both the local and parent dictionaries'''
297*179860b2SJed Brown    keyList = dict.keys(self)
298*179860b2SJed Brown    if not self.parent is None:
299*179860b2SJed Brown      keyList.extend(self.send())
300*179860b2SJed Brown    return keyList
301*179860b2SJed Brown
302*179860b2SJed Brown  def update(self, d):
303*179860b2SJed Brown    '''Update the dictionary with the contents of d'''
304*179860b2SJed Brown    for k in d:
305*179860b2SJed Brown      self[k] = d[k]
306*179860b2SJed Brown    return
307*179860b2SJed Brown
308*179860b2SJed Brown  def updateTypes(self, d):
309*179860b2SJed Brown    '''Update types locally, which is equivalent to the dict.update() method'''
310*179860b2SJed Brown    return dict.update(self, d)
311*179860b2SJed Brown
312*179860b2SJed Brown  def insertArg(self, key, value, arg):
313*179860b2SJed Brown    '''Insert a (key, value) pair into the dictionary. If key is None, arg is put into the target list.'''
314*179860b2SJed Brown    if not key is None:
315*179860b2SJed Brown      self[key] = value
316*179860b2SJed Brown    else:
317*179860b2SJed Brown      if not self.target == ['default']:
318*179860b2SJed Brown        self.target.append(arg)
319*179860b2SJed Brown      else:
320*179860b2SJed Brown        self.target = [arg]
321*179860b2SJed Brown    return
322*179860b2SJed Brown
323*179860b2SJed Brown  def insertArgs(self, args):
324*179860b2SJed Brown    '''Insert some text arguments into the dictionary (list and dictionaries are recognized)'''
325*179860b2SJed Brown    import UserDict
326*179860b2SJed Brown
327*179860b2SJed Brown    if isinstance(args, list):
328*179860b2SJed Brown      for arg in args:
329*179860b2SJed Brown        (key, value) = nargs.Arg.parseArgument(arg)
330*179860b2SJed Brown        self.insertArg(key, value, arg)
331*179860b2SJed Brown    # Necessary since os.environ is a UserDict
332*179860b2SJed Brown    elif isinstance(args, dict) or isinstance(args, UserDict.UserDict):
333*179860b2SJed Brown      for key in args.keys():
334*179860b2SJed Brown        if isinstance(args[key], str):
335*179860b2SJed Brown          value = nargs.Arg.parseValue(args[key])
336*179860b2SJed Brown        else:
337*179860b2SJed Brown          value = args[key]
338*179860b2SJed Brown        self.insertArg(key, value, None)
339*179860b2SJed Brown    elif isinstance(args, str):
340*179860b2SJed Brown        (key, value) = nargs.Arg.parseArgument(args)
341*179860b2SJed Brown        self.insertArg(key, value, args)
342*179860b2SJed Brown    return
343*179860b2SJed Brown
344*179860b2SJed Brown  def hasParent(self):
345*179860b2SJed Brown    '''Return True if this RDict has a parent dictionary'''
346*179860b2SJed Brown    return not self.parent is None
347*179860b2SJed Brown
348*179860b2SJed Brown  def getServerAddr(self, dir):
349*179860b2SJed Brown    '''Read the server socket address (in pickled form) from a file, usually RDict.loc
350*179860b2SJed Brown       - If we fail to connect to the server specified in the file, we spawn it using startServer()'''
351*179860b2SJed Brown    filename = os.path.join(dir, self.addrFilename)
352*179860b2SJed Brown    if not os.path.exists(filename):
353*179860b2SJed Brown      self.startServer(filename)
354*179860b2SJed Brown    if not os.path.exists(filename):
355*179860b2SJed Brown      raise RuntimeError('Server address file does not exist: '+filename)
356*179860b2SJed Brown    try:
357*179860b2SJed Brown      f    = open(filename, 'r')
358*179860b2SJed Brown      addr = cPickle.load(f)
359*179860b2SJed Brown      f.close()
360*179860b2SJed Brown      return addr
361*179860b2SJed Brown    except Exception, e:
362*179860b2SJed Brown      self.writeLogLine('CLIENT: Exception during server address determination: '+str(e.__class__)+': '+str(e))
363*179860b2SJed Brown    raise RuntimeError('Could not get server address in '+filename)
364*179860b2SJed Brown
365*179860b2SJed Brown  def writeServerAddr(self, server):
366*179860b2SJed Brown    '''Write the server socket address (in pickled form) to a file, usually RDict.loc.'''
367*179860b2SJed Brown    f = file(self.addrFilename, 'w')
368*179860b2SJed Brown    cPickle.dump(server.server_address, f)
369*179860b2SJed Brown    f.close()
370*179860b2SJed Brown    self.writeLogLine('SERVER: Wrote lock file '+os.path.abspath(self.addrFilename))
371*179860b2SJed Brown    return
372*179860b2SJed Brown
373*179860b2SJed Brown  def startServer(self, addrFilename):
374*179860b2SJed Brown    '''Spawn a new RDict server in the parent directory'''
375*179860b2SJed Brown    import RDict # Need this to locate server script
376*179860b2SJed Brown    import sys
377*179860b2SJed Brown    import time
378*179860b2SJed Brown    import distutils.sysconfig
379*179860b2SJed Brown
380*179860b2SJed Brown    self.writeLogLine('CLIENT: Spawning a new server with lock file '+os.path.abspath(addrFilename))
381*179860b2SJed Brown    if os.path.exists(addrFilename):
382*179860b2SJed Brown      os.remove(addrFilename)
383*179860b2SJed Brown    oldDir      = os.getcwd()
384*179860b2SJed Brown    source      = os.path.join(os.path.dirname(os.path.abspath(sys.modules['RDict'].__file__)), 'RDict.py')
385*179860b2SJed Brown    interpreter = os.path.join(distutils.sysconfig.get_config_var('BINDIR'), distutils.sysconfig.get_config_var('PYTHON'))
386*179860b2SJed Brown    if not os.path.isfile(interpreter):
387*179860b2SJed Brown      interpreter = 'python'
388*179860b2SJed Brown    os.chdir(os.path.dirname(addrFilename))
389*179860b2SJed Brown    self.writeLogLine('CLIENT: Executing '+interpreter+' '+source+' server"')
390*179860b2SJed Brown    try:
391*179860b2SJed Brown      os.spawnvp(os.P_NOWAIT, interpreter, [interpreter, source, 'server'])
392*179860b2SJed Brown    except:
393*179860b2SJed Brown      self.writeLogLine('CLIENT: os.spawnvp failed.\n \
394*179860b2SJed Brown      This is a typical problem on CYGWIN systems.  If you are using CYGWIN,\n \
395*179860b2SJed Brown      you can fix this problem by running /bin/rebaseall.  If you do not have\n \
396*179860b2SJed Brown      this program, you can install it with the CYGWIN installer in the package\n \
397*179860b2SJed Brown      Rebase, under the category System.  You must run /bin/rebaseall after\n \
398*179860b2SJed Brown      turning off all cygwin services -- in particular sshd, if any such services\n \
399*179860b2SJed Brown      are running.  For more information about rebase, go to http://www.cygwin.com')
400*179860b2SJed Brown      print '\n \
401*179860b2SJed Brown      This is a typical problem on CYGWIN systems.  If you are using CYGWIN,\n \
402*179860b2SJed Brown      you can fix this problem by running /bin/rebaseall.  If you do not have\n \
403*179860b2SJed Brown      this program, you can install it with the CYGWIN installer in the package\n \
404*179860b2SJed Brown      Rebase, under the category System.  You must run /bin/rebaseall after\n \
405*179860b2SJed Brown      turning off all cygwin services -- in particular sshd, if any such services\n \
406*179860b2SJed Brown      are running.  For more information about rebase, go to http://www.cygwin.com\n'
407*179860b2SJed Brown      raise
408*179860b2SJed Brown    os.chdir(oldDir)
409*179860b2SJed Brown    timeout = 1
410*179860b2SJed Brown    for i in range(10):
411*179860b2SJed Brown      time.sleep(timeout)
412*179860b2SJed Brown      timeout *= 2
413*179860b2SJed Brown      if timeout > 100: timeout = 100
414*179860b2SJed Brown      if os.path.exists(addrFilename): return
415*179860b2SJed Brown    self.writeLogLine('CLIENT: Could not start server')
416*179860b2SJed Brown    return
417*179860b2SJed Brown
418*179860b2SJed Brown  def connectParent(self, addr, dir):
419*179860b2SJed Brown    '''Try to connect to a parent RDict server
420*179860b2SJed Brown       - If addr and dir are both None, this operation fails
421*179860b2SJed Brown       - If addr is None, check for an address file in dir'''
422*179860b2SJed Brown    if addr is None:
423*179860b2SJed Brown      if dir is None: return 0
424*179860b2SJed Brown      addr = self.getServerAddr(dir)
425*179860b2SJed Brown
426*179860b2SJed Brown    import socket
427*179860b2SJed Brown    import errno
428*179860b2SJed Brown    connected = 0
429*179860b2SJed Brown    s         = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
430*179860b2SJed Brown    timeout   = 1
431*179860b2SJed Brown    for i in range(10):
432*179860b2SJed Brown      try:
433*179860b2SJed Brown        self.writeLogLine('CLIENT: Trying to connect to '+str(addr))
434*179860b2SJed Brown        s.connect(addr)
435*179860b2SJed Brown        connected = 1
436*179860b2SJed Brown        break
437*179860b2SJed Brown      except socket.error, e:
438*179860b2SJed Brown        self.writeLogLine('CLIENT: Failed to connect: '+str(e))
439*179860b2SJed Brown        if e[0] == errno.ECONNREFUSED:
440*179860b2SJed Brown          try:
441*179860b2SJed Brown            import time
442*179860b2SJed Brown            time.sleep(timeout)
443*179860b2SJed Brown            timeout *= 2
444*179860b2SJed Brown            if timeout > 100: timeout = 100
445*179860b2SJed Brown          except KeyboardInterrupt:
446*179860b2SJed Brown            break
447*179860b2SJed Brown          # Try to spawn parent
448*179860b2SJed Brown          if dir:
449*179860b2SJed Brown            filename = os.path.join(dir, self.addrFilename)
450*179860b2SJed Brown            if os.path.isfile(filename):
451*179860b2SJed Brown              os.remove(filename)
452*179860b2SJed Brown            self.startServer(filename)
453*179860b2SJed Brown      except Exception, e:
454*179860b2SJed Brown        self.writeLogLine('CLIENT: Failed to connect: '+str(e.__class__)+': '+str(e))
455*179860b2SJed Brown    if not connected:
456*179860b2SJed Brown      self.writeLogLine('CLIENT: Failed to connect to parent')
457*179860b2SJed Brown      return 0
458*179860b2SJed Brown    self.parent = s
459*179860b2SJed Brown    self.writeLogLine('CLIENT: Connected to '+str(self.parent))
460*179860b2SJed Brown    return 1
461*179860b2SJed Brown
462*179860b2SJed Brown  def sendPacket(self, s, packet, source = 'Unknown', isPickled = 0):
463*179860b2SJed Brown    '''Pickle the input packet. Send first the size of the pickled string in 32-bit integer, and then the string itself'''
464*179860b2SJed Brown    self.writeLogLine(source+': Sending packet '+str(packet))
465*179860b2SJed Brown    if isPickled:
466*179860b2SJed Brown      p = packet
467*179860b2SJed Brown    else:
468*179860b2SJed Brown      p = cPickle.dumps(packet)
469*179860b2SJed Brown    self.packer.reset()
470*179860b2SJed Brown    self.packer.pack_uint(len(p))
471*179860b2SJed Brown    if hasattr(s, 'write'):
472*179860b2SJed Brown      s.write(self.packer.get_buffer())
473*179860b2SJed Brown      s.write(p)
474*179860b2SJed Brown    else:
475*179860b2SJed Brown      s.sendall(self.packer.get_buffer())
476*179860b2SJed Brown      s.sendall(p)
477*179860b2SJed Brown    self.writeLogLine(source+': Sent packet')
478*179860b2SJed Brown    return
479*179860b2SJed Brown
480*179860b2SJed Brown  def recvPacket(self, s, source = 'Unknown'):
481*179860b2SJed Brown    '''Receive first the size of the pickled string in a 32-bit integer, and then the string itself. Return the unpickled object'''
482*179860b2SJed Brown    self.writeLogLine(source+': Receiving packet')
483*179860b2SJed Brown    if hasattr(s, 'read'):
484*179860b2SJed Brown      s.read(4)
485*179860b2SJed Brown      value = cPickle.load(s)
486*179860b2SJed Brown    else:
487*179860b2SJed Brown      # I probably need to check that it actually read these 4 bytes
488*179860b2SJed Brown      self.unpacker.reset(s.recv(4))
489*179860b2SJed Brown      length    = self.unpacker.unpack_uint()
490*179860b2SJed Brown      objString = ''
491*179860b2SJed Brown      while len(objString) < length:
492*179860b2SJed Brown        objString += s.recv(length - len(objString))
493*179860b2SJed Brown      value = cPickle.loads(objString)
494*179860b2SJed Brown    self.writeLogLine(source+': Received packet '+str(value))
495*179860b2SJed Brown    return value
496*179860b2SJed Brown
497*179860b2SJed Brown  def send(self, key = None, value = None, operation = None):
498*179860b2SJed Brown    '''Send a request to the parent'''
499*179860b2SJed Brown    import inspect
500*179860b2SJed Brown
501*179860b2SJed Brown    objString = ''
502*179860b2SJed Brown    for i in range(3):
503*179860b2SJed Brown      try:
504*179860b2SJed Brown        packet = []
505*179860b2SJed Brown        if operation is None:
506*179860b2SJed Brown          operation = inspect.stack()[1][3]
507*179860b2SJed Brown        packet.append(operation)
508*179860b2SJed Brown        if not key is None:
509*179860b2SJed Brown          packet.append(key)
510*179860b2SJed Brown          if not value is None:
511*179860b2SJed Brown            packet.append(value)
512*179860b2SJed Brown        self.sendPacket(self.parent, tuple(packet), source = 'CLIENT')
513*179860b2SJed Brown        response = self.recvPacket(self.parent, source = 'CLIENT')
514*179860b2SJed Brown        break
515*179860b2SJed Brown      except IOError, e:
516*179860b2SJed Brown        self.writeLogLine('CLIENT: IOError '+str(e))
517*179860b2SJed Brown        if e.errno == 32:
518*179860b2SJed Brown          self.connectParent(self.parentAddr, self.parentDirectory)
519*179860b2SJed Brown      except Exception, e:
520*179860b2SJed Brown        self.writeLogLine('CLIENT: Exception '+str(e)+' '+str(e.__class__))
521*179860b2SJed Brown    try:
522*179860b2SJed Brown      if isinstance(response, Exception):
523*179860b2SJed Brown        self.writeLogLine('CLIENT: Got an exception '+str(response))
524*179860b2SJed Brown        raise response
525*179860b2SJed Brown      else:
526*179860b2SJed Brown        self.writeLogLine('CLIENT: Received value '+str(response)+' '+str(type(response)))
527*179860b2SJed Brown    except UnboundLocalError:
528*179860b2SJed Brown      self.writeLogLine('CLIENT: Could not unpickle response')
529*179860b2SJed Brown      response  = None
530*179860b2SJed Brown    return response
531*179860b2SJed Brown
532*179860b2SJed Brown  def serve(self):
533*179860b2SJed Brown    '''Start a server'''
534*179860b2SJed Brown    import socket
535*179860b2SJed Brown    import SocketServer
536*179860b2SJed Brown
537*179860b2SJed Brown    if not useThreads:
538*179860b2SJed Brown      raise RuntimeError('Cannot run a server if threads are disabled')
539*179860b2SJed Brown
540*179860b2SJed Brown    class ProcessHandler(SocketServer.StreamRequestHandler):
541*179860b2SJed Brown      def handle(self):
542*179860b2SJed Brown        import time
543*179860b2SJed Brown
544*179860b2SJed Brown        self.server.rdict.lastAccess = time.time()
545*179860b2SJed Brown        self.server.rdict.writeLogLine('SERVER: Started new handler')
546*179860b2SJed Brown        while 1:
547*179860b2SJed Brown          try:
548*179860b2SJed Brown            value = self.server.rdict.recvPacket(self.rfile, source = 'SERVER')
549*179860b2SJed Brown          except EOFError, e:
550*179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: EOFError receiving packet '+str(e)+' '+str(e.__class__))
551*179860b2SJed Brown            return
552*179860b2SJed Brown          except Exception, e:
553*179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: Error receiving packet '+str(e)+' '+str(e.__class__))
554*179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER')
555*179860b2SJed Brown            continue
556*179860b2SJed Brown          if value[0] == 'stop': break
557*179860b2SJed Brown          try:
558*179860b2SJed Brown            response = getattr(self.server.rdict, value[0])(*value[1:])
559*179860b2SJed Brown          except Exception, e:
560*179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: Error executing operation '+str(e)+' '+str(e.__class__))
561*179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER')
562*179860b2SJed Brown          else:
563*179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, response, source = 'SERVER')
564*179860b2SJed Brown        return
565*179860b2SJed Brown
566*179860b2SJed Brown    # check if server is running
567*179860b2SJed Brown    if os.path.exists(self.addrFilename):
568*179860b2SJed Brown      rdict     = RDict(parentDirectory = '.')
569*179860b2SJed Brown      hasParent = rdict.hasParent()
570*179860b2SJed Brown      del rdict
571*179860b2SJed Brown      if hasParent:
572*179860b2SJed Brown        self.writeLogLine('SERVER: Another server is already running')
573*179860b2SJed Brown        raise RuntimeError('Server already running')
574*179860b2SJed Brown
575*179860b2SJed Brown    # Daemonize server
576*179860b2SJed Brown    self.writeLogLine('SERVER: Daemonizing server')
577*179860b2SJed Brown    if os.fork(): # Launch child
578*179860b2SJed Brown      os._exit(0) # Kill off parent, so we are not a process group leader and get a new PID
579*179860b2SJed Brown    os.setsid()   # Set session ID, so that we have no controlling terminal
580*179860b2SJed Brown    # We choose to leave cwd at RDict.py: os.chdir('/') # Make sure root directory is not on a mounted drive
581*179860b2SJed Brown    os.umask(077) # Fix creation mask
582*179860b2SJed Brown    for i in range(3): # Crappy stopgap for closing descriptors
583*179860b2SJed Brown      try:
584*179860b2SJed Brown        os.close(i)
585*179860b2SJed Brown      except OSError, e:
586*179860b2SJed Brown        if e.errno != errno.EBADF:
587*179860b2SJed Brown          raise RuntimeError('Could not close default descriptor '+str(i))
588*179860b2SJed Brown
589*179860b2SJed Brown    # wish there was a better way to get a usable socket
590*179860b2SJed Brown    self.writeLogLine('SERVER: Establishing socket server')
591*179860b2SJed Brown    basePort = 8000
592*179860b2SJed Brown    flag     = 'nosocket'
593*179860b2SJed Brown    p        = 1
594*179860b2SJed Brown    while p < 1000 and flag == 'nosocket':
595*179860b2SJed Brown      try:
596*179860b2SJed Brown        server = SocketServer.ThreadingTCPServer((socket.gethostname(), basePort+p), ProcessHandler)
597*179860b2SJed Brown        flag   = 'socket'
598*179860b2SJed Brown      except Exception, e:
599*179860b2SJed Brown        p = p + 1
600*179860b2SJed Brown    if flag == 'nosocket':
601*179860b2SJed Brown      p = 1
602*179860b2SJed Brown      while p < 1000 and flag == 'nosocket':
603*179860b2SJed Brown        try:
604*179860b2SJed Brown          server = SocketServer.ThreadingTCPServer(('localhost', basePort+p), ProcessHandler)
605*179860b2SJed Brown          flag   = 'socket'
606*179860b2SJed Brown        except Exception, e:
607*179860b2SJed Brown          p = p + 1
608*179860b2SJed Brown    if flag == 'nosocket':
609*179860b2SJed Brown      self.writeLogLine('SERVER: Could not established socket server on port '+str(basePort+p))
610*179860b2SJed Brown      raise RuntimeError,'Cannot get available socket'
611*179860b2SJed Brown    self.writeLogLine('SERVER: Established socket server on port '+str(basePort+p))
612*179860b2SJed Brown
613*179860b2SJed Brown    self.isServer = 1
614*179860b2SJed Brown    self.writeServerAddr(server)
615*179860b2SJed Brown    self.serverShutdown(os.getpid())
616*179860b2SJed Brown
617*179860b2SJed Brown    server.rdict = self
618*179860b2SJed Brown    self.writeLogLine('SERVER: Started server')
619*179860b2SJed Brown    server.serve_forever()
620*179860b2SJed Brown    return
621*179860b2SJed Brown
622*179860b2SJed Brown  def load(self):
623*179860b2SJed Brown    '''Load the saved dictionary'''
624*179860b2SJed Brown    if not self.parentDirectory is None and os.path.samefile(os.getcwd(), self.parentDirectory):
625*179860b2SJed Brown      return
626*179860b2SJed Brown    self.saveFilename = os.path.abspath(self.saveFilename)
627*179860b2SJed Brown    if os.path.exists(self.saveFilename):
628*179860b2SJed Brown      try:
629*179860b2SJed Brown        dbFile = file(self.saveFilename)
630*179860b2SJed Brown        data   = cPickle.load(dbFile)
631*179860b2SJed Brown        self.updateTypes(data)
632*179860b2SJed Brown        dbFile.close()
633*179860b2SJed Brown        self.writeLogLine('Loaded dictionary from '+self.saveFilename)
634*179860b2SJed Brown      except Exception, e:
635*179860b2SJed Brown        self.writeLogLine('Problem loading dictionary from '+self.saveFilename+'\n--> '+str(e))
636*179860b2SJed Brown    else:
637*179860b2SJed Brown      self.writeLogLine('No dictionary to load in this file: '+self.saveFilename)
638*179860b2SJed Brown    return
639*179860b2SJed Brown
640*179860b2SJed Brown  def save(self, force = 0):
641*179860b2SJed Brown    '''Save the dictionary after 5 seconds, ignoring all subsequent calls until the save
642*179860b2SJed Brown       - Giving force = True will cause an immediate save'''
643*179860b2SJed Brown    if self.readonly: return
644*179860b2SJed Brown    if force or not useThreads:
645*179860b2SJed Brown      self.saveTimer = None
646*179860b2SJed Brown      # This should be a critical section
647*179860b2SJed Brown      dbFile = file(self.saveFilename, 'w')
648*179860b2SJed Brown      data   = dict(filter(lambda i: not i[1].getTemporary(), self.localitems()))
649*179860b2SJed Brown      cPickle.dump(data, dbFile)
650*179860b2SJed Brown      dbFile.close()
651*179860b2SJed Brown      self.writeLogLine('Saved local dictionary to '+os.path.abspath(self.saveFilename))
652*179860b2SJed Brown    elif not self.saveTimer:
653*179860b2SJed Brown      import threading
654*179860b2SJed Brown      self.saveTimer = threading.Timer(5, self.save, [], {'force': 1})
655*179860b2SJed Brown      self.saveTimer.setDaemon(1)
656*179860b2SJed Brown      self.saveTimer.start()
657*179860b2SJed Brown    return
658*179860b2SJed Brown
659*179860b2SJed Brown  def shutdown(self):
660*179860b2SJed Brown    '''Shutdown the dictionary, writing out changes and notifying parent'''
661*179860b2SJed Brown    if self.saveTimer:
662*179860b2SJed Brown      self.saveTimer.cancel()
663*179860b2SJed Brown      self.save(force = 1)
664*179860b2SJed Brown    if self.isServer and os.path.isfile(self.addrFilename):
665*179860b2SJed Brown      os.remove(self.addrFilename)
666*179860b2SJed Brown    if not self.parent is None:
667*179860b2SJed Brown      self.sendPacket(self.parent, self.stopCmd, isPickled = 1)
668*179860b2SJed Brown      self.parent.close()
669*179860b2SJed Brown      self.parent = None
670*179860b2SJed Brown    self.writeLogLine('Shutting down')
671*179860b2SJed Brown    self.logFile.close()
672*179860b2SJed Brown    return
673*179860b2SJed Brown
674*179860b2SJed Brown  def serverShutdown(self, pid, delay = shutdownDelay):
675*179860b2SJed Brown    if self.shutdownTimer is None:
676*179860b2SJed Brown      import threading
677*179860b2SJed Brown
678*179860b2SJed Brown      self.shutdownTimer = threading.Timer(delay, self.serverShutdown, [pid], {'delay': 0})
679*179860b2SJed Brown      self.shutdownTimer.setDaemon(1)
680*179860b2SJed Brown      self.shutdownTimer.start()
681*179860b2SJed Brown      self.writeLogLine('SERVER: Set shutdown timer for process '+str(pid)+' at '+str(delay)+' seconds')
682*179860b2SJed Brown    else:
683*179860b2SJed Brown      try:
684*179860b2SJed Brown        import signal
685*179860b2SJed Brown        import time
686*179860b2SJed Brown
687*179860b2SJed Brown        idleTime = time.time() - self.lastAccess
688*179860b2SJed Brown        self.writeLogLine('SERVER: Last access '+str(self.lastAccess))
689*179860b2SJed Brown        self.writeLogLine('SERVER: Idle time '+str(idleTime))
690*179860b2SJed Brown        if idleTime < RDict.shutdownDelay:
691*179860b2SJed Brown          self.writeLogLine('SERVER: Extending shutdown timer for '+str(pid)+' by '+str(RDict.shutdownDelay - idleTime)+' seconds')
692*179860b2SJed Brown          self.shutdownTimer = None
693*179860b2SJed Brown          self.serverShutdown(pid, RDict.shutdownDelay - idleTime)
694*179860b2SJed Brown        else:
695*179860b2SJed Brown          self.writeLogLine('SERVER: Killing server '+str(pid))
696*179860b2SJed Brown          os.kill(pid, signal.SIGTERM)
697*179860b2SJed Brown      except Exception, e:
698*179860b2SJed Brown        self.writeLogLine('SERVER: Exception killing server: '+str(e))
699*179860b2SJed Brown    return
700*179860b2SJed Brown
701*179860b2SJed Brownif __name__ ==  '__main__':
702*179860b2SJed Brown  import sys
703*179860b2SJed Brown  try:
704*179860b2SJed Brown    if len(sys.argv) < 2:
705*179860b2SJed Brown      print 'RDict.py [server | client | clear | insert | remove] [parent]'
706*179860b2SJed Brown    else:
707*179860b2SJed Brown      action = sys.argv[1]
708*179860b2SJed Brown      parent = None
709*179860b2SJed Brown      if len(sys.argv) > 2:
710*179860b2SJed Brown        if not sys.argv[2] == 'None': parent = sys.argv[2]
711*179860b2SJed Brown      if action == 'server':
712*179860b2SJed Brown        RDict(parentDirectory = parent).serve()
713*179860b2SJed Brown      elif action == 'client':
714*179860b2SJed Brown        print 'Entries in server dictionary'
715*179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
716*179860b2SJed Brown        for key in rdict.types():
717*179860b2SJed Brown          if not key.startswith('cacheKey') and not key.startswith('stamp-'):
718*179860b2SJed Brown            print str(key)+' '+str(rdict.getType(key))
719*179860b2SJed Brown      elif action == 'cacheClient':
720*179860b2SJed Brown        print 'Cache entries in server dictionary'
721*179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
722*179860b2SJed Brown        for key in rdict.types():
723*179860b2SJed Brown          if key.startswith('cacheKey'):
724*179860b2SJed Brown            print str(key)+' '+str(rdict.getType(key))
725*179860b2SJed Brown      elif action == 'stampClient':
726*179860b2SJed Brown        print 'Stamp entries in server dictionary'
727*179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
728*179860b2SJed Brown        for key in rdict.types():
729*179860b2SJed Brown          if key.startswith('stamp-'):
730*179860b2SJed Brown            print str(key)+' '+str(rdict.getType(key))
731*179860b2SJed Brown      elif action == 'clear':
732*179860b2SJed Brown        print 'Clearing all dictionaries'
733*179860b2SJed Brown        RDict(parentDirectory = parent).clear()
734*179860b2SJed Brown      elif action == 'insert':
735*179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
736*179860b2SJed Brown        rdict[sys.argv[3]] = sys.argv[4]
737*179860b2SJed Brown      elif action == 'remove':
738*179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
739*179860b2SJed Brown        del rdict[sys.argv[3]]
740*179860b2SJed Brown      else:
741*179860b2SJed Brown        sys.exit('Unknown action: '+action)
742*179860b2SJed Brown  except Exception, e:
743*179860b2SJed Brown    import traceback
744*179860b2SJed Brown    print traceback.print_tb(sys.exc_info()[2])
745*179860b2SJed Brown    sys.exit(str(e))
746*179860b2SJed Brown  sys.exit(0)
747