xref: /petsc/config/BuildSystem/RDict.py (revision 31de54b548664dc8b1b1b4a049385191b26d917b)
1179860b2SJed Brown#!/usr/bin/env python
2179860b2SJed Brown'''A remote dictionary server
3179860b2SJed Brown
4179860b2SJed Brown    RDict is a typed, hierarchical, persistent dictionary intended to manage
5179860b2SJed Brown    all arguments or options for a program. The interface remains exactly the
6179860b2SJed Brown    same as dict, but the storage is more complicated.
7179860b2SJed Brown
8179860b2SJed Brown    Argument typing is handled by wrapping all values stored in the dictionary
9179860b2SJed Brown    with nargs.Arg or a subclass. A user can call setType() to set the type of
10179860b2SJed Brown    an argument without any value being present. Whenever __getitem__() or
11179860b2SJed Brown    __setitem__() is called, values are extracted or replaced in the wrapper.
12179860b2SJed Brown    These wrappers can be accessed directly using getType(), setType(), and
13179860b2SJed Brown    types().
14179860b2SJed Brown
15179860b2SJed Brown    Hierarchy is allowed using a single "parent" dictionary. All operations
16179860b2SJed Brown    cascade to the parent. For instance, the length of the dictionary is the
17179860b2SJed Brown    number of local keys plus the number of keys in the parent, and its
18179860b2SJed Brown    parent, etc. Also, a dictionary need not have a parent. If a key does not
19179860b2SJed Brown    appear in the local dicitonary, the call if passed to the parent. However,
20179860b2SJed Brown    in this case we see that local keys can shadow those in a parent.
21179860b2SJed Brown    Communication with the parent is handled using sockets, with the parent
22179860b2SJed Brown    being a server and the interactive dictionary a client.
23179860b2SJed Brown
24179860b2SJed Brown    The default persistence mechanism is a pickle file, RDict.db, written
25179860b2SJed Brown    whenever an argument is changed locally. A timer thread is created after
26179860b2SJed Brown    an initial change, so that many rapid changes do not cause many writes.
27179860b2SJed Brown    Each dictionary only saves its local entries, so all parents also
28179860b2SJed Brown    separately save data in different RDict.db files. Each time a dictionary
29179860b2SJed Brown    is created, the current directory is searched for an RDict.db file, and
30179860b2SJed Brown    if found the contents are loaded into the dictionary.
31179860b2SJed Brown
32179860b2SJed Brown    This script also provides some default actions:
33179860b2SJed Brown
34179860b2SJed Brown      - server [parent]
35179860b2SJed Brown        Starts a server in the current directory with an optional parent. This
36179860b2SJed Brown        server will accept socket connections from other dictionaries and act
37179860b2SJed Brown        as a parent.
38179860b2SJed Brown
39179860b2SJed Brown      - client [parent]
40179860b2SJed Brown        Creates a dictionary in the current directory with an optional parent
41179860b2SJed Brown        and lists the contents. Notice that the contents may come from either
42179860b2SJed Brown        an RDict.db file in the current directory, or from the parent.
43179860b2SJed Brown
44179860b2SJed Brown      - clear [parent]
45179860b2SJed Brown        Creates a dictionary in the current directory with an optional parent
46179860b2SJed Brown        and clears the contents. Notice that this will also clear the parent.
47179860b2SJed Brown
48179860b2SJed Brown      - insert <parent> <key> <value>
49179860b2SJed Brown        Creates a dictionary in the current directory with a parent, and inserts
50179860b2SJed Brown        the key-value pair. If "parent" is "None", no parent is assigned.
51179860b2SJed Brown
52179860b2SJed Brown      - remove <parent> <key>
53179860b2SJed Brown        Creates a dictionary in the current directory with a parent, and removes
54179860b2SJed Brown        the given key. If "parent" is "None", no parent is assigned.
55179860b2SJed Brown'''
56179860b2SJed Browntry:
57179860b2SJed Brown  import project          # This is necessary for us to create Project objects on load
58179860b2SJed Brown  import build.buildGraph # This is necessary for us to create BuildGraph objects on load
59179860b2SJed Brownexcept ImportError:
60179860b2SJed Brown  pass
61179860b2SJed Brownimport nargs
62179860b2SJed Brown
63179860b2SJed Brownimport cPickle
64179860b2SJed Brownimport os
65179860b2SJed Brownimport sys
66179860b2SJed BrownuseThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:])
67179860b2SJed Brownif useThreads is None:
68*31de54b5SMatthew G. Knepley  useThreads = 1
69179860b2SJed Brownelse:
70179860b2SJed Brown  useThreads = int(useThreads)
71179860b2SJed Brown
72179860b2SJed Brownclass RDict(dict):
73179860b2SJed Brown  '''An RDict is a typed dictionary, which may be hierarchically composed. All elements derive from the
74179860b2SJed BrownArg class, which wraps the usual value.'''
75179860b2SJed Brown  # The server will self-shutdown after this many seconds
76179860b2SJed Brown  shutdownDelay = 60*60*5
77179860b2SJed Brown
78179860b2SJed Brown  def __init__(self, parentAddr = None, parentDirectory = None, load = 1, autoShutdown = 1, readonly = False):
79179860b2SJed Brown    import atexit
80179860b2SJed Brown    import time
81179860b2SJed Brown    import xdrlib
82179860b2SJed Brown
83179860b2SJed Brown    self.logFile         = None
84179860b2SJed Brown    self.setupLogFile()
85179860b2SJed Brown    self.target          = ['default']
86179860b2SJed Brown    self.parent          = None
87179860b2SJed Brown    self.saveTimer       = None
88179860b2SJed Brown    self.shutdownTimer   = None
89179860b2SJed Brown    self.lastAccess      = time.time()
90179860b2SJed Brown    self.saveFilename    = 'RDict.db'
91179860b2SJed Brown    self.addrFilename    = 'RDict.loc'
92179860b2SJed Brown    self.parentAddr      = parentAddr
93179860b2SJed Brown    self.isServer        = 0
94179860b2SJed Brown    self.readonly        = readonly
95179860b2SJed Brown    self.parentDirectory = parentDirectory
96179860b2SJed Brown    self.packer          = xdrlib.Packer()
97179860b2SJed Brown    self.unpacker        = xdrlib.Unpacker('')
98179860b2SJed Brown    self.stopCmd         = cPickle.dumps(('stop',))
99179860b2SJed Brown    self.writeLogLine('Greetings')
100179860b2SJed Brown    self.connectParent(self.parentAddr, self.parentDirectory)
101179860b2SJed Brown    if load: self.load()
102179860b2SJed Brown    if autoShutdown and useThreads:
103179860b2SJed Brown      atexit.register(self.shutdown)
104179860b2SJed Brown    self.writeLogLine('SERVER: Last access '+str(self.lastAccess))
105179860b2SJed Brown    return
106179860b2SJed Brown
107179860b2SJed Brown  def __getstate__(self):
108179860b2SJed Brown    '''Remove any parent socket object, the XDR translators, and the log file from the dictionary before pickling'''
109179860b2SJed Brown    self.writeLogLine('Pickling RDict')
110179860b2SJed Brown    d = self.__dict__.copy()
111179860b2SJed Brown    if 'parent'    in d: del d['parent']
112179860b2SJed Brown    if 'saveTimer' in d: del d['saveTimer']
113179860b2SJed Brown    if '_setCommandLine' in d: del d['_setCommandLine']
114179860b2SJed Brown    del d['packer']
115179860b2SJed Brown    del d['unpacker']
116179860b2SJed Brown    del d['logFile']
117179860b2SJed Brown    return d
118179860b2SJed Brown
119179860b2SJed Brown  def __setstate__(self, d):
120179860b2SJed Brown    '''Reconnect the parent socket object, recreate the XDR translators and reopen the log file after unpickling'''
121179860b2SJed Brown    self.logFile  = file('RDict.log', 'a')
122179860b2SJed Brown    self.writeLogLine('Unpickling RDict')
123179860b2SJed Brown    self.__dict__.update(d)
124179860b2SJed Brown    import xdrlib
125179860b2SJed Brown    self.packer   = xdrlib.Packer()
126179860b2SJed Brown    self.unpacker = xdrlib.Unpacker('')
127179860b2SJed Brown    self.connectParent(self.parentAddr, self.parentDirectory)
128179860b2SJed Brown    return
129179860b2SJed Brown
130179860b2SJed Brown  def setupLogFile(self, filename = 'RDict.log'):
131179860b2SJed Brown    if not self.logFile is None:
132179860b2SJed Brown      self.logFile.close()
133179860b2SJed Brown    if os.path.isfile(filename) and os.stat(filename).st_size > 10*1024*1024:
134179860b2SJed Brown      if os.path.isfile(filename+'.bkp'):
135179860b2SJed Brown        os.remove(filename+'.bkp')
136179860b2SJed Brown      os.rename(filename, filename+'.bkp')
137179860b2SJed Brown      self.logFile = file(filename, 'w')
138179860b2SJed Brown    else:
139179860b2SJed Brown      self.logFile = file(filename, 'a')
140179860b2SJed Brown    return
141179860b2SJed Brown
142179860b2SJed Brown  def writeLogLine(self, message):
143179860b2SJed Brown    '''Writes the message to the log along with the current time'''
144179860b2SJed Brown    import time
145179860b2SJed Brown    self.logFile.write('('+str(os.getpid())+')('+str(id(self))+')'+message+' ['+time.asctime(time.localtime())+']\n')
146179860b2SJed Brown    self.logFile.flush()
147179860b2SJed Brown    return
148179860b2SJed Brown
149179860b2SJed Brown  def __len__(self):
150179860b2SJed Brown    '''Returns the length of both the local and parent dictionaries'''
151179860b2SJed Brown    length = dict.__len__(self)
152179860b2SJed Brown    if not self.parent is None:
153179860b2SJed Brown      length = length + self.send()
154179860b2SJed Brown    return length
155179860b2SJed Brown
156179860b2SJed Brown  def getType(self, key):
157179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Returns the Arg object or None if not found.'''
158179860b2SJed Brown    if dict.has_key(self, key):
159179860b2SJed Brown      self.writeLogLine('getType: Getting local type for '+key+' '+str(dict.__getitem__(self, key)))
160179860b2SJed Brown      return dict.__getitem__(self, key)
161179860b2SJed Brown    elif not self.parent is None:
162179860b2SJed Brown      return self.send(key)
163179860b2SJed Brown    return None
164179860b2SJed Brown
165179860b2SJed Brown  def __getitem__(self, key):
166179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Returns the value of the Arg.
167179860b2SJed Brown       - If the value has not been set, the user will be prompted for input'''
168179860b2SJed Brown    if dict.has_key(self, key):
169179860b2SJed Brown      self.writeLogLine('__getitem__: '+key+' has local type')
170179860b2SJed Brown      pass
171179860b2SJed Brown    elif not self.parent is None:
172179860b2SJed Brown      self.writeLogLine('__getitem__: Checking parent value')
173179860b2SJed Brown      if self.send(key, operation = 'has_key'):
174179860b2SJed Brown        self.writeLogLine('__getitem__: Parent has value')
175179860b2SJed Brown        return self.send(key)
176179860b2SJed Brown      else:
177179860b2SJed Brown        self.writeLogLine('__getitem__: Checking parent type')
178179860b2SJed Brown        arg = self.send(key, operation = 'getType')
179179860b2SJed Brown        if not arg:
180179860b2SJed Brown          self.writeLogLine('__getitem__: Parent has no type')
181179860b2SJed Brown          arg = nargs.Arg(key)
182179860b2SJed Brown        try:
183179860b2SJed Brown          value = arg.getValue()
184179860b2SJed Brown        except AttributeError, e:
185179860b2SJed Brown          self.writeLogLine('__getitem__: Parent had invalid entry: '+str(e))
186179860b2SJed Brown          arg   = nargs.Arg(key)
187179860b2SJed Brown          value = arg.getValue()
188179860b2SJed Brown        self.writeLogLine('__getitem__: Setting parent value '+str(value))
189179860b2SJed Brown        self.send(key, value, operation = '__setitem__')
190179860b2SJed Brown        return value
191179860b2SJed Brown    else:
192179860b2SJed Brown      self.writeLogLine('__getitem__: Setting local type for '+key)
193179860b2SJed Brown      dict.__setitem__(self, key, nargs.Arg(key))
194179860b2SJed Brown      self.save()
195179860b2SJed Brown    self.writeLogLine('__getitem__: Setting local value for '+key)
196179860b2SJed Brown    return dict.__getitem__(self, key).getValue()
197179860b2SJed Brown
198179860b2SJed Brown  def setType(self, key, value, forceLocal = 0):
199179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Sets the type for this key.
200179860b2SJed Brown       - If a value for the key already exists, it is converted to the new type'''
201179860b2SJed Brown    if not isinstance(value, nargs.Arg):
202179860b2SJed Brown      raise TypeError('An argument type must be a subclass of Arg')
203179860b2SJed Brown    value.setKey(key)
204179860b2SJed Brown    if forceLocal or self.parent is None or dict.has_key(self, key):
205179860b2SJed Brown      if dict.has_key(self, key):
206179860b2SJed Brown        v = dict.__getitem__(self, key)
207179860b2SJed Brown        if v.isValueSet():
208179860b2SJed Brown          try:
209179860b2SJed Brown            value.setValue(v.getValue())
210179860b2SJed Brown          except TypeError: pass
211179860b2SJed Brown      dict.__setitem__(self, key, value)
212179860b2SJed Brown      self.save()
213179860b2SJed Brown    else:
214179860b2SJed Brown      return self.send(key, value)
215179860b2SJed Brown    return
216179860b2SJed Brown
217179860b2SJed Brown  def __setitem__(self, key, value):
218179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Sets the value of the Arg.'''
219179860b2SJed Brown    if not dict.has_key(self, key):
220179860b2SJed Brown      if not self.parent is None:
221179860b2SJed Brown        return self.send(key, value)
222179860b2SJed Brown      else:
223179860b2SJed Brown        dict.__setitem__(self, key, nargs.Arg(key))
224179860b2SJed Brown    dict.__getitem__(self, key).setValue(value)
225179860b2SJed Brown    self.writeLogLine('__setitem__: Set value for '+key+' to '+str(dict.__getitem__(self, key)))
226179860b2SJed Brown    self.save()
227179860b2SJed Brown    return
228179860b2SJed Brown
229179860b2SJed Brown  def __delitem__(self, key):
230179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Deletes the Arg completely.'''
231179860b2SJed Brown    if dict.has_key(self, key):
232179860b2SJed Brown      dict.__delitem__(self, key)
233179860b2SJed Brown      self.save()
234179860b2SJed Brown    elif not self.parent is None:
235179860b2SJed Brown      self.send(key)
236179860b2SJed Brown    return
237179860b2SJed Brown
238179860b2SJed Brown  def clear(self):
239179860b2SJed Brown    '''Clears both the local and parent dictionaries'''
240179860b2SJed Brown    if dict.__len__(self):
241179860b2SJed Brown      dict.clear(self)
242179860b2SJed Brown      self.save()
243179860b2SJed Brown    if not self.parent is None:
244179860b2SJed Brown      self.send()
245179860b2SJed Brown    return
246179860b2SJed Brown
247179860b2SJed Brown  def __contains__(self, key):
248179860b2SJed Brown    '''This method just calls self.has_key(key)'''
249179860b2SJed Brown    return self.has_key(key)
250179860b2SJed Brown
251179860b2SJed Brown  def has_key(self, key):
252179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Then checks whether the value has been set'''
253179860b2SJed Brown    if dict.has_key(self, key):
254179860b2SJed Brown      if dict.__getitem__(self, key).isValueSet():
255179860b2SJed Brown        self.writeLogLine('has_key: Have value for '+key)
256179860b2SJed Brown      else:
257179860b2SJed Brown        self.writeLogLine('has_key: Do not have value for '+key)
258179860b2SJed Brown      return dict.__getitem__(self, key).isValueSet()
259179860b2SJed Brown    elif not self.parent is None:
260179860b2SJed Brown      return self.send(key)
261179860b2SJed Brown    return 0
262179860b2SJed Brown
263179860b2SJed Brown  def get(self, key, default=None):
264179860b2SJed Brown    if self.has_key(key):
265179860b2SJed Brown      return self.__getitem__(key)
266179860b2SJed Brown    else:
267179860b2SJed Brown      return default
268179860b2SJed Brown
269179860b2SJed Brown  def hasType(self, key):
270179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Then checks whether the type has been set'''
271179860b2SJed Brown    if dict.has_key(self, key):
272179860b2SJed Brown      return 1
273179860b2SJed Brown    elif not self.parent is None:
274179860b2SJed Brown      return self.send(key)
275179860b2SJed Brown    return 0
276179860b2SJed Brown
277179860b2SJed Brown  def items(self):
278179860b2SJed Brown    '''Return a list of all accessible items, as (key, value) pairs.'''
279179860b2SJed Brown    l = dict.items(self)
280179860b2SJed Brown    if not self.parent is None:
281179860b2SJed Brown      l.extend(self.send())
282179860b2SJed Brown    return l
283179860b2SJed Brown
284179860b2SJed Brown  def localitems(self):
285179860b2SJed Brown    '''Return a list of all the items stored locally, as (key, value) pairs.'''
286179860b2SJed Brown    return dict.items(self)
287179860b2SJed Brown
288179860b2SJed Brown  def keys(self):
289179860b2SJed Brown    '''Returns the list of keys in both the local and parent dictionaries'''
290179860b2SJed Brown    keyList = filter(lambda key: dict.__getitem__(self, key).isValueSet(), dict.keys(self))
291179860b2SJed Brown    if not self.parent is None:
292179860b2SJed Brown      keyList.extend(self.send())
293179860b2SJed Brown    return keyList
294179860b2SJed Brown
295179860b2SJed Brown  def types(self):
296179860b2SJed Brown    '''Returns the list of keys for which types are defined in both the local and parent dictionaries'''
297179860b2SJed Brown    keyList = dict.keys(self)
298179860b2SJed Brown    if not self.parent is None:
299179860b2SJed Brown      keyList.extend(self.send())
300179860b2SJed Brown    return keyList
301179860b2SJed Brown
302179860b2SJed Brown  def update(self, d):
303179860b2SJed Brown    '''Update the dictionary with the contents of d'''
304179860b2SJed Brown    for k in d:
305179860b2SJed Brown      self[k] = d[k]
306179860b2SJed Brown    return
307179860b2SJed Brown
308179860b2SJed Brown  def updateTypes(self, d):
309179860b2SJed Brown    '''Update types locally, which is equivalent to the dict.update() method'''
310179860b2SJed Brown    return dict.update(self, d)
311179860b2SJed Brown
312179860b2SJed Brown  def insertArg(self, key, value, arg):
313179860b2SJed Brown    '''Insert a (key, value) pair into the dictionary. If key is None, arg is put into the target list.'''
314179860b2SJed Brown    if not key is None:
315179860b2SJed Brown      self[key] = value
316179860b2SJed Brown    else:
317179860b2SJed Brown      if not self.target == ['default']:
318179860b2SJed Brown        self.target.append(arg)
319179860b2SJed Brown      else:
320179860b2SJed Brown        self.target = [arg]
321179860b2SJed Brown    return
322179860b2SJed Brown
323179860b2SJed Brown  def insertArgs(self, args):
324179860b2SJed Brown    '''Insert some text arguments into the dictionary (list and dictionaries are recognized)'''
325179860b2SJed Brown    import UserDict
326179860b2SJed Brown
327179860b2SJed Brown    if isinstance(args, list):
328179860b2SJed Brown      for arg in args:
329179860b2SJed Brown        (key, value) = nargs.Arg.parseArgument(arg)
330179860b2SJed Brown        self.insertArg(key, value, arg)
331179860b2SJed Brown    # Necessary since os.environ is a UserDict
332179860b2SJed Brown    elif isinstance(args, dict) or isinstance(args, UserDict.UserDict):
333179860b2SJed Brown      for key in args.keys():
334179860b2SJed Brown        if isinstance(args[key], str):
335179860b2SJed Brown          value = nargs.Arg.parseValue(args[key])
336179860b2SJed Brown        else:
337179860b2SJed Brown          value = args[key]
338179860b2SJed Brown        self.insertArg(key, value, None)
339179860b2SJed Brown    elif isinstance(args, str):
340179860b2SJed Brown        (key, value) = nargs.Arg.parseArgument(args)
341179860b2SJed Brown        self.insertArg(key, value, args)
342179860b2SJed Brown    return
343179860b2SJed Brown
344179860b2SJed Brown  def hasParent(self):
345179860b2SJed Brown    '''Return True if this RDict has a parent dictionary'''
346179860b2SJed Brown    return not self.parent is None
347179860b2SJed Brown
348179860b2SJed Brown  def getServerAddr(self, dir):
349179860b2SJed Brown    '''Read the server socket address (in pickled form) from a file, usually RDict.loc
350179860b2SJed Brown       - If we fail to connect to the server specified in the file, we spawn it using startServer()'''
351179860b2SJed Brown    filename = os.path.join(dir, self.addrFilename)
352179860b2SJed Brown    if not os.path.exists(filename):
353179860b2SJed Brown      self.startServer(filename)
354179860b2SJed Brown    if not os.path.exists(filename):
355179860b2SJed Brown      raise RuntimeError('Server address file does not exist: '+filename)
356179860b2SJed Brown    try:
357179860b2SJed Brown      f    = open(filename, 'r')
358179860b2SJed Brown      addr = cPickle.load(f)
359179860b2SJed Brown      f.close()
360179860b2SJed Brown      return addr
361179860b2SJed Brown    except Exception, e:
362179860b2SJed Brown      self.writeLogLine('CLIENT: Exception during server address determination: '+str(e.__class__)+': '+str(e))
363179860b2SJed Brown    raise RuntimeError('Could not get server address in '+filename)
364179860b2SJed Brown
365179860b2SJed Brown  def writeServerAddr(self, server):
366179860b2SJed Brown    '''Write the server socket address (in pickled form) to a file, usually RDict.loc.'''
367179860b2SJed Brown    f = file(self.addrFilename, 'w')
368179860b2SJed Brown    cPickle.dump(server.server_address, f)
369179860b2SJed Brown    f.close()
370179860b2SJed Brown    self.writeLogLine('SERVER: Wrote lock file '+os.path.abspath(self.addrFilename))
371179860b2SJed Brown    return
372179860b2SJed Brown
373179860b2SJed Brown  def startServer(self, addrFilename):
374179860b2SJed Brown    '''Spawn a new RDict server in the parent directory'''
375179860b2SJed Brown    import RDict # Need this to locate server script
376179860b2SJed Brown    import sys
377179860b2SJed Brown    import time
378179860b2SJed Brown    import distutils.sysconfig
379179860b2SJed Brown
380179860b2SJed Brown    self.writeLogLine('CLIENT: Spawning a new server with lock file '+os.path.abspath(addrFilename))
381179860b2SJed Brown    if os.path.exists(addrFilename):
382179860b2SJed Brown      os.remove(addrFilename)
383179860b2SJed Brown    oldDir      = os.getcwd()
384179860b2SJed Brown    source      = os.path.join(os.path.dirname(os.path.abspath(sys.modules['RDict'].__file__)), 'RDict.py')
385179860b2SJed Brown    interpreter = os.path.join(distutils.sysconfig.get_config_var('BINDIR'), distutils.sysconfig.get_config_var('PYTHON'))
386179860b2SJed Brown    if not os.path.isfile(interpreter):
387179860b2SJed Brown      interpreter = 'python'
388179860b2SJed Brown    os.chdir(os.path.dirname(addrFilename))
389179860b2SJed Brown    self.writeLogLine('CLIENT: Executing '+interpreter+' '+source+' server"')
390179860b2SJed Brown    try:
391179860b2SJed Brown      os.spawnvp(os.P_NOWAIT, interpreter, [interpreter, source, 'server'])
392179860b2SJed Brown    except:
393179860b2SJed Brown      self.writeLogLine('CLIENT: os.spawnvp failed.\n \
394179860b2SJed Brown      This is a typical problem on CYGWIN systems.  If you are using CYGWIN,\n \
395179860b2SJed Brown      you can fix this problem by running /bin/rebaseall.  If you do not have\n \
396179860b2SJed Brown      this program, you can install it with the CYGWIN installer in the package\n \
397179860b2SJed Brown      Rebase, under the category System.  You must run /bin/rebaseall after\n \
398179860b2SJed Brown      turning off all cygwin services -- in particular sshd, if any such services\n \
399179860b2SJed Brown      are running.  For more information about rebase, go to http://www.cygwin.com')
400179860b2SJed Brown      print '\n \
401179860b2SJed Brown      This is a typical problem on CYGWIN systems.  If you are using CYGWIN,\n \
402179860b2SJed Brown      you can fix this problem by running /bin/rebaseall.  If you do not have\n \
403179860b2SJed Brown      this program, you can install it with the CYGWIN installer in the package\n \
404179860b2SJed Brown      Rebase, under the category System.  You must run /bin/rebaseall after\n \
405179860b2SJed Brown      turning off all cygwin services -- in particular sshd, if any such services\n \
406179860b2SJed Brown      are running.  For more information about rebase, go to http://www.cygwin.com\n'
407179860b2SJed Brown      raise
408179860b2SJed Brown    os.chdir(oldDir)
409179860b2SJed Brown    timeout = 1
410179860b2SJed Brown    for i in range(10):
411179860b2SJed Brown      time.sleep(timeout)
412179860b2SJed Brown      timeout *= 2
413179860b2SJed Brown      if timeout > 100: timeout = 100
414179860b2SJed Brown      if os.path.exists(addrFilename): return
415179860b2SJed Brown    self.writeLogLine('CLIENT: Could not start server')
416179860b2SJed Brown    return
417179860b2SJed Brown
418179860b2SJed Brown  def connectParent(self, addr, dir):
419179860b2SJed Brown    '''Try to connect to a parent RDict server
420179860b2SJed Brown       - If addr and dir are both None, this operation fails
421179860b2SJed Brown       - If addr is None, check for an address file in dir'''
422179860b2SJed Brown    if addr is None:
423179860b2SJed Brown      if dir is None: return 0
424179860b2SJed Brown      addr = self.getServerAddr(dir)
425179860b2SJed Brown
426179860b2SJed Brown    import socket
427179860b2SJed Brown    import errno
428179860b2SJed Brown    connected = 0
429179860b2SJed Brown    s         = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
430179860b2SJed Brown    timeout   = 1
431179860b2SJed Brown    for i in range(10):
432179860b2SJed Brown      try:
433179860b2SJed Brown        self.writeLogLine('CLIENT: Trying to connect to '+str(addr))
434179860b2SJed Brown        s.connect(addr)
435179860b2SJed Brown        connected = 1
436179860b2SJed Brown        break
437179860b2SJed Brown      except socket.error, e:
438179860b2SJed Brown        self.writeLogLine('CLIENT: Failed to connect: '+str(e))
439179860b2SJed Brown        if e[0] == errno.ECONNREFUSED:
440179860b2SJed Brown          try:
441179860b2SJed Brown            import time
442179860b2SJed Brown            time.sleep(timeout)
443179860b2SJed Brown            timeout *= 2
444179860b2SJed Brown            if timeout > 100: timeout = 100
445179860b2SJed Brown          except KeyboardInterrupt:
446179860b2SJed Brown            break
447179860b2SJed Brown          # Try to spawn parent
448179860b2SJed Brown          if dir:
449179860b2SJed Brown            filename = os.path.join(dir, self.addrFilename)
450179860b2SJed Brown            if os.path.isfile(filename):
451179860b2SJed Brown              os.remove(filename)
452179860b2SJed Brown            self.startServer(filename)
453179860b2SJed Brown      except Exception, e:
454179860b2SJed Brown        self.writeLogLine('CLIENT: Failed to connect: '+str(e.__class__)+': '+str(e))
455179860b2SJed Brown    if not connected:
456179860b2SJed Brown      self.writeLogLine('CLIENT: Failed to connect to parent')
457179860b2SJed Brown      return 0
458179860b2SJed Brown    self.parent = s
459179860b2SJed Brown    self.writeLogLine('CLIENT: Connected to '+str(self.parent))
460179860b2SJed Brown    return 1
461179860b2SJed Brown
462179860b2SJed Brown  def sendPacket(self, s, packet, source = 'Unknown', isPickled = 0):
463179860b2SJed Brown    '''Pickle the input packet. Send first the size of the pickled string in 32-bit integer, and then the string itself'''
464179860b2SJed Brown    self.writeLogLine(source+': Sending packet '+str(packet))
465179860b2SJed Brown    if isPickled:
466179860b2SJed Brown      p = packet
467179860b2SJed Brown    else:
468179860b2SJed Brown      p = cPickle.dumps(packet)
469179860b2SJed Brown    self.packer.reset()
470179860b2SJed Brown    self.packer.pack_uint(len(p))
471179860b2SJed Brown    if hasattr(s, 'write'):
472179860b2SJed Brown      s.write(self.packer.get_buffer())
473179860b2SJed Brown      s.write(p)
474179860b2SJed Brown    else:
475179860b2SJed Brown      s.sendall(self.packer.get_buffer())
476179860b2SJed Brown      s.sendall(p)
477179860b2SJed Brown    self.writeLogLine(source+': Sent packet')
478179860b2SJed Brown    return
479179860b2SJed Brown
480179860b2SJed Brown  def recvPacket(self, s, source = 'Unknown'):
481179860b2SJed Brown    '''Receive first the size of the pickled string in a 32-bit integer, and then the string itself. Return the unpickled object'''
482179860b2SJed Brown    self.writeLogLine(source+': Receiving packet')
483179860b2SJed Brown    if hasattr(s, 'read'):
484179860b2SJed Brown      s.read(4)
485179860b2SJed Brown      value = cPickle.load(s)
486179860b2SJed Brown    else:
487179860b2SJed Brown      # I probably need to check that it actually read these 4 bytes
488179860b2SJed Brown      self.unpacker.reset(s.recv(4))
489179860b2SJed Brown      length    = self.unpacker.unpack_uint()
490179860b2SJed Brown      objString = ''
491179860b2SJed Brown      while len(objString) < length:
492179860b2SJed Brown        objString += s.recv(length - len(objString))
493179860b2SJed Brown      value = cPickle.loads(objString)
494179860b2SJed Brown    self.writeLogLine(source+': Received packet '+str(value))
495179860b2SJed Brown    return value
496179860b2SJed Brown
497179860b2SJed Brown  def send(self, key = None, value = None, operation = None):
498179860b2SJed Brown    '''Send a request to the parent'''
499179860b2SJed Brown    import inspect
500179860b2SJed Brown
501179860b2SJed Brown    objString = ''
502179860b2SJed Brown    for i in range(3):
503179860b2SJed Brown      try:
504179860b2SJed Brown        packet = []
505179860b2SJed Brown        if operation is None:
506179860b2SJed Brown          operation = inspect.stack()[1][3]
507179860b2SJed Brown        packet.append(operation)
508179860b2SJed Brown        if not key is None:
509179860b2SJed Brown          packet.append(key)
510179860b2SJed Brown          if not value is None:
511179860b2SJed Brown            packet.append(value)
512179860b2SJed Brown        self.sendPacket(self.parent, tuple(packet), source = 'CLIENT')
513179860b2SJed Brown        response = self.recvPacket(self.parent, source = 'CLIENT')
514179860b2SJed Brown        break
515179860b2SJed Brown      except IOError, e:
516179860b2SJed Brown        self.writeLogLine('CLIENT: IOError '+str(e))
517179860b2SJed Brown        if e.errno == 32:
518179860b2SJed Brown          self.connectParent(self.parentAddr, self.parentDirectory)
519179860b2SJed Brown      except Exception, e:
520179860b2SJed Brown        self.writeLogLine('CLIENT: Exception '+str(e)+' '+str(e.__class__))
521179860b2SJed Brown    try:
522179860b2SJed Brown      if isinstance(response, Exception):
523179860b2SJed Brown        self.writeLogLine('CLIENT: Got an exception '+str(response))
524179860b2SJed Brown        raise response
525179860b2SJed Brown      else:
526179860b2SJed Brown        self.writeLogLine('CLIENT: Received value '+str(response)+' '+str(type(response)))
527179860b2SJed Brown    except UnboundLocalError:
528179860b2SJed Brown      self.writeLogLine('CLIENT: Could not unpickle response')
529179860b2SJed Brown      response  = None
530179860b2SJed Brown    return response
531179860b2SJed Brown
532179860b2SJed Brown  def serve(self):
533179860b2SJed Brown    '''Start a server'''
534179860b2SJed Brown    import socket
535179860b2SJed Brown    import SocketServer
536179860b2SJed Brown
537179860b2SJed Brown    if not useThreads:
538179860b2SJed Brown      raise RuntimeError('Cannot run a server if threads are disabled')
539179860b2SJed Brown
540179860b2SJed Brown    class ProcessHandler(SocketServer.StreamRequestHandler):
541179860b2SJed Brown      def handle(self):
542179860b2SJed Brown        import time
543179860b2SJed Brown
544179860b2SJed Brown        self.server.rdict.lastAccess = time.time()
545179860b2SJed Brown        self.server.rdict.writeLogLine('SERVER: Started new handler')
546179860b2SJed Brown        while 1:
547179860b2SJed Brown          try:
548179860b2SJed Brown            value = self.server.rdict.recvPacket(self.rfile, source = 'SERVER')
549179860b2SJed Brown          except EOFError, e:
550179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: EOFError receiving packet '+str(e)+' '+str(e.__class__))
551179860b2SJed Brown            return
552179860b2SJed Brown          except Exception, e:
553179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: Error receiving packet '+str(e)+' '+str(e.__class__))
554179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER')
555179860b2SJed Brown            continue
556179860b2SJed Brown          if value[0] == 'stop': break
557179860b2SJed Brown          try:
558179860b2SJed Brown            response = getattr(self.server.rdict, value[0])(*value[1:])
559179860b2SJed Brown          except Exception, e:
560179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: Error executing operation '+str(e)+' '+str(e.__class__))
561179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER')
562179860b2SJed Brown          else:
563179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, response, source = 'SERVER')
564179860b2SJed Brown        return
565179860b2SJed Brown
566179860b2SJed Brown    # check if server is running
567179860b2SJed Brown    if os.path.exists(self.addrFilename):
568179860b2SJed Brown      rdict     = RDict(parentDirectory = '.')
569179860b2SJed Brown      hasParent = rdict.hasParent()
570179860b2SJed Brown      del rdict
571179860b2SJed Brown      if hasParent:
572179860b2SJed Brown        self.writeLogLine('SERVER: Another server is already running')
573179860b2SJed Brown        raise RuntimeError('Server already running')
574179860b2SJed Brown
575179860b2SJed Brown    # Daemonize server
576179860b2SJed Brown    self.writeLogLine('SERVER: Daemonizing server')
577179860b2SJed Brown    if os.fork(): # Launch child
578179860b2SJed Brown      os._exit(0) # Kill off parent, so we are not a process group leader and get a new PID
579179860b2SJed Brown    os.setsid()   # Set session ID, so that we have no controlling terminal
580179860b2SJed Brown    # We choose to leave cwd at RDict.py: os.chdir('/') # Make sure root directory is not on a mounted drive
581179860b2SJed Brown    os.umask(077) # Fix creation mask
582179860b2SJed Brown    for i in range(3): # Crappy stopgap for closing descriptors
583179860b2SJed Brown      try:
584179860b2SJed Brown        os.close(i)
585179860b2SJed Brown      except OSError, e:
586179860b2SJed Brown        if e.errno != errno.EBADF:
587179860b2SJed Brown          raise RuntimeError('Could not close default descriptor '+str(i))
588179860b2SJed Brown
589179860b2SJed Brown    # wish there was a better way to get a usable socket
590179860b2SJed Brown    self.writeLogLine('SERVER: Establishing socket server')
591179860b2SJed Brown    basePort = 8000
592179860b2SJed Brown    flag     = 'nosocket'
593179860b2SJed Brown    p        = 1
594179860b2SJed Brown    while p < 1000 and flag == 'nosocket':
595179860b2SJed Brown      try:
596179860b2SJed Brown        server = SocketServer.ThreadingTCPServer((socket.gethostname(), basePort+p), ProcessHandler)
597179860b2SJed Brown        flag   = 'socket'
598179860b2SJed Brown      except Exception, e:
599179860b2SJed Brown        p = p + 1
600179860b2SJed Brown    if flag == 'nosocket':
601179860b2SJed Brown      p = 1
602179860b2SJed Brown      while p < 1000 and flag == 'nosocket':
603179860b2SJed Brown        try:
604179860b2SJed Brown          server = SocketServer.ThreadingTCPServer(('localhost', basePort+p), ProcessHandler)
605179860b2SJed Brown          flag   = 'socket'
606179860b2SJed Brown        except Exception, e:
607179860b2SJed Brown          p = p + 1
608179860b2SJed Brown    if flag == 'nosocket':
609179860b2SJed Brown      self.writeLogLine('SERVER: Could not established socket server on port '+str(basePort+p))
610179860b2SJed Brown      raise RuntimeError,'Cannot get available socket'
611179860b2SJed Brown    self.writeLogLine('SERVER: Established socket server on port '+str(basePort+p))
612179860b2SJed Brown
613179860b2SJed Brown    self.isServer = 1
614179860b2SJed Brown    self.writeServerAddr(server)
615179860b2SJed Brown    self.serverShutdown(os.getpid())
616179860b2SJed Brown
617179860b2SJed Brown    server.rdict = self
618179860b2SJed Brown    self.writeLogLine('SERVER: Started server')
619179860b2SJed Brown    server.serve_forever()
620179860b2SJed Brown    return
621179860b2SJed Brown
622179860b2SJed Brown  def load(self):
623179860b2SJed Brown    '''Load the saved dictionary'''
624179860b2SJed Brown    if not self.parentDirectory is None and os.path.samefile(os.getcwd(), self.parentDirectory):
625179860b2SJed Brown      return
626179860b2SJed Brown    self.saveFilename = os.path.abspath(self.saveFilename)
627179860b2SJed Brown    if os.path.exists(self.saveFilename):
628179860b2SJed Brown      try:
629179860b2SJed Brown        dbFile = file(self.saveFilename)
630179860b2SJed Brown        data   = cPickle.load(dbFile)
631179860b2SJed Brown        self.updateTypes(data)
632179860b2SJed Brown        dbFile.close()
633179860b2SJed Brown        self.writeLogLine('Loaded dictionary from '+self.saveFilename)
634179860b2SJed Brown      except Exception, e:
635179860b2SJed Brown        self.writeLogLine('Problem loading dictionary from '+self.saveFilename+'\n--> '+str(e))
636179860b2SJed Brown    else:
637179860b2SJed Brown      self.writeLogLine('No dictionary to load in this file: '+self.saveFilename)
638179860b2SJed Brown    return
639179860b2SJed Brown
640179860b2SJed Brown  def save(self, force = 0):
641179860b2SJed Brown    '''Save the dictionary after 5 seconds, ignoring all subsequent calls until the save
642179860b2SJed Brown       - Giving force = True will cause an immediate save'''
643179860b2SJed Brown    if self.readonly: return
64408eb64ffSMatthew G. Knepley    if force:
645179860b2SJed Brown      self.saveTimer = None
646179860b2SJed Brown      # This should be a critical section
647179860b2SJed Brown      dbFile = file(self.saveFilename, 'w')
648179860b2SJed Brown      data   = dict(filter(lambda i: not i[1].getTemporary(), self.localitems()))
649179860b2SJed Brown      cPickle.dump(data, dbFile)
650179860b2SJed Brown      dbFile.close()
651179860b2SJed Brown      self.writeLogLine('Saved local dictionary to '+os.path.abspath(self.saveFilename))
652179860b2SJed Brown    elif not self.saveTimer:
653179860b2SJed Brown      import threading
654179860b2SJed Brown      self.saveTimer = threading.Timer(5, self.save, [], {'force': 1})
655179860b2SJed Brown      self.saveTimer.setDaemon(1)
656179860b2SJed Brown      self.saveTimer.start()
657179860b2SJed Brown    return
658179860b2SJed Brown
659179860b2SJed Brown  def shutdown(self):
660179860b2SJed Brown    '''Shutdown the dictionary, writing out changes and notifying parent'''
661179860b2SJed Brown    if self.saveTimer:
662179860b2SJed Brown      self.saveTimer.cancel()
663179860b2SJed Brown      self.save(force = 1)
664179860b2SJed Brown    if self.isServer and os.path.isfile(self.addrFilename):
665179860b2SJed Brown      os.remove(self.addrFilename)
666179860b2SJed Brown    if not self.parent is None:
667179860b2SJed Brown      self.sendPacket(self.parent, self.stopCmd, isPickled = 1)
668179860b2SJed Brown      self.parent.close()
669179860b2SJed Brown      self.parent = None
670179860b2SJed Brown    self.writeLogLine('Shutting down')
671179860b2SJed Brown    self.logFile.close()
672179860b2SJed Brown    return
673179860b2SJed Brown
674179860b2SJed Brown  def serverShutdown(self, pid, delay = shutdownDelay):
675179860b2SJed Brown    if self.shutdownTimer is None:
676179860b2SJed Brown      import threading
677179860b2SJed Brown
678179860b2SJed Brown      self.shutdownTimer = threading.Timer(delay, self.serverShutdown, [pid], {'delay': 0})
679179860b2SJed Brown      self.shutdownTimer.setDaemon(1)
680179860b2SJed Brown      self.shutdownTimer.start()
681179860b2SJed Brown      self.writeLogLine('SERVER: Set shutdown timer for process '+str(pid)+' at '+str(delay)+' seconds')
682179860b2SJed Brown    else:
683179860b2SJed Brown      try:
684179860b2SJed Brown        import signal
685179860b2SJed Brown        import time
686179860b2SJed Brown
687179860b2SJed Brown        idleTime = time.time() - self.lastAccess
688179860b2SJed Brown        self.writeLogLine('SERVER: Last access '+str(self.lastAccess))
689179860b2SJed Brown        self.writeLogLine('SERVER: Idle time '+str(idleTime))
690179860b2SJed Brown        if idleTime < RDict.shutdownDelay:
691179860b2SJed Brown          self.writeLogLine('SERVER: Extending shutdown timer for '+str(pid)+' by '+str(RDict.shutdownDelay - idleTime)+' seconds')
692179860b2SJed Brown          self.shutdownTimer = None
693179860b2SJed Brown          self.serverShutdown(pid, RDict.shutdownDelay - idleTime)
694179860b2SJed Brown        else:
695179860b2SJed Brown          self.writeLogLine('SERVER: Killing server '+str(pid))
696179860b2SJed Brown          os.kill(pid, signal.SIGTERM)
697179860b2SJed Brown      except Exception, e:
698179860b2SJed Brown        self.writeLogLine('SERVER: Exception killing server: '+str(e))
699179860b2SJed Brown    return
700179860b2SJed Brown
701179860b2SJed Brownif __name__ ==  '__main__':
702179860b2SJed Brown  import sys
703179860b2SJed Brown  try:
704179860b2SJed Brown    if len(sys.argv) < 2:
705179860b2SJed Brown      print 'RDict.py [server | client | clear | insert | remove] [parent]'
706179860b2SJed Brown    else:
707179860b2SJed Brown      action = sys.argv[1]
708179860b2SJed Brown      parent = None
709179860b2SJed Brown      if len(sys.argv) > 2:
710179860b2SJed Brown        if not sys.argv[2] == 'None': parent = sys.argv[2]
711179860b2SJed Brown      if action == 'server':
712179860b2SJed Brown        RDict(parentDirectory = parent).serve()
713179860b2SJed Brown      elif action == 'client':
714179860b2SJed Brown        print 'Entries in server dictionary'
715179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
716179860b2SJed Brown        for key in rdict.types():
717179860b2SJed Brown          if not key.startswith('cacheKey') and not key.startswith('stamp-'):
718179860b2SJed Brown            print str(key)+' '+str(rdict.getType(key))
719179860b2SJed Brown      elif action == 'cacheClient':
720179860b2SJed Brown        print 'Cache entries in server dictionary'
721179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
722179860b2SJed Brown        for key in rdict.types():
723179860b2SJed Brown          if key.startswith('cacheKey'):
724179860b2SJed Brown            print str(key)+' '+str(rdict.getType(key))
725179860b2SJed Brown      elif action == 'stampClient':
726179860b2SJed Brown        print 'Stamp entries in server dictionary'
727179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
728179860b2SJed Brown        for key in rdict.types():
729179860b2SJed Brown          if key.startswith('stamp-'):
730179860b2SJed Brown            print str(key)+' '+str(rdict.getType(key))
731179860b2SJed Brown      elif action == 'clear':
732179860b2SJed Brown        print 'Clearing all dictionaries'
733179860b2SJed Brown        RDict(parentDirectory = parent).clear()
734179860b2SJed Brown      elif action == 'insert':
735179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
736179860b2SJed Brown        rdict[sys.argv[3]] = sys.argv[4]
737179860b2SJed Brown      elif action == 'remove':
738179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
739179860b2SJed Brown        del rdict[sys.argv[3]]
740179860b2SJed Brown      else:
741179860b2SJed Brown        sys.exit('Unknown action: '+action)
742179860b2SJed Brown  except Exception, e:
743179860b2SJed Brown    import traceback
744179860b2SJed Brown    print traceback.print_tb(sys.exc_info()[2])
745179860b2SJed Brown    sys.exit(str(e))
746179860b2SJed Brown  sys.exit(0)
747