X-Git-Url: https://git.distorted.org.uk/~mdw/catacomb-python/blobdiff_plain/053c2659b82af978e28dfb18a9bcdcfc317a01f3..1d93191e19f43bce5b8f3bd3447c755c1d615f56:/catacomb/pwsafe.py

diff --git a/catacomb/pwsafe.py b/catacomb/pwsafe.py
index 8b68182..cfbac7e 100644
--- a/catacomb/pwsafe.py
+++ b/catacomb/pwsafe.py
@@ -28,10 +28,72 @@
 
 from __future__ import with_statement
 
+import errno as _E
 import os as _OS
+from cStringIO import StringIO as _StringIO
 
 import catacomb as _C
-import gdbm as _G
+
+###--------------------------------------------------------------------------
+### Text encoding utilities.
+
+def _literalp(s):
+  """
+  Answer whether S can be represented literally.
+
+  If True, then S can be stored literally, as a metadata item name or
+  value; if False, then S requires some kind of encoding.
+  """
+  return all(ch.isalnum() or ch in '-_:' for ch in s)
+
+def _enc_metaname(name):
+  """Encode NAME as a metadata item name, returning the result."""
+  if _literalp(name):
+    return name
+  else:
+    sio = _StringIO()
+    sio.write('!')
+    for ch in name:
+      if _literalp(ch): sio.write(ch)
+      elif ch == ' ': sio.write('+')
+      else: sio.write('%%%02x' % ord(ch))
+    return sio.getvalue()
+
+def _dec_metaname(name):
+  """Decode NAME as a metadata item name, returning the result."""
+  if not name.startswith('!'):
+    return name
+  else:
+    sio = _StringIO()
+    i, n = 1, len(name)
+    while i < n:
+      ch = name[i]
+      i += 1
+      if ch == '+':
+        sio.write(' ')
+      elif ch == '%':
+        sio.write(chr(int(name[i:i + 2], 16)))
+        i += 2
+      else:
+        sio.write(ch)
+    return sio.getvalue()
+
+def _b64(s):
+  """Encode S as base64, without newlines, and trimming `=' padding."""
+  return s.encode('base64').translate(None, '\n=')
+def _unb64(s):
+  """Decode S as base64 with trimmed `=' padding."""
+  return (s + '='*((4 - len(s))%4)).decode('base64')
+
+def _enc_metaval(val):
+  """Encode VAL as a metadata item value, returning the result."""
+  if _literalp(val): return val
+  else: return '?' + _b64(val)
+
+def _dec_metaval(val):
+  """Decode VAL as a metadata item value, returning the result."""
+  if not val.startswith('?'): return val
+  else: return _unb64(val[1:])
 
 ###--------------------------------------------------------------------------
 ### Underlying cryptography.
@@ -410,7 +472,7 @@ class StorageBackend (object):
     database is open for writing.
     """
     me._check_write()
-    me._del_passwd(label, payload)
+    me._del_passwd(label)
 
   def iter_passwds(me):
     """
@@ -422,59 +484,470 @@ class StorageBackend (object):
     me._check_live()
     return me._iter_passwds()
 
-class GDBMStorageBackend (StorageBackend):
+try: import gdbm as _G
+except ImportError: pass
+else:
+  class GDBMStorageBackend (StorageBackend):
+    """
+    My instances store password data in a GDBM database.
+
+    Metadata and password entries are mixed into the same database.  The key
+    for a metadata item is simply its name; the key for a password entry is
+    the entry's label prefixed by `$', since we're guaranteed that no
+    metadata item name begins with `$'.
+    """
+
+    NAME = 'gdbm'
+
+    def _open(me, file, writep):
+      try: me._db = _G.open(file, writep and 'w' or 'r')
+      except _G.error, e: raise StorageBackendRefusal, e
+
+    def _create(me, file):
+      me._db = _G.open(file, 'n', 0600)
+
+    def _close(me, abruptp):
+      me._db.close()
+      me._db = None
+
+    def _get_meta(me, name, default):
+      try: return me._db[name]
+      except KeyError: return default
+
+    def _put_meta(me, name, value):
+      me._db[name] = value
+
+    def _del_meta(me, name):
+      del me._db[name]
+
+    def _iter_meta(me):
+      k = me._db.firstkey()
+      while k is not None:
+        if not k.startswith('$'): yield k, me._db[k]
+        k = me._db.nextkey(k)
+
+    def _get_passwd(me, label):
+      return me._db['$' + label]
+
+    def _put_passwd(me, label, payload):
+      me._db['$' + label] = payload
+
+    def _del_passwd(me, label):
+      del me._db['$' + label]
+
+    def _iter_passwds(me):
+      k = me._db.firstkey()
+      while k is not None:
+        if k.startswith('$'): yield k[1:], me._db[k]
+        k = me._db.nextkey(k)
+
+try: import sqlite3 as _Q
+except ImportError: pass
+else:
+  class SQLiteStorageBackend (StorageBackend):
+    """
+    I represent a password database stored in SQLite.
+
+    Metadata and password items are stored in separate tables, so there's no
+    conflict.  Some additional metadata is stored in the `meta' table, with
+    names beginning with `$' so as not to conflict with clients:
+
+    $version            The schema version of the table.
+    """
+
+    NAME = 'sqlite'
+    VERSION = 0
+
+    def _open(me, file, writep):
+      try:
+        me._db = _Q.connect(file)
+        ver = me._query_scalar(
+          "SELECT value FROM meta WHERE name = '$version'",
+          "version check")
+      except (_Q.DatabaseError, _Q.OperationalError), e:
+        raise StorageBackendRefusal, e
+      if ver is None: raise ValueError, 'database broken (missing $version)'
+      elif ver < me.VERSION: me._upgrade(ver)
+      elif ver > me.VERSION:
+        raise ValueError, 'unknown database schema version (%d > %d)' % \
+            (ver, me.VERSION)
+
+    def _create(me, file):
+      fd = _OS.open(file, _OS.O_WRONLY | _OS.O_CREAT | _OS.O_EXCL, 0600)
+      _OS.close(fd)
+      try:
+        me._db = _Q.connect(file)
+        c = me._db.cursor()
+        c.execute("""
+          CREATE TABLE meta (
+                  name TEXT PRIMARY KEY NOT NULL,
+                  value BLOB NOT NULL);
+        """)
+        c.execute("""
+          CREATE TABLE passwd (
+                  label BLOB PRIMARY KEY NOT NULL,
+                  payload BLOB NOT NULL);
+        """)
+        c.execute("""
+          INSERT INTO meta (name, value) VALUES ('$version', ?);
+        """, [me.VERSION])
+      except:
+        try: _OS.unlink(file)
+        except OSError: pass
+        raise
+
+    def _upgrade(me, ver):
+      """Upgrade the database from schema version VER."""
+      assert False, 'how embarrassing'
+
+    def _close(me, abruptp):
+      if not abruptp: me._db.commit()
+      me._db.close()
+      me._db = None
+
+    def _fetch_scalar(me, c, what, default = None):
+      try: row = next(c)
+      except StopIteration: val = default
+      else: val, = row
+      try: row = next(c)
+      except StopIteration: pass
+      else: raise ValueError, 'multiple matching records for %s' % what
+      return val
+
+    def _query_scalar(me, query, what, default = None, args = []):
+      c = me._db.cursor()
+      c.execute(query, args)
+      return me._fetch_scalar(c, what, default)
+
+    def _get_meta(me, name, default):
+      v = me._query_scalar("SELECT value FROM meta WHERE name = ?",
+                           "metadata item `%s'" % name,
+                           default = default, args = [name])
+      if v is default: return v
+      else: return str(v)
+
+    def _put_meta(me, name, value):
+      c = me._db.cursor()
+      c.execute("INSERT OR REPLACE INTO meta (name, value) VALUES (?, ?)",
+                [name, buffer(value)])
+
+    def _del_meta(me, name):
+      c = me._db.cursor()
+      c.execute("DELETE FROM meta WHERE name = ?", [name])
+      if not c.rowcount: raise KeyError, name
+
+    def _iter_meta(me):
+      c = me._db.cursor()
+      c.execute("SELECT name, value FROM meta WHERE name NOT LIKE '$%'")
+      for k, v in c: yield k, str(v)
+
+    def _get_passwd(me, label):
+      pld = me._query_scalar("SELECT payload FROM passwd WHERE label = ?",
+                             "password", default = None,
+                             args = [buffer(label)])
+      if pld is None: raise KeyError, label
+      return str(pld)
+
+    def _put_passwd(me, label, payload):
+      c = me._db.cursor()
+      c.execute("INSERT OR REPLACE INTO passwd (label, payload) "
+                "VALUES (?, ?)",
+                [buffer(label), buffer(payload)])
+
+    def _del_passwd(me, label):
+      c = me._db.cursor()
+      c.execute("DELETE FROM passwd WHERE label = ?", [label])
+      if not c.rowcount: raise KeyError, label
+
+    def _iter_passwds(me):
+      c = me._db.cursor()
+      c.execute("SELECT label, payload FROM passwd")
+      for k, v in c: yield str(k), str(v)
+
+class PlainTextBackend (StorageBackend):
   """
-  My instances store password data in a GDBM database.
+  I'm a utility base class for storage backends which use plain text files.
 
-  Metadata and password entries are mixed into the same database.  The key
-  for a metadata item is simply its name; the key for a password entry is
-  the entry's label prefixed by `$', since we're guaranteed that no
-  metadata item name begins with `$'.
+  I provide subclasses with the following capabilities.
+
+    * Creating files, with given modes, optionally ensuring that the file
+      doesn't exist already.
+
+    * Parsing flat text files, checking leading magic, skipping comments, and
+      providing standard encodings of troublesome characters and binary
+      strings in metadata and password records.  See below.
+
+    * Maintenance of metadata and password records in in-memory dictionaries,
+      with ready implementations of the necessary StorageBackend subclass
+      responsibility methods.  (Subclasses can override these if they want to
+      make different arrangements.)
+
+  Metadata records are written with an optional prefix string chosen by the
+  caller, followed by a `NAME=VALUE' pair.  The NAME is form-urlencoded and
+  prefixed with `!' if it contains strange characters; the VALUE is base64-
+  encoded (without the pointless trailing `=' padding) and prefixed with `?'
+  if necessary.
+
+  Password records are written with an optional prefix string chosen by the
+  caller, followed by a LABEL=PAYLOAD pair, both of which are base64-encoded
+  (without padding).
+
+  The following attributes are available for subclasses:
+
+  _meta         Dictionary mapping metadata item names to their values.
+                Populated by `_parse_meta' and managed by `_get_meta' and
+                friends.
+
+  _pw           Dictionary mapping password labels to encrypted payloads.
+                Populated by `_parse_passwd' and managed by `_get_passwd' and
+                friends.
+
+  _dirtyp       Boolean: set if either of the dictionaries has been modified.
   """
 
-  NAME = 'gdbm'
+  def __init__(me, *args, **kw):
+    """
+    Hook for initialization.
+
+    Sets up the published instance attributes.
+    """
+    me._meta = {}
+    me._pw = {}
+    me._dirtyp = False
+    super(PlainTextBackend, me).__init__(*args, **kw)
+
+  def _create_file(me, file, mode = 0600, freshp = False):
+    """
+    Make sure FILE exists, creating it with the given MODE if necessary.
+
+    If FRESHP is true, then make sure the file did not exist previously.
+    Return a file object for the newly created file.
+    """
+    flags = _OS.O_CREAT | _OS.O_WRONLY
+    if freshp: flags |= _OS.O_EXCL
+    else: flags |= _OS.O_TRUNC
+    fd = _OS.open(file, flags, mode)
+    return _OS.fdopen(fd, 'w')
+
+  def _mark_dirty(me):
+    """
+    Set the `_dirtyp' flag.
+
+    Subclasses might find it useful to intercept this method.
+    """
+    me._dirtyp = True
+
+  def _eqsplit(me, line):
+    """
+    Extract the KEY, VALUE pair from a LINE of the form `KEY=VALUE'.
+
+    Raise `ValueError' if there is no `=' in the LINE.
+    """
+    eq = line.index('=')
+    return line[:eq], line[eq + 1:]
+
+  def _parse_file(me, file, magic = None):
+    """
+    Parse a FILE.
+
+    Specifically:
+
+      * Raise `StorageBackendRefusal' if that the first line doesn't match
+        MAGIC (if provided).  MAGIC should not contain the terminating
+        newline.
+
+      * Ignore comments (beginning `#') and blank lines.
+
+      * Call `_parse_line' (provided by the subclass) for other lines.
+    """
+    with open(file, 'r') as f:
+      if magic is not None:
+        if f.readline().rstrip('\n') != magic: raise StorageBackendRefusal
+      for line in f:
+        line = line.rstrip('\n')
+        if not line or line.startswith('#'): continue
+        me._parse_line(line)
+
+  def _write_file(me, file, writebody, mode = 0600, magic = None):
+    """
+    Update FILE atomically.
+
+    The newly created file will have the given MODE.  If MAGIC is given, then
+    write that as the first line.  Calls WRITEBODY(F) to write the main body
+    of the file where F is a file object for the new file.
+    """
+    new = file + '.new'
+    with me._create_file(new, mode) as f:
+      if magic is not None: f.write(magic + '\n')
+      writebody(f)
+    _OS.rename(new, file)
+
+  def _parse_meta(me, line):
+    """Parse LINE as a metadata NAME=VALUE pair, and updates `_meta'."""
+    k, v = me._eqsplit(line)
+    me._meta[_dec_metaname(k)] = _dec_metaval(v)
+
+  def _write_meta(me, f, prefix = ''):
+    """Write the metadata records to F, each with the given PREFIX."""
+    f.write('\n## Metadata.\n')
+    for k, v in me._meta.iteritems():
+      f.write('%s%s=%s\n' % (prefix, _enc_metaname(k), _enc_metaval(v)))
+
+  def _get_meta(me, name, default):
+    return me._meta.get(name, default)
+  def _put_meta(me, name, value):
+    me._mark_dirty()
+    me._meta[name] = value
+  def _del_meta(me, name):
+    me._mark_dirty()
+    del me._meta[name]
+  def _iter_meta(me):
+    return me._meta.iteritems()
+
+  def _parse_passwd(me, line):
+    """Parse LINE as a password LABEL=PAYLOAD pair, and updates `_pw'."""
+    k, v = me._eqsplit(line)
+    me._pw[_unb64(k)] = _unb64(v)
+
+  def _write_passwd(me, f, prefix = ''):
+    """Write the password records to F, each with the given PREFIX."""
+    f.write('\n## Password data.\n')
+    for k, v in me._pw.iteritems():
+      f.write('%s%s=%s\n' % (prefix, _b64(k), _b64(v)))
+
+  def _get_passwd(me, label):
+    return me._pw[str(label)]
+  def _put_passwd(me, label, payload):
+    me._mark_dirty()
+    me._pw[str(label)] = payload
+  def _del_passwd(me, label):
+    me._mark_dirty()
+    del me._pw[str(label)]
+  def _iter_passwds(me):
+    return me._pw.iteritems()
+
+class FlatFileStorageBackend (PlainTextBackend):
+  """
+  I maintain a password database in a plain text file.
+
+  The text file consists of lines, as follows.
+
+    * Empty lines, and lines beginning with `#' (in the leftmost column only)
+      are ignored.
+
+    * Lines of the form `$LABEL=PAYLOAD' store password data.  Both LABEL and
+      PAYLOAD are base64-encoded, without `=' padding.
+
+    * Lines of the form `NAME=VALUE' store metadata.  If the NAME contains
+      characters other than alphanumerics, hyphens, underscores, and colons,
+      then it is form-urlencoded, and prefixed wth `!'.  If the VALUE
+      contains such characters, then it is base64-encoded, without `='
+      padding, and prefixed with `?'.
+
+    * Other lines are erroneous.
+
+  The file is rewritten from scratch when it's changed: any existing
+  commentary is lost, and items may be reordered.  There is no file locking,
+  but the file is updated atomically, by renaming.
+
+  It is expected that the FlatFileStorageBackend is used mostly for
+  diagnostics and transfer, rather than for a live system.
+  """
+
+  NAME = 'flat'
+  PRIO = 0
+  MAGIC = '### pwsafe password database'
 
   def _open(me, file, writep):
-    try: me._db = _G.open(file, writep and 'w' or 'r')
-    except _G.error, e: raise StorageBackendRefusal, e
+    if not _OS.path.isfile(file): raise StorageBackendRefusal
+    me._parse_file(file, magic = me.MAGIC)
+  def _parse_line(me, line):
+    if line.startswith('$'): me._parse_passwd(line[1:])
+    else: me._parse_meta(line)
 
   def _create(me, file):
-    me._db = _G.open(file, 'n', 0600)
+    with me._create_file(file, freshp = True) as f: pass
+    me._file = file
+    me._mark_dirty()
 
   def _close(me, abruptp):
-    me._db.close()
-    me._db = None
+    if not abruptp and me._dirtyp:
+      me._write_file(me._file, me._write_body, magic = me.MAGIC)
 
-  def _get_meta(me, name, default):
-    try: return me._db[name]
-    except KeyError: return default
+  def _write_body(me, f):
+    me._write_meta(f)
+    me._write_passwd(f, '$')
 
-  def _put_meta(me, name, value):
-    me._db[name] = value
+class DirectoryStorageBackend (PlainTextBackend):
+  """
+  I maintain a password database in a directory, with one file per password.
 
-  def _del_meta(me, name):
-    del me._db[name]
+  This makes password databases easy to maintain in a revision-control system
+  such as Git.
 
-  def _iter_meta(me):
-    k = me._db.firstkey()
-    while k is not None:
-      if not k.startswith('$'): yield k, me._db[k]
-      k = me._db.nextkey(k)
+  The directory is structured as follows.
 
-  def _get_passwd(me, label):
-    return me._db['$' + label]
+  dir/meta      Contains metadata, similar to the `FlatFileBackend'.
 
-  def _put_passwd(me, label, payload):
-    me._db['$' + label] = payload
+  dir/pw/LABEL  Contains the (raw binary) payload for the given password
+                LABEL (base64-encoded, without the useless `=' padding, and
+                with `/' replaced by `.').
 
-  def _del_passwd(me, label):
-    del me._db['$' + label]
+  dir/tmp/      Contains temporary files used by the implementation.
+  """
+
+  NAME = 'dir'
+  METAMAGIC = '### pwsafe password directory metadata'
+
+  def _open(me, file, writep):
+    if not _OS.path.isdir(file) or \
+          not _OS.path.isdir(_OS.path.join(file, 'pw')) or \
+          not _OS.path.isdir(_OS.path.join(file, 'tmp')) or \
+          not _OS.path.isfile(_OS.path.join(file, 'meta')):
+      raise StorageBackendRefusal
+    me._dir = file
+    me._parse_file(_OS.path.join(file, 'meta'), magic = me.METAMAGIC)
+  def _parse_line(me, line):
+    me._parse_meta(line)
 
+  def _create(me, file):
+    _OS.mkdir(file, 0700)
+    _OS.mkdir(_OS.path.join(file, 'pw'), 0700)
+    _OS.mkdir(_OS.path.join(file, 'tmp'), 0700)
+    me._mark_dirty()
+    me._dir = file
+
+  def _close(me, abruptp):
+    if not abruptp and me._dirtyp:
+      me._write_file(_OS.path.join(me._dir, 'meta'),
+                     me._write_meta, magic = me.METAMAGIC)
+
+  def _pwfile(me, label, dir = 'pw'):
+    return _OS.path.join(me._dir, dir, _b64(label).replace('/', '.'))
+  def _get_passwd(me, label):
+    try:
+      f = open(me._pwfile(label), 'rb')
+    except (OSError, IOError), e:
+      if e.errno == _E.ENOENT: raise KeyError, label
+      else: raise
+    with f: return f.read()
+  def _put_passwd(me, label, payload):
+    new = me._pwfile(label, 'tmp')
+    fd = _OS.open(new, _OS.O_WRONLY | _OS.O_CREAT | _OS.O_TRUNC, 0600)
+    _OS.close(fd)
+    with open(new, 'wb') as f: f.write(payload)
+    _OS.rename(new, me._pwfile(label))
+  def _del_passwd(me, label):
+    try:
+      _OS.remove(me._pwfile(label))
+    except (OSError, IOError), e:
+      if e == _E.ENOENT: raise KeyError, label
+      else: raise
   def _iter_passwds(me):
-    k = me._db.firstkey()
-    while k is not None:
-      if k.startswith('$'): yield k[1:], me._db[k]
-      k = me._db.nextkey(k)
+    pw = _OS.path.join(me._dir, 'pw')
+    for i in _OS.listdir(pw):
+      with open(_OS.path.join(pw, i), 'rb') as f: pld = f.read()
+      yield _unb64(i.replace('.', '/')), pld
 
 ###--------------------------------------------------------------------------
 ### Password storage.