*.py: Hack around Python exception-catching syntax mismatch.

[catacomb-python] / catacomb / pwsafe.py

### -*-python-*-
###
### Management of a secure password database
###
### (c) 2005 Straylight/Edgeware
###

###----- Licensing notice ---------------------------------------------------
###
### This file is part of the Python interface to Catacomb.
###
### Catacomb/Python is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation; either version 2 of the License, or
### (at your option) any later version.
###
### Catacomb/Python is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
### You should have received a copy of the GNU General Public License along
### with Catacomb/Python; if not, write to the Free Software Foundation,
### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

###--------------------------------------------------------------------------
### Imported modules.

from __future__ import with_statement

import errno as _E
import os as _OS
from cStringIO import StringIO as _StringIO

import catacomb as _C

###--------------------------------------------------------------------------
### Python version portability.

def _excval(): return _SYS.exc_info()[1]

###--------------------------------------------------------------------------
### Text encoding utilities.

def _literalp(s):
  """
  Answer whether S can be represented literally.

  If True, then S can be stored literally, as a metadata item name or
  value; if False, then S requires some kind of encoding.
  """
  return all(ch.isalnum() or ch in '-_:' for ch in s)

def _enc_metaname(name):
  """Encode NAME as a metadata item name, returning the result."""
  if _literalp(name):
    return name
  else:
    sio = _StringIO()
    sio.write('!')
    for ch in name:
      if _literalp(ch): sio.write(ch)
      elif ch == ' ': sio.write('+')
      else: sio.write('%%%02x' % ord(ch))
    return sio.getvalue()

def _dec_metaname(name):
  """Decode NAME as a metadata item name, returning the result."""
  if not name.startswith('!'):
    return name
  else:
    sio = _StringIO()
    i, n = 1, len(name)
    while i < n:
      ch = name[i]
      i += 1
      if ch == '+':
        sio.write(' ')
      elif ch == '%':
        sio.write(chr(int(name[i:i + 2], 16)))
        i += 2
      else:
        sio.write(ch)
    return sio.getvalue()

def _b64(s):
  """Encode S as base64, without newlines, and trimming `=' padding."""
  return s.encode('base64').replace('\n', '').rstrip('=')
def _unb64(s):
  """Decode S as base64 with trimmed `=' padding."""
  return (s + '='*((4 - len(s))%4)).decode('base64')

def _enc_metaval(val):
  """Encode VAL as a metadata item value, returning the result."""
  if _literalp(val): return val
  else: return '?' + _b64(val)

def _dec_metaval(val):
  """Decode VAL as a metadata item value, returning the result."""
  if not val.startswith('?'): return val
  else: return _unb64(val[1:])

###--------------------------------------------------------------------------
### Underlying cryptography.

class DecryptError (Exception):
  """
  I represent a failure to decrypt a message.

  Usually this means that someone used the wrong key, though it can also
  mean that a ciphertext has been modified.
  """
  pass

class Crypto (object):
  """
  I represent a symmetric crypto transform.

  There's currently only one transform implemented, which is the obvious
  generic-composition construction: given a message m, and keys K0 and K1, we
  choose an IV v, and compute:

    * y = v || E(K0, v; m)
    * t = M(K1; y)

  The final ciphertext is t || y.
  """

  def __init__(me, c, h, m, ck, mk):
    """
    Initialize the Crypto object with a given algorithm selection and keys.

    We need a GCipher subclass C, a GHash subclass H, a GMAC subclass M, and
    keys CK and MK for C and M respectively.
    """
    me.c = c(ck)
    me.m = m(mk)
    me.h = h

  def encrypt(me, pt):
    """
    Encrypt the message PT and return the resulting ciphertext.
    """
    blksz = me.c.__class__.blksz
    b = _C.WriteBuffer()
    if blksz:
      iv = _C.rand.block(blksz)
      me.c.setiv(iv)
      b.put(iv)
    b.put(me.c.encrypt(pt))
    t = me.m().hash(b).done()
    return t + str(buffer(b))

  def decrypt(me, ct):
    """
    Decrypt the ciphertext CT, returning the plaintext.

    Raises DecryptError if anything goes wrong.
    """
    blksz = me.c.__class__.blksz
    tagsz = me.m.__class__.tagsz
    b = _C.ReadBuffer(ct)
    t = b.get(tagsz)
    h = me.m()
    if blksz:
      iv = b.get(blksz)
      me.c.setiv(iv)
      h.hash(iv)
    x = b.get(b.left)
    h.hash(x)
    if t != h.done(): raise DecryptError
    return me.c.decrypt(x)

class PPK (Crypto):
  """
  I represent a crypto transform whose keys are derived from a passphrase.

  The password is salted and hashed; the salt is available as the `salt'
  attribute.
  """

  def __init__(me, pp, c, h, m, salt = None):
    """
    Initialize the PPK object with a passphrase and algorithm selection.

    We want a passphrase PP, a GCipher subclass C, a GHash subclass H, a GMAC
    subclass M, and a SALT.  The SALT may be None, if we're generating new
    keys, indicating that a salt should be chosen randomly.
    """
    if not salt: salt = _C.rand.block(h.hashsz)
    tag = '%s\0%s' % (pp, salt)
    Crypto.__init__(me, c, h, m,
                    h().hash('cipher:' + tag).done(),
                    h().hash('mac:' + tag).done())
    me.salt = salt

###--------------------------------------------------------------------------
### Backend storage.

class StorageBackendRefusal (Exception):
  """
  I signify that a StorageBackend subclass has refused to open a file.

  This is used by the StorageBackend.open class method.
  """
  pass

class StorageBackendClass (type):
  """
  I am a metaclass for StorageBackend classes.

  My main feature is that I register my concrete instances (with a `NAME'
  which is not `None') with the StorageBackend class.
  """
  def __init__(me, name, supers, dict):
    """
    Register a new concrete StorageBackend subclass.
    """
    super(StorageBackendClass, me).__init__(name, supers, dict)
    if me.NAME is not None: StorageBackend.register_concrete_subclass(me)

class StorageBackend (object):
  """
  I provide basic protocol for password storage backends.

  I'm an abstract class: you want one of my subclasses if you actually want
  to do something useful.  But I maintain a list of my subclasses and can
  choose an appropriate one to open a database file you've found lying about.

  Backends are responsible for storing and retrieving stuff, but not for the
  cryptographic details.  Backends need to store two kinds of information:

    * metadata, consisting of a number of property names and their values;
      and

    * password mappings, consisting of a number of binary labels and
      payloads.

  Backends need to implement the following ordinary methods.  See the calling
  methods for details of the subclass responsibilities.

  BE._create(FILE)      Create a new database in FILE; used by `create'.

  BE._open(FILE, WRITEP)
                        Open the existing database FILE; used by `open'.

  BE._close(ABRUPTP)    Close the database, freeing up any resources.  If
                        ABRUPTP then don't try to commit changes.

  BE._get_meta(NAME, DEFAULT)
                        Return the value of the metadata item with the given
                        NAME, or DEFAULT if it doesn't exist; used by
                        `get_meta'.

  BE._put_meta(NAME, VALUE)
                        Set the VALUE of the metadata item with the given
                        NAME, creating one if necessary; used by `put_meta'.

  BE._del_meta(NAME)    Forget the metadata item with the given NAME; raise
                        `KeyError' if there is no such item; used by
                        `del_meta'.

  BE._iter_meta()       Return an iterator over the metadata (NAME, VALUE)
                        pairs; used by `iter_meta'.

  BE._get_passwd(LABEL)
                        Return the password payload stored with the (binary)
                        LABEL; used by `get_passwd'.

  BE._put_passwd(LABEL, PAYLOAD)
                        Associate the (binary) PAYLOAD with the LABEL,
                        forgetting any previous payload for that LABEL; used
                        by `put_passwd'.

  BE._del_passwd(LABEL) Forget the password record with the given LABEL; used
                        by `_del_passwd'.

  BE._iter_passwds()    Return an iterator over the password (LABEL, PAYLOAD)
                        pairs; used by `iter_passwds'.

  Also, concrete subclasses should define the following class attributes.

  NAME                  The name of the backend, so that the user can select
                        it when creating a new database.

  PRIO                  An integer priority: backends are tried in decreasing
                        priority order when opening an existing database.
  """

  __metaclass__ = StorageBackendClass
  NAME = None
  PRIO = 10

  ## The registry of subclasses.
  CLASSES = {}

  FAIL = ['FAIL']

  @staticmethod
  def register_concrete_subclass(sub):
    """Register a concrete subclass, so that `open' can try it."""
    StorageBackend.CLASSES[sub.NAME] = sub

  @staticmethod
  def byname(name):
    """
    Return the concrete subclass with the given NAME.

    Raise `KeyError' if the name isn't found.
    """
    return StorageBackend.CLASSES[name]

  @staticmethod
  def classes():
    """Return an iterator over the concrete subclasses."""
    return StorageBackend.CLASSES.itervalues()

  @staticmethod
  def open(file, writep = False):
    """Open a database FILE, using some appropriate backend."""
    _OS.stat(file)
    for cls in sorted(StorageBackend.CLASSES.values(), reverse = True,
                      key = lambda cls: cls.PRIO):
      try: return cls(file, writep)
      except StorageBackendRefusal: pass
    raise StorageBackendRefusal

  @classmethod
  def create(cls, file):
    """
    Create a new database in the named FILE, using this backend.

    Subclasses must implement the `_create' instance method.
    """
    return cls(writep = True, _magic = lambda me: me._create(file))

  def __init__(me, file = None, writep = False, _magic = None, *args, **kw):
    """
    Main constructor.

    Subclasses are not, in general, expected to override this: there's a
    somewhat hairy protocol between the constructor and some of the class
    methods.  Instead, the main hook for customization is the subclass's
    `_open' method, which is invoked in the usual case.
    """
    super(StorageBackend, me).__init__(*args, **kw)
    if me.NAME is None: raise ValueError('abstract class')
    if _magic is not None: _magic(me)
    elif file is None: raise ValueError('missing file parameter')
    else: me._open(file, writep)
    me._writep = writep
    me._livep = True

  def close(me, abruptp = False):
    """
    Close the database.

    It is harmless to attempt to close a database which has been closed
    already.  Calls the subclass's `_close' method.
    """
    if me._livep:
      me._livep = False
      me._close(abruptp)

  ## Utilities.

  def _check_live(me):
    """Raise an error if the receiver has been closed."""
    if not me._livep: raise ValueError('database is closed')

  def _check_write(me):
    """Raise an error if the receiver is not open for writing."""
    me._check_live()
    if not me._writep: raise ValueError('database is read-only')

  def _check_meta_name(me, name):
    """
    Raise an error unless NAME is a valid name for a metadata item.

    Metadata names may not start with `$': such names are reserved for
    password storage.
    """
    if name.startswith('$'):
      raise ValueError("invalid metadata key `%s'" % name)

  ## Context protocol.

  def __enter__(me):
    """Context protocol: make sure the database is closed on exit."""
    return me
  def __exit__(me, exctype, excvalue, exctb):
    """Context protocol: see `__enter__'."""
    me.close(excvalue is not None)

  ## Metadata.

  def get_meta(me, name, default = FAIL):
    """
    Fetch the value for the metadata item NAME.

    If no such item exists, then return DEFAULT if that was set; otherwise
    raise a `KeyError'.

    This calls the subclass's `_get_meta' method, which should return the
    requested item or return the given DEFAULT value.  It may assume that the
    name is valid and the database is open.
    """
    me._check_meta_name(name)
    me._check_live()
    value = me._get_meta(name, default)
    if value is StorageBackend.FAIL: raise KeyError(name)
    return value

  def put_meta(me, name, value):
    """
    Store VALUE in the metadata item called NAME.

    This calls the subclass's `_put_meta' method, which may assume that the
    name is valid and the database is open for writing.
    """
    me._check_meta_name(name)
    me._check_write()
    me._put_meta(name, value)

  def del_meta(me, name):
    """
    Forget about the metadata item with the given NAME.

    This calls the subclass's `_del_meta' method, which may assume that the
    name is valid and the database is open for writing.
    """
    me._check_meta_name(name)
    me._check_write()
    me._del_meta(name)

  def iter_meta(me):
    """
    Return an iterator over the name/value metadata items.

    This calls the subclass's `_iter_meta' method, which may assume that the
    database is open.
    """
    me._check_live()
    return me._iter_meta()

  def get_passwd(me, label):
    """
    Fetch and return the payload stored with the (opaque, binary) LABEL.

    If there is no such payload then raise `KeyError'.

    This calls the subclass's `_get_passwd' method, which may assume that the
    database is open.
    """
    me._check_live()
    return me._get_passwd(label)

  def put_passwd(me, label, payload):
    """
    Associate the (opaque, binary) PAYLOAD with the (opaque, binary) LABEL.

    Any previous payload for LABEL is forgotten.

    This calls the subclass's `_put_passwd' method, which may assume that the
    database is open for writing.
    """
    me._check_write()
    me._put_passwd(label, payload)

  def del_passwd(me, label):
    """
    Forget any PAYLOAD associated with the (opaque, binary) LABEL.

    If there is no such payload then raise `KeyError'.

    This calls the subclass's `_del_passwd' method, which may assume that the
    database is open for writing.
    """
    me._check_write()
    me._del_passwd(label)

  def iter_passwds(me):
    """
    Return an iterator over the stored password label/payload pairs.

    This calls the subclass's `_iter_passwds' method, which may assume that
    the database is open.
    """
    me._check_live()
    return me._iter_passwds()

try: import gdbm as _G
except ImportError: pass
else:
  class GDBMStorageBackend (StorageBackend):
    """
    My instances store password data in a GDBM database.

    Metadata and password entries are mixed into the same database.  The key
    for a metadata item is simply its name; the key for a password entry is
    the entry's label prefixed by `$', since we're guaranteed that no
    metadata item name begins with `$'.
    """

    NAME = 'gdbm'

    def _open(me, file, writep):
      try: me._db = _G.open(file, writep and 'w' or 'r')
      except _G.error: raise StorageBackendRefusal(_excval())

    def _create(me, file):
      me._db = _G.open(file, 'n', 0600)

    def _close(me, abruptp):
      me._db.close()
      me._db = None

    def _get_meta(me, name, default):
      try: return me._db[name]
      except KeyError: return default

    def _put_meta(me, name, value):
      me._db[name] = value

    def _del_meta(me, name):
      del me._db[name]

    def _iter_meta(me):
      k = me._db.firstkey()
      while k is not None:
        if not k.startswith('$'): yield k, me._db[k]
        k = me._db.nextkey(k)

    def _get_passwd(me, label):
      return me._db['$' + label]

    def _put_passwd(me, label, payload):
      me._db['$' + label] = payload

    def _del_passwd(me, label):
      del me._db['$' + label]

    def _iter_passwds(me):
      k = me._db.firstkey()
      while k is not None:
        if k.startswith('$'): yield k[1:], me._db[k]
        k = me._db.nextkey(k)

try: import sqlite3 as _Q
except ImportError: pass
else:
  class SQLiteStorageBackend (StorageBackend):
    """
    I represent a password database stored in SQLite.

    Metadata and password items are stored in separate tables, so there's no
    conflict.  Some additional metadata is stored in the `meta' table, with
    names beginning with `$' so as not to conflict with clients:

    $version            The schema version of the table.
    """

    NAME = 'sqlite'
    VERSION = 0

    def _open(me, file, writep):
      try:
        me._db = _Q.connect(file)
        ver = me._query_scalar(
          "SELECT value FROM meta WHERE name = '$version'",
          "version check")
      except (_Q.DatabaseError, _Q.OperationalError):
        raise StorageBackendRefusal(_excval())
      if ver is None: raise ValueError('database broken (missing $version)')
      elif ver < me.VERSION: me._upgrade(ver)
      elif ver > me.VERSION: raise ValueError \
        ('unknown database schema version (%d > %d)' % (ver, me.VERSION))

    def _create(me, file):
      fd = _OS.open(file, _OS.O_WRONLY | _OS.O_CREAT | _OS.O_EXCL, 0600)
      _OS.close(fd)
      try:
        me._db = _Q.connect(file)
        c = me._db.cursor()
        c.execute("""
          CREATE TABLE meta (
                  name TEXT PRIMARY KEY NOT NULL,
                  value BLOB NOT NULL);
        """)
        c.execute("""
          CREATE TABLE passwd (
                  label BLOB PRIMARY KEY NOT NULL,
                  payload BLOB NOT NULL);
        """)
        c.execute("""
          INSERT INTO meta (name, value) VALUES ('$version', ?);
        """, [me.VERSION])
      except:
        try: _OS.unlink(file)
        except OSError: pass
        raise

    def _upgrade(me, ver):
      """Upgrade the database from schema version VER."""
      assert False, 'how embarrassing'

    def _close(me, abruptp):
      if not abruptp: me._db.commit()
      me._db.close()
      me._db = None

    def _fetch_scalar(me, c, what, default = None):
      try: row = next(c)
      except StopIteration: val = default
      else: val, = row
      try: row = next(c)
      except StopIteration: pass
      else: raise ValueError('multiple matching records for %s' % what)
      return val

    def _query_scalar(me, query, what, default = None, args = []):
      c = me._db.cursor()
      c.execute(query, args)
      return me._fetch_scalar(c, what, default)

    def _get_meta(me, name, default):
      v = me._query_scalar("SELECT value FROM meta WHERE name = ?",
                           "metadata item `%s'" % name,
                           default = default, args = [name])
      if v is default: return v
      else: return str(v)

    def _put_meta(me, name, value):
      c = me._db.cursor()
      c.execute("INSERT OR REPLACE INTO meta (name, value) VALUES (?, ?)",
                [name, buffer(value)])

    def _del_meta(me, name):
      c = me._db.cursor()
      c.execute("DELETE FROM meta WHERE name = ?", [name])
      if not c.rowcount: raise KeyError(name)

    def _iter_meta(me):
      c = me._db.cursor()
      c.execute("SELECT name, value FROM meta WHERE name NOT LIKE '$%'")
      for k, v in c: yield k, str(v)

    def _get_passwd(me, label):
      pld = me._query_scalar("SELECT payload FROM passwd WHERE label = ?",
                             "password", default = None,
                             args = [buffer(label)])
      if pld is None: raise KeyError(label)
      return str(pld)

    def _put_passwd(me, label, payload):
      c = me._db.cursor()
      c.execute("INSERT OR REPLACE INTO passwd (label, payload) "
                "VALUES (?, ?)",
                [buffer(label), buffer(payload)])

    def _del_passwd(me, label):
      c = me._db.cursor()
      c.execute("DELETE FROM passwd WHERE label = ?", [label])
      if not c.rowcount: raise KeyError(label)

    def _iter_passwds(me):
      c = me._db.cursor()
      c.execute("SELECT label, payload FROM passwd")
      for k, v in c: yield str(k), str(v)

class PlainTextBackend (StorageBackend):
  """
  I'm a utility base class for storage backends which use plain text files.

  I provide subclasses with the following capabilities.

    * Creating files, with given modes, optionally ensuring that the file
      doesn't exist already.

    * Parsing flat text files, checking leading magic, skipping comments, and
      providing standard encodings of troublesome characters and binary
      strings in metadata and password records.  See below.

    * Maintenance of metadata and password records in in-memory dictionaries,
      with ready implementations of the necessary StorageBackend subclass
      responsibility methods.  (Subclasses can override these if they want to
      make different arrangements.)

  Metadata records are written with an optional prefix string chosen by the
  caller, followed by a `NAME=VALUE' pair.  The NAME is form-urlencoded and
  prefixed with `!' if it contains strange characters; the VALUE is base64-
  encoded (without the pointless trailing `=' padding) and prefixed with `?'
  if necessary.

  Password records are written with an optional prefix string chosen by the
  caller, followed by a LABEL=PAYLOAD pair, both of which are base64-encoded
  (without padding).

  The following attributes are available for subclasses:

  _meta         Dictionary mapping metadata item names to their values.
                Populated by `_parse_meta' and managed by `_get_meta' and
                friends.

  _pw           Dictionary mapping password labels to encrypted payloads.
                Populated by `_parse_passwd' and managed by `_get_passwd' and
                friends.

  _dirtyp       Boolean: set if either of the dictionaries has been modified.
  """

  def __init__(me, *args, **kw):
    """
    Hook for initialization.

    Sets up the published instance attributes.
    """
    me._meta = {}
    me._pw = {}
    me._dirtyp = False
    super(PlainTextBackend, me).__init__(*args, **kw)

  def _create_file(me, file, mode = 0600, freshp = False):
    """
    Make sure FILE exists, creating it with the given MODE if necessary.

    If FRESHP is true, then make sure the file did not exist previously.
    Return a file object for the newly created file.
    """
    flags = _OS.O_CREAT | _OS.O_WRONLY
    if freshp: flags |= _OS.O_EXCL
    else: flags |= _OS.O_TRUNC
    fd = _OS.open(file, flags, mode)
    return _OS.fdopen(fd, 'w')

  def _mark_dirty(me):
    """
    Set the `_dirtyp' flag.

    Subclasses might find it useful to intercept this method.
    """
    me._dirtyp = True

  def _eqsplit(me, line):
    """
    Extract the KEY, VALUE pair from a LINE of the form `KEY=VALUE'.

    Raise `ValueError' if there is no `=' in the LINE.
    """
    eq = line.index('=')
    return line[:eq], line[eq + 1:]

  def _parse_file(me, file, magic = None):
    """
    Parse a FILE.

    Specifically:

      * Raise `StorageBackendRefusal' if that the first line doesn't match
        MAGIC (if provided).  MAGIC should not contain the terminating
        newline.

      * Ignore comments (beginning `#') and blank lines.

      * Call `_parse_line' (provided by the subclass) for other lines.
    """
    with open(file, 'r') as f:
      if magic is not None:
        if f.readline().rstrip('\n') != magic: raise StorageBackendRefusal
      for line in f:
        line = line.rstrip('\n')
        if not line or line.startswith('#'): continue
        me._parse_line(line)

  def _write_file(me, file, writebody, mode = 0600, magic = None):
    """
    Update FILE atomically.

    The newly created file will have the given MODE.  If MAGIC is given, then
    write that as the first line.  Calls WRITEBODY(F) to write the main body
    of the file where F is a file object for the new file.
    """
    new = file + '.new'
    with me._create_file(new, mode) as f:
      if magic is not None: f.write(magic + '\n')
      writebody(f)
    _OS.rename(new, file)

  def _parse_meta(me, line):
    """Parse LINE as a metadata NAME=VALUE pair, and updates `_meta'."""
    k, v = me._eqsplit(line)
    me._meta[_dec_metaname(k)] = _dec_metaval(v)

  def _write_meta(me, f, prefix = ''):
    """Write the metadata records to F, each with the given PREFIX."""
    f.write('\n## Metadata.\n')
    for k, v in me._meta.iteritems():
      f.write('%s%s=%s\n' % (prefix, _enc_metaname(k), _enc_metaval(v)))

  def _get_meta(me, name, default):
    return me._meta.get(name, default)
  def _put_meta(me, name, value):
    me._mark_dirty()
    me._meta[name] = value
  def _del_meta(me, name):
    me._mark_dirty()
    del me._meta[name]
  def _iter_meta(me):
    return me._meta.iteritems()

  def _parse_passwd(me, line):
    """Parse LINE as a password LABEL=PAYLOAD pair, and updates `_pw'."""
    k, v = me._eqsplit(line)
    me._pw[_unb64(k)] = _unb64(v)

  def _write_passwd(me, f, prefix = ''):
    """Write the password records to F, each with the given PREFIX."""
    f.write('\n## Password data.\n')
    for k, v in me._pw.iteritems():
      f.write('%s%s=%s\n' % (prefix, _b64(k), _b64(v)))

  def _get_passwd(me, label):
    return me._pw[str(label)]
  def _put_passwd(me, label, payload):
    me._mark_dirty()
    me._pw[str(label)] = payload
  def _del_passwd(me, label):
    me._mark_dirty()
    del me._pw[str(label)]
  def _iter_passwds(me):
    return me._pw.iteritems()

class FlatFileStorageBackend (PlainTextBackend):
  """
  I maintain a password database in a plain text file.

  The text file consists of lines, as follows.

    * Empty lines, and lines beginning with `#' (in the leftmost column only)
      are ignored.

    * Lines of the form `$LABEL=PAYLOAD' store password data.  Both LABEL and
      PAYLOAD are base64-encoded, without `=' padding.

    * Lines of the form `NAME=VALUE' store metadata.  If the NAME contains
      characters other than alphanumerics, hyphens, underscores, and colons,
      then it is form-urlencoded, and prefixed wth `!'.  If the VALUE
      contains such characters, then it is base64-encoded, without `='
      padding, and prefixed with `?'.

    * Other lines are erroneous.

  The file is rewritten from scratch when it's changed: any existing
  commentary is lost, and items may be reordered.  There is no file locking,
  but the file is updated atomically, by renaming.

  It is expected that the FlatFileStorageBackend is used mostly for
  diagnostics and transfer, rather than for a live system.
  """

  NAME = 'flat'
  PRIO = 0
  MAGIC = '### pwsafe password database'

  def _open(me, file, writep):
    if not _OS.path.isfile(file): raise StorageBackendRefusal
    me._parse_file(file, magic = me.MAGIC)
  def _parse_line(me, line):
    if line.startswith('$'): me._parse_passwd(line[1:])
    else: me._parse_meta(line)

  def _create(me, file):
    with me._create_file(file, freshp = True) as f: pass
    me._file = file
    me._mark_dirty()

  def _close(me, abruptp):
    if not abruptp and me._dirtyp:
      me._write_file(me._file, me._write_body, magic = me.MAGIC)

  def _write_body(me, f):
    me._write_meta(f)
    me._write_passwd(f, '$')

class DirectoryStorageBackend (PlainTextBackend):
  """
  I maintain a password database in a directory, with one file per password.

  This makes password databases easy to maintain in a revision-control system
  such as Git.

  The directory is structured as follows.

  dir/meta      Contains metadata, similar to the `FlatFileBackend'.

  dir/pw/LABEL  Contains the (raw binary) payload for the given password
                LABEL (base64-encoded, without the useless `=' padding, and
                with `/' replaced by `.').

  dir/tmp/      Contains temporary files used by the implementation.
  """

  NAME = 'dir'
  METAMAGIC = '### pwsafe password directory metadata'

  def _open(me, file, writep):
    if not _OS.path.isdir(file) or \
          not _OS.path.isdir(_OS.path.join(file, 'pw')) or \
          not _OS.path.isdir(_OS.path.join(file, 'tmp')) or \
          not _OS.path.isfile(_OS.path.join(file, 'meta')):
      raise StorageBackendRefusal
    me._dir = file
    me._parse_file(_OS.path.join(file, 'meta'), magic = me.METAMAGIC)
  def _parse_line(me, line):
    me._parse_meta(line)

  def _create(me, file):
    _OS.mkdir(file, 0700)
    _OS.mkdir(_OS.path.join(file, 'pw'), 0700)
    _OS.mkdir(_OS.path.join(file, 'tmp'), 0700)
    me._mark_dirty()
    me._dir = file

  def _close(me, abruptp):
    if not abruptp and me._dirtyp:
      me._write_file(_OS.path.join(me._dir, 'meta'),
                     me._write_meta, magic = me.METAMAGIC)

  def _pwfile(me, label, dir = 'pw'):
    return _OS.path.join(me._dir, dir, _b64(label).replace('/', '.'))
  def _get_passwd(me, label):
    try:
      f = open(me._pwfile(label), 'rb')
    except (OSError, IOError):
      if _excval().errno == _E.ENOENT: raise KeyError(label)
      else: raise
    with f: return f.read()
  def _put_passwd(me, label, payload):
    new = me._pwfile(label, 'tmp')
    fd = _OS.open(new, _OS.O_WRONLY | _OS.O_CREAT | _OS.O_TRUNC, 0600)
    _OS.close(fd)
    with open(new, 'wb') as f: f.write(payload)
    _OS.rename(new, me._pwfile(label))
  def _del_passwd(me, label):
    try:
      _OS.remove(me._pwfile(label))
    except (OSError, IOError):
      if _excval().errno == _E.ENOENT: raise KeyError(label)
      else: raise
  def _iter_passwds(me):
    pw = _OS.path.join(me._dir, 'pw')
    for i in _OS.listdir(pw):
      with open(_OS.path.join(pw, i), 'rb') as f: pld = f.read()
      yield _unb64(i.replace('.', '/')), pld

###--------------------------------------------------------------------------
### Password storage.

class PW (object):
  """
  I represent a secure (ish) password store.

  I can store short secrets, associated with textual names, in a way which
  doesn't leak too much information about them.

  I implement (some of) the Python mapping protocol.

  I keep track of everything using a StorageBackend object.  This contains
  password entries, identified by cryptographic labels, and a number of
  metadata items.

  cipher        Names the Catacomb cipher selected.

  hash          Names the Catacomb hash function selected.

  key           Cipher and MAC keys, each prefixed by a 16-bit big-endian
                length and concatenated, encrypted using the master
                passphrase.

  mac           Names the Catacomb message authentication code selected.

  magic         A magic string for obscuring password tag names.

  salt          The salt for hashing the passphrase.

  tag           The master passphrase's tag, for the Pixie's benefit.

  Password entries are assigned labels of the form `$' || H(MAGIC || TAG);
  the corresponding value consists of a pair (TAG, PASSWD), prefixed with
  16-bit lengths, concatenated, padded to a multiple of 256 octets, and
  encrypted using the stored keys.
  """

  def __init__(me, file, writep = False):
    """
    Initialize a PW object from the database in FILE.

    If WRITEP is false (the default) then the database is opened read-only;
    if true then it may be written.  Requests the database password from the
    Pixie, which may cause interaction.
    """

    ## Open the database.
    me.db = StorageBackend.open(file, writep)

    ## Find out what crypto to use.
    c = _C.gcciphers[me.db.get_meta('cipher')]
    h = _C.gchashes[me.db.get_meta('hash')]
    m = _C.gcmacs[me.db.get_meta('mac')]

    ## Request the passphrase and extract the master keys.
    tag = me.db.get_meta('tag')
    ppk = PPK(_C.ppread(tag), c, h, m, me.db.get_meta('salt'))
    try:
      b = _C.ReadBuffer(ppk.decrypt(me.db.get_meta('key')))
    except DecryptError:
      _C.ppcancel(tag)
      raise
    me.ck = b.getblk16()
    me.mk = b.getblk16()
    if not b.endp: raise ValueError('trailing junk')

    ## Set the key, and stash it and the tag-hashing secret.
    me.k = Crypto(c, h, m, me.ck, me.mk)
    me.magic = me.k.decrypt(me.db.get_meta('magic'))

  @classmethod
  def create(cls, dbcls, file, tag, c, h, m):
    """
    Create and initialize a new database FILE using StorageBackend DBCLS.

    We want a GCipher subclass C, a GHash subclass H, and a GMAC subclass M;
    and a Pixie passphrase TAG.

    This doesn't return a working object: it just creates the database file
    and gets out of the way.
    """

    ## Set up the cryptography.
    pp = _C.ppread(tag, _C.PMODE_VERIFY)
    ppk = PPK(pp, c, h, m)
    ck = _C.rand.block(c.keysz.default)
    mk = _C.rand.block(c.keysz.default)
    k = Crypto(c, h, m, ck, mk)

    ## Set up and initialize the database.
    kct = ppk.encrypt(_C.WriteBuffer().putblk16(ck).putblk16(mk))
    with dbcls.create(file) as db:
      db.put_meta('tag', tag)
      db.put_meta('salt', ppk.salt)
      db.put_meta('cipher', c.name)
      db.put_meta('hash', h.name)
      db.put_meta('mac', m.name)
      db.put_meta('key', kct)
      db.put_meta('magic', k.encrypt(_C.rand.block(h.hashsz)))

  def keyxform(me, key):
    """Transform the KEY (actually a password tag) into a password label."""
    return me.k.h().hash(me.magic).hash(key).done()

  def changepp(me):
    """
    Change the database password.

    Requests the new password from the Pixie, which will probably cause
    interaction.
    """
    tag = me.db.get_meta('tag')
    _C.ppcancel(tag)
    ppk = PPK(_C.ppread(tag, _C.PMODE_VERIFY),
              me.k.c.__class__, me.k.h, me.k.m.__class__)
    kct = ppk.encrypt(_C.WriteBuffer().putblk16(me.ck).putblk16(me.mk))
    me.db.put_meta('key', kct)
    me.db.put_meta('salt', ppk.salt)

  def pack(me, key, value):
    """Pack the KEY and VALUE into a ciphertext, and return it."""
    b = _C.WriteBuffer()
    b.putblk16(key).putblk16(value)
    b.zero(((b.size + 255) & ~255) - b.size)
    return me.k.encrypt(b)

  def unpack(me, ct):
    """
    Unpack a ciphertext CT and return a (KEY, VALUE) pair.

    Might raise DecryptError, of course.
    """
    b = _C.ReadBuffer(me.k.decrypt(ct))
    key = b.getblk16()
    value = b.getblk16()
    return key, value

  ## Mapping protocol.

  def __getitem__(me, key):
    """Return the password for the given KEY."""
    try: return me.unpack(me.db.get_passwd(me.keyxform(key)))[1]
    except KeyError: raise KeyError(key)

  def __setitem__(me, key, value):
    """Associate the password VALUE with the KEY."""
    me.db.put_passwd(me.keyxform(key), me.pack(key, value))

  def __delitem__(me, key):
    """Forget all about the KEY."""
    try: me.db.del_passwd(me.keyxform(key))
    except KeyError: raise KeyError(key)

  def __iter__(me):
    """Iterate over the known password tags."""
    for _, pld in me.db.iter_passwds():
      yield me.unpack(pld)[0]

  ## Context protocol.

  def __enter__(me):
    return me
  def __exit__(me, excty, excval, exctb):
    me.db.close(excval is not None)

###----- That's all, folks --------------------------------------------------