bin/chroot-maint: Program for maintaining chroots.

[distorted-chroot] / bin / chroot-maint

#! /usr/bin/python
###
### Create, upgrade, and maintain (native and cross-) chroots
###
### (c) 2018 Mark Wooding
###

###----- Licensing notice ---------------------------------------------------
###
### This file is part of the distorted.org.uk chroot maintenance tools.
###
### distorted-chroot is free software: you can redistribute it and/or
### modify it under the terms of the GNU General Public License as
### published by the Free Software Foundation; either version 2 of the
### License, or (at your option) any later version.
###
### distorted-chroot is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
### General Public License for more details.
###
### You should have received a copy of the GNU General Public License
### along with distorted-chroot.  If not, write to the Free Software
### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
### USA.

## still to do:
##   tidy up

import contextlib as CTX
import errno as E
import fcntl as FC
import fnmatch as FM
import glob as GLOB
import itertools as I
import optparse as OP
import os as OS
import random as R
import re as RX
import signal as SIG
import select as SEL
import stat as ST
from cStringIO import StringIO
import sys as SYS
import time as T
import traceback as TB

import jobclient as JC

QUIS = OS.path.basename(SYS.argv[0])
TODAY = T.strftime("%Y-%m-%d")
NOW = T.time()

###--------------------------------------------------------------------------
### Random utilities.

RC = 0
def moan(msg):
  """Print MSG to stderr as a warning."""
  if not OPT.silent: OS.write(2, "%s: %s\n" % (QUIS, msg))
def error(msg):
  """Print MSG to stderr, and remember to exit nonzero."""
  global RC
  moan(msg)
  RC = 2

class ExpectedError (Exception):
  """A fatal error which shouldn't print a backtrace."""
  pass

@CTX.contextmanager
def toplevel_handler():
  """Catch `ExpectedError's and report Unixish error messages."""
  try: yield None
  except ExpectedError, err: moan(err); SYS.exit(2)

def spew(msg):
  """Print MSG to stderr as a debug trace."""
  if OPT.debug: OS.write(2, ";; %s\n" % msg)

class Tag (object):
  """Unique objects with no internal structure."""
  def __init__(me, label): me._label = label
  def __str__(me): return '#<%s %s>' % (me.__class__.__name__, me._label)
  def __repr__(me): return '#<%s %s>' % (me.__class__.__name__, me._label)

class Struct (object):
  def __init__(me, **kw): me.__dict__.update(kw)

class Cleanup (object):
  """
  A context manager for stacking other context managers.

  By itself, it does nothing.  Attach other context managers with `enter' or
  loose cleanup functions with `add'.  On exit, contexts are left and
  cleanups performed in reverse order.
  """
  def __init__(me):
    me._cleanups = []
  def __enter__(me):
    return me
  def __exit__(me, exty, exval, extb):
    trap = False
    for c in reversed(me._cleanups):
      if c(exty, exval, extb): trap = True
    return trap
  def enter(me, ctx):
    v = ctx.__enter__()
    me._cleanups.append(ctx.__exit__)
    return v
  def add(me, func):
    me._cleanups.append(lambda exty, exval, extb: func())

def zulu(t = None):
  """Return the time T (default now) as a string."""
  return T.strftime("%Y-%m-%dT%H:%M:%SZ", T.gmtime(t))

R_ZULU = RX.compile(r"^(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)Z$")
def unzulu(z):
  """Convert the time string Z back to a Unix time."""
  m = R_ZULU.match(z)
  if not m: raise ValueError("bad time spec `%s'" % z)
  yr, mo, dy, hr, mi, se = map(int, m.groups())
  return T.mktime((yr, mo, dy, hr, mi, se, 0, 0, 0))

###--------------------------------------------------------------------------
### Simple select(2) utilities.

class BaseSelector (object):
  """
  A base class for hooking into `select_loop'.

  See `select_loop' for details of the protocol.
  """
  def preselect(me, rfds, wfds): pass
  def postselect_read(me, fd): pass
  def postselect_write(me, fd): pass

class WriteLinesSelector (BaseSelector):
  """Write whole lines to an output file descriptor."""

  def __init__(me, fd, nextfn = None, *args, **kw):
    """
    Initialize the WriteLinesSelector to write to the file descriptor FD.

    The FD is marked non-blocking.

    The lines are produced by the NEXTFN, which is called without arguments.
    It can affect the output in three ways:

      * It can return a string (or almost any other kind of object, which
        will be converted into a string by `str'), which will be written to
        the descriptor followed by a newline.  Lines are written in the order
        in which they are produced.

      * It can return `None', which indicates that there are no more items to
        be written for the moment.  The function will be called again from
        time to time, to see if it has changed its mind.  This is the right
        thing to do in order to stall output temporarily.

      * It can raise `StopIteration', which indicates that there will never
        be any more items.  The file descriptor will be closed.

    Subclasses can override this behaviour by defining a method `_next' and
    passing `None' as the NEXTFN.
    """
    super(WriteLinesSelector, me).__init__(*args, **kw)
    set_nonblocking(fd)
    me._fd = fd
    if nextfn is not None: me._next = nextfn

    ## Selector state.
    ##
    ##  * `_buf' contains a number of output items, already formatted, and
    ##    ready for output in a single batch.  It might be empty.
    ##
    ##  * `_pos' is the current output position in `_buf'.
    ##
    ##  * `_more' is set unless the `_next' function has raised
    ##    `StopIteration': it indicates that we should close the descriptor
    ##    once the all of the remaining data in the buffer has been sent.
    me._buf = ""
    me._pos = 0
    me._more = True

  def _refill(me):
    """Refill `_buf' by calling `_next'."""
    sio = StringIO(); n = 0
    while n < 4096:
      try: item = me._next()
      except StopIteration: me._more = False; break
      if item is None: break
      item = str(item)
      sio.write(item); sio.write("\n"); n += len(item) + 1
    me._buf = sio.getvalue(); me._pos = 0

  def preselect(me, rfds, wfds):
    if me._fd == -1: return
    if me._buf == "" and me._more: me._refill()
    if me._buf != "" or not me._more: wfds.append(me._fd)

  def postselect_write(me, fd):
    if fd != me._fd: return
    while True:
      if me._pos >= len(me._buf):
        if me._more: me._refill()
        if not me._more: OS.close(me._fd); me._fd = -1; break
        if not me._buf: break
      try: n = OS.write(me._fd, me._buf[me._pos:])
      except OSError, err:
        if err.errno == E.EAGAIN or err.errno == E.WOULDBLOCK: break
        elif err.errno == E.EPIPE: OS.close(me._fd); me._fd = -1; break
        else: raise
      me._pos += n

class ReadLinesSelector (BaseSelector):
  """Report whole lines from an input file descriptor as they arrive."""

  def __init__(me, fd, linefn = None, *args, **kw):
    """
    Initialize the ReadLinesSelector to read from the file descriptor FD.

    The FD is marked non-blocking.

    For each whole line, and the final partial line (if any), the selector
    calls LINEFN with the line as an argument (without the terminating
    newline, if any).

    Subclasses can override this behaviour by defining a method `_line' and
    passing `None' as the LINEFN.
    """
    super(ReadLinesSelector, me).__init__(*args, **kw)
    set_nonblocking(fd)
    me._fd = fd
    me._buf = ""
    if linefn is not None: me._line = linefn

  def preselect(me, rfds, wfds):
    if me._fd != -1: rfds.append(me._fd)

  def postselect_read(me, fd):
    if fd != me._fd: return
    while True:
      try: buf = OS.read(me._fd, 4096)
      except OSError, err:
        if err.errno == E.EAGAIN or err.errno == E.WOULDBLOCK: break
        else: raise
      if buf == "":
        OS.close(me._fd); me._fd = -1
        if me._buf: me._line(me._buf)
        break
      buf = me._buf + buf
      i = 0
      while True:
        try: j = buf.index("\n", i)
        except ValueError: break
        me._line(buf[i:j])
        i = j + 1
      me._buf = buf[i:]

def select_loop(selectors):
  """
  Multiplex I/O between the various SELECTORS.

  A `selector' SEL is an object which implements the selector protocol, which
  consists of three methods.

    * SEL.preselect(RFDS, WFDS) -- add any file descriptors which the
      selector is interested in reading from to the list RFDS, and add file
      descriptors it's interested in writing to to the list WFDS.

    * SEL.postselect_read(FD) -- informs the selector that FD is ready for
      reading.

    * SEL.postselect_write(FD) -- informs the selector that FD is ready for
      writing.

  The `select_loop' function loops as follows.

    * It calls the `preselect' method on each SELECTOR to determine what I/O
      events it thinks are interesting.

    * It waits for some interesting event to happen.

    * It calls the `postselect_read' and/or `postselect_write' methods on all
      of the selectors for each file descriptor which is ready.

  The loop ends when no selector is interested in any events.  This is simple
  but rather inefficient.
  """
  while True:
    rfds, wfds = [], []
    for sel in selectors: sel.preselect(rfds, wfds)
    if not rfds and not wfds: break
    rfds, wfds, _ = SEL.select(rfds, wfds, [])
    for fd in rfds:
      for sel in selectors: sel.postselect_read(fd)
    for fd in wfds:
      for sel in selectors: sel.postselect_write(fd)

###--------------------------------------------------------------------------
### Running subprocesses.

def wait_outcome(st):
  """
  Given a ST from `waitpid' (or similar), return a human-readable outcome.
  """
  if OS.WIFSIGNALED(st): return "killed by signal %d" % OS.WTERMSIG(st)
  elif OS.WIFEXITED(st):
    rc = OS.WEXITSTATUS(st)
    if rc: return "failed: rc = %d" % rc
    else: return "completed successfully"
  else: return "died with incomprehensible status 0x%04x" % st

class SubprocessFailure (Exception):
  """An exception indicating that a subprocess failed."""
  def __init__(me, what, st):
    me.st = st
    me.what = what
    if OS.WIFEXITED(st): me.rc, me.sig = OS.WEXITSTATUS(st), None
    elif OS.WIFSIGNALED(st): me.rc, me.sig = None, OS.WTERMSIG(st)
    else: me.rc, me.sig = None, None
  def __str__(me):
    return "subprocess `%s' %s" % (me.what, wait_outcome(me.st))

INHERIT = Tag('INHERIT')
PIPE = Tag('PIPE')
DISCARD = Tag('DISCARD')
@CTX.contextmanager
def subprocess(command,
               stdin = INHERIT, stdout = INHERIT, stderr = INHERIT,
               cwd = INHERIT, jobserver = DISCARD):
  """
  Hairy context manager for running subprocesses.

  The COMMAND is a list of arguments; COMMAND[0] names the program to be
  invoked.  (There's currently no way to run a program with an unusual
  `argv[0]'.)

  The keyword arguments `stdin', `stdout', and `stderr' explain what to do
  with the standard file descriptors.

    * `INHERIT' means that they should be left alone: the child will use a
      copy of the parent's descriptor.  This is the default.

    * `DISCARD' means that the descriptor should be re-opened onto
      `/dev/null' (for reading or writing as appropriate).

    * `PIPE' means that the descriptor should be re-opened as (the read or
      write end, as appropriate, of) a pipe, and the other end returned to
      the context body.

  Simiarly, the JOBSERVER may be `INHERIT' to pass the jobserver descriptors
  and environment variable down to the child, or `DISCARD' to close it.  The
  default is `DISCARD'.

  The CWD may be `INHERIT' to run the child with the same working directory
  as the parent, or a pathname to change to an explicitly given working
  directory.

  The context is returned three values, which are file descriptors for other
  pipe ends for stdin, stdout, and stderr respectively, or -1 if there is no
  pipe.

  The context owns the pipe descriptors, and is expected to close them
  itself.  (Timing of closure is significant, particularly for `stdin'.)
  """

  ## Set up.
  r_in, w_in = -1, -1
  r_out, w_out = -1, -1
  r_err, w_err = -1, -1
  spew("running subprocess `%s'" % " ".join(command))

  ## Clean up as necessary...
  try:

    ## Set up stdin.
    if stdin is PIPE: r_in, w_in = OS.pipe()
    elif stdin is DISCARD: r_in = OS.open("/dev/null", OS.O_RDONLY)
    elif stdin is not INHERIT:
      raise ValueError("bad `stdin' value `%r'" % stdin)

    ## Set up stdout.
    if stdout is PIPE: r_out, w_out = OS.pipe()
    elif stdout is DISCARD: w_out = OS.open("/dev/null", OS.O_WRONLY)
    elif stdout is not INHERIT:
      raise ValueError("bad `stderr' value `%r'" % stdout)

    ## Set up stderr.
    if stderr is PIPE: r_err, w_err = OS.pipe()
    elif stderr is DISCARD: w_err = OS.open("/dev/null", OS.O_WRONLY)
    elif stderr is not INHERIT:
      raise ValueError("bad `stderr' value `%r'" % stderr)

    ## Start up the child.
    kid = OS.fork()

    if kid == 0:
      ## Child process.

      ## Fix up stdin.
      if r_in != -1: OS.dup2(r_in, 0); OS.close(r_in)
      if w_in != -1: OS.close(w_in)

      ## Fix up stdout.
      if w_out != -1: OS.dup2(w_out, 1); OS.close(w_out)
      if r_out != -1: OS.close(r_out)

      ## Fix up stderr.
      if w_err != -1: OS.dup2(w_err, 2); OS.close(w_err)
      if r_err != -1: OS.close(r_err)

      ## Change directory.
      if cwd is not INHERIT: OS.chdir(cwd)

      ## Fix up the jobserver.
      if jobserver is DISCARD: SCHED.close_jobserver()

      ## Run the program.
      try: OS.execvp(command[0], command)
      except OSError, err:
        moan("failed to run `%s': %s" % err.strerror)
        OS._exit(127)

    ## Close the other ends of the pipes.
    if r_in != -1: OS.close(r_in); r_in = -1
    if w_out != -1: OS.close(w_out); w_out = -1
    if w_err != -1: OS.close(w_err); w_err = -1

    ## Return control to the context body.  Remember not to close its pipes.
    yield w_in, r_out, r_err
    w_in = r_out = r_err = -1

    ## Collect the child process's exit status.
    _, st = OS.waitpid(kid, 0)
    spew("subprocess `%s' %s" % (" ".join(command), wait_outcome(st)))
    if st: raise SubprocessFailure(" ".join(command), st)

  ## Tidy up.
  finally:

    ## Close any left-over file descriptors.
    for fd in [r_in, w_in, r_out, w_out, r_err, w_err]:
      if fd != -1: OS.close(fd)

def set_nonblocking(fd):
  """Mark the descriptor FD as non-blocking."""
  FC.fcntl(fd, FC.F_SETFL, FC.fcntl(fd, FC.F_GETFL) | OS.O_NONBLOCK)

class DribbleOut (BaseSelector):
  """A simple selector to feed a string to a descriptor, in pieces."""
  def __init__(me, fd, string, *args, **kw):
    super(DribbleOut, me).__init__(*args, **kw)
    me._fd = fd
    me._string = string
    me._i = 0
    set_nonblocking(me._fd)
    me.result = None
  def preselect(me, rfds, wfds):
    if me._fd != -1: wfds.append(me._fd)
  def postselect_write(me, fd):
    if fd != me._fd: return
    try: n = OS.write(me._fd, me._string)
    except OSError, err:
      if err.errno == E.EAGAIN or err.errno == E.EWOULDBLOCK: return
      elif err.errno == E.EPIPE: OS.close(me._fd); me._fd = -1; return
      else: raise
    if n == len(me._string): OS.close(me._fd); me._fd = -1
    else: me._string = me._string[n:]

class DribbleIn (BaseSelector):
  """A simple selector to collect all the input as a big string."""
  def __init__(me, fd, *args, **kw):
    super(DribbleIn, me).__init__(*args, **kw)
    me._fd = fd
    me._buf = StringIO()
    set_nonblocking(me._fd)
  def preselect(me, rfds, wfds):
    if me._fd != -1: rfds.append(me._fd)
  def postselect_read(me, fd):
    if fd != me._fd: return
    while True:
      try: buf = OS.read(me._fd, 4096)
      except OSError, err:
        if err.errno == E.EAGAIN or err.errno == E.EWOULDBLOCK: break
        else: raise
      if buf == "": OS.close(me._fd); me._fd = -1; break
      else: me._buf.write(buf)
  @property
  def result(me): return me._buf.getvalue()

RETURN = Tag('RETURN')
def run_program(command,
                stdin = INHERIT, stdout = INHERIT, stderr = INHERIT,
                *args, **kwargs):
  """
  A simplifying wrapper around `subprocess'.

  The COMMAND is a list of arguments; COMMAND[0] names the program to be
  invoked, as for `subprocess'.

  The keyword arguments `stdin', `stdout', and `stderr' explain what to do
  with the standard file descriptors.

    * `INHERIT' means that they should be left alone: the child will use a
      copy of the parent's descriptor.

    * `DISCARD' means that the descriptor should be re-opened onto
      `/dev/null' (for reading or writing as appropriate).

    * `RETURN', for an output descriptor, means that all of the output
      produced on that descriptor should be collected and returned as a
      string.

    * A string, for stdin, means that the string should be provided on the
      child's standard input.

  (The value `PIPE' is not permitted here.)

  Other arguments are passed on to `subprocess'.

  If no descriptors are marked `RETURN', then the function returns `None'; if
  exactly one descriptor is so marked, then the function returns that
  descriptor's output as a string; otherwise, it returns a tuple of strings
  for each such descriptor, in the usual order.
  """
  kw = dict(); kw.update(kwargs)
  selfn = []

  if isinstance(stdin, basestring):
    kw['stdin'] = PIPE; selfn.append(lambda fds: DribbleOut(fds[0], stdin))
  elif stdin is INHERIT or stdin is DISCARD:
    kw['stdin'] = stdin
  else:
    raise ValueError("bad `stdin' value `%r'" % stdin)

  if stdout is RETURN:
    kw['stdout'] = PIPE; selfn.append(lambda fds: DribbleIn(fds[1]))
  elif stdout is INHERIT or stdout is DISCARD:
    kw['stdout'] = stdout
  else:
    raise ValueError("bad `stdout' value `%r'" % stdout)

  if stderr is RETURN:
    kw['stderr'] = PIPE; selfn.append(lambda fds: DribbleIn(fds[2]))
  elif stderr is INHERIT or stderr is DISCARD:
    kw['stderr'] = stderr
  else:
    raise ValueError("bad `stderr' value `%r'" % stderr)

  with subprocess(command, *args, **kw) as fds:
    sel = [fn(fds) for fn in selfn]
    select_loop(sel)
  rr = []
  for s in sel:
    r = s.result
    if r is not None: rr.append(r)
  if len(rr) == 0: return None
  if len(rr) == 1: return rr[0]
  else: return tuple(rr)

###--------------------------------------------------------------------------
### Other system-ish utilities.

@CTX.contextmanager
def safewrite(path):
  """
  Context manager for writing to a file.

  A new file, named `PATH.new', is opened for writing, and the file object
  provided to the context body.  If the body completes normally, the file is
  closed and renamed to PATH.  If the body raises an exception, the file is
  still closed, but not renamed into place.
  """
  new = path + ".new"
  with open(new, "w") as f: yield f
  OS.rename(new, path)

@CTX.contextmanager
def safewrite_root(path, mode = None, uid = None, gid = None):
  """
  Context manager for writing to a file with root privileges.

  This is as for `safewrite', but the file is opened and written as root.
  """
  new = path + ".new"
  with subprocess(C.ROOTLY + ["tee", new],
                  stdin = PIPE, stdout = DISCARD) as (fd_in, _, _):
    pipe = OS.fdopen(fd_in, 'w')
    try: yield pipe
    finally: pipe.close()
  if mode is not None: run_program(C.ROOTLY + ["chmod", mode, new])
  if uid is not None:
    run_program(C.ROOTLY + ["chown",
                            uid + (gid is not None and ":" + gid or ""),
                            new])
  elif gid is not None:
    run_program(C.ROOTLY + ["chgrp", gid, new])
  run_program(C.ROOTLY + ["mv", new, path])

def mountpoint_p(dir):
  """Return true if DIR is a mountpoint."""

  ## A mountpoint can be distinguished because it is a directory whose device
  ## number differs from its parent.
  try: st1 = OS.stat(dir)
  except OSError, err:
    if err.errno == E.ENOENT: return False
    else: raise
  if not ST.S_ISDIR(st1.st_mode): return False
  st0 = OS.stat(OS.path.join(dir, ".."))
  return st0.st_dev != st1.st_dev

def mkdir_p(dir, mode = 0777):
  """
  Make a directory DIR, and any parents, as necessary.

  Unlike `OS.makedirs', this doesn't fail if DIR already exists.
  """
  d = ""
  for p in dir.split("/"):
    d = OS.path.join(d, p)
    if d == "": continue
    try: OS.mkdir(d, mode)
    except OSError, err:
      if err.errno == E.EEXIST: pass
      else: raise

def umount(fs):
  """
  Unmount the filesystem FS.

  The FS may be the block device holding the filesystem, or (more usually)
  the mount point.
  """

  ## Sometimes random things can prevent unmounting.  Be persistent.
  for i in xrange(5):
    try: run_program(C.ROOTLY + ["umount", fs], stderr = DISCARD)
    except SubprocessFailure, err:
      if err.rc == 32: pass
      else: raise
    else: return
    T.sleep(0.2)
  run_program(C.ROOTLY + ["umount", fs], stderr = DISCARD)

@CTX.contextmanager
def lockfile(lock, exclp = True, waitp = True):
  """
  Acquire an exclusive lock on a named file LOCK while executing the body.

  If WAITP is true, wait until the lock is available; if false, then fail
  immediately if the lock can't be acquired.
  """
  fd = -1
  flag = 0
  if exclp: flag |= FC.LOCK_EX
  else: flag |= FC.LOCK_SH
  if not waitp: flag |= FC.LOCK_NB
  spew("acquiring %s lock on `%s'" %
       (exclp and "exclusive" or "shared", lock))
  try:
    while True:

      ## Open the file and take note of which file it is.
      fd = OS.open(lock, OS.O_RDWR | OS.O_CREAT, 0666)
      st0 = OS.fstat(fd)

      ## Acquire the lock, waiting if necessary.
      FC.lockf(fd, flag)

      ## Check that the lock file is still the same one.  It's permissible
      ## for the lock holder to release the lock by unlinking or renaming the
      ## lock file, in which case there might be a different lockfile there
      ## now which we need to acquire instead.
      ##
      ## It's tempting to `optimize' this code by opening a new file
      ## descriptor here so as to elide the additional call to fstat(2)
      ## above.  But this doesn't work: if we successfully acquire the lock,
      ## we then have two file descriptors open on the lock file, so we have
      ## to close one -- but, under the daft fcntl(2) rules, even closing
      ## `nfd' will release the lock immediately.
      try:
        st1 = OS.stat(lock)
      except OSError, err:
        if err.errno == E.ENOENT: pass
        else: raise
      if st0.st_dev == st1.st_dev and st0.st_ino == st1.st_ino: break
      OS.close(fd)

    ## We have the lock, so away we go.
    spew("lock `%s' acquired" % lock)
    yield None
    spew("lock `%s' released" % lock)

  finally:
    if fd != -1: OS.close(fd)

def block_device_p(dev):
  """Return true if DEV names a block device."""
  try: st = OS.stat(dev)
  except OSError, err:
    if err.errno == E.ENOENT: return False
    else: raise
  else: return ST.S_ISBLK(st.st_mode)

###--------------------------------------------------------------------------
### Running parallel jobs.

## Return codes from `check'
SLEEP = Tag('SLEEP')
READY = Tag('READY')
FAILED = Tag('FAILED')
DONE = Tag('DONE')

class BaseJob (object):
  """
  Base class for jobs.

  Subclasses must implement `run' and `_mkname', and probably ought to extend
  `prepare' and `check'.
  """

  ## A magic token to prevent sneaky uninterned jobs.
  _MAGIC = Tag('MAGIC')

  ## A map from job names to objects.
  _MAP = {}

  ## Number of tail lines of the log to print on failure.
  LOGLINES = 20

  def __init__(me, _token, *args, **kw):
    """
    Initialize a job.

    Jobs are interned!  Don't construct instances (of subclasses) directly:
    use the `ensure' class method.
    """
    assert _token is me._MAGIC
    super(BaseJob, me).__init__(*args, **kw)

    ## Dependencies on other jobs.
    me._deps = None
    me._waiting = set()

    ## Attributes maintained by the JobServer.
    me.done = False
    me.started = False
    me.win = None
    me._token = None
    me._known = False
    me._st = None
    me._logkid = -1
    me._logfile = None

  def prepare(me):
    """
    Establish any prerequisite jobs.

    Delaying this allows command-line settings to override those chosen by
    dependent jobs.
    """
    pass

  @classmethod
  def ensure(cls, *args, **kw):
    """
    Return the unique job with the given parameters.

    If a matching job already exists, then return it.  Otherwise, create the
    new job, register it in the table, and notify the scheduler about it.
    """
    me = cls(_token = cls._MAGIC, *args, **kw)
    try:
      job = cls._MAP[me.name]
    except KeyError:
      cls._MAP[me.name] = me
      SCHED.add(me)
      return me
    else:
      return job

  ## Naming.
  @property
  def name(me):
    """Return the job's name, as calculated by `_mkname'."""
    try: name = me._name
    except AttributeError: name = me._name = me._mkname()
    return name

  ## Subclass responsibilities.
  def _mkname(me):
    """
    Return the job's name.

    By default, this is an unhelpful string which is distinct for every job.
    Subclasses should normally override this method to return a name as an
    injective function of the job parameters.
    """
    return "%s.%x" % (me.__class__.__name__, id(me))

  def check(me):
    """
    Return whether the job is ready to run.

    Returns a pair STATE, REASON.  The REASON is a human-readable string
    explaining what's going on, or `None' if it's not worth explaining.  The
    STATE is one of the following.

      * `READY' -- the job can be run at any time.

      * `FAILED' -- the job can't be started.  Usually, this means that some
        prerequisite job failed, there was some error in the job's
        parameters, or the environment is unsuitable for the job to run.

      * `DONE' -- the job has nothing to do.  Usually, this means that the
        thing the job acts on is already up-to-date.  It's bad form to do
        even minor work in `check'.

      * `SLEEP' -- the job can't be run right now.  It has arranged to be
        retried if conditions change.  (Spurious wakeups are permitted and
        must be handled correctly.)

    The default behaviour checks the set of dependencies, as built by the
    `await' method, and returns `SLEEP' or `FAILED' as appropriate, or
    `READY' if all the prerequisite jobs have completed successfully.
    """
    for job in me._deps:
      if not job.done:
        job._waiting.add(me)
        return SLEEP, "waiting for job `%s'" % job.name
      elif not job.win and not OPT.ignerr:
        return FAILED, "dependent on failed job `%s'" % job.name
    return READY, None

  ## Subclass utilities.
  def await(me, job):
    """Make sure that JOB completes before allowing this job to start."""
    me._deps.add(job)

  def _logtail(me):
    """
    Dump the last `LOGLINES' lines of the logfile.

    This is called if the job fails and was being run quietly, to provide the
    user with some context for the failure.
    """

    ## Gather blocks from the end of the log until we have enough lines.
    with open(me._logfile, 'r') as f:
      nlines = 0
      bufs = []
      bufsz = 4096
      f.seek(0, 2); off = f.tell()
      spew("start: off = %d" % off)
      while nlines <= me.LOGLINES and off > 0:
        off = max(0, off - bufsz)
        f.seek(off, 0)
        spew("try at off = %d" % off)
        buf = f.read(bufsz)
        nlines += buf.count("\n")
        spew("now lines = %d" % nlines)
        bufs.append(buf)
    buf = ''.join(reversed(bufs))

    ## We probably overshot.  Skip the extra lines from the start.
    i = 0
    while nlines > me.LOGLINES: i = buf.index("\n", i) + 1; nlines -= 1

    ## If we ended up trimming the log, print an ellipsis.
    if off > 0 or i > 0: print "%-*s * [...]" % (TAGWD, me.name)

    ## Print the log tail.
    lines = buf[i:].split("\n")
    if lines and lines[-1] == '': lines.pop()
    for line in lines: print "%-*s %s" % (TAGWD, me.name, line)

class BaseJobToken (object):
  """
  A job token is the authorization for a job to be run.

  Subclasses must implement `recycle' to allow some other job to use the
  token.
  """
  pass

class TrivialJobToken (BaseJobToken):
  """
  A trivial reusable token, for when issuing jobs in parallel without limit.

  There only needs to be one of these.
  """
  def recycle(me):
    spew("no token needed; nothing to recycle")
TRIVIAL_TOKEN = TrivialJobToken()

class JobServerToken (BaseJobToken):
  """A job token storing a byte from the jobserver pipe."""
  def __init__(me, char, pipefd, *args, **kw):
    super(JobServerToken, me).__init__(*args, **kw)
    me._char = char
    me._fd = pipefd
  def recycle(me):
    spew("returning token to jobserver pipe")
    OS.write(me._fd, me._char)

class PrivateJobToken (BaseJobToken):
  """
  The private job token belonging to a scheduler.

  When running under a GNU Make jobserver, there is a token for each byte in
  the pipe, and an additional one which represents the slot we're actually
  running in.  This class represents that additional token.
  """
  def __init__(me, sched, *args, **kw):
    super(PrivateJobToken, me).__init__(*args, **kw)
    me._sched = sched
  def recycle(me):
    assert me._sched._privtoken is None
    spew("recycling private token")
    me._sched._privtoken = me

TAGWD = 29
LOGKEEP = 20

class JobScheduler (object):
  """
  The main machinery for running and ordering jobs.

  This handles all of the details of job scheduling.
  """

  def __init__(me, rfd = -1, wfd = -1, npar = 1):
    """
    Initialize a scheduler.

      * RFD and WFD are the read and write ends of the jobserver pipe, as
        determined from the `MAKEFLAGS' environment variable, or -1.

      * NPAR is the maximum number of jobs to run in parallel, or `True' if
        there is no maximum (i.e., we're in `forkbomb' mode).
    """

    ## Set the parallelism state.  The `_rfd' and `_wfd' are the read and
    ## write ends of the jobserver pipe, or -1 if there is no jobserver.
    ## `_par' is true if we're meant to run jobs in parallel.  The case _par
    ## and _rfd = -1 means unconstrained parallelism.
    ##
    ## The jobserver pipe contains a byte for each shared job slot.  A
    ## scheduler reads a byte from the pipe for each job it wants to run
    ## (nearly -- see `_privtoken' below), and puts the byte back when the
    ## job finishes.  The GNU Make jobserver protocol specification insists
    ## that we preserve the value of the byte in the pipe (though doesn't
    ## currently make any use of this flexibility), so we record it in a
    ## `JobToken' object's `_char' attribute.
    me._par = rfd != -1 or npar is True or npar != 1
    spew("par is %r" % me._par)
    if rfd == -1 and npar > 1:
      rfd, wfd = OS.pipe()
      OS.write(wfd, (npar - 1)*'+')
      OS.environ["MAKEFLAGS"] = \
        (" -j --jobserver-auth=%(rfd)d,%(wfd)d " +
         "--jobserver-fds=%(rfd)d,%(wfd)d") % dict(rfd = rfd, wfd = wfd)
    me._rfd = rfd; me._wfd = wfd

    ## The scheduler state.  A job starts in the `_check' list.  Each
    ## iteration of the scheduler loop will inspect the jobs here and see
    ## whether it's ready to run: if not, it gets put in the `_sleep' list,
    ## where it will languish until something moves it back; if it is ready,
    ## it gets moved to the `_ready' list to wait for a token from the
    ## jobserver.  At that point the job can be started, and it moves to the
    ## `_kidmap', which associates a process-id with each running job.
    ## Finally, jobs which have completed are simply forgotten.  The `_njobs'
    ## counter keeps track of how many jobs are outstanding, so that we can
    ## stop when there are none left.
    me._check = set()
    me._sleep = set()
    me._ready = set()
    me._kidmap = {}
    me._logkidmap = {}
    me._njobs = 0

    ## As well as the jobserver pipe, we implicitly have one extra job slot,
    ## which is the one we took when we were started by our parent.  The
    ## right to do processing in this slot is represnted by the `private
    ## token' here, distinguished from tokens from the jobserver pipe by
    ## having `None' as its `_char' value.
    me._privtoken = PrivateJobToken(me)

  def add(me, job):
    """Notice a new job and arrange for it to (try to) run."""
    if job._known: return
    spew("adding new job `%s'" % job.name)
    job._known = True
    me._check.add(job)
    me._njobs += 1

  def close_jobserver(me):
    """
    Close the jobserver file descriptors.

    This should be called within child processes to prevent them from messing
    with the jobserver.
    """
    if me._rfd != -1: OS.close(me._rfd); me._rfd = -1
    if me._wfd != -1: OS.close(me._wfd); me._wfd = -1
    try: del OS.environ["MAKEFLAGS"]
    except KeyError: pass

  def _killall(me):
    """Zap all jobs which aren't yet running."""
    for jobset in [me._sleep, me._check, me._ready]:
      while jobset:
        job = jobset.pop()
        job.done = True
        job.win = False
        me._njobs -= 1

  def _retire(me, job, win, outcome):
    """
    Declare that a job has stopped, and deal with the consequences.

    JOB is the completed job, which should not be on any of the job queues.
    WIN is true if the job succeeded, and false otherwise.  OUTCOME is a
    human-readable string explaining how the job came to its end, or `None'
    if no message should be reported.
    """

    global RC

    ## Return the job's token to the pool.
    if job._token is not None: job._token.recycle()
    job._token = None
    me._njobs -= 1

    ## Update and maybe report the job's status.
    job.done = True
    job.win = win
    if outcome is not None and not OPT.silent:
      if OPT.quiet and not job.win and job._logfile: job._logtail()
      if not job.win or not OPT.quiet:
        print "%-*s %c (%s)" % \
          (TAGWD, job.name, job.win and '|' or '*', outcome)

    ## If the job failed, and we care, arrange to exit nonzero.
    if not win and not OPT.ignerr: RC = 2

    ## If the job failed, and we're supposed to give up after the first
    ## error, then zap all of the waiting jobs.
    if not job.win and not OPT.keepon and not OPT.ignerr: me._killall()

    ## If this job has dependents then wake them up and see whether they're
    ## ready to run.
    for j in job._waiting:
      try: me._sleep.remove(j)
      except KeyError: pass
      else:
        spew("waking dependent job `%s'" % j.name)
        me._check.add(j)

  def _reap(me, kid, st):
    """
    Deal with the child with process-id KID having exited with status ST.
    """

    ## Figure out what kind of child this is.  Note that it has finished.
    try: job = me._kidmap[kid]
    except KeyError:
      try: job = me._logkidmap[kid]
      except KeyError:
        spew("unknown child %d exits with status 0x%04x" % (kid, st))
        return
      else:
        ## It's a logging child.
        del me._logkidmap[kid]
        job._logkid = DONE
        spew("logging process for job `%s' exits with status 0x%04x" %
             (job.name, st))
    else:
      job._st = st
      del me._kidmap[kid]
      spew("main process for job `%s' exits with status 0x%04x" %
           (job.name, st))

    ## If either of the job's associated processes is still running then we
    ## should stop now and give the other one a chance.
    if job._st is None or job._logkid is not DONE:
      spew("deferring retirement for job `%s'" % job.name)
      return
    spew("completing deferred retirement for job `%s'" % job.name)

    ## Update and (maybe) report the job status.
    if job._st == 0: win = True; outcome = None
    else: win = False; outcome = wait_outcome(job._st)

    ## Retire the job.
    me._retire(job, win, outcome)

  def _reapkids(me):
    """Reap all finished child processes."""
    while True:
      try: kid, st = OS.waitpid(-1, OS.WNOHANG)
      except OSError, err:
        if err.errno == E.ECHILD: break
        else: raise
      if kid == 0: break
      me._reap(kid, st)

  def run_job(me, job):
    """Start running the JOB."""

    job.started = True
    if OPT.dryrun: return None, None

    ## Make pipes to collect the job's output and error reports.
    r_out, w_out = OS.pipe()
    r_err, w_err = OS.pipe()

    ## Find a log file to write.  Avoid races over the log names; but this
    ## means that the log descriptor needs to be handled somewhat carefully.
    logdir = OS.path.join(C.STATE, "log"); mkdir_p(logdir)
    logseq = 1
    while True:
      logfile = OS.path.join(logdir, "%s-%s#%d" % (job.name, TODAY, logseq))
      try:
        logfd = OS.open(logfile, OS.O_WRONLY | OS.O_CREAT | OS.O_EXCL, 0666)
      except OSError, err:
        if err.errno == E.EEXIST: logseq += 1; continue
        else: raise
      else:
        break
    job._logfile = logfile

    ## Make sure there's no pending output, or we might get two copies.  (I
    ## don't know how to flush all output streams in Python, but this is good
    ## enough for our purposes.)
    SYS.stdout.flush()

    ## Set up the logging child first.  If we can't, take down the whole job.
    try: job._logkid = OS.fork()
    except OSError, err: OS.close(logfd); return None, err
    if not job._logkid:
      ## The main logging loop.

      ## Close the jobserver descriptors, and the write ends of the pipes.
      me.close_jobserver()
      OS.close(w_out); OS.close(w_err)

      ## Capture the job's stdout and stderr and wait for everything to
      ## happen.
      def log_lines(fd, marker):
        def fn(line):
          if not OPT.quiet:
            OS.write(1, "%-*s %s %s\n" % (TAGWD, job.name, marker, line))
          OS.write(logfd, "%s %s\n" % (marker, line))
        return ReadLinesSelector(fd, fn)
      select_loop([log_lines(r_out, "|"), log_lines(r_err, "*")])

      ## We're done.  (Closing the descriptors here would be like polishing
      ## the floors before the building is demolished.)
      OS._exit(0)

    ## Back in the main process: record the logging child.  At this point we
    ## no longer need the logfile descriptor.
    me._logkidmap[job._logkid] = job
    OS.close(logfd)

    ## Start the main job process.
    try: kid = OS.fork()
    except OSError, err: return None, err
    if not kid:
      ## The main job.

      ## Close the read ends of the pipes, and move the write ends to the
      ## right places.  (This will go wrong if we were started without enough
      ## descriptors.  Fingers crossed.)
      OS.dup2(w_out, 1); OS.dup2(w_err, 2)
      OS.close(r_out); OS.close(w_out)
      OS.close(r_err); OS.close(w_err)
      spew("running job `%s' as pid %d" % (job.name, OS.getpid()))

      ## Run the job, catching nonlocal flow.
      try:
        job.run()
      except ExpectedError, err:
        moan(str(err))
        OS._exit(2)
      except Exception, err:
        TB.print_exc(SYS.stderr)
        OS._exit(3)
      except BaseException, err:
        moan("caught unexpected exception: %r" % err)
        OS._exit(112)
      else:
        spew("job `%s' ran to completion" % job.name)

        ## Clean up old logs.
        match = []
        pat = RX.compile(r"^%s-(\d{4})-(\d{2})-(\d{2})\#(\d+)$" %
                         RX.escape(job.name))
        for f in OS.listdir(logdir):
          m = pat.match(f)
          if m: match.append((f, int(m.group(1)), int(m.group(2)),
                              int(m.group(3)), int(m.group(4))))
        match.sort(key = lambda (_, y, m, d, q): (y, m, d, q))
        if len(match) > LOGKEEP:
          for (f, _, _, _, _) in match[:-LOGKEEP]:
            try: OS.unlink(OS.path.join(logdir, f))
            except OSError, err:
              if err.errno == E.ENOENT: pass
              else: raise

        ## All done.
        OS._exit(0)

    ## Back in the main process: close both the pipes and return the child
    ## process.
    OS.close(r_out); OS.close(w_out)
    OS.close(r_err); OS.close(w_err)
    if OPT.quiet: print "%-*s | (started)" % (TAGWD, job.name)
    return kid, None

  def run(me):
    """Run the scheduler."""

    spew("JobScheduler starts")

    while True:
      ## The main scheduler loop.  We go through three main phases:
      ##
      ##   * Inspect the jobs in the `check' list to see whether they can
      ##     run.  After this, the `check' list will be empty.
      ##
      ##   * If there are running jobs, check to see whether any of them have
      ##     stopped, and deal with the results.  Also, if there are jobs
      ##     ready to start and a job token has become available, then
      ##     retrieve the token.  (Doing these at the same time is the tricky
      ##     part.)
      ##
      ##   * If there is a job ready to run, and we retrieved a token, then
      ##     start running the job.

      ## Check the pending jobs to see if they can make progress: run each
      ## job's `check' method and move it to the appropriate queue.  (It's OK
      ## if `check' methods add more jobs to the list, as long as things
      ## settle down eventually.)
      while True:
        try: job = me._check.pop()
        except KeyError: break
        if job._deps is None:
          job._deps = set()
          job.prepare()
        state, reason = job.check()
        tail = reason is not None and ": %s" % reason or ""
        if state == READY:
          spew("job `%s' ready to run%s" % (job.name, tail))
          me._ready.add(job)
        elif state is FAILED:
          spew("job `%s' refused to run%s" % (job.name, tail))
          me._retire(job, False, "refused to run%s" % tail)
        elif state is DONE:
          spew("job `%s' has nothing to do%s" % (job.name, tail))
          me._retire(job, True, reason)
        elif state is SLEEP:
          spew("job `%s' can't run yet%s" % (job.name, tail))
          me._sleep.add(job)
        else:
          raise ValueError("unexpected job check from `%s': %r, %r" %
                           (job.name, state, reason))

      ## If there are no jobs left, then we're done.
      if not me._njobs:
        spew("all jobs completed")
        break

      ## Make sure we can make progress.  There are no jobs on the check list
      ## any more, because we just cleared it.  We assume that jobs which are
      ## ready to run will eventually receive a token.  So we only end up in
      ## trouble if there are jobs asleep, but none running or ready to run.
      ##spew("#jobs = %d" % me._njobs)
      ##spew("sleeping: %s" % ", ".join([j.name for j in me._sleep]))
      ##spew("ready: %s" % ", ".join([j.name for j in me._ready]))
      ##spew("running: %s" % ", ".join([j.name for j in me._kidmap.itervalues()]))
      assert not me._sleep or me._kidmap or me._logkidmap or me._ready

      ## Wait for something to happen.
      if not me._ready or (not me._par and me._privtoken is None):
        ## If we have no jobs ready to run, then we must wait for an existing
        ## child to exit.  Hopefully, a sleeping job will be able to make
        ## progress after this.
        ##
        ## Alternatively, if we're not supposed to be running jobs in
        ## parallel and we don't have the private token, then we have no
        ## choice but to wait for the running job to complete.
        ##
        ## There's no check here for `ECHILD'.  We really shouldn't be here
        ## if there are no children to wait for.  (The check list must be
        ## empty because we just drained it.  If the ready list is empty,
        ## then all of the jobs must be running or sleeping; but the
        ## assertion above means that either there are no jobs at all, in
        ## which case we should have stopped, or at least one is running, in
        ## which case it's safe to wait for it.  The other case is that we're
        ## running jobs sequentially, and one is currently running, so
        ## there's nothing for it but to wait for it -- and hope that it will
        ## wake up one of the sleeping jobs.  The remaining possibility is
        ## that we've miscounted somewhere, which will cause a crash.)
        if not me._ready:
          spew("no new jobs ready: waiting for outstanding jobs to complete")
        else:
          spew("job running without parallelism: waiting for it to finish")
        kid, st = OS.waitpid(-1, 0)
        me._reap(kid, st)
        me._reapkids()
        continue

      ## We have jobs ready to run, so try to acquire a token.
      if me._rfd == -1 and me._par:
        ## We're running with unlimited parallelism, so we don't need a token
        ## to run a job.
        spew("running new job without token")
        token = TRIVIAL_TOKEN
      elif me._privtoken:
        ## Our private token is available, so we can use that to start
        ## a new job.
        spew("private token available: assigning to new job")
        token = me._privtoken
        me._privtoken = None
      else:
        ## We have to read from the jobserver pipe.  Unfortunately, we're not
        ## allowed to set the pipe nonblocking, because make is also using it
        ## and will get into a serious mess.  And we must deal with `SIGCHLD'
        ## arriving at any moment.  We use the same approach as GNU Make.  We
        ## start by making a copy of the jobserver descriptor: it's this
        ## descriptor we actually try to read from.  We set a signal handler
        ## to close this descriptor if a child exits.  And we try one last
        ## time to reap any children which have exited just before we try
        ## reading the jobserver pipe.  This way we're covered:
        ##
        ##   * If a child exits during the main loop, before we establish the
        ##     descriptor copy then we'll notice when we try reaping
        ##     children.
        ##
        ##   * If a child exits between the last-chance reap and the read,
        ##     the signal handler will close the descriptor and the `read'
        ##     call will fail with `EBADF'.
        ##
        ##   * If a child exits while we're inside the `read' system call,
        ##     then the syscall will fail with `EINTR'.
        ##
        ## The only problem is that we can't do this from Python, because
        ## Python signal handlers are delayed.  This is what the `jobclient'
        ## module is for.
        ##
        ## The `jobclient' function is called as
        ##
        ##      jobclient(FD)
        ##
        ## It returns a tuple of three values: TOKEN, PID, STATUS.  If TOKEN
        ## is not `None', then reading the pipe succeeded; if TOKEN is empty,
        ## then the pipe returned EOF, so we should abort; otherwise, TOKEN
        ## is a singleton string holding the token character.  If PID is not
        ## `None', then PID is the process id of a child which exited, and
        ## STATUS is its exit status.
        spew("waiting for token from jobserver")
        tokch, kid, st = JC.jobclient(me._rfd)

        if kid is not None:
          me._reap(kid, st)
          me._reapkids()
        if tokch is None:
          spew("no token; trying again")
          continue
        elif token == '':
          error("jobserver pipe closed; giving up")
          me._killall()
          continue
        spew("received token from jobserver")
        token = JobServerToken(tokch, me._wfd)

      ## We have a token, so we should start up the job.
      job = me._ready.pop()
      job._token = token
      spew("start new job `%s'" % job.name)
      kid, err = me.run_job(job)
      if err is not None:
        me._retire(job, False, "failed to fork: %s" % err)
        continue
      if kid is None: me._retire(job, True, "dry run")
      else: me._kidmap[kid] = job

    ## We ran out of work to do.
    spew("JobScheduler done")

###--------------------------------------------------------------------------
### Configuration.

R_CONFIG = RX.compile(r"^([a-zA-Z0-9_]+)='(.*)'$")

class Config (object):

  def _conv_str(s): return s
  def _conv_list(s): return s.split()
  def _conv_set(s): return set(s.split())

  _CONVERT = {
    "ROOTLY": _conv_list,
    "DISTS": _conv_set,
    "MYARCH": _conv_set,
    "NATIVE_ARCHS": _conv_set,
    "FOREIGN_ARCHS": _conv_set,
    "FOREIGN_GNUARCHS": _conv_list,
    "ALL_ARCHS": _conv_set,
    "NATIVE_CHROOTS": _conv_set,
    "FOREIGN_CHROOTS": _conv_set,
    "ALL_CHROOTS": _conv_set,
    "BASE_PACKAGES": _conv_list,
    "EXTRA_PACKAGES": _conv_list,
    "CROSS_PACKAGES": _conv_list,
    "CROSS_PATHS": _conv_list,
    "APTCONF": _conv_list,
    "LOCALPKGS": _conv_list,
    "SCHROOT_COPYFILES": _conv_list,
    "SCHROOT_NSSDATABASES": _conv_list
  }

  _CONV_MAP = {
    "*_APTCONFSRC": ("APTCONFSRC", _conv_str),
    "*_DEPS": ("PKGDEPS", _conv_list),
    "*_QEMUHOST": ("QEMUHOST", _conv_str),
    "*_QEMUARCH": ("QEMUARCH", _conv_str),
    "*_ALIASES": ("DISTALIAS", _conv_str)
  }

  _conv_str = staticmethod(_conv_str)
  _conv_list = staticmethod(_conv_list)
  _conv_set = staticmethod(_conv_set)

  def __init__(me):
    raw = r"""
    """; raw = open('state/config.sh').read(); _ignore = """ @@@config@@@
    """
    me._conf = {}
    for line in raw.split("\n"):
      line = line.strip()
      if not line or line.startswith('#'): continue
      m = R_CONFIG.match(line)
      if not m: raise ExpectedError("bad config line `%s'" % line)
      k, v = m.group(1), m.group(2).replace("'\\''", "'")
      d = me._conf
      try: conv = me._CONVERT[k]
      except KeyError:
        i = 0
        while True:
          try: i = k.index("_", i + 1)
          except ValueError: conv = me._conv_str; break
          try: map, conv = me._CONV_MAP["*" + k[i:]]
          except KeyError: pass
          else:
            d = me._conf.setdefault(map, dict())
            k = k[:i]
            if k.startswith("_"): k = k[1:]
            break
      d[k] = conv(v)

  def __getattr__(me, attr):
    try: return me._conf[attr]
    except KeyError, err: raise AttributeError(err.args[0])

with toplevel_handler(): C = Config()

###--------------------------------------------------------------------------
### Chroot maintenance utilities.

CREATE = Tag("CREATE")
FORCE = Tag("FORCE")

def check_fresh(fresh, update):
  """
  Compare a refresh mode FRESH against an UPDATE time.

  Return a (STATUS, REASON) pair, suitable for returning from a job `check'
  method.

  The FRESH argument may be one of the following:

    * `CREATE' is satisfied if the thing exists at all: it returns `READY' if
      the thing doesn't yet exist (UPDATE is `None'), or `DONE' otherwise.

    * `FORCE' is never satisfied: it always returns `READY'.

    * an integer N is satisfied if UPDATE time is at most N seconds earlier
      than the present: if returns `READY' if the UPDATE is too old, or
      `DONE' otherwise.
  """
  if update is None: return READY, "must create"
  elif fresh is FORCE: return READY, "update forced"
  elif fresh is CREATE: return DONE, "already created"
  elif NOW - unzulu(update) > fresh: return READY, "too stale: updating"
  else: return DONE, "already sufficiently up-to-date"

def lockfile_path(file):
  """
  Return the full path for a lockfile named FILE.

  Create the lock directory if necessary.
  """
  lockdir = OS.path.join(C.STATE, "lock"); mkdir_p(lockdir)
  return OS.path.join(lockdir, file)

def chroot_src_lockfile(dist, arch):
  """
  Return the lockfile for the source-chroot for DIST on ARCH.

  It is not allowed to acquire a source-chroot lock while holding any other
  locks.
  """
  return lockfile_path("source.%s-%s" % (dist, arch))

def chroot_src_lv(dist, arch):
  """
  Return the logical volume name for the source-chroot for DIST on ARCH.
  """
  return "%s%s-%s" % (C.LVPREFIX, dist, arch)

def chroot_src_blkdev(dist, arch):
  """
  Return the block-device name for the source-chroot for DIST on ARCH.
  """
  return OS.path.join("/dev", C.VG, chroot_src_lv(dist, arch))

def chroot_src_mntpt(dist, arch):
  """
  Return mountpoint path for setting up the source-chroot for DIST on ARCH.

  Note that this is not the mountpoint that schroot(1) uses.
  """
  mnt = OS.path.join(C.STATE, "mnt", "%s-%s" % (dist, arch))
  mkdir_p(mnt)
  return mnt

def chroot_session_mntpt(session):
  """Return the mountpoint for an schroot session."""
  return OS.path.join("/schroot", session)

def crosstools_lockfile(dist, arch):
  """
  Return the lockfile for the cross-build tools for DIST, hosted by ARCH.

  When locking multiple cross-build tools, you must acquire the locks in
  lexicographically ascending order.
  """
  return lockfile_path("cross-tools.%s-%s" % (dist, arch))

def switch_prefix(string, map):
  """
  Replace the prefix of a STRING, according to the given MAP.

  MAP is a sequence of (OLD, NEW) pairs.  For each such pair in turn, test
  whether STRING starts with OLD: if so, return STRING, but with the prefix
  OLD replaced by NEW.  If no OLD prefix matches, then raise a `ValueError'.
  """
  for old, new in map:
    if string.startswith(old): return new + string[len(old):]
  raise ValueError("expected `%s' to start with one of %s" %
                   ", ".join(["`%s'" % old for old, new in map]))

def host_to_chroot(path):
  """
  Convert a host path under `C.LOCAL' to the corresponding chroot path under
  `/usr/local.schroot'.
  """
  return switch_prefix(path, [(C.LOCAL + "/", "/usr/local.schroot/")])

def chroot_to_host(path):
  """
  Convert a chroot path under `/usr/local.schroot' to the corresponding
  host path under `C.LOCAL'.
  """
  return switch_prefix(path, [("/usr/local.schroot/", C.LOCAL + "/")])

def split_dist_arch(spec):
  """Split a SPEC of the form `DIST-ARCH' into the pair (DIST, ARCH)."""
  dash = spec.index("-")
  return spec[:dash], spec[dash + 1:]

def elf_binary_p(arch, path):
  """Return whether PATH is an ELF binary for ARCH."""
  if not OS.path.isfile(path): return False
  with open(path, 'rb') as f: magic = f.read(20)
  if magic[0:4] != "\x7fELF": return False
  if magic[8:16] != 8*"\0": return False
  if arch == "i386":
    if magic[4:7] != "\x01\x01\x01": return False
    if magic[18:20] != "\x03\x00": return False
  elif arch == "amd64":
    if magic[4:7] != "\x02\x01\x01": return False
    if magic[18:20] != "\x3e\x00": return False
  else:
    raise ValueError("unsupported donor architecture `%s'" % arch)
  return True

def progress(msg):
  """
  Print a progress message MSG.

  This is intended to be called within a job's `run' method, so it doesn't
  check `OPT.quiet' or `OPT.silent'.
  """
  OS.write(1, ";; %s\n" % msg)

class NoSuchChroot (Exception):
  """
  Exception indicating that a chroot does not exist.

  Specifically, it means that it doesn't even have a logical volume.
  """
  def __init__(me, dist, arch):
    me.dist = dist
    me.arch = arch
  def __str__(me):
    return "chroot for `%s' on `%s' not found" % (me.dist, me.arch)

@CTX.contextmanager
def mount_chroot_src(dist, arch):
  """
  Context manager for mounting the source-chroot for DIST on ARCH.

  The context manager automatically unmounts the filesystem again when the
  body exits.  You must hold the appropriate source-chroot lock before
  calling this routine.
  """
  dev = chroot_src_blkdev(dist, arch)
  if not block_device_p(dev): raise NoSuchChroot(dist, arch)
  mnt = chroot_src_mntpt(dist, arch)
  try:
    run_program(C.ROOTLY + ["mount", dev, mnt])
    yield mnt
  finally:
    umount(mnt)

@CTX.contextmanager
def chroot_session(dist, arch, sourcep = False):
  """
  Context manager for running an schroot(1) session.

  Returns the (ugly, automatically generated) session name to the context
  body.  By default, a snapshot session is started: set SOURCEP true to start
  a source-chroot session.  You must hold the appropriate source-chroot lock
  before starting a source-chroot session.

  The context manager automatically closes the session again when the body
  exits.
  """
  chroot = chroot_src_lv(dist, arch)
  if sourcep: chroot = "source:" + chroot
  session = run_program(["schroot", "-uroot", "-b", "-c", chroot],
                        stdout = RETURN).rstrip("\n")
  try:
    root = OS.path.join(chroot_session_mntpt(session), "fs")
    yield session, root
  finally:
    run_program(["schroot", "-e", "-c", session])

def run_root(command, **kw):
  """Run a COMMAND as root.  Arguments are as for `run_program'."""
  return run_program(C.ROOTLY + command, **kw)

def run_schroot_session(session, command, rootp = False, **kw):
  """
  Run a COMMAND within an schroot(1) session.

  Arguments are as for `run_program'.
  """
  if rootp:
    return run_program(["schroot", "-uroot", "-r",
                        "-c", session, "--"] + command, **kw)
  else:
    return run_program(["schroot", "-r",
                        "-c", session, "--"] + command, **kw)

def run_schroot_source(dist, arch, command, **kw):
  """
  Run a COMMAND through schroot(1), in the source-chroot for DIST on ARCH.

  Arguments are as for `run_program'.  You must hold the appropriate source-
  chroot lock before calling this routine.
  """
  return run_program(["schroot", "-uroot",
                      "-c", "source:%s" % chroot_src_lv(dist, arch),
                      "--"] + command, **kw)

###--------------------------------------------------------------------------
### Metadata files.

class MetadataClass (type):
  """
  Metaclass for metadata classes.

  Notice a `VARS' attribute in the class dictionary, and augment it with a
  `_VARSET' attribute, constructed as a set containing the same items.  (We
  need them both: the set satisfies fast lookups, while the original sequence
  remembers the ordering.)
  """
  def __new__(me, name, supers, dict):
    try: vars = dict['VARS']
    except KeyError: pass
    else: dict['_VARSET'] = set(vars)
    return super(MetadataClass, me).__new__(me, name, supers, dict)

class BaseMetadata (object):
  """
  Base class for metadate objects.

  Metadata bundles are simple collections of key/value pairs.  Keys should
  usually be Python identifiers because they're used to name attributes.
  Values are strings, but shouldn't have leading or trailing whitespace, and
  can't contain newlines.

  Metadata bundles are written to files.  The format is simple enough: empty
  lines and lines starting with `#' are ignored; otherwise, the line must
  have the form

        KEY = VALUE

  where KEY does not contain `='; spaces around the `=' are optional, and
  spaces around the KEY and VALUE are stripped.  The order of keys is
  unimportant; keys are always written in a standard order on output.
  """
  __metaclass__ = MetadataClass

  def __init__(me, **kw):
    """Initialize a metadata bundle from keyword arguments."""
    for k, v in kw.iteritems():
      setattr(me, k, v)
    for v in me.VARS:
      try: getattr(me, v)
      except AttributeError: setattr(me, v, None)

  def __setattr__(me, attr, value):
    """
    Try to set an attribute.

    Only attribute names listed in the `VARS' class attribute are permitted.
    """
    if attr not in me._VARSET: raise AttributeError, attr
    super(BaseMetadata, me).__setattr__(attr, value)

  @classmethod
  def read(cls, path):
    """Return a new metadata bundle read from a named PATH."""
    map = {}
    with open(path) as f:
      for line in f:
        line = line.strip()
        if line == "" or line.startswith("#"): continue
        k, v = line.split("=", 1)
        map[k.strip()] = v.strip()
    return cls(**map)

  def _write(me, file):
    """
    Write the metadata bundle to the FILE (a file-like object).

    This is intended for use by subclasses which want to override the default
    I/O behaviour of the main `write' method.
    """
    file.write("### -*-conf-*-\n")
    for k in me.VARS:
      try: v = getattr(me, k)
      except AttributeError: pass
      else:
        if v is not None: file.write("%s = %s\n" % (k, v))

  def write(me, path):
    """
    Write the metadata bundle to a given PATH.

    The file is replaced atomically.
    """
    with safewrite(path) as f: me._write(f)

  def __repr__(me):
    return "#<%s: %s>" % (me.__class__.__name__,
                          ", ".join("%s=%r" % (k, getattr(me, k, None))
                                    for k in me.VARS))

class ChrootMetadata (BaseMetadata):
  VARS = ['dist', 'arch', 'update']

  @classmethod
  def read(cls, dist, arch):
    try:
      with lockfile(chroot_src_lockfile(dist, arch), exclp = False):
        with mount_chroot_src(dist, arch) as mnt:
          return super(ChrootMetadata, cls).read(OS.path.join(mnt, "META"))
    except IOError, err:
      if err.errno == E.ENOENT: pass
      else: raise
    except NoSuchChroot: pass
    return cls(dist = dist, arch = arch)

  def write(me):
    with mount_chroot_src(me.dist, me.arch) as mnt:
      with safewrite_root(OS.path.join(mnt, "META")) as f:
        me._write(f)

class CrossToolsMetadata (BaseMetadata):
  VARS = ['dist', 'arch', 'update']

  @classmethod
  def read(cls, dist, arch):
    try:
      return super(CrossToolsMetadata, cls)\
        .read(OS.path.join(C.LOCAL, "cross", "%s-%s" % (dist, arch), "META"))
    except IOError, err:
      if err.errno == E.ENOENT: pass
      else: raise
    return cls(dist = dist, arch = arch)

  def write(me, dir = None):
    if dir is None:
      dir = OS.path.join(C.LOCAL, "cross", "%s-%s" % (me.dist, me.arch))
    with safewrite_root(OS.path.join(dir, "META")) as f:
      me._write(f)

###--------------------------------------------------------------------------
### Constructing a chroot.

R_DIVERT = RX.compile(r"^diversion of (.*) to .* by install-cross-tools$")

class ChrootJob (BaseJob):
  """
  Create or update a chroot.
  """

  SPECS = C.ALL_CHROOTS

  def __init__(me, spec, fresh = CREATE, *args, **kw):
    super(ChrootJob, me).__init__(*args, **kw)
    me._dist, me._arch = split_dist_arch(spec)
    me._fresh = fresh
    me._meta = ChrootMetadata.read(me._dist, me._arch)
    me._tools_chroot = me._qemu_chroot = None

  def _mkname(me): return "chroot.%s-%s" % (me._dist, me._arch)

  def prepare(me):
    if me._arch in C.FOREIGN_ARCHS:
      me._tools_chroot = CrossToolsJob.ensure\
        ("%s-%s" % (me._dist, C.TOOLSARCH), FRESH)
      me._qemu_chroot = CrossToolsJob.ensure\
        ("%s-%s" % (me._dist, C.QEMUHOST[me._arch]), FRESH)
      me.await(me._tools_chroot)
      me.await(me._qemu_chroot)

  def check(me):
    status, reason = super(ChrootJob, me).check()
    if status is not READY: return status, reason
    if (me._tools_chroot is not None and me._tools_chroot.started) or \
       (me._qemu_chroot is not None and me._qemu_chroot.started):
      return READY, "prerequisites run"
    return check_fresh(me._fresh, me._meta.update)

  def _install_cross_tools(me):
    """
    Install or refresh cross-tools in the source-chroot.

    This function version assumes that the source-chroot lock is already
    held.

    Note that there isn't a job class corresponding to this function.  It's
    done automatically as part of source-chroot setup and update for foreign
    architectures.
    """
    with Cleanup() as clean:

      dist, arch = me._dist, me._arch

      mymulti = run_program(["dpkg-architecture", "-a", C.TOOLSARCH,
                             "-qDEB_HOST_MULTIARCH"],
                            stdout = RETURN).rstrip("\n")
      gnuarch = run_program(["dpkg-architecture", "-A", arch,
                             "-qDEB_TARGET_GNU_TYPE"],
                            stdout = RETURN).rstrip("\n")

      crossdir = OS.path.join(C.LOCAL, "cross",
                              "%s-%s" % (dist, C.TOOLSARCH))

      qarch, qhost = C.QEMUARCH[arch], C.QEMUHOST[arch]
      qemudir = OS.path.join(C.LOCAL, "cross",
                             "%s-%s" % (dist, qhost), "QEMU")

      ## Acquire lockfiles in a canonical order to prevent deadlocks.
      donors = [C.TOOLSARCH]
      if qarch != C.TOOLSARCH: donors.append(qarch)
      donors.sort()
      for a in donors:
        clean.enter(lockfile(crosstools_lockfile(dist, a), exclp = False))

      ## Open a session.
      session, root = clean.enter(chroot_session(dist, arch, sourcep = True))

      ## Search the cross-tools tree for tools, to decide what to do with
      ## each file.  Make lists:
      ##
      ##   * `want_div' is simply a set of all files in the chroot which need
      ##     dpkg diversions to prevent foreign versions of the tools from
      ##     clobbering our native versions.
      ##
      ##   * `want_link' is a dictionary mapping paths which need symbolic
      ##     links into the cross-tools trees to their link destinations.
      progress("scan cross-tools tree")
      want_div = set()
      want_link = dict()
      cross_prefix = crossdir + "/"
      qemu_prefix = qemudir + "/"
      toolchain_prefix = OS.path.join(crossdir, "TOOLCHAIN", gnuarch) + "/"
      def examine(path):
        dest = switch_prefix(path, [(qemu_prefix, "/usr/bin/"),
                                    (toolchain_prefix, "/usr/bin/"),
                                    (cross_prefix, "/")])
        if OS.path.islink(path): src = OS.readlink(path)
        else: src = host_to_chroot(path)
        want_link[dest] = src
        if not OS.path.isdir(path): want_div.add(dest)
      examine(OS.path.join(qemudir, "qemu-%s-static" % qarch))
      examine(OS.path.join(crossdir, "lib", mymulti))
      examine(OS.path.join(crossdir, "usr/lib", mymulti))
      examine(OS.path.join(crossdir, "usr/lib/gcc-cross"))
      def visit(_, dir, files):
        ff = []
        for f in files:
          if f == "META" or f == "QEMU" or f == "TOOLCHAIN" or \
             (dir.endswith("/lib") and (f == mymulti or f == "gcc-cross")):
            continue
          ff.append(f)
          path = OS.path.join(dir, f)
          if not OS.path.isdir(path): examine(path)
        files[:] = ff
      OS.path.walk(crossdir, visit, None)
      OS.path.walk(OS.path.join(crossdir, "TOOLCHAIN", gnuarch),
                   visit, None)

      ## Build the set `have_div' of paths which already have diversions.
      progress("scan chroot")
      have_div = set()
      with subprocess(["schroot", "-uroot", "-r", "-c", session, "--",
                       "dpkg-divert", "--list"],
                      stdout = PIPE) as (_, fd_out, _):
        try:
          f = OS.fdopen(fd_out)
          for line in f:
            m = R_DIVERT.match(line.rstrip("\n"))
            if m: have_div.add(m.group(1))
        finally:
          f.close()

      ## Build a dictionary `have_link' of symbolic links into the cross-
      ## tools trees.  Also, be sure to collect all of the relative symbolic
      ## links which are in the cross-tools tree.
      have_link = dict()
      with subprocess(["schroot", "-uroot", "-r", "-c", session, "--",
                       "sh", "-e", "-c", """
        find / -xdev -lname "/usr/local.schroot/cross/*" -printf "%p %l\n"
      """], stdout = PIPE) as (_, fd_out, _):
        try:
          f = OS.fdopen(fd_out)
          for line in f:
            dest, src = line.split()
            have_link[dest] = src
        finally:
          f.close()
      for path in want_link.iterkeys():
        real = root + path
        if not OS.path.islink(real): continue
        have_link[path] = OS.readlink(real)

      ## Add diversions for the paths which need one, but don't have one.
      ## There's a hack here because the `--no-rename' option was required in
      ## the same version in which it was introduced, so there's no single
      ## incantation that will work across the boundary.
      progress("add missing diversions")
      with subprocess(["schroot", "-uroot", "-r", "-c", session, "--",
                       "sh", "-e", "-c", """
        a="%(arch)s"

        if dpkg-divert >/dev/null 2>&1 --no-rename --help
        then no_rename=--no-rename
        else no_rename=
        fi

        while read path; do
          dpkg-divert --package "install-cross-tools" $no_rename \
            --divert "$path.$a" --add "$path"
        done
      """ % dict(arch = arch)], stdin = PIPE) as (fd_in, _, _):
        try:
          f = OS.fdopen(fd_in, 'w')
          for path in want_div:
            if path not in have_div: f.write(path + "\n")
        finally:
          f.close()

      ## Go through each diverted tool, and, if it hasn't been moved aside,
      ## then /link/ it across now.  If we rename it, then the chroot will
      ## stop working -- which is why we didn't allow `dpkg-divert' to do the
      ## rename.  We can tell a tool that hasn't been moved, because it's a
      ## symlink into one of the cross trees.
      progress("preserve existing foreign files")
      chroot_cross_prefix = host_to_chroot(crossdir) + "/"
      chroot_qemu_prefix = host_to_chroot(qemudir) + "/"
      for path in want_div:
        real = root + path; div = real + "." + arch; cross = crossdir + path
        if OS.path.exists(div): continue
        if not OS.path.exists(real): continue
        if OS.path.islink(real):
          realdest = OS.readlink(real)
          if realdest.startswith(chroot_cross_prefix) or \
             realdest.startswith(chroot_qemu_prefix):
            continue
          if OS.path.islink(cross) and realdest == OS.readlink(cross):
            continue
        progress("preserve existing foreign file `%s'" % path)
        run_root(["ln", real, div])

      ## Update all of the symbolic links which are currently wrong: add
      ## links which are missing, delete ones which are obsolete, and update
      ## ones which have the wrong target.
      progress("update symlinks")
      for path, src in want_link.iteritems():
        real = root + path
        try: old_src = have_link[path]
        except KeyError: pass
        else:
          if src == old_src: continue
        new = real + ".new"
        progress("link `%s' -> `%s'" % (path, src))
        dir = OS.path.dirname(real)
        if not OS.path.isdir(dir): run_root(["mkdir", "-p", dir])
        if OS.path.exists(new): run_root(["rm", "-f", new])
        run_root(["ln", "-s", src, new])
        run_root(["mv", new, real])
      for path in have_link.iterkeys():
        if path in want_link: continue
        progress("remove obsolete link `%s' -> `%s'" % path)
        real = root + path
        run_root(["rm", "-f", real])

      ## Remove diversions from paths which don't need them any more.  Here
      ## it's safe to rename, because either the tool isn't there, in which
      ## case it obviously wasn't important, or it is, and `dpkg-divert' will
      ## atomically replace our link with the foreign version.
      progress("remove obsolete diversions")
      with subprocess(["schroot", "-uroot", "-r", "-c", session, "--",
                       "sh", "-e", "-c", """
        a="%(arch)s"

        while read path; do
          dpkg-divert --package "install-cross-tools" --rename \
            --divert "$path.$a" --remove "$path"
        done
      """ % dict(arch = arch)], stdin = PIPE) as (fd_in, _, _):
        try:
          f = OS.fdopen(fd_in, 'w')
          for path in have_div:
            if path not in want_div: f.write(path + "\n")
        finally:
          f.close()

  def _make_chroot(me):
    """
    Create the source-chroot with chroot metadata META.

    This will recreate a source-chroot from scratch, destroying the existing
    logical volume if necessary.
    """
    with Cleanup() as clean:

      dist, arch = me._dist, me._arch
      clean.enter(lockfile(chroot_src_lockfile(dist, arch)))

      mnt = chroot_src_mntpt(dist, arch)
      dev = chroot_src_blkdev(dist, arch)
      lv = chroot_src_lv(dist, arch)
      newlv = lv + ".new"

      ## Clean up any leftover debris.
      if mountpoint_p(mnt): umount(mnt)
      if block_device_p(dev):
        run_root(["lvremove", "-f", "%s/%s" % (C.VG, lv)])

      ## Create the logical volume and filesystem.  It's important that the
      ## logical volume not have its official name until after it contains a
      ## mountable filesystem.
      progress("create filesystem")
      run_root(["lvcreate", "--yes", C.LVSZ, "-n", newlv, C.VG])
      run_root(["mkfs", "-j", "-L%s-%s" % (dist, arch),
                OS.path.join("/dev", C.VG, newlv)])
      run_root(["lvrename", C.VG, newlv, lv])

      ## Start installing the chroot.
      with mount_chroot_src(dist, arch) as mnt:

        ## Set the basic structure.
        run_root(["mkdir", "-m755", OS.path.join(mnt, "fs")])
        run_root(["chmod", "750", mnt])

        ## Install the base system.
        progress("install base system")
        run_root(["eatmydata", "debootstrap"] +
                 (arch in C.FOREIGN_ARCHS and ["--foreign"] or []) +
                 ["--arch=" + arch, "--variant=minbase",
                  "--include=" + ",".join(C.BASE_PACKAGES),
                  dist, OS.path.join(mnt, "fs"), C.DEBMIRROR])

        ## If this is a cross-installation, then install the necessary `qemu'
        ## and complete the installation.
        if arch in C.FOREIGN_ARCHS:
          qemu = OS.path.join("cross", "%s-%s" % (dist, C.QEMUHOST[arch]),
                              "QEMU", "qemu-%s-static" % C.QEMUARCH[arch])
          run_root(["install", OS.path.join(C.LOCAL, qemu),
                    OS.path.join(mnt, "fs/usr/bin")])
          run_root(["chroot", OS.path.join(mnt, "fs"),
                    "/debootstrap/debootstrap", "--second-stage"])
          run_root(["ln", "-sf",
                    OS.path.join("/usr/local.schroot", qemu),
                    OS.path.join(mnt, "fs/usr/bin")])

        ## Set up `/usr/local'.
        progress("install `/usr/local' symlink")
        run_root(["rm", "-rf", OS.path.join(mnt, "fs/usr/local")])
        run_root(["ln", "-s",
                  OS.path.join("local.schroot", arch),
                  OS.path.join(mnt, "fs/usr/local")])

        ## Install the `apt' configuration.
        progress("configure package manager")
        run_root(["rm", "-f", OS.path.join(mnt, "fs/etc/apt/sources.list")])
        for c in C.APTCONF:
          run_root(["ln", "-s",
                    OS.path.join("/usr/local.schroot/etc/apt/apt.conf.d", c),
                    OS.path.join(mnt, "fs/etc/apt/apt.conf.d")])
        run_root(["ln", "-s",
                  "/usr/local.schroot/etc/apt/sources.%s" % dist,
                  OS.path.join(mnt, "fs/etc/apt/sources.list")])

        with safewrite_root\
             (OS.path.join(mnt, "fs/etc/apt/apt.conf.d/20arch")) as f:
          f.write("""\
  ### -*-conf-*-

  APT {
          Architecture "%s";
  };
  """ % arch)

        ## Set up the locale and time zone from the host system.
        progress("configure locales and timezone")
        run_root(["cp", "/etc/locale.gen", "/etc/timezone",
                  OS.path.join(mnt, "fs/etc")])
        with open("/etc/timezone") as f: tz = f.readline().strip()
        run_root(["ln", "-sf",
                  OS.path.join("/usr/share/timezone", tz),
                  OS.path.join(mnt, "fs/etc/localtime")])
        run_root(["cp", "/etc/default/locale",
                  OS.path.join(mnt, "fs/etc/default")])

        ## Fix `/etc/mtab'.
        progress("set `/etc/mtab'")
        run_root(["ln", "-sf", "/proc/mounts",
                  OS.path.join(mnt, "fs/etc/mtab")])

        ## Prevent daemons from starting within the chroot.
        progress("inhibit daemon startup")
        with safewrite_root(OS.path.join(mnt, "fs/usr/sbin/policy-rc.d"),
                            mode = "755") as f:
          f.write("""\
  #! /bin/sh
  echo >&2 "policy-rc.d: Services disabled by policy."
  exit 101
  """)

        ## Hack the dynamic linker to prefer libraries in `/usr' over
        ## `/usr/local'.  This prevents `dpkg-shlibdeps' from becoming
        ## confused.
        progress("configure dynamic linker")
        with safewrite_root\
             (OS.path.join(mnt, "fs/etc/ld.so.conf.d/libc.conf")) as f:
          f.write("# libc default configuration")
        with safewrite_root\
             (OS.path.join(mnt, "fs/etc/ld.so.conf.d/zzz-local.conf")) as f:
          f.write("""\
  ### -*-conf-*-
  ### Local hack to make /usr/local/ late.
  /usr/local/lib
  """)

      ## If this is a foreign architecture then we need to set it up.
      if arch in C.FOREIGN_ARCHS:

        ## Keep the chroot's native Qemu out of our way: otherwise we'll stop
        ## being able to run programs in the chroot.  There's a hack here
        ## because the `--no-rename' option was required in the same version
        ## in which is was introduced, so there's no single incantation that
        ## will work across the boundary.
        progress("divert emulator")
        run_schroot_source(dist, arch, ["eatmydata", "sh", "-e", "-c", """
          if dpkg-divert >/dev/null 2>&1 --no-rename --help
          then no_rename=--no-rename
          else no_rename=
          fi

          dpkg-divert --package install-cross-tools $no_rename \
            --divert /usr/bin/%(qemu)s.%(arch)s --add /usr/bin/%(qemu)s
        """ % dict(arch = arch, qemu = "qemu-%s-static" % C.QEMUARCH[arch])])

        ## Install faster native tools.
        me._install_cross_tools()

      ## Finishing touches.
      progress("finishing touches")
      run_schroot_source(dist, arch, ["eatmydata", "sh", "-e", "-c", """
        apt-get update
        apt-get -y upgrade
        apt-get -y install "$@"
        ldconfig
        apt-get -y autoremove
        apt-get clean
      """, "."] + C.EXTRA_PACKAGES, stdin = DISCARD)

      ## Mark the chroot as done.
      me._meta.update = zulu()
      me._meta.write()

  def _update_chroot(me):
    """Refresh the source-chroot with chroot metadata META."""
    with Cleanup() as clean:
      dist, arch = me._dist, me._arch
      clean.enter(lockfile(chroot_src_lockfile(dist, arch)))
      run_schroot_source(dist, arch, ["eatmydata", "sh", "-e", "-c", """
        apt-get update
        apt-get -y dist-upgrade
        apt-get -y autoremove
        apt-get -y clean
      """], stdin = DISCARD)
      if arch in C.FOREIGN_ARCHS: me._install_cross_tools()
      me._meta.update = zulu(); me._meta.write()

  def run(me):
    if me._meta.update is not None: me._update_chroot()
    else: me._make_chroot()

###--------------------------------------------------------------------------
### Extracting the cross tools.

class CrossToolsJob (BaseJob):
  """Extract cross-tools from a donor chroot."""

  SPECS = C.NATIVE_CHROOTS

  def __init__(me, spec, fresh = CREATE, *args, **kw):
    super(CrossToolsJob, me).__init__(*args, **kw)
    me._dist, me._arch = split_dist_arch(spec)
    me._meta = CrossToolsMetadata.read(me._dist, me._arch)
    me._fresh = fresh
    me._chroot = None

  def _mkname(me): return "cross-tools.%s-%s" % (me._dist, me._arch)

  def prepare(me):
    st, r = check_fresh(me._fresh, me._meta.update)
    if st is DONE: return
    me._chroot = ChrootJob.ensure("%s-%s" % (me._dist, me._arch), FRESH)
    me.await(me._chroot)

  def check(me):
    status, reason = super(CrossToolsJob, me).check()
    if status is not READY: return status, reason
    if me._chroot is not None and me._chroot.started:
      return READY, "prerequisites run"
    return check_fresh(me._fresh, me._meta.update)

  def run(me):
    with Cleanup() as clean:

      dist, arch = me._dist, me._arch

      mymulti = run_program(["dpkg-architecture", "-a" + arch,
                             "-qDEB_HOST_MULTIARCH"],
                            stdout = RETURN).rstrip("\n")
      crossarchs = [run_program(["dpkg-architecture", "-A" + a,
                                 "-qDEB_TARGET_GNU_TYPE"],
                                stdout = RETURN).rstrip("\n")
                    for a in C.FOREIGN_ARCHS]

      crossdir = OS.path.join(C.LOCAL, "cross", "%s-%s" % (dist, arch))
      crossold = crossdir + ".old"; crossnew = crossdir + ".new"
      usrbin = OS.path.join(crossnew, "usr/bin")

      clean.enter(lockfile(crosstools_lockfile(dist, arch)))
      run_program(["rm", "-rf", crossnew])
      mkdir_p(crossnew)

      ## Open a session to the donor chroot.
      progress("establish snapshot")
      session, root = clean.enter(chroot_session(dist, arch))

      ## Make sure the donor tree is up-to-date, and install the extra
      ## packages we need.
      progress("install tools packages")
      run_schroot_session(session, ["eatmydata", "sh", "-e", "-c", """
        apt-get update
        apt-get -y upgrade
        apt-get -y install "$@"
      """, "."] + C.CROSS_PACKAGES, rootp = True, stdin = DISCARD)

      def chase(path):
        dest = ""

        ## Work through the remaining components of the PATH.
        while path != "":
          try: sl = path.index("/")
          except ValueError: step = path; path = ""
          else: step, path = path[:sl], path[sl + 1:]

          ## Split off and analyse the first component.
          if step == "" or step == ".":
            ## A redundant `/' or `./'.  Skip it.
            pass
          elif step == "..":
            ## A `../'.  Strip off the trailing component of DEST.
            dest = dest[:dest.rindex("/")]
          else:
            ## Something else.  Transfer the component name to DEST.
            dest += "/" + step

          ## If DEST refers to something in the cross-tools tree then we're
          ## good.
          crossdest = crossnew + dest
          try: st = OS.lstat(crossdest)
          except OSError, err:
            if err.errno == E.ENOENT:
              ## No.  We need to copy something from the donor tree so that
              ## the name works.

              st = OS.lstat(root + dest)
              if ST.S_ISDIR(st.st_mode):
                OS.mkdir(crossdest)
              else:
                progress("copy `%s'" % dest)
                run_program(["rsync", "-aHR",
                             "%s/.%s" % (root, dest),
                             crossnew])
            else:
              raise

          ## If DEST refers to a symbolic link, then prepend the link target
          ## to PATH so that we can be sure the link will work.
          if ST.S_ISLNK(st.st_mode):
            link = OS.readlink(crossdest)
            if link.startswith("/"): dest = ""; link = link[1:]
            else:
              try: dest = dest[:dest.rindex("/")]
              except ValueError: dest = ""
            if path == "": path = link
            else: path = "%s/%s" % (path, link)

      ## Work through the shopping list, copying the things it names into the
      ## cross-tools tree.
      scan = []
      for pat in C.CROSS_PATHS:
        pat = pat.replace("MULTI", mymulti)
        any = False
        for rootpath in GLOB.iglob(root + pat):
          any = True
          path = rootpath[len(root):]
          progress("copy `%s'" % path)
          run_program(["rsync", "-aHR", "%s/.%s" % (root, path), crossnew])
        if not any:
          raise RuntimeError("no matches for cross-tool pattern `%s'" % pat)

      ## Scan the new tree: chase down symbolic links, copying extra stuff
      ## that we'll need; and examine ELF binaries to make sure we get the
      ## necessary shared libraries.
      def visit(_, dir, files):
        for f in files:
          path = OS.path.join(dir, f)
          inside = switch_prefix(path, [(crossnew + "/", "/")])
          if OS.path.islink(path): chase(inside)
          if elf_binary_p(arch, path): scan.append(inside)
      OS.path.walk(crossnew, visit, None)

      ## Work through the ELF binaries in `scan', determining which shared
      ## libraries they'll need.
      ##
      ## The rune running in the chroot session reads ELF binary names on
      ## stdin, one per line, and runs `ldd' on them to discover the binary's
      ## needed libraries and resolve them into pathnames.  Each pathname is
      ## printed to stderr as a line `+PATHNAME', followed by a final line
      ## consisting only of `-' as a terminator.  This is necessary so that
      ## we can tell when we've finished, because newly discovered libraries
      ## need to be fed back to discover their recursive dependencies.  (This
      ## is why the `WriteLinesSelector' interface is quite so hairy.)
      with subprocess(["schroot", "-r", "-c", session, "--",
                       "sh", "-e", "-c", """
        while read path; do
          ldd "$path" | while read a b c d; do
            case $a:$b:$c:$d in
              not:a:dynamic:executable) ;;
              statically:linked::) ;;
              /*) echo "+$a" ;;
              *:=\\>:/*) echo "+$c" ;;
              linux-*) ;;
              *) echo >&2 "failed to find shared library \\`$a'"; exit 2 ;;
            esac
          done
          echo -
        done
      """], stdin = PIPE, stdout = PIPE) as (fd_in, fd_out, _):

        ## Keep track of the number of binaries we've reported to the `ldd'
        ## process for which we haven't yet seen all of their dependencies.
        ## (This is wrapped in a `Struct' because of Python's daft scoping
        ## rules.)
        v = Struct(n = 0)

        def line_in():
          ## Provide a line in., so raise `StopIteration' to signal this.

          try:
            ## See if there's something to scan.
            path = scan.pop()

          except IndexError:
            ## There's nothing currently waiting to be scanned.
            if v.n:
              ## There are still outstanding replies, so stall.
              return None
            else:
              ## There are no outstanding replies left, and we have nothing
              ## more to scan, then we must be finished.
              raise StopIteration

          else:
            ## The `scan' list isn't empty, so return an item from that, and
            ## remember that there's one more thing we expect to see answers
            ## from.
            v.n += 1; return path

        def line_out(line):
          ## We've received a line from the `ldd' process.

          if line == "-":
            ## It's finished processing one of our binaries.  Note this.
            ## Maybe it's time to stop
            v.n -= 1
            return

          ## Strip the leading marker (which is just there so that the
          ## terminating `-' is unambiguous).
          assert line.startswith("+")
          lib = line[1:]

          ## If we already have this binary then we'll already have submitted
          ## it.
          path = crossnew + lib
          try: OS.lstat(path)
          except OSError, err:
            if err.errno == E.ENOENT: pass
            else: raise
          else: return

          ## Copy it into the tools tree, together with any symbolic links
          ## along the path.
          chase(lib)

          ## If this is an ELF binary (and it ought to be!) then submit it
          ## for further scanning.
          if elf_binary_p(arch, path):
            scan.append(switch_prefix(path, [(crossnew + "/", "/")]))

        ## And run this entire contraption.  When this is done, we should
        ## have all of the library dependencies for all of our binaries.
        select_loop([WriteLinesSelector(fd_in, line_in),
                     ReadLinesSelector(fd_out, line_out)])

      ## Set up the cross-compiler and emulator.  Start by moving the cross
      ## compilers and emulator into their specific places, so they don't end
      ## up cluttering chroots for non-matching architectures.
      progress("establish TOOLCHAIN and QEMU")
      OS.mkdir(OS.path.join(crossnew, "TOOLCHAIN"))
      qemudir = OS.path.join(crossnew, "QEMU")
      OS.mkdir(qemudir)
      for gnu in C.FOREIGN_GNUARCHS:
        OS.mkdir(OS.path.join(crossnew, "TOOLCHAIN", gnu))
      for f in OS.listdir(usrbin):
        for gnu in C.FOREIGN_GNUARCHS:
          gnuprefix = gnu + "-"
          if f.startswith(gnuprefix):
            tooldir = OS.path.join(crossnew, "TOOLCHAIN", gnu)
            OS.rename(OS.path.join(usrbin, f), OS.path.join(tooldir, f))
            OS.symlink(f, OS.path.join(tooldir, f[len(gnuprefix):]))
            break
        else:
          if f.startswith("qemu-") and f.endswith("-static"):
            OS.rename(OS.path.join(usrbin, f), OS.path.join(qemudir, f))

      ## The GNU cross compilers try to find their additional pieces via a
      ## relative path, which isn't going to end well.  Add a symbolic link
      ## at the right place to where the things are actually going to live.
      toollib = OS.path.join(crossnew, "TOOLCHAIN", "lib")
      OS.mkdir(toollib)
      OS.symlink("../../usr/lib/gcc-cross",
                 OS.path.join(toollib, "gcc-cross"))

      ## We're done.  Replace the old cross-tools with our new one.
      me._meta.update = zulu()
      me._meta.write(crossnew)
      if OS.path.exists(crossdir): run_program(["mv", crossdir, crossold])
      OS.rename(crossnew, crossdir)
      run_program(["rm", "-rf", crossold])

###--------------------------------------------------------------------------
### Buliding and installing local packages.

def pkg_metadata_lockfile(pkg):
  return lockfile_path("pkg-meta.%s" % pkg)

def pkg_srcdir_lockfile(pkg, ver):
  return lockfile_path("pkg-source.%s-%s" % (pkg, ver))

def pkg_srcdir(pkg, ver):
  return OS.path.join(C.LOCAL, "src", "%s-%s" % (pkg, ver))

def pkg_builddir(pkg, ver, arch):
  return OS.path.join(pkg_srcdir(pkg, ver), "build.%s" % arch)

class PackageMetadata (BaseMetadata):
  VARS = ["pkg"] + list(C.ALL_ARCHS)

  @classmethod
  def read(cls, pkg):
    try:
      return super(PackageMetadata, cls)\
        .read(OS.path.join(C.LOCAL, "src", "META.%s" % pkg))
    except IOError, err:
      if err.errno == E.ENOENT: pass
      else: raise
    return cls(pkg = pkg)

  def write(me):
    super(PackageMetadata, me)\
      .write(OS.path.join(C.LOCAL, "src", "META.%s" % me.pkg))

class PackageSourceJob (BaseJob):

  SPECS = C.LOCALPKGS

  def __init__(me, pkg, fresh = CREATE, *args, **kw):
    super(PackageSourceJob, me).__init__(*args, **kw)
    me._pkg = pkg
    tar = None; ver = None
    r = RX.compile("^%s-(\d.*)\.tar.(?:Z|z|gz|bz2|xz|lzma)$" %
                   RX.escape(pkg))
    for f in OS.listdir("pkg"):
      m = r.match(f)
      if not m: pass
      elif tar is not None:
        raise ExpectedError("multiple source tarballs of package `%s'" % pkg)
      else: tar, ver = f, m.group(1)
    me.version = ver
    me.tarball = OS.path.join("pkg", tar)

  def _mkname(me): return "pkg-source.%s" % me._pkg

  def check(me):
    status, reason = super(PackageSourceJob, me).check()
    if status is not READY: return status, reason
    if OS.path.isdir(pkg_srcdir(me._pkg, me.version)):
      return DONE, "already unpacked"
    else:
      return READY, "no source tree"

  def run(me):
    with Cleanup() as clean:
      pkg, ver, tar = me._pkg, me.version, me.tarball
      srcdir = pkg_srcdir(pkg, ver)
      newdir = srcdir + ".new"

      progress("unpack `%s'" % me.tarball)
      clean.enter(lockfile(pkg_srcdir_lockfile(pkg, ver)))
      run_program(["rm", "-rf", newdir])
      mkdir_p(newdir)
      run_program(["tar", "xf", OS.path.join(OS.getcwd(), me.tarball)],
                  cwd = newdir)
      things = OS.listdir(newdir)
      if len(things) == 1:
        OS.rename(OS.path.join(newdir, things[0]), srcdir)
        OS.rmdir(newdir)
      else:
        OS.rename(newdir, srcdir)

class PackageBuildJob (BaseJob):

  SPECS = ["%s:%s" % (pkg, arch)
           for pkg in C.LOCALPKGS
           for arch in C.ALL_ARCHS]

  def __init__(me, spec, fresh = CREATE, *args, **kw):
    super(PackageBuildJob, me).__init__(*args, **kw)
    colon = spec.index(":")
    me._pkg, me._arch = spec[:colon], spec[colon + 1:]

  def _mkname(me): return "pkg-build.%s:%s" % (me._pkg, me._arch)

  def prepare(me):
    me.await(ChrootJob.ensure("%s-%s" % (C.PRIMARY_DIST, me._arch), CREATE))
    me._meta = PackageMetadata.read(me._pkg)
    me._src = PackageSourceJob.ensure(me._pkg, FRESH); me.await(me._src)
    me._prereq = [PackageBuildJob.ensure("%s:%s" % (prereq, me._arch), FRESH)
                  for prereq in C.PKGDEPS[me._pkg]]
    for j in me._prereq: me.await(j)

  def check(me):
    status, reason = super(PackageBuildJob, me).check()
    if status is not READY: return status, reason
    if me._src.started: return READY, "fresh source directory"
    for j in me._prereq:
      if j.started:
        return READY, "dependency `%s' freshly installed" % j._pkg
    if getattr(me._meta, me._arch) == me._src.version:
      return DONE, "already installed"
    return READY, "not yet installed"

  def run(me):
    with Cleanup() as clean:
      pkg, ver, arch = me._pkg, me._src.version, me._arch

      session, _ = clean.enter(chroot_session(C.PRIMARY_DIST, arch))
      builddir = OS.path.join(pkg_srcdir(pkg, ver), "build.%s" % arch)
      chroot_builddir = host_to_chroot(builddir)
      run_program(["rm", "-rf", builddir])
      OS.mkdir(builddir)

      progress("prepare %s chroot" % (arch))
      run_schroot_session(session,
                          ["eatmydata", "apt-get", "update"],
                          rootp = True, stdin = DISCARD)
      run_schroot_session(session,
                          ["eatmydata", "apt-get", "-y", "upgrade"],
                          rootp = True, stdin = DISCARD)
      run_schroot_session(session,
                          ["eatmydata", "apt-get", "-y",
                           "install", "pkg-config"],
                          rootp = True, stdin = DISCARD)
      run_schroot_session(session,
                          ["mount", "-oremount,rw", "/usr/local.schroot"],
                          rootp = True, stdin = DISCARD)

      progress("configure `%s' %s for %s" % (pkg, ver, arch))
      run_schroot_session(session, ["sh", "-e", "-c", """
        cd "$1" &&
        ../configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig.hidden
      """, ".", chroot_builddir])

      progress("compile `%s' %s for %s" % (pkg, ver, arch))
      run_schroot_session(session, ["sh", "-e", "-c", """
        cd "$1" && make -j4 && make -j4 check
      """, ".", chroot_builddir])

      existing = getattr(me._meta, arch, None)
      if existing is not None and existing != ver:
        progress("uninstall existing `%s' %s for %s" % (pkg, existing, arch))
        run_schroot_session(session, ["sh", "-e", "-c", """
          cd "$1" && make uninstall
        """, ".", OS.path.join(pkg_srcdir(pkg, existing),
                               "build.%s" % arch)],
                            rootp = True)

      progress("install `%s' %s for %s" % (pkg, existing, arch))
      run_schroot_session(session, ["sh", "-e", "-c", """
        cd "$1" && make install
        mkdir -p /usr/local/lib/pkgconfig.hidden
        mv /usr/local/lib/pkgconfig/*.pc /usr/local/lib/pkgconfig.hidden || :
      """, ".", chroot_builddir], rootp = True)

      clean.enter(lockfile(pkg_metadata_lockfile(pkg)))
      me._meta = PackageMetadata.read(pkg)
      setattr(me._meta, arch, ver); me._meta.write()

    with lockfile(chroot_src_lockfile(C.PRIMARY_DIST, arch)):
      run_schroot_source(C.PRIMARY_DIST, arch, ["ldconfig"])

###--------------------------------------------------------------------------
### Process the configuration and options.

OPTIONS = OP.OptionParser\
  (usage = "chroot-maint [-diknqs] [-fFRESH] [-jN] JOB[.SPEC,...] ...")
for short, long, props in [
  ("-d", "--debug", {
    'dest': 'debug', 'default': False, 'action': 'store_true',
    'help': "print lots of debugging drivel" }),
  ("-f", "--fresh", {
    'dest': 'fresh', 'metavar': 'FRESH', 'default': "create",
    'help': "how fresh (`create', `force', or `N[s|m|h|d|w]')" }),
  ("-i", "--ignore-errors", {
    'dest': 'ignerr', 'default': False, 'action': 'store_true',
    'help': "ignore all errors encountered while processing" }),
  ("-j", "--jobs", {
    'dest': 'njobs', 'metavar': 'N', 'default': 1, 'type': 'int',
    'help': 'run up to N jobs in parallel' }),
  ("-J", "--forkbomb", {
    'dest': 'njobs', 'action': 'store_true',
    'help': 'run as many jobs in parallel as possible' }),
  ("-k", "--keep-going", {
    'dest': 'keepon', 'default': False, 'action': 'store_true',
    'help': "keep going even if independent jobs fail" }),
  ("-n", "--dry-run", {
    'dest': 'dryrun', 'default': False, 'action': 'store_true',
    'help': "don't actually do anything" }),
  ("-q", "--quiet", {
    'dest': 'quiet', 'default': False, 'action': 'store_true',
    'help': "don't print the output from successful jobs" }),
  ("-s", "--silent", {
    'dest': 'silent', 'default': False, 'action': 'store_true',
    'help': "don't print progress messages" })]:
  OPTIONS.add_option(short, long, **props)

###--------------------------------------------------------------------------
### Main program.

R_JOBSERV = RX.compile(r'^--jobserver-(?:fds|auth)=(\d+),(\d+)$')

JOBMAP = { "chroot": ChrootJob,
           "cross-tools": CrossToolsJob,
           "pkg-source": PackageSourceJob,
           "pkg-build": PackageBuildJob }

R_FRESH = RX.compile(r"^(?:create|force|(\d+)(|[smhdw]))$")

def parse_fresh(spec):
  m = R_FRESH.match(spec)
  if not m: raise ExpectedError("bad freshness `%s'" % spec)
  if spec == "create": fresh = CREATE
  elif spec == "force": fresh = FORCE
  else:
    n, u = int(m.group(1)), m.group(2)
    if u == "" or u == "s": fresh = n
    elif u == "m": fresh = 60*n
    elif u == "h": fresh = 3600*n
    elif u == "d": fresh = 86400*n
    elif u == "w": fresh = 604800*n
    else: assert False
  return fresh

with toplevel_handler():
  OPT, args = OPTIONS.parse_args()
  rfd, wfd = -1, -1
  njobs = OPT.njobs
  try: mkflags = OS.environ['MAKEFLAGS']
  except KeyError: pass
  else:
    ff = mkflags.split()
    for f in ff:
      if f == "--": break
      m = R_JOBSERV.match(f)
      if m: rfd, wfd = int(m.group(1)), int(m.group(2))
      elif f == '-j': njobs = None
      elif not f.startswith('-'):
        for ch in f:
          if ch == 'i': OPT.ignerr = True
          elif ch == 'k': OPT.keepon = True
          elif ch == 'n': OPT.dryrun = True
          elif ch == 's': OPT.silent = True
  if OPT.njobs < 1:
    raise ExpectedError("running no more than %d jobs is silly" % OPT.njobs)

  FRESH = parse_fresh(OPT.fresh)

  SCHED = JobScheduler(rfd, wfd, njobs)
  OS.environ["http_proxy"] = C.PROXY

  jobs = []
  if not args: OPTIONS.print_usage(SYS.stderr); SYS.exit(2)
  for arg in args:
    try: sl = arg.index("/")
    except ValueError: fresh = FRESH
    else: arg, fresh = arg[:sl], parse_fresh(arg[sl + 1:])
    try: dot = arg.index(".")
    except ValueError: jty, pats = arg, "*"
    else: jty, pats = arg[:dot], arg[dot + 1:]
    try: jcls = JOBMAP[jty]
    except KeyError: raise ExpectedError("unknown job type `%s'" % jty)
    specs = []
    for pat in pats.split(","):
      any = False
      for s in jcls.SPECS:
        if FM.fnmatch(s, pat): specs.append(s); any = True
      if not any: raise ExpectedError("no match for `%s'" % pat)
    for s in specs:
      jobs.append(jcls.ensure(s, fresh))

  SCHED.run()

SYS.exit(RC)

###----- That's all, folks --------------------------------------------------