--- /dev/null
+#! /usr/bin/python
+###
+### Convert a directory tree of audio files
+###
+### (c) 2010 Mark Wooding
+###
+
+###----- Licensing notice ---------------------------------------------------
+###
+### This program is free software; you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation; either version 2 of the License, or
+### (at your option) any later version.
+###
+### This program is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with this program; if not, write to the Free Software Foundation,
+### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+###--------------------------------------------------------------------------
+### External dependencies.
+
+## Language features.
+from __future__ import with_statement
+
+## Standard Python libraries.
+import sys as SYS
+import os as OS
+import errno as E
+import time as T
+import unicodedata as UD
+import fnmatch as FN
+import re as RX
+import shutil as SH
+import optparse as OP
+import threading as TH
+import shlex as L
+from math import sqrt
+from contextlib import contextmanager
+
+## eyeD3 tag fettling.
+import eyeD3 as E3
+
+## Gstreamer. It picks up command-line arguments -- most notably `--help' --
+## and processes them itself. Of course, its help is completely wrong. This
+## kludge is due to Jonas Wagner.
+_argv, SYS.argv = SYS.argv, []
+import gobject as G
+import gio as GIO
+import gst as GS
+SYS.argv = _argv
+
+## Python Imaging.
+from PIL import Image as I
+
+## Python parsing.
+import pyparsing as P
+
+###--------------------------------------------------------------------------
+### Special initialization.
+
+VERSION = '1.0.0~pre'
+
+## GLib.
+G.threads_init()
+
+###--------------------------------------------------------------------------
+### Eyecandy progress reports.
+
+def charwidth(s):
+ """
+ Return the width of S, in characters.
+
+ Specifically, this is the number of backspace characters required to
+ overprint the string S. If the current encoding for `stdout' appears to be
+ Unicode then do a complicated Unicode thing; otherwise assume that
+ characters take up one cell each.
+
+ None of this handles tab characters in any kind of useful way. Sorry.
+ """
+
+ ## If there's no encoding for stdout then we're doing something stupid.
+ if SYS.stdout.encoding is None: return len(s)
+
+ ## Turn the string into Unicode so we can hack on it properly. Maybe that
+ ## won't work out, in which case fall back to being stupid.
+ try: u = s.decode(SYS.stdout.encoding)
+ except UnicodeError: return len(s)
+
+ ## Our main problem is combining characters, but we should also try to
+ ## handle wide (mostly Asian) characters, and zero-width ones. This hack
+ ## is taken mostly from http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+ w = 0
+ for ch in u:
+ cd = ord(ch)
+ if UD.category(ch) in ['Cf', 'Me', 'Mn'] or \
+ 0x1160 <= cd <= 0x11ff: pass
+ elif UD.east_asian_width(ch) in ['F', 'W']: w += 2
+ else: w += 1
+
+ ## Done.
+ #print ';; %r -> %d' % (s, w)
+ return w
+
+class StatusLine (object):
+ """
+ Maintains a status line containing ephemeral progress information.
+
+ The status line isn't especially important, but it keeps interactive users
+ amused.
+
+ There should be only one status line object in your program; otherwise
+ they'll interfere with each other and get confused.
+
+ The update algorithm (in `set') is fairly careful to do the right thing
+ with long status `lines', and to work properly in an Emacs `shell' buffer.
+ """
+
+ def __init__(me):
+ "Initialize the status line."
+ me._last = ''
+ me._lastlen = 0
+ me.eyecandyp = OS.isatty(SYS.stdout.fileno())
+
+ def set(me, line):
+ """
+ Set the status line contents to LINE, replacing what was there before.
+
+ This only produces actual output if stdout is interactive.
+ """
+ n = len(line)
+
+ ## Eyecandy update.
+ if me.eyecandyp:
+ #print
+ #print ';; new status %r' % line
+
+ ## If the old line was longer, we need to clobber its tail, so work out
+ ## what that involves.
+ if n < me._lastlen:
+ b = charwidth(me._last[n:])
+ pre = '\b'*b + ' '*b
+ else:
+ pre = ''
+
+ ## Now figure out the length of the common prefix between what we had
+ ## before and what we have now. This reduces the amount of I/O done,
+ ## which keeps network traffic down on SSH links, and keeps down the
+ ## amount of work slow terminal emulators like Emacs have to do.
+ i = 0
+ m = min(n, me._lastlen)
+ while i < m and line[i] == me._last[i]:
+ i += 1
+
+ ## Actually do the output, all in one syscall.
+ b = charwidth(me._last[i:])
+ SYS.stdout.write(pre + '\b'*b + line[i:])
+ #print ';; => %r' % (pre + '\b'*b + line[i:])
+ SYS.stdout.flush()
+
+ ## Update our idea of what's gone on.
+ me._lastlen = n
+ me._last = line
+
+ def clear(me):
+ "Clear the status line. Just like set('')."
+ me.set('')
+
+ def commit(me, line = None):
+ """
+ Commit the current status line, and maybe the string LINE.
+
+ If the current status line is nonempty, then commit it to the transcript.
+ If LINE is not None, then commit that to the transcript too.
+
+ After all of this, we clear the status line to get back to a clean state.
+ """
+ if me._last:
+ if me.eyecandyp:
+ SYS.stdout.write('\n')
+ else:
+ SYS.stdout.write(me._last + '\n')
+ if line is not None:
+ SYS.stdout.write(line + '\n')
+ me._lastlen = 0
+ me._last = ''
+
+STATUS = StatusLine()
+
+def filestatus(file, status):
+ return '%s%s: %s' % (' '*8, OS.path.basename(file), status)
+
+class ProgressEyecandy (object):
+ """
+ Provide amusement while something big and complicated is happening.
+
+ This is an abstract class. Subclasses must provide a method `progress'
+ returning a pair (CURRENT, MAX) indicating the current progress through the
+ operation.
+ """
+
+ def __init__(me, what, silentp = False):
+ """
+ Initialize a progress meter.
+
+ WHAT is a prefix string to be written before the progress eyecandy
+ itself.
+ """
+ me._what = what
+ me._silentp = silentp
+ me._spinner = 0
+ me._start = T.time()
+
+ def _fmt_time(me, t):
+ "Format T as a time, in (maybe hours) minutes and seconds."
+ s, t = t % 60, int(t/60)
+ m, h = t % 60, int(t/60)
+ if h > 0:
+ return '%d:%02d:%02d' % (h, m, s)
+ else:
+ return '%02d:%02d' % (m, s)
+
+ def show(me):
+ "Show the current level of progress."
+
+ ## If we're not showing pointless frippery, don't bother at all.
+ if not STATUS.eyecandyp:
+ return
+
+ ## Update the spinner index.
+ me._spinner = (me._spinner + 1)%4
+
+ ## Fetch the current progress information. Note that we always fetch
+ ## both the current and maximum levels, because both might change if an
+ ## operation revises its idea of how much work needs doing.
+ cur, max = me.progress()
+
+ ## If we couldn't get progress information, display something vaguely
+ ## amusing anyway.
+ if cur is None or max is None:
+ STATUS.set('%s %c [unknown progress]' %
+ (me._what, r'/-\|'[me._spinner]))
+ return
+
+ ## Work out -- well, guess -- the time remaining.
+ if cur:
+ t = T.time()
+ eta = me._fmt_time((t - me._start)*(max - cur)/cur)
+ else:
+ eta = '???'
+
+ ## Set the status bar.
+ n = 40*cur/max
+ STATUS.set('%s %c [%s%s] %3d%% (%s)' % \
+ (me._what,
+ r'/-\|'[me._spinner],
+ '='*n, ' '*(40 - n),
+ 100*cur/max,
+ eta))
+
+ def done(me, win = True):
+ "Show a completion notice, or a failure if WIN is false."
+ if not win:
+ STATUS.set('%s FAILED!' % me._what)
+ elif not me._silentp:
+ STATUS.set('%s done (%s)' %
+ (me._what,
+ me._fmt_time(T.time() - me._start)))
+ else:
+ return
+ STATUS.commit()
+
+###--------------------------------------------------------------------------
+### Timeout handling.
+
+KILLSWITCH = TH.Event()
+
+def timeout(t0, t1):
+ T.sleep(t0)
+ KILLSWITCH.set()
+ T.sleep(t1)
+ moan('dying messily due to timeout')
+ OS._exit(3)
+
+###--------------------------------------------------------------------------
+### Parsing utilities.
+
+## Allow hyphens in identifiers.
+IDCHARS = P.alphanums + '-_'
+P.Keyword.setDefaultKeywordChars(IDCHARS)
+
+## Some common kinds of tokens.
+Name = P.Word(IDCHARS)
+Num = P.Word(P.nums).setParseAction(lambda toks: map(int, toks))
+String = P.QuotedString('"', '\\')
+
+## Handy abbreviations for constructed parser elements.
+def K(k): return P.Keyword(k).suppress()
+def D(d): return P.Literal(d).suppress()
+##R = P.ZeroOrMore
+def R(p): return P.ZeroOrMore(p).setParseAction(lambda s, l, t: [t])
+O = P.Optional
+
+###--------------------------------------------------------------------------
+### Format identification and conversion.
+
+class IdentificationFailure (Exception):
+ pass
+
+class FileCategory (object):
+ """
+ A FileCategory represents a class of files.
+
+ For example, it's sensible to consider audio, or image files as a
+ category. A file category knows how to recognize member files from
+ MIME content types.
+ """
+
+ def __init__(me, name, mime_pats, ident):
+ """
+ Construct a new category.
+
+ The PATS are a list of `fnmatch' patterns to be compared with a MIME
+ type. The IDENT is a function which produces an identification object
+ given a file's name and first-guess MIME type. The object is passed to a
+ Format's `check' method to see whether a file needs re-encoding, and to
+ `convert' to assist with the conversion.
+
+ An identification object must have an attribute `mime' which is a set of
+ possible MIME types accumulated for the object.
+ """
+ me.name = name
+ me._mime_pats = mime_pats
+ me._ident = ident
+ CATEGORYMAP[name] = me
+
+ def identify(me, file, mime):
+ """
+ Attempt to identify FILE, given its apparent MIME type.
+
+ If identification succeeds, return an identification object which can be
+ used by associated file formats; otherwise return None.
+ """
+ for p in me._mime_pats:
+ if not FN.fnmatchcase(mime, p):
+ continue
+ try:
+ return me._ident(file, mime)
+ except IdentificationFailure:
+ pass
+ return None
+
+class BaseFormat (object):
+ """
+ A BaseFormat object represents a particular encoding and parameters.
+
+ The object can verify (the `check' method) whether a particular file
+ matches its requirements, and if necessary (`encode') re-encode a file.
+
+ Subclasses should define the following methods.
+
+ check(ID)
+ Answer whether the file identified by ID is acceptable according to
+ the receiver's parameters.
+
+ convert(MASTER, ID, TARGET)
+ Convert the file MASTER, which has been identified as ID, according
+ to the receiver's parameters, writing the output to TARGET.
+
+ Subclasses should also provide these attributes.
+
+ CATEGORY
+ A FileCategory object for the category of files that this format
+ lives within.
+
+ EXT A file extension to be applied to encoded output files.
+
+ NAME A user-facing name for the format.
+
+ PROPS A parser element to parse a property definition. It should produce
+ a pair NAME, VALUE to be stored in a dictionary.
+
+ Subclasses for different kinds of file may introduce more subclass
+ protocol.
+ """
+
+ def fixup(me, path):
+ """Post-encoding fixups."""
+ pass
+
+FORMATMAP = {}
+CATEGORYMAP = {}
+
+def defformat(name, cls):
+ "Define a format NAME using class CLS."
+ if not hasattr(cls, 'NAME'):
+ raise ValueError, 'abstract class'
+ if not hasattr(cls, 'CATEGORY'):
+ raise ValueError, 'no category'
+ FORMATMAP[name] = cls
+
+class FormatParser (P.ParserElement):
+ """
+ Parse a format specifier:
+
+ format-spec ::= string [format-properties]
+ format-properties ::= `{' format-property (`,' format-property)* `}'
+
+ The syntax of a format-property is determined by the PROPS attribute on the
+ named format and its superclasses.
+ """
+
+ ## We cache the parser elements we generate to avoid enormous consing.
+ CACHE = {}
+
+ def parseImpl(me, s, loc, actp = True):
+
+ ## Firstly, determine the format name.
+ loc, r = Name._parse(s, loc, actp)
+ fmt = r[0]
+
+ ## Look up the format class.
+ try: fcls = FORMATMAP[fmt]
+ except KeyError:
+ raise P.ParseException(s, loc, "Unknown format `%s'" % fmt)
+
+ ## Fetch the property-list parser from the cache, if possible; else
+ ## construct it.
+ try:
+ pp = me.CACHE[fmt]
+ except KeyError:
+ seen = set()
+ prop = None
+ for c in fcls.mro():
+ try: p = c.PROPS
+ except AttributeError: continue
+ if p in seen: continue
+ if prop is None: prop = p
+ else: prop |= p
+ seen.add(p)
+ if prop is None:
+ pp = me.CACHE[fmt] = None
+ else:
+ props = P.delimitedList(prop)
+ props.setParseAction(lambda s, l, t: dict(t.asList()))
+ pp = me.CACHE[fmt] = O(D('{') - props - D('}'))
+
+ ## Parse the properties.
+ if pp is None:
+ pd = {}
+ else:
+ loc, r = pp._parse(s, loc, actp)
+ if r: pd = r[0]
+ else: pd = {}
+
+ ## Construct the format object and return it.
+ return loc, fcls(**pd)
+
+Format = FormatParser()
+
+def prop(kw, pval, tag = None):
+ if tag is None: tag = kw
+ if pval is None:
+ p = K(kw)
+ p.setParseAction(lambda s, l, t: (tag, True))
+ else:
+ p = K(kw) + D('=') + pval
+ p.setParseAction(lambda s, l, t: (tag, t[0]))
+ return p
+
+###--------------------------------------------------------------------------
+### Policies and actions.
+
+class Action (object):
+ """
+ An Action object represents a conversion action to be performed.
+
+ This class isn't intended to be instantiated directly. It exists to define
+ some protocol common to all Action objects.
+
+ Action objects have the following attributes.
+
+ master The name of the master (source) file.
+
+ target The name of the target (destination) file.
+
+ PRIORITY The priority of the action, for deciding which of two actions
+ to perform. Higher priorities are more likely to win.
+
+ Converting an Action to a string describes the action in a simple
+ user-readable manner. The `perform' method actually carries the action
+ out.
+ """
+
+ PRIORITY = 0
+
+ def __init__(me, master):
+ "Stash the MASTER file name for later."
+ me.master = master
+
+ def choose(me, him):
+ "Choose either ME or HIM and return one."
+ if him is None or me.PRIORITY > him.PRIORITY:
+ return me
+ else:
+ return him
+
+class CopyAction (Action):
+ """
+ An Action object for simply copying a file.
+
+ Actually we try to hardlink it first, falling back to a copy later. This
+ is both faster and more efficient with regard to disk space.
+ """
+
+ ## Copying is good. Linking is really good, but we can't tell the
+ ## difference at this stage.
+ PRIORITY = 10
+
+ def __init__(me, master, targetdir):
+ "Initialize a CopyAction, from MASTER to the TARGETDIR directory."
+ Action.__init__(me, master)
+ me.target = OS.path.join(targetdir, OS.path.basename(master))
+
+ def __str__(me):
+ return 'copy/link'
+
+ def perform(me):
+ "Actually perform a CopyAction."
+ try:
+ STATUS.set(filestatus(me.master, 'link'))
+ OS.link(me.master, me.target)
+ except OSError, err:
+ if err.errno != E.EXDEV:
+ raise
+ STATUS.set(filestatus(me.master, 'copy'))
+ new = me.target + '.new'
+ SH.copyfile(me.master, new)
+ OS.rename(new, me.target)
+ STATUS.commit()
+
+class ConvertAction (Action):
+ """
+ An Action object for converting a file to a given format.
+
+ Additional attributes:
+
+ id The identification object for the master file.
+
+ format The format to which we're meant to conver the master.
+ """
+
+ def __init__(me, master, targetdir, id, format):
+ "Initialize a ConvertAction."
+ Action.__init__(me, master)
+ stem, ext = OS.path.splitext(OS.path.basename(master))
+ me.target = OS.path.join(targetdir, stem + '.' + format.EXT)
+ me.id = id
+ me.format = format
+
+ def __str__(me):
+ return 'convert to %s' % me.format.NAME
+
+ def perform(me):
+ "Acually perform a ConvertAction."
+ STATUS.set(filestatus(me.master, me))
+ me.format.convert(me.master, me.id, me.target)
+
+Policy = P.Forward()
+
+class FormatPolicy (object):
+ """
+ A FormatPolicy object represents a set of rules for how to convert files.
+
+ Given a master file, the FormatPolicy will identify it and return a list of
+ actions to be performed. The methods required of a FormatPolicy are:
+
+ setcategory(CAT)
+ Store CAT as the policy's category. Check that this is consistent
+ with the policy as stored.
+
+ actions(MASTER, TARGETDIR, ID, COHORT)
+ Given a MASTER file, identified as ID, a target directory
+ TARGETDIR, and a list COHORT of (FILE, ID) pairs for other files
+ of the same category in the same directory, return a list of
+ actions to be performed to get the target directory into the right
+ form. The list might be empty if the policy object /rejects/ the
+ file.
+ """
+
+class AndPolicy (FormatPolicy):
+ """
+ A FormatPolicy which does the union of a bunch of other policies.
+
+ Each subsidiary policy is invoked in turn. The highest-priority action for
+ each target file is returned.
+ """
+
+ def __init__(me, policies):
+ me._policies = policies
+
+ def setcategory(me, cat):
+ me.cat = cat
+ for p in me._policies:
+ p.setcategory(cat)
+
+ def actions(me, master, targetdir, id, cohort):
+ tmap = {}
+ for p in me._policies:
+ for a in p.actions(master, targetdir, id, cohort):
+ if a.target in tmap:
+ tmap[a.target] = a.choose(tmap.get(a.target))
+ else:
+ tmap[a.target] = a
+ return tmap.values()
+
+And = K('and') - D('{') - R(Policy) - D('}')
+And.setParseAction(lambda s, l, t: AndPolicy(t[0]))
+
+class OrPolicy (FormatPolicy):
+ """
+ A FormatPolicy which tries other policies and uses the first that accepts.
+
+ Each subsidiary policy is invoked in turn. If any accepts, the actions it
+ proposes are turned and no further policies are invoked. If none accepts
+ then the file is rejected.
+ """
+
+ def __init__(me, policies):
+ me._policies = policies
+
+ def setcategory(me, cat):
+ me.cat = cat
+ for p in me._policies:
+ p.setcategory(cat)
+
+ def actions(me, master, targetdir, id, cohort):
+ for p in me._policies:
+ aa = p.actions(master, targetdir, id, cohort)
+ if aa:
+ return aa
+ else:
+ return []
+
+Or = K('or') - D('{') - R(Policy) - D('}')
+Or.setParseAction(lambda s, l, t: OrPolicy(t[0]))
+
+class AcceptPolicy (FormatPolicy):
+ """
+ A FormatPolicy which copies files in a particular format.
+
+ If all of the files in a cohort are recognized as being in a particular
+ format (including this one), then accept it with a CopyAction; otherwise
+ reject.
+ """
+
+ def __init__(me, format):
+ me._format = format
+
+ def setcategory(me, cat):
+ if me._format.CATEGORY is not cat:
+ raise ValueError, \
+ "Accept format `%s' has category `%s', not `%s'" % \
+ (me._format.__class__.__name__,
+ me._format.CATEGORY.name, cat.name)
+ me.cat = cat
+
+ def actions(me, master, targetdir, id, cohort):
+ if me._format.check(id) and \
+ all(me._format.check(cid) for f, cid in cohort):
+ return [CopyAction(master, targetdir)]
+ else:
+ return []
+
+Accept = K('accept') - Format
+Accept.setParseAction(lambda s, l, t: AcceptPolicy(t[0]))
+
+class ConvertPolicy (FormatPolicy):
+ """
+ A FormatPolicy which copies files in a particular format or converts if
+ necessary.
+ """
+ def __init__(me, format):
+ me._format = format
+
+ def setcategory(me, cat):
+ if me._format.CATEGORY is not cat:
+ raise ValueError, \
+ "Accept format `%s' has category `%s', not `%s'" % \
+ (me._format.__class__.__name__,
+ me._format.CATEGORY.name, cat.name)
+ me.cat = cat
+
+ def actions(me, master, targetdir, id, cohort):
+ if me._format.check(id):
+ return [CopyAction(master, targetdir)]
+ else:
+ return [ConvertAction(master, targetdir, id, me._format)]
+
+Convert = K('convert') - Format
+Convert.setParseAction(lambda s, l, t: ConvertPolicy(t[0]))
+
+Policy << (And | Or | Accept | Convert)
+
+###--------------------------------------------------------------------------
+### Audio handling, based on GStreamer.
+
+def make_element(factory, name = None, **props):
+ "Return a new element from the FACTORY with the given NAME and PROPS."
+ elt = GS.element_factory_make(factory, name)
+ elt.set_properties(**props)
+ return elt
+
+class GStreamerProgressEyecandy (ProgressEyecandy):
+ """
+ Provide amusement while GStreamer is busy doing something.
+
+ The GStreamerProgressEyecandy object is a context manager. Wrap it round
+ your GStreamer loop to provide progress information for an operation.
+ """
+
+ def __init__(me, what, elt, **kw):
+ """
+ Initialize a progress meter.
+
+ WHAT is a prefix string to be written before the progress eyecandy
+ itself. ELT is a GStreamer element to interrogate to find the progress
+ information.
+ """
+ me._elt = elt
+ ProgressEyecandy.__init__(me, what, **kw)
+
+ def _update(me):
+ "Called by GLib main event loop to update the eyecandy."
+ me.show()
+ return True
+
+ def _timer(me):
+ """
+ Update the progress meter.
+
+ This is called periodically by the GLib main event-processing loop.
+ """
+ me.show()
+ return True
+
+ def progress(me):
+ "Return the current progress as a pair (CURRENT, MAX)."
+
+ ## Fetch the current progress information. We get the duration each
+ ## time, because (particularly with VBR-encoded MP3 inputs) the estimated
+ ## duration can change as we progress. Hopefully it settles down fairly
+ ## soon.
+ try:
+ t, hunoz = me._elt.query_position(GS.FORMAT_TIME)
+ end, hukairz = me._elt.query_duration(GS.FORMAT_TIME)
+ return t, end
+ except GS.QueryError:
+ return None, None
+
+ def __enter__(me):
+ "Enter context: attach progress meter display."
+
+ ## If we're not showing pointless frippery, don't bother at all.
+ if not STATUS.eyecandyp:
+ return
+
+ ## Update regularly. The pipeline runs asynchronously.
+ me._id = G.timeout_add(200, me._update)
+
+ def __exit__(me, ty, val, tb):
+ "Leave context: remove display and report completion or failure."
+
+ ## If we're not showing pointless frippery, there's nothing to remove.
+ if STATUS.eyecandyp:
+ G.source_remove(me._id)
+
+ ## Report completion anyway.
+ me.done(ty is None)
+
+ ## As you were.
+ return False
+
+class AudioIdentifier (object):
+ """
+ Analyses and identifies an audio file.
+
+ Important properties are:
+
+ cap A capabilities structure describing the audio file data. The most
+ interesting thing in here is probably its name, which is a MIME
+ type describing the data.
+
+ dcap A capabilities structure describing the decoded audio data. This
+ is of interest during conversion.
+
+ tags A dictionary containing metadata tags from the file. These are in
+ GStreamer's encoding-independent format.
+
+ bitrate An approximation to the stream's bitrate, in kilobits per second.
+ This might be slow to work out for some files so it's computed on
+ demand.
+ """
+
+ def __init__(me, file, mime):
+ "Initialize the object suitably for identifying FILE."
+
+ ## Make some initial GStreamer objects. We'll want the pipeline later if
+ ## we need to analyse a poorly tagged MP3 stream, so save it away.
+ me._pipe = GS.Pipeline()
+ me._file = file
+ bus = me._pipe.get_bus()
+ bus.add_signal_watch()
+ loop = G.MainLoop()
+
+ ## The basic recognition kit is based around `decodebin'. We must keep
+ ## it happy by giving it sinks for the streams it's found, which it
+ ## announces asynchronously.
+ source = make_element('filesrc', 'file', location = file)
+ decoder = make_element('decodebin', 'decode')
+ sink = make_element('fakesink')
+ def decoder_pad_arrived(elt, pad):
+ if pad.get_caps()[0].get_name().startswith('audio/'):
+ elt.link_pads(pad.get_name(), sink, 'sink')
+ dpaid = decoder.connect('pad-added', decoder_pad_arrived)
+ me._pipe.add(source, decoder, sink)
+ GS.element_link_many(source, decoder)
+
+ ## Arrange to collect tags from the pipeline's bus as they're reported.
+ ## If we reuse the pipeline later, we'll want different bus-message
+ ## handling, so make sure we can take the signal handler away.
+ tags = {}
+ fail = []
+ def bus_message(bus, msg):
+ if msg.type == GS.MESSAGE_ERROR:
+ fail[:] = (ValueError, msg.structure['debug'], None)
+ loop.quit()
+ elif msg.type == GS.MESSAGE_STATE_CHANGED:
+ if msg.structure['new-state'] == GS.STATE_PAUSED and \
+ msg.src == me._pipe:
+ loop.quit()
+ elif msg.type == GS.MESSAGE_TAG:
+ tags.update(msg.structure)
+ bmid = bus.connect('message', bus_message)
+
+ ## We want to identify the kind of stream this is. (Hmm. The MIME type
+ ## recognizer has already done this work, but GStreamer is probably more
+ ## reliable.) The `decodebin' has a `typefind' element inside which will
+ ## announce the identified media type. All we need to do is find it and
+ ## attach a signal handler. (Note that the handler might be run in the
+ ## thread context of the pipeline element, but Python's GIL will keep
+ ## things from being too awful.)
+ me.cap = None
+ me.dcap = None
+ for e in decoder.elements():
+ if e.get_factory().get_name() == 'typefind':
+ tfelt = e
+ break
+ else:
+ assert False, 'failed to find typefind element'
+
+ ## Crank up most of the heavy machinery. The message handler will stop
+ ## the loop when things seem to be sufficiently well underway.
+ me._pipe.set_state(GS.STATE_PAUSED)
+ loop.run()
+ bus.disconnect(bmid)
+ decoder.disconnect(dpaid)
+ if fail:
+ me._pipe.set_state(GS.STATE_NULL)
+ raise fail[0], fail[1], fail[2]
+
+ ## Store the collected tags.
+ me.tags = tags
+
+ ## Gather the capabilities. The `typefind' element knows the input data
+ ## type. The 'decodebin' knows the raw data type.
+ me.cap = tfelt.get_pad('src').get_negotiated_caps()[0]
+ me.mime = set([mime, me.cap.get_name()])
+ me.dcap = sink.get_pad('sink').get_negotiated_caps()[0]
+
+ ## If we found a plausible bitrate then stash it. Otherwise note that we
+ ## failed. If anybody asks then we'll work it out then.
+ if 'nominal-bitrate' in tags:
+ me._bitrate = tags['nominal-bitrate']/1000
+ elif 'bitrate' in tags and tags['bitrate'] >= 80000:
+ me._bitrate = tags['bitrate']/1000
+ else:
+ me._bitrate = None
+
+ ## The bitrate computation wants the file size. Ideally we'd want the
+ ## total size of the frames' contents, but that seems hard to dredge
+ ## out. If the framing overhead is small, this should be close enough
+ ## for our purposes.
+ me._bytes = OS.stat(file).st_size
+
+ def __del__(me):
+ "Close the pipeline down so we don't leak file descriptors."
+ me._pipe.set_state(GS.STATE_NULL)
+
+ @property
+ def bitrate(me):
+ """
+ Return the approximate bit-rate of the input file.
+
+ This might take a while if we have to work it out the hard way.
+ """
+
+ ## If we already know the answer then just return it.
+ if me._bitrate is not None:
+ return me._bitrate
+
+ ## Make up a new main loop.
+ loop = G.MainLoop()
+
+ ## Watch for bus messages. We'll stop when we reach the end of the
+ ## stream: then we'll have a clear idea of how long the track was.
+ fail = []
+ def bus_message(bus, msg):
+ if msg.type == GS.MESSAGE_ERROR:
+ fail[:] = (ValueError, msg.structure['debug'], None)
+ loop.quit()
+ elif msg.type == GS.MESSAGE_EOS:
+ loop.quit()
+ bus = me._pipe.get_bus()
+ bmid = bus.connect('message', bus_message)
+
+ ## Get everything moving, and keep the user amused while we work.
+ me._pipe.set_state(GS.STATE_PLAYING)
+ with GStreamerProgressEyecandy(filestatus(file, 'measure bitrate') %
+ me._pipe,
+ silentp = True):
+ loop.run()
+ bus.disconnect(bmid)
+ if fail:
+ me._pipe.set_state(GS.STATE_NULL)
+ raise fail[0], fail[1], fail[2]
+
+ ## Now we should be able to find out our position accurately and work out
+ ## a bitrate. Cache it in case anybody asks again.
+ t, hukairz = me._pipe.query_position(GS.FORMAT_TIME)
+ me._bitrate = int(8*me._bytes*1e6/t)
+
+ ## Done.
+ return me._bitrate
+
+class AudioFormat (BaseFormat):
+ """
+ An AudioFormat is a kind of Format specialized for audio files.
+
+ Format checks are done on an AudioIdentifier object.
+ """
+
+ PROPS = prop('bitrate', Num)
+
+ ## libmagic reports `application/ogg' for Ogg Vorbis files. We've switched
+ ## to GIO now, which reports either `audio/ogg' or `audio/x-vorbis+ogg'
+ ## depending on how thorough it's trying to be. Still, it doesn't do any
+ ## harm here; the main risk is picking up Ogg Theora files by accident, and
+ ## we'll probably be able to extract the audio from them anyway.
+ CATEGORY = FileCategory('audio', ['audio/*', 'application/ogg'],
+ AudioIdentifier)
+
+ def __init__(me, bitrate = None):
+ "Construct an object, requiring an approximate bitrate."
+ me.bitrate = bitrate
+
+ def check(me, id):
+ """
+ Return whether the AudioIdentifier ID is suitable for our purposes.
+
+ Subclasses can either override this method or provide a property
+ `MIMETYPES', which is a list (other thing that implements `__contains__')
+ of GStreamer MIME types matching this format.
+ """
+ return id.mime & me.MIMETYPES and \
+ (me.bitrate is None or id.bitrate <= me.bitrate * sqrt(2))
+
+ def encoder(me):
+ """
+ Constructs a GStreamer element to encode audio input.
+
+ Subclasses can either override this method (or replace `encode'
+ entirely), or provide a method `encoder_chain' which returns a list of
+ elements to be linked together in sequence. The first element in the
+ chain must have a pad named `sink' and the last must have a pad named
+ `src'.
+ """
+ elts = me.encoder_chain()
+ bin = GS.Bin()
+ bin.add(*elts)
+ GS.element_link_many(*elts)
+ bin.add_pad(GS.GhostPad('sink', elts[0].get_pad('sink')))
+ bin.add_pad(GS.GhostPad('src', elts[-1].get_pad('src')))
+ return bin
+
+ def convert(me, master, id, target):
+ """
+ Encode audio from MASTER, already identified as ID, writing it to TARGET.
+
+ See `encoder' for subclasses' responsibilities.
+ """
+
+ ## Construct the necessary equipment.
+ pipe = GS.Pipeline()
+ bus = pipe.get_bus()
+ bus.add_signal_watch()
+ loop = G.MainLoop()
+
+ ## Make sure that there isn't anything in the way of our output. We're
+ ## going to write to a scratch file so that we don't get confused by
+ ## half-written rubbish left by a crashed program.
+ new = target + '.new'
+ try:
+ OS.unlink(new)
+ except OSError, err:
+ if err.errno != E.ENOENT:
+ raise
+
+ ## Piece together our pipeline. The annoying part is that the
+ ## `decodebin' doesn't have any source pads yet, so our chain is in two
+ ## halves for now.
+ source = make_element('filesrc', 'source', location = master)
+ decoder = make_element('decodebin', 'decode')
+ convert = make_element('audioconvert', 'convert')
+ encoder = me.encoder()
+ sink = make_element('filesink', 'sink', location = new)
+ pipe.add(source, decoder, convert, encoder, sink)
+ GS.element_link_many(source, decoder)
+ GS.element_link_many(convert, encoder, sink)
+
+ ## Some decoders (e.g., the AC3 decoder) include channel-position
+ ## indicators in their output caps. The Vorbis encoder interferes with
+ ## this, and you end up with a beautifully encoded mono signal from a
+ ## stereo source. From a quick butchers at the `vorbisenc' source, I
+ ## /think/ that this is only a problem with stereo signals: mono signals
+ ## are mono already, and `vorbisenc' accepts channel positions if there
+ ## are more than two channels.
+ ##
+ ## So we have this bodge. We already collected the decoded audio caps
+ ## during identification. So if we see 2-channel audio with channel
+ ## positions, we strip the positions off forcibly by adding a filter.
+ if id.dcap.get_name().startswith('audio/x-raw-') and \
+ id.dcap.has_field('channels') and \
+ id.dcap['channels'] == 2 and \
+ id.dcap.has_field('channel-positions'):
+ dcap = GS.Caps()
+ c = id.dcap.copy()
+ c.remove_field('channel-positions')
+ dcap.append(c)
+ else:
+ dcap = None
+
+ ## Hook onto the `decodebin' so we can link together the two halves of
+ ## our encoding chain. For now, we'll hope that there's only one audio
+ ## stream in there, and just throw everything else away.
+ def decoder_pad_arrived(elt, pad):
+ if pad.get_caps()[0].get_name().startswith('audio/'):
+ if dcap:
+ elt.link_pads_filtered(pad.get_name(), convert, 'sink', dcap)
+ else:
+ elt.link_pads(pad.get_name(), convert, 'sink')
+ decoder.connect('pad-added', decoder_pad_arrived)
+
+ ## Watch the bus for completion messages.
+ fail = []
+ def bus_message(bus, msg):
+ if msg.type == GS.MESSAGE_ERROR:
+ fail[:] = (ValueError, msg.structure['debug'], None)
+ loop.quit()
+ elif msg.type == GS.MESSAGE_EOS:
+ loop.quit()
+ bmid = bus.connect('message', bus_message)
+
+ ## Get everything ready and let it go.
+ pipe.set_state(GS.STATE_PLAYING)
+ with GStreamerProgressEyecandy(filestatus(master,
+ 'convert to %s' % me.NAME),
+ pipe):
+ loop.run()
+ pipe.set_state(GS.STATE_NULL)
+ if fail:
+ raise fail[0], fail[1], fail[2]
+
+ ## Fix up the output file if we have to.
+ me.fixup(new)
+
+ ## We're done.
+ OS.rename(new, target)
+
+class OggVorbisFormat (AudioFormat):
+ "AudioFormat object for Ogg Vorbis."
+
+ ## From http://en.wikipedia.org/wiki/Vorbis
+ QMAP = [(-1, 45), ( 0, 64), ( 1, 80), ( 2, 96),
+ ( 3, 112), ( 4, 128), ( 5, 160), ( 6, 192),
+ ( 7, 224), ( 8, 256), ( 9, 320), (10, 500)]
+
+ NAME = 'Ogg Vorbis'
+ MIMETYPES = set(['application/ogg', 'audio/x-vorbis', 'audio/ogg',
+ 'audio/x-vorbis+ogg'])
+ EXT = 'ogg'
+
+ def encoder_chain(me):
+ for q, br in me.QMAP:
+ if br >= me.bitrate:
+ break
+ else:
+ raise ValueError, 'no suitable quality setting found'
+ return [make_element('vorbisenc',
+ quality = q/10.0),
+ make_element('oggmux')]
+
+defformat('ogg-vorbis', OggVorbisFormat)
+
+class MP3Format (AudioFormat):
+ "AudioFormat object for MP3."
+
+ NAME = 'MP3'
+ MIMETYPES = set(['audio/mpeg'])
+ EXT = 'mp3'
+
+ def encoder_chain(me):
+ return [make_element('lame',
+ vbr_mean_bitrate = me.bitrate,
+ vbr = 4),
+ make_element('xingmux'),
+ make_element('id3v2mux')]
+
+ def fixup(me, path):
+ """
+ Fix up MP3 files.
+
+ GStreamer produces ID3v2 tags, but not ID3v1. This seems unnecessarily
+ unkind to stupid players.
+ """
+ tag = E3.Tag()
+ tag.link(path)
+ tag.setTextEncoding(E3.UTF_8_ENCODING)
+ try:
+ tag.update(E3.ID3_V1_1)
+ except (UnicodeEncodeError, E3.tag.GenreException):
+ pass
+
+defformat('mp3', MP3Format)
+
+###--------------------------------------------------------------------------
+### Image handling, based on the Python Imaging Library.
+
+class ImageIdentifier (object):
+ """
+ Analyses and identifies an image file.
+
+ Simply leaves an Image object in the `img' property which can be inspected.
+ """
+
+ def __init__(me, file, mime):
+
+ ## Get PIL to open the file. It will magically work out what kind of
+ ## file it is.
+ try:
+ me.img = I.open(file)
+ except IOError, exc:
+
+ ## Unhelpful thing to raise on identification failure. We can
+ ## distinguish this from an actual I/O error because it doesn't have an
+ ## `errno'.
+ if exc.errno is None:
+ raise IdentificationFailure
+ raise
+
+ me.mime = set([mime])
+
+class ImageFormat (BaseFormat):
+ """
+ An ImageFormat is a kind of Format specialized for image files.
+
+ Subclasses don't need to provide anything other than the properties
+ required by all concrete Format subclasses. However, there is a
+ requirement that the `NAME' property match PIL's `format' name for the
+ format.
+ """
+
+ PROPS = prop('size', Num)
+ CATEGORY = FileCategory('image', ['image/*'], ImageIdentifier)
+
+ def __init__(me, size = None, **kw):
+ """
+ Initialize an ImageFormat object.
+
+ Additional keywords are used when encoding, and may be recognized by
+ enhanced `check' methods in subclasses.
+ """
+ me._size = size
+ me._props = kw
+
+ def check(me, id):
+ "Check whether the ImageIdentifier ID matches our requirements."
+ return id.img.format == me.NAME and \
+ (me._size is None or
+ (id.img.size[0] <= me._size and
+ id.img.size[1] <= me._size))
+
+ def convert(me, master, id, target):
+ "Encode the file MASTER, identified as ID, writing the result to TARGET."
+
+ ## Write to a scratch file.
+ new = target + '.new'
+
+ ## The ImageIdentifier already contains a copy of the open file. It
+ ## would be wasteful not to use it.
+ img = id.img
+ STATUS.set(filestatus(master, 'convert to %s' % me.NAME))
+
+ ## If there's a stated maximum size then scale the image down to match.
+ ## But thumbnailing clobbers the original, so take a copy.
+ if me._size is not None and \
+ (img.size[0] > me._size or img.size[1] > me._size):
+ img = img.copy()
+ img.thumbnail((me._size, me._size), I.ANTIALIAS)
+
+ ## Write the output image.
+ img.save(new, me.NAME, **me._props)
+
+ ## Fix it up if necessary.
+ me.fixup(new)
+
+ ## We're done.
+ OS.rename(new, target)
+ STATUS.commit()
+
+class JPEGFormat (ImageFormat):
+ """
+ Image format for JPEG (actually JFIF) files.
+
+ Interesting properties to set:
+
+ optimize
+ If present, take a second pass to select optimal encoder settings.
+
+ progression
+ If present, make a progressive file.
+
+ quality Integer from 1--100 (worst to best); default is 75.
+ """
+ EXT = 'jpg'
+ NAME = 'JPEG'
+ PROPS = prop('optimize', None) \
+ | prop('progressive', None, 'progression') \
+ | prop('quality', Num)
+
+defformat('jpeg', JPEGFormat)
+
+class PNGFormat (ImageFormat):
+ """
+ Image format for PNG files.
+
+ Interesting properties:
+
+ optimize
+ If present, make a special effort to minimize the output file.
+ """
+ EXT = 'png'
+ NAME = 'PNG'
+ PROPS = prop('optimize', None)
+
+defformat('png', PNGFormat)
+
+class BMPFormat (ImageFormat):
+ """
+ Image format for Windows BMP files, as used by RockBox.
+
+ No additional properties.
+ """
+ NAME = 'BMP'
+ EXT = 'bmp'
+
+defformat('bmp', BMPFormat)
+
+###--------------------------------------------------------------------------
+### The directory grobbler.
+
+class Grobbler (object):
+ """
+ The directory grobbler copies a directory tree, converting files.
+ """
+
+ def __init__(me, policies, noact = False):
+ """
+ Create a new Grobbler, working with the given POLICIES.
+ """
+ me._pmap = {}
+ me._noact = noact
+ for p in policies:
+ me._pmap.setdefault(p.cat, []).append(p)
+ me._dirs = []
+
+ def _grobble_file(me, master, targetdir, cohorts):
+ """
+ Convert MASTER, writing the result to TARGETDIR.
+
+ The COHORTS are actually (CAT, ID, COHORT) triples, where a COHORT is a
+ list of (FILENAME, ID) pairs.
+
+ Since this function might convert the MASTER file, the caller doesn't
+ know the name of the output files, so we return then as a list.
+ """
+
+ done = set()
+ st_m = OS.stat(master)
+
+ ## Work through each category listed and apply its policy.
+ for cat, id, cohort in cohorts:
+
+ ## Go through the category's policies and see if any match. If we fail
+ ## here, see if there are more categories to try.
+ for pol in me._pmap[cat]:
+ acts = pol.actions(master, targetdir, id, cohort)
+ if acts: break
+ else:
+ continue
+
+ ## Work through the targets one by one.
+ for a in acts:
+ done.add(a.target)
+
+ ## Find out whether the target file already exists and is up-to-date
+ ## with respect to the master. (Caution here with low-resolution
+ ## timestamps.) If it's OK, then just move on.
+ try:
+ st_t = OS.stat(a.target)
+ if st_m.st_mtime < st_t.st_mtime or \
+ (st_m.st_ino, st_m.st_dev) == (st_t.st_ino, st_t.st_dev):
+ continue
+ except OSError, err:
+ if err.errno not in (E.ENOENT, E.ENOTDIR):
+ raise
+
+ ## We have real work to do. If there's a current status message,
+ ## it's the containing directory so flush it so that people know
+ ## where we are.
+ STATUS.commit()
+
+ ## Remove the target. (A hardlink will fail if the target already
+ ## exists.)
+ if not me._noact:
+ try:
+ OS.unlink(a.target)
+ except OSError, err:
+ if err.errno not in (E.ENOENT, E.ENOTDIR):
+ raise
+
+ ## Do whatever it is we decided to do.
+ if me._noact:
+ STATUS.commit(filestatus(master, a))
+ else:
+ a.perform()
+
+ ## We're done. Return the names of the targets.
+ return list(done)
+
+ @contextmanager
+ def _wrap(me, masterfile):
+ """
+ Handle exceptions found while trying to convert a particular file or
+ directory.
+ """
+
+ try:
+ yield masterfile
+
+ ## Something bad happened. Report the error, but continue. (This list
+ ## of exceptions needs a lot of work.)
+ except (IOError, OSError), exc:
+ STATUS.clear()
+ STATUS.commit(filestatus(masterfile, 'failed (%s)' % exc))
+ me._broken.append((masterfile, exc))
+
+ def _grobble_dir(me, master, target):
+ """
+ Recursively convert files in MASTER, writing them to TARGET.
+ """
+
+ ## Make sure the TARGET exists and is a directory. It's a fundamental
+ ## assumption of this program that the entire TARGET tree is disposable,
+ ## so if something exists but isn't a directory, we should kill it.
+ if OS.path.isdir(target):
+ pass
+ else:
+ if OS.path.exists(target):
+ STATUS.commit(filestatus(target, 'clear nondirectory'))
+ if not me._noact:
+ OS.unlink(target)
+ STATUS.commit(filestatus(target, 'create directory'))
+ if not me._noact:
+ OS.mkdir(target)
+
+ ## Keep a list of things in the target. As we convert files, we'll check
+ ## them off. Anything left over is rubbish and needs to be deleted.
+ checklist = {}
+ try:
+ for i in OS.listdir(target):
+ checklist[i] = False
+ except OSError, err:
+ if err.errno not in (E.ENOENT, E.ENOTDIR):
+ raise
+
+ ## Keep track of the files in each category.
+ catmap = {}
+ todo = []
+ done = []
+
+ ## Work through the master files.
+ for f in sorted(OS.listdir(master)):
+
+ ## If the killswitch has been pulled then stop. The whole idea is that
+ ## we want to cause a clean shutdown if possible, so we don't want to
+ ## do it in the middle of encoding because the encoding effort will
+ ## have been wasted. This is the only place we need to check. If
+ ## we've exited the loop, then clearing old files will probably be
+ ## fast, and we'll either end up here when the recursive call returns
+ ## or we'll be in the same boat as before, clearing old files, only up
+ ## a level. If worst comes to worst, we'll be killed forcibly
+ ## somewhere inside `SH.rmtree', and that can continue where it left
+ ## off.
+ if KILLSWITCH.is_set():
+ return
+
+ ## Do something with the file.
+ with me._wrap(OS.path.join(master, f)) as masterfile:
+
+ ## If it's a directory then grobble it recursively. Keep the user
+ ## amused by telling him where we are in the tree.
+ if OS.path.isdir(masterfile):
+ me._dirs.append(f)
+ STATUS.set('/'.join(me._dirs))
+ try:
+ done += me._grobble_dir(masterfile, OS.path.join(target, f))
+ finally:
+ me._dirs.pop()
+ STATUS.set('/'.join(me._dirs))
+
+ ## Otherwise it's a file. Work out what kind, and stash it under
+ ## the appropriate categories. Later, we'll apply policy to the
+ ## files, by category, and work out what to do with them all.
+ else:
+ gf = GIO.File(masterfile)
+ mime = gf.query_info('standard::content-type').get_content_type()
+ cats = []
+ for cat in me._pmap.iterkeys():
+ id = cat.identify(masterfile, mime)
+ if id is None: continue
+ catmap.setdefault(cat, []).append((masterfile, id))
+ cats.append((cat, id))
+ if not cats:
+ catmap.setdefault(None, []).append((masterfile, id))
+ todo.append((masterfile, cats))
+
+ ## Work through the categorized files to see what actions to do for
+ ## them.
+ for masterfile, cats in todo:
+ with me._wrap(masterfile):
+ done += me._grobble_file(masterfile, target,
+ [(cat, id, catmap[cat])
+ for cat, id in cats])
+
+ ## Check the results off the list so that we don't clear it later.
+ for f in done:
+ checklist[OS.path.basename(f)] = True
+
+ ## Maybe there's stuff in the target which isn't accounted for. Delete
+ ## it: either the master has changed, or the policy for this target has
+ ## changed. Either way, the old files aren't wanted.
+ for f in checklist:
+ if not checklist[f]:
+ STATUS.commit(filestatus(f, 'clear bogus file'))
+ if not me._noact:
+ bogus = OS.path.join(target, f)
+ try:
+ if OS.path.isdir(bogus):
+ SH.rmtree(bogus)
+ else:
+ OS.unlink(bogus)
+ except OSError, err:
+ if err.errno != E.ENOENT:
+ raise
+
+ ## Return the target name, so that it can be checked off.
+ return [target]
+
+ def grobble(me, master, target):
+ """
+ Convert MASTER, writing a directory tree TARGET.
+
+ Returns a list of files which couldn't be converted.
+ """
+ try:
+ me._broken = []
+ me._grobble_dir(master, target)
+ return me._broken
+ finally:
+ del me._broken
+
+###--------------------------------------------------------------------------
+### Remaining parsing machinery.
+
+Type = K('type') - Name - D('{') - R(Policy) - D('}')
+def build_type(s, l, t):
+ try:
+ cat = CATEGORYMAP[t[0]]
+ except KeyError:
+ raise P.ParseException(s, loc, "Unknown category `%s'" % t[0])
+ pols = t[1]
+ if len(pols) == 1: pol = pols[0]
+ else: pol = AndPolicy(pols)
+ pol.setcategory(cat)
+ return pol
+Type.setParseAction(build_type)
+
+TARGETS = []
+class TargetJob (object):
+ def __init__(me, targetdir, policies):
+ me.targetdir = targetdir
+ me.policies = policies
+ def perform(me):
+ TARGETS.append(me)
+
+Target = K('target') - String - D('{') - R(Type) - D('}')
+def build_target(s, l, t):
+ return TargetJob(t[0], t[1])
+Target.setParseAction(build_target)
+
+VARS = { 'master': None }
+class VarsJob (object):
+ def __init__(me, vars):
+ me.vars = vars
+ def perform(me):
+ for k, v in me.vars:
+ VARS[k] = v
+
+Var = prop('master', String)
+Vars = K('vars') - D('{') - R(Var) - D('}')
+def build_vars(s, l, t):
+ return VarsJob(t[0])
+Vars.setParseAction(build_vars)
+
+TopLevel = Vars | Target
+Config = R(TopLevel)
+Config.ignore(P.pythonStyleComment)
+
+###--------------------------------------------------------------------------
+### Command-line interface.
+
+QUIS = OS.path.basename(SYS.argv[0])
+
+def moan(msg):
+ "Report a warning message to the user."
+ SYS.stderr.write('%s: %s\n' % (QUIS, msg))
+
+def die(msg):
+ "Report a fatal error message to the user."
+ moan(msg)
+ SYS.exit(1)
+
+def parse_opts(args):
+ """
+ Parse command-line arguments in ARGS.
+
+ Returns a Grobbler object and the MASTER and TARGET directories to be
+ grobbled.
+ """
+
+ ## Build the option parser object.
+ op = OP.OptionParser(prog = QUIS, version = VERSION,
+ usage = '%prog [-t TIMEOUT] CONFIG',
+ description = """\
+Convert a directory tree of files according to the configuration file
+CONFIG.
+""")
+
+ ## Timeout handling.
+ def cb_time(opt, ostr, arg, op):
+ m = RX.match(r'\s*(\d+)\s*([dhms]?)\s*', arg)
+ if not m:
+ raise OP.OptionValueerror, 'bad time value `%s\'' % arg
+ t, u = m.groups()
+ t = int(t) * { '': 1, 's': 1, 'm': 60, 'h': 3600, 'd': 86400 }[u]
+ setattr(op.values, opt.dest, t)
+ op.add_option('-t', '--timeout', type = 'string', metavar = 'SECS',
+ dest = 'timeout',
+ help = 'stop processing nicely after SECS',
+ action = 'callback', callback = cb_time)
+ op.add_option('-T', '--timeout-nasty', type = 'string', metavar = 'SECS',
+ dest = 'timeout_nasty',
+ help = 'stop processing unpleasantly after further SECS',
+ action = 'callback', callback = cb_time)
+
+ ## Other options.
+ op.add_option('-i', '--interactive', action = 'store_true', dest = 'tty',
+ help = 'provide progress information')
+ op.add_option('-n', '--no-act', action = 'store_true', dest = 'noact',
+ help = 'don\'t actually modify the filesystem')
+
+ ## Ready to rock.
+ op.set_defaults(formats = [], noact = False,
+ timeout = None, timeout_nasty = 300)
+ opts, args = op.parse_args(args)
+
+ ## Check that we got the non-option arguments that we want.
+ if len(args) != 1:
+ op.error('wrong number of arguments')
+
+ ## Act on the options.
+ if opts.tty:
+ STATUS.eyecandyp = True
+ if opts.timeout is not None:
+ to = TH.Thread(target = timeout,
+ args = (opts.timeout, opts.timeout_nasty))
+ to.daemon = True
+ to.start()
+
+ ## Parse the configuration file.
+ with open(args[0]) as conf:
+ jobs, = Config.parseFile(conf, True)
+ for j in jobs:
+ j.perform()
+
+ return opts
+
+if __name__ == '__main__':
+ opts = parse_opts(SYS.argv[1:])
+ if 'master' not in VARS:
+ die("no master directory set")
+ broken = []
+ for t in TARGETS:
+ g = Grobbler(t.policies, opts.noact)
+ b = g.grobble(VARS['master'], t.targetdir)
+ broken += b
+ if broken:
+ moan('failed to convert some files:')
+ for file, exc in broken:
+ moan('%s: %s' % (file, exc))
+ SYS.exit(1)
+
+ ## This is basically a successful completion: we did what we were asked to
+ ## do. It seems polite to report a message, though.
+ ##
+ ## Why don't we have a nonzero exit status? The idea would be that a
+ ## calling script would be interested that we used up all of our time, and
+ ## not attempt to convert some other directory as well. But that doesn't
+ ## quite work. Such a script would need to account correctly for time we
+ ## had spent even if we complete successfully. And if the script is having
+ ## to watch the clock itself, it can do that without our help here.
+ if KILLSWITCH.is_set():
+ moan('killed by timeout')
+
+###----- That's all, folks --------------------------------------------------
~mdw / autoys / commitdiff