X-Git-Url: https://git.distorted.org.uk/~mdw/chopwood/blobdiff_plain/a2916c0635fec5b45ad742904db9f5769b48f53d..e3295bed428adbf5e1863f2d9395adac33dbb071:/agpl.py diff --git a/agpl.py b/agpl.py index b89330c..a35ea3f 100644 --- a/agpl.py +++ b/agpl.py @@ -24,31 +24,61 @@ ### . import contextlib as CTX +import grp as GR import os as OS +import pwd as PW import shlex as SL import shutil as SH import subprocess as SUB import sys as SYS import tarfile as TAR import tempfile as TF +import time as T + +from cStringIO import StringIO from auto import PACKAGE, VERSION import util as U +###-------------------------------------------------------------------------- +### Initial utilities. + @CTX.contextmanager def tempdir(): + """ + Context manager: create and return the name of a temporary directory. + + The directory will be deleted automatically on exit from the body. + """ d = TF.mkdtemp() try: yield d finally: SH.rmtree(d, ignore_errors = True) +###-------------------------------------------------------------------------- +### Determining which files to include. + def dirs_to_dump(): + """ + Return a list of directories containing used Python modules. + + Directories under `/usr/' but outside `/usr/local/' are excluded, since + they are assumed to be covered by the AGPL exception for parts of the + operating system. + """ + + ## Collect a set of directories. dirs = set() + + ## Work through the list of known modules, adding them to the list. for m in SYS.modules.itervalues(): try: f = m.__file__ except AttributeError: continue d = OS.path.realpath(OS.path.dirname(f)) if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue dirs.add(d) + + ## Now go through the directories again, and remove any which are wholly + ## included within other entries. dirs = sorted(dirs) last = '!' dump = [] @@ -56,57 +86,192 @@ def dirs_to_dump(): if d.startswith(last): continue dump.append(d) last = d + + ## We're done: return the filtered list. return dump +### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE +### is a function of one argument -- a directory name -- which returns true +### if the LISTERS should be used to enumerate that directory. The LISTERS +### are a list of functions of one argument -- again, the directory name -- +### which should return an iterable of files within that directory, relative +### to its top-level. Lister functions should not return the root directory, +### because it should obviously only be included once. Instead, the root is +### handled separately by `dump_dir'. + def exists_subdir(subdir): + """ + Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR. + + This is mainly useful for detecting working trees subject to version + control. + """ return lambda dir: OS.path.isdir(OS.path.join(dir, subdir)) def filez(cmd): + """ + Lister for `DUMPERS': generate the null-terminated items output by CMD. + + Run CMD, a string containing words with shell-like quoting (expected to be + a literal in the code, so security concerns don't arise) in the directory + of interest, yielding the invidual null-terminated strings which the + command writes to its standard output. + """ def _(dir): + + ## Start the command, kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir) + + ## Collect and return the null-terminated items. Strip off any leading + ## `./' and exclude the root directory because that gets handled + ## separately. left = '' while True: + + ## Read a new bufferload of stuff. If there's nothing left then we're + ## done. buf = kid.stdout.read(16384) if not buf: break + + ## Tack whatever was left over from last time on the front, and carve + ## into null-terminated pieces. buf = left + buf i = 0 while True: z = buf.find('\0', i) if z < 0: break f = buf[i:z] + i = z + 1 + if f.rstrip('/') == '.': continue if f.startswith('./'): f = f[2:] yield f - i = z + 1 + + ## Whatever's left over will be dealt with next time through. left = buf[i:] + + ## Make sure the command actually completed successfully. + if kid.wait(): + rc = kid.returncode + raise U.ExpectedError, \ + (500, "lister command `%s' failed (%s) in `%s'" % ( + cmd, + (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc, + dir)) + + ## If there's trailing junk left over then we should complain. if left: raise U.ExpectedError, \ (500, "trailing junk from `%s' in `%s'" % (cmd, dir)) + + ## Return the listing function. return _ +## The list of predicates and listers. DUMPERS = [ (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'), filez('find .git -print0')]), (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])] -def dump_dir(dir, tf, root): +###-------------------------------------------------------------------------- +### Actually dumping files. + +def dump_dir(name, dir, dirmap, tf, root): + """ + Add the contents of directory DIR to the tarfile TF, under the given NAME. + + The ROOT names the toplevel of the tarball (we're not in the business of + making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME) + pairs being dumped, used for fixing up symbolic links between directories. + """ + + ## Find an appropriate `DUMPERS' list entry. for test, listers in DUMPERS: if test(dir): break else: raise U.ExpectedError, (500, "no dumper for `%s'" % dir) + + ## Write a tarfile entry for the toplevel. + tf.add(dir, OS.path.join(root, name), recursive = False) + + ## Work through all of the listers. for lister in listers: - base = OS.path.basename(dir) + + ## Work through each file. for file in lister(dir): - tf.add(OS.path.join(dir, file), OS.path.join(root, base, file), - recursive = False) + with U.Escape() as skip: + full = OS.path.join(dir, file) + tarname = OS.path.join(root, name, file) + + ## Check for symbolic links. If we find one that points to another + ## directory we're going to dump separately then fiddle it so that it + ## works in the directory tree we're going to make. + if OS.path.islink(full): + dest = OS.path.realpath(full) + for d, local in dirmap: + if dest.startswith(d): + fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]), + OS.path.join('/', name, + OS.path.dirname(file))) + st = OS.stat(full) + ti = tf.gettarinfo(full, tarname) + ti.linkname = fix + tf.addfile(ti) + skip() + + ## Nothing special, so just dump the file. Or whatever it is. + tf.add(full, tarname, recursive = False) def source(out): + """ + Write a tarball for the program's source code to OUT. + + This function automatically dumps all of the program's dependencies except + for those covered by the operating-system exemption. + """ + + ## Make a tarfile writer. There's an annoying incompatibility to bodge + ## around. if SYS.version_info >= (2, 6): tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT) else: tf = TAR.open(fileobj = out, mode = 'w|gz') tf.posix = True - for d in dirs_to_dump(): - dump_dir(d, tf, '%s-%s' % (PACKAGE, VERSION)) + + ## First of all, find out what needs to be dumped, and assign names to all + ## of the various directories. + root = '%s-%s' % (PACKAGE, VERSION) + seen = set() + dirmap = [] + festout = StringIO() + for dir in dirs_to_dump(): + dir = dir.rstrip('/') + base = OS.path.basename(dir) + if base not in seen: + name = base + else: + for i in I.count(): + name = '%s.%d' % (base, i) + if name not in seen: break + dirmap.append((dir + '/', name)) + festout.write('%s = %s\n' % (name, dir)) + + ## Write a map of where things were in the filesystem. This may help a + ## user figure out how to deploy the thing. + fest = festout.getvalue() + ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST')) + ti.size = len(fest) + ti.mtime = T.time() + ti.mode = 0664 + ti.type = TAR.REGTYPE + uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name + gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name + tf.addfile(ti, fileobj = StringIO(fest)) + + ## Now actually dump all of the individual directories. + for dir, name in dirmap: + dump_dir(name, dir, dirmap, tf, root) + + ## We're done. tf.close() ###----- That's all, folks --------------------------------------------------