From a98c9dba94d44ada3a7e7f3b4ce8b6df6911a2d2 Mon Sep 17 00:00:00 2001 From: Mark Wooding Date: Wed, 11 Sep 2019 17:52:48 +0100 Subject: [PATCH] bin/chroot-maint: Program for maintaining chroots. --- Makefile | 256 ++--- bin/chroot-maint | 2809 +++++++++++++++++++++++++++++++++++++++++++++++ bin/install-cross-tools | 2 +- bin/mkbuildchroot | 14 +- bin/mkchrootconf | 1 + 5 files changed, 2921 insertions(+), 161 deletions(-) create mode 100755 bin/chroot-maint diff --git a/Makefile b/Makefile index dbd1714..6870ddc 100644 --- a/Makefile +++ b/Makefile @@ -62,15 +62,14 @@ APTSRC = etc/aptsrc.conf $(wildcard etc/aptsrc.local.conf) ## APT configuration fragment names. These will be linked into ## `/etc/apt/apt.conf.d' in each chroot. To put a fragment f in a surprising -## place, set $($f_APTCONFSRC). -CONFIG_VARS += APTCONF -APTCONF_DIR = etc/apt-conf.d -APTCONF = $(notdir $(wildcard $(APTCONF_DIR)/[0-9]*[!~])) +## place, set $(_$f_APTCONFSRC). +CONFIG_VARS += APTCONF $(foreach f,$(APTCONF),_$f_APTCONFSRC) +APTCONF = $(notdir $(wildcard etc/apt-conf.d/[0-9]*[!~])) ## Proxy setting. CONFIG_VARS += PROXY PROXY := $(shell \ - eval $$(apt-config $(foreach a,$(APTCONF), -c$(APTCONF_DIR)/$a) \ + eval $$(apt-config $(foreach a,$(APTCONF),-cetc/apt-conf.d/$a) \ shell proxy Acquire::http::proxy); \ case $${proxy+t} in (t) echo "$$proxy" ;; (*) echo nil ;; esac) @@ -90,6 +89,7 @@ CONFIG_VARS += FOREIGN_ARCHS FOREIGN_ARCHS = ## Master lists of chroots to build and maintain. +CONFIG_VARS += NATIVE_CHROOTS FOREIGN_CHROOTS NATIVE_CHROOTS = $(foreach a,$(NATIVE_ARCHS), \ $(foreach d,$(or $($a_DISTS) $(DISTS)), \ $d-$a)) @@ -97,18 +97,76 @@ FOREIGN_CHROOTS = $(foreach a,$(FOREIGN_ARCHS), \ $(foreach d,$(or $($a_DISTS) $(DISTS)), \ $d-$a)) -## Extra packages to be installed in chroots. -CONFIG_VARS += BASE_PACKAGES NATIVE_BASE_PACKAGES FOREIGN_BASE_PACKAGES -BASE_PACKAGES = ccache -BASE_PACKAGES += eatmydata fakeroot -BASE_PACKAGES += locales tzdata -BASE_PACKAGES += libfile-fcntllock-perl -NATIVE_BASE_PACKAGES = build-essential -FOREIGN_BASE_PACKAGES = +## Extra packages to be installed in chroots. `BASE_PACKAGES' are installed +## through `debootstrap'; `EXTRA_PACKAGES' are installed later, using Apt, +## which is faster in foreign chroots. +CONFIG_VARS += BASE_PACKAGES EXTRA_PACKAGES +BASE_PACKAGES = eatmydata +EXTRA_PACKAGES = build-essential +EXTRA_PACKAGES += ccache +EXTRA_PACKAGES += fakeroot +EXTRA_PACKAGES += libfile-fcntllock-perl +EXTRA_PACKAGES += locales + +## Extra packages from which to install the cross tools. +CONFIG_VARS += CROSS_PACKAGES +CROSS_PACKAGES = bash coreutils dash findutils +CROSS_PACKAGES += gzip m4 mawk sed tar xz-utils +CROSS_PACKAGES += apt ccache eatmydata fakeroot make +CROSS_PACKAGES += qemu-user-static +CROSS_PACKAGES += $(foreach a,$(FOREIGN_GNUARCHS),\ + gcc-$a g++-$a binutils-$a) + +## Native files to install in place of the foreign versions. `MULTI' here +## stands for the donor's GNU multiarch name. +CONFIG_VARS += CROSS_PATHS +CROSS_PATHS += \ + $(addprefix /usr/bin/, \ + apt apt-cache apt-config apt-get apt-key apt-mark) \ + $(addprefix /usr/lib/apt/, \ + methods/ solvers/) \ + $(addprefix /bin/, \ + cat chgrp chown cp date dd df dir echo false ln ls mkdir \ + mknod mktemp mv pwd readlink rm rmdir sleep stty sync touch \ + true uname vdir) \ + $(addprefix /usr/bin/, \ + [ arch b2sum base32 base64 basename chcon cksum comm \ + csplit cut dircolors dirname du env expand expr factor fmt \ + fold groups head hostid id install join link logname md5sum \ + mkfifo nice nl nohup nproc numfmt od paste pathchk pinky pr \ + printenv printf ptx realpath runcon seq sha1sum sha224sum \ + sha256sum sha384sum sha512sum shred shuf sort split stat \ + stdbuf sum tac tail tee test timeout tr truncate tsort tty \ + unexpand uniq unlink users wc who whoami yes) \ + /usr/lib/MULTI/coreutils/ \ + $(addprefix /lib/MULTI/, \ + libnsl.so.* libnss_*.so.*) \ + /usr/bin/gpgv \ + /usr/bin/qemu-*-static \ + $(addprefix /bin/, \ + bash dash gzip sed tar) \ + $(addprefix /usr/bin/, \ + ccache find m4 make mawk xargs xz) \ + $(addprefix /usr/lib/MULTI/, \ + libeatmydata.so* libfakeroot/) \ + $(addprefix /etc/ld.so.conf.d/, \ + MULTI.conf fakeroot*.conf) \ + $(foreach a,$(FOREIGN_GNUARCHS), \ + $(addprefix /usr/bin/$a-, \ + addr2line ar as c++filt dwp elfedit gprof ld ld.* \ + nm objcopy objdump ranlib readelf size strings \ + strip) \ + $(addprefix /usr/bin/$a-, \ + cpp gcc g++ gcov gcov-dump gcov-tool gprof \ + gcc-ar gcc-nm gcc-ranlib) \ + /usr/lib/gcc-cross/$a/) ## Local packages to be compiled and installed in chroots. Archives can be ## found in `pkg/'. +CONFIG_VARS += LOCALPKGS $(foreach p,$(LOCALPKGS),$p_DEPS) LOCALPKGS = mLib checkpath +mLib_DEPS = +checkpath_DEPS = mLib ## Which host architecture to use for foreign architectures. It turns out ## that it's best to use a Qemu with the same host bitness as the target @@ -117,7 +175,7 @@ LOCALPKGS = mLib checkpath 32BIT_QEMUHOST = $(or $(filter i386,$(NATIVE_ARCHS)),$(TOOLSARCH)) 64BIT_QEMUHOST = $(or $(filter amd64,$(NATIVE_ARCHS)),$(TOOLSARCH)) -CONFIG_VARS += $(foreach a,$(FOREIGN_ARCHS), $a_QEMUHOST) +CONFIG_VARS += $(foreach a,$(FOREIGN_ARCHS),$a_QEMUHOST) armel_QEMUHOST = $(32BIT_QEMUHOST) armhf_QEMUHOST = $(32BIT_QEMUHOST) arm64_QEMUHOST = $(64BIT_QEMUHOST) @@ -126,7 +184,7 @@ amd64_QEMUHOST = $(64BIT_QEMUHOST) ## Qemu architecture names. These tell us which Qemu binary to use for a ## particular Debian architecture. -CONFIG_VARS += $(foreach a,$(FOREIGN_ARCHS), $a_QEMUARCH) +CONFIG_VARS += $(foreach a,$(FOREIGN_ARCHS),$a_QEMUARCH) armel_QEMUARCH = arm armhf_QEMUARCH = arm arm64_QEMUARCH = aarch64 @@ -134,10 +192,10 @@ i386_QEMUARCH = i386 amd64_QEMUARCH = x86_64 ## Alias mapping for chroots. -CONFIG_VARS += $(foreach d,$(DISTS), $d_ALIASES) +CONFIG_VARS += $(foreach d,$(DISTS),$d_ALIASES) stretch_ALIASES = oldstable buster_ALIASES = stable -bullseye_ALIASE = testing +bullseye_ALIASES = testing sid_ALIASES = unstable ## Which host architecture to use for commonly used tools in foreign chroots. @@ -154,6 +212,7 @@ CONFIG_VARS += LOCAL LOCAL = local.schroot ## How to run a command as a privileged user. +CONFIG_VARS += ROOTLY ROOTLY = sudo ## Files to be copied into a chroot from the host. @@ -168,6 +227,12 @@ CONFIG_VARS += ALL_ARCHS ALL_CHROOTS ALL_ARCHS = $(NATIVE_ARCHS) $(FOREIGN_ARCHS) ALL_CHROOTS = $(NATIVE_CHROOTS) $(FOREIGN_CHROOTS) +## GNU names for foreign architectures. +CONFIG_VARS += FOREIGN_GNUARCHS +FOREIGN_GNUARCHS := $(foreach a,$(FOREIGN_ARCHS),\ + $(shell dpkg-architecture -A$a \ + -qDEB_TARGET_GNU_TYPE)) + ###-------------------------------------------------------------------------- ### Utilities. @@ -198,6 +263,11 @@ CLEANFILES += log/*.log SILENCE_LVM = \ LVM_SUPPRESS_FD_WARNINGS=1; export LVM_SUPPRESS_FD_WARNINGS +## $(call definedp,VAR) +## +## Expand non-empty if and only if VAR is defined (but possibly empty). +definedp = $(filter-out undefined,$(origin $1)) + ## $(call catchrc,...$(call throwrc,CMD)...) ## ## Catch the exit status of some subpart of a complicated shell rune. @@ -351,6 +421,7 @@ $(PYMODULES): $(STATE)/lib/python/%.so: $$(call c-object,$$($$*_SOURCES)) ###-------------------------------------------------------------------------- ### Scripts. +SCRIPTS += chroot-maint SCRIPTS += mkbuildchroot SCRIPTS += mkchrootconf SCRIPTS += install-cross-tools update-cross-tools @@ -397,7 +468,7 @@ CLEANFILES += $(APT_SOURCES) APT_CONFIGS = $(addprefix $(LOCAL)/etc/apt/apt.conf.d/,$(APTCONF)) all:: $(APT_CONFIGS) $(APT_CONFIGS): $(LOCAL)/etc/apt/apt.conf.d/%: \ - $$(or $$($$*_APTCONFSRC) $$(APTCONF_DIR)/$$*) + $$(or $$(_$$*_APTCONFSRC) etc/apt-conf.d/$$*) $(V_AT)mkdir -p $(dir $@) $(call v_tag,COPY)cp $< $@.new && mv $@.new $@ clean::; rm -f $(APT_CONFIGS) @@ -437,7 +508,8 @@ check::; $(call check-symlink,ERR,/schroot,/run/schroot/mount) %print-varlist = { \ echo "\#\#\# -*-sh-*- GENERATED by distorted-chroot: do not edit"; \ - $(foreach v,$1, echo $v=\'$(call squote,$($v))\';) \ + $(foreach v,$1,$(if $(call definedp,$v),\ + echo $v=\'$(call squote,$($v))\';)) \ } schroot-config_HASH := \ $(shell $(call %print-varlist,$(CONFIG_VARS)) | \ @@ -552,139 +624,21 @@ $(foreach a,$(ALL_ARCHS),\ $(call v_tag,SYMLINK)ln -sf ../$(notdir $(patsubst %/,%,$@)) $(patsubst %/,%,$@) ###-------------------------------------------------------------------------- -### Constructing chroots. - -chroot-stamp = $(addprefix $(STATE)/stamp/chroot.,$1) -BUILD_CHROOTS = $(addprefix chroot/,$(ALL_CHROOTS)) -CHROOT_STAMPS = $(call chroot-stamp,$(ALL_CHROOTS)) -setup-chroots: $(BUILD_CHROOTS) -$(BUILD_CHROOTS): chroot/%: $(STATE)/stamp/chroot.% -.PHONY: setup-chroots $(BUILD_CHROOTS) - -$(CHROOT_STAMPS): $(STATE)/stamp/chroot.%: - $(V_AT)mkdir -p $(dir $@) log/ - $(MAKE) \ - $(STATE)/bin/mkbuildchroot $(STATE)/bin/install-cross-tools \ - $(STATE)/etc/schroot/sbuild.schroot \ - $$(call chroot-deps,$(STATE)/stamp/cross-tools.,$$*) - $(call v_tag,CHROOT)$(call v_log,setup-chroot.$*, \ - $(SILENCE_LVM); \ - $(ROOTLY) $(STATE)/bin/mkbuildchroot \ - $(call chroot-dist,$*) $(call chroot-arch,$*)) - $(V_AT)touch $@ - -UPDATE_CHROOTS = $(addprefix update/,$(ALL_CHROOTS)) -update-chroots: $(UPDATE_CHROOTS) -$(UPDATE_CHROOTS): update/%: $(STATE)/stamp/chroot.% - $(V_AT)mkdir -p log/ - $(MAKE) $(STATE)/bin/install-cross-tools - $(call v_tag,UPDATE)$(call v_log,update-chroot.$*, { \ - schroot -uroot -csource:$(LVPREFIX)$* -- \ - apt-get update && \ - schroot -uroot -csource:$(LVPREFIX)$* -- \ - apt-get -y dist-upgrade && \ - schroot -uroot -csource:$(LVPREFIX)$* -- \ - apt-get -y autoremove && \ - schroot -uroot -csource:$(LVPREFIX)$* -- \ - apt-get -y clean && \ - $(if $(filter $*,$(FOREIGN_CHROOTS)), \ - $(ROOTLY) $(STATE)/bin/install-cross-tools \ - $(call chroot-dist,$*) \ - $(call chroot-arch,$*), \ - :); \ - }) -.PHONY: update-chroots $(UPDATE_CHROOTS) - -cross-tools-stamp = $(addprefix $(STATE)/stamp/cross-tools.,$1) -CROSS_TOOLS = $(addprefix cross-tools/,$(NATIVE_CHROOTS)) -UPDATE_CROSS_TOOLS = $(addprefix update-cross-tools/,$(NATIVE_CHROOTS)) -cross-tools: $(CROSS_TOOLS) -update-cross-tools: $(UPDATE_CROSS_TOOLS) -$(CROSS_TOOLS): cross-tools/%: $(STATE)/stamp/cross-tools.% -define updcross - $(V_AT)mkdir -p log/ - $(MAKE) $(STATE)/bin/update-cross-tools - $(call v_tag,UPDCROSS)$(call v_log,update-cross-tools.$*, \ - $(STATE)/bin/update-cross-tools \ - $(call chroot-dist,$*) \ - $(call chroot-arch,$*)) - $(V_AT)touch $(call cross-tools-stamp,$*) -endef -$(call cross-tools-stamp,$(NATIVE_CHROOTS)): $(STATE)/stamp/cross-tools.%: \ - $$(call chroot-stamp,$$*) - $(V_AT)mkdir -p $(dir $@) - $(updcross) -$(UPDATE_CROSS_TOOLS): update-cross-tools/%: \ - $$(call chroot-stamp,$$*) _force - $(updcross) -.PHONY: cross-tools update-cross-tools $(CROSS_TOOLS) $(UPDATE_CROSS_TOOLS) +### Main chroot maintenance. -###-------------------------------------------------------------------------- -### Installing basic custom software. - -$(foreach p,$(LOCALPKGS), $(eval $p_VERSION := $(shell \ - set -- pkg/$p-[0-9]*.tar.gz; \ - case $$#,$$1 in \ - (1,*\**) echo "NOT-FOUND"; exit 2 ;; \ - (1,*) v=$${1#pkg/$p-}; v=$${v%.tar.gz}; echo "$$v" ;; \ - (*) echo "AMBIGUOUS"; exit 2 ;; \ - esac))) - -pkg-stamp = \ - $(foreach p,$1,$(STATE)/stamp/package.$p-$($p_VERSION).$2) -unpack-pkg-stamp = \ - $(foreach p,$1,$(STATE)/stamp/unpack.$p-$($p_VERSION)) -PACKAGE_STAMPS = \ - $(foreach a,$(ALL_ARCHS),$(call pkg-stamp,$(LOCALPKGS),$a)) -INSTALL_PACKAGES = $(addprefix install/,$(LOCALPKGS)) -install-packages: $(INSTALL_PACKAGES) -$(INSTALL_PACKAGES): install/%: \ - $$(foreach a,$$(ALL_ARCHS),$$(call pkg-stamp,$$*,$$a)) - -$(foreach p,$(LOCALPKGS),$(call unpack-pkg-stamp,$p)): \ - $(STATE)/stamp/unpack.%: pkg/%.tar.gz - $(V_AT)mkdir -p $(dir $@) $(LOCAL)/src/ - $(call v_tag,UNPACK){ \ - set -e; \ - p=$(call package-dir-name,$*); \ - v=$(call package-dir-version,$*); \ - cd $(LOCAL)/src/; \ - $(ROOTLY) rm -rf $$p-*; \ - mkdir $$p-$$v.unpack; \ - (cd $$p-$$v.unpack && tar xf $(HERE)/$<); \ - mv $$p-$$v.unpack/$$p-$$v $$p-$$v; \ - rmdir $$p-$$v.unpack/; \ - cd $(HERE); \ - touch $@; \ - } - -$(PACKAGE_STAMPS): $(STATE)/stamp/package.%: \ - $$(call unpack-pkg-stamp,$$(call package-name,$$*)) \ - $$(call chroot-stamp,$$(PRIMARY_DIST)-$$(call package-arch,$$*)) - $(V_AT)mkdir -p $(dir $@) log/ - $(call v_tag,BUILD)$(call v_log,install-package.$*, { \ - $(SILENCE_LVM); \ - schroot -uroot -c$(LVPREFIX)$(PRIMARY_DIST)-$(call package-arch,$*) -- \ - sh -exc ' \ - mount -oremount$(comma)rw /usr/local.schroot; \ - eatmydata apt-get update; \ - eatmydata apt-get -y install pkg-config; \ - p=$(call package-name,$*); \ - v=$(call package-version,$*); \ - a=$(call package-arch,$*); \ - cd /usr/local/src/$$p-$$v/; \ - rm -rf build.$$a/; \ - mkdir build.$$a/; \ - cd build.$$a/; \ - ../configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig.hidden; \ - make -j4; \ - make install; \ - mkdir -p /usr/local/lib/pkgconfig.hidden; \ - mv /usr/local/lib/pkgconfig/*.pc /usr/local/lib/pkgconfig.hidden || :' && \ - schroot -uroot -csource:$(LVPREFIX)$(PRIMARY_DIST)-$(call package-arch,$*) -- \ - ldconfig; \ - }) - $(V_AT)touch $@ +OPTS = +FRESH = create +JOBS = chroot cross-tools pkg-build + +MAINTQ_ = -q +MAINTQ_0 = -q +MAINT = +$(call v_tag,RUN)\ + PYTHONPATH=$(STATE)/lib/python $(STATE)/bin/chroot-maint \ + $(MAINTQ_$V) $(OPTS) + +maint: all check + $(MAINT) -f$(FRESH) $(JOBS) +.PHONY: maint ###-------------------------------------------------------------------------- ### Other maintenance targets. diff --git a/bin/chroot-maint b/bin/chroot-maint new file mode 100755 index 0000000..5b63037 --- /dev/null +++ b/bin/chroot-maint @@ -0,0 +1,2809 @@ +#! /usr/bin/python +### +### Create, upgrade, and maintain (native and cross-) chroots +### +### (c) 2018 Mark Wooding +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of the distorted.org.uk chroot maintenance tools. +### +### distorted-chroot is free software: you can redistribute it and/or +### modify it under the terms of the GNU General Public License as +### published by the Free Software Foundation; either version 2 of the +### License, or (at your option) any later version. +### +### distorted-chroot is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +### General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with distorted-chroot. If not, write to the Free Software +### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, +### USA. + +## still to do: +## tidy up + +import contextlib as CTX +import errno as E +import fcntl as FC +import fnmatch as FM +import glob as GLOB +import itertools as I +import optparse as OP +import os as OS +import random as R +import re as RX +import signal as SIG +import select as SEL +import stat as ST +from cStringIO import StringIO +import sys as SYS +import time as T +import traceback as TB + +import jobclient as JC + +QUIS = OS.path.basename(SYS.argv[0]) +TODAY = T.strftime("%Y-%m-%d") +NOW = T.time() + +###-------------------------------------------------------------------------- +### Random utilities. + +RC = 0 +def moan(msg): + """Print MSG to stderr as a warning.""" + if not OPT.silent: OS.write(2, "%s: %s\n" % (QUIS, msg)) +def error(msg): + """Print MSG to stderr, and remember to exit nonzero.""" + global RC + moan(msg) + RC = 2 + +class ExpectedError (Exception): + """A fatal error which shouldn't print a backtrace.""" + pass + +@CTX.contextmanager +def toplevel_handler(): + """Catch `ExpectedError's and report Unixish error messages.""" + try: yield None + except ExpectedError, err: moan(err); SYS.exit(2) + +def spew(msg): + """Print MSG to stderr as a debug trace.""" + if OPT.debug: OS.write(2, ";; %s\n" % msg) + +class Tag (object): + """Unique objects with no internal structure.""" + def __init__(me, label): me._label = label + def __str__(me): return '#<%s %s>' % (me.__class__.__name__, me._label) + def __repr__(me): return '#<%s %s>' % (me.__class__.__name__, me._label) + +class Struct (object): + def __init__(me, **kw): me.__dict__.update(kw) + +class Cleanup (object): + """ + A context manager for stacking other context managers. + + By itself, it does nothing. Attach other context managers with `enter' or + loose cleanup functions with `add'. On exit, contexts are left and + cleanups performed in reverse order. + """ + def __init__(me): + me._cleanups = [] + def __enter__(me): + return me + def __exit__(me, exty, exval, extb): + trap = False + for c in reversed(me._cleanups): + if c(exty, exval, extb): trap = True + return trap + def enter(me, ctx): + v = ctx.__enter__() + me._cleanups.append(ctx.__exit__) + return v + def add(me, func): + me._cleanups.append(lambda exty, exval, extb: func()) + +def zulu(t = None): + """Return the time T (default now) as a string.""" + return T.strftime("%Y-%m-%dT%H:%M:%SZ", T.gmtime(t)) + +R_ZULU = RX.compile(r"^(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)Z$") +def unzulu(z): + """Convert the time string Z back to a Unix time.""" + m = R_ZULU.match(z) + if not m: raise ValueError("bad time spec `%s'" % z) + yr, mo, dy, hr, mi, se = map(int, m.groups()) + return T.mktime((yr, mo, dy, hr, mi, se, 0, 0, 0)) + +###-------------------------------------------------------------------------- +### Simple select(2) utilities. + +class BaseSelector (object): + """ + A base class for hooking into `select_loop'. + + See `select_loop' for details of the protocol. + """ + def preselect(me, rfds, wfds): pass + def postselect_read(me, fd): pass + def postselect_write(me, fd): pass + +class WriteLinesSelector (BaseSelector): + """Write whole lines to an output file descriptor.""" + + def __init__(me, fd, nextfn = None, *args, **kw): + """ + Initialize the WriteLinesSelector to write to the file descriptor FD. + + The FD is marked non-blocking. + + The lines are produced by the NEXTFN, which is called without arguments. + It can affect the output in three ways: + + * It can return a string (or almost any other kind of object, which + will be converted into a string by `str'), which will be written to + the descriptor followed by a newline. Lines are written in the order + in which they are produced. + + * It can return `None', which indicates that there are no more items to + be written for the moment. The function will be called again from + time to time, to see if it has changed its mind. This is the right + thing to do in order to stall output temporarily. + + * It can raise `StopIteration', which indicates that there will never + be any more items. The file descriptor will be closed. + + Subclasses can override this behaviour by defining a method `_next' and + passing `None' as the NEXTFN. + """ + super(WriteLinesSelector, me).__init__(*args, **kw) + set_nonblocking(fd) + me._fd = fd + if nextfn is not None: me._next = nextfn + + ## Selector state. + ## + ## * `_buf' contains a number of output items, already formatted, and + ## ready for output in a single batch. It might be empty. + ## + ## * `_pos' is the current output position in `_buf'. + ## + ## * `_more' is set unless the `_next' function has raised + ## `StopIteration': it indicates that we should close the descriptor + ## once the all of the remaining data in the buffer has been sent. + me._buf = "" + me._pos = 0 + me._more = True + + def _refill(me): + """Refill `_buf' by calling `_next'.""" + sio = StringIO(); n = 0 + while n < 4096: + try: item = me._next() + except StopIteration: me._more = False; break + if item is None: break + item = str(item) + sio.write(item); sio.write("\n"); n += len(item) + 1 + me._buf = sio.getvalue(); me._pos = 0 + + def preselect(me, rfds, wfds): + if me._fd == -1: return + if me._buf == "" and me._more: me._refill() + if me._buf != "" or not me._more: wfds.append(me._fd) + + def postselect_write(me, fd): + if fd != me._fd: return + while True: + if me._pos >= len(me._buf): + if me._more: me._refill() + if not me._more: OS.close(me._fd); me._fd = -1; break + if not me._buf: break + try: n = OS.write(me._fd, me._buf[me._pos:]) + except OSError, err: + if err.errno == E.EAGAIN or err.errno == E.WOULDBLOCK: break + elif err.errno == E.EPIPE: OS.close(me._fd); me._fd = -1; break + else: raise + me._pos += n + +class ReadLinesSelector (BaseSelector): + """Report whole lines from an input file descriptor as they arrive.""" + + def __init__(me, fd, linefn = None, *args, **kw): + """ + Initialize the ReadLinesSelector to read from the file descriptor FD. + + The FD is marked non-blocking. + + For each whole line, and the final partial line (if any), the selector + calls LINEFN with the line as an argument (without the terminating + newline, if any). + + Subclasses can override this behaviour by defining a method `_line' and + passing `None' as the LINEFN. + """ + super(ReadLinesSelector, me).__init__(*args, **kw) + set_nonblocking(fd) + me._fd = fd + me._buf = "" + if linefn is not None: me._line = linefn + + def preselect(me, rfds, wfds): + if me._fd != -1: rfds.append(me._fd) + + def postselect_read(me, fd): + if fd != me._fd: return + while True: + try: buf = OS.read(me._fd, 4096) + except OSError, err: + if err.errno == E.EAGAIN or err.errno == E.WOULDBLOCK: break + else: raise + if buf == "": + OS.close(me._fd); me._fd = -1 + if me._buf: me._line(me._buf) + break + buf = me._buf + buf + i = 0 + while True: + try: j = buf.index("\n", i) + except ValueError: break + me._line(buf[i:j]) + i = j + 1 + me._buf = buf[i:] + +def select_loop(selectors): + """ + Multiplex I/O between the various SELECTORS. + + A `selector' SEL is an object which implements the selector protocol, which + consists of three methods. + + * SEL.preselect(RFDS, WFDS) -- add any file descriptors which the + selector is interested in reading from to the list RFDS, and add file + descriptors it's interested in writing to to the list WFDS. + + * SEL.postselect_read(FD) -- informs the selector that FD is ready for + reading. + + * SEL.postselect_write(FD) -- informs the selector that FD is ready for + writing. + + The `select_loop' function loops as follows. + + * It calls the `preselect' method on each SELECTOR to determine what I/O + events it thinks are interesting. + + * It waits for some interesting event to happen. + + * It calls the `postselect_read' and/or `postselect_write' methods on all + of the selectors for each file descriptor which is ready. + + The loop ends when no selector is interested in any events. This is simple + but rather inefficient. + """ + while True: + rfds, wfds = [], [] + for sel in selectors: sel.preselect(rfds, wfds) + if not rfds and not wfds: break + rfds, wfds, _ = SEL.select(rfds, wfds, []) + for fd in rfds: + for sel in selectors: sel.postselect_read(fd) + for fd in wfds: + for sel in selectors: sel.postselect_write(fd) + +###-------------------------------------------------------------------------- +### Running subprocesses. + +def wait_outcome(st): + """ + Given a ST from `waitpid' (or similar), return a human-readable outcome. + """ + if OS.WIFSIGNALED(st): return "killed by signal %d" % OS.WTERMSIG(st) + elif OS.WIFEXITED(st): + rc = OS.WEXITSTATUS(st) + if rc: return "failed: rc = %d" % rc + else: return "completed successfully" + else: return "died with incomprehensible status 0x%04x" % st + +class SubprocessFailure (Exception): + """An exception indicating that a subprocess failed.""" + def __init__(me, what, st): + me.st = st + me.what = what + if OS.WIFEXITED(st): me.rc, me.sig = OS.WEXITSTATUS(st), None + elif OS.WIFSIGNALED(st): me.rc, me.sig = None, OS.WTERMSIG(st) + else: me.rc, me.sig = None, None + def __str__(me): + return "subprocess `%s' %s" % (me.what, wait_outcome(me.st)) + +INHERIT = Tag('INHERIT') +PIPE = Tag('PIPE') +DISCARD = Tag('DISCARD') +@CTX.contextmanager +def subprocess(command, + stdin = INHERIT, stdout = INHERIT, stderr = INHERIT, + cwd = INHERIT, jobserver = DISCARD): + """ + Hairy context manager for running subprocesses. + + The COMMAND is a list of arguments; COMMAND[0] names the program to be + invoked. (There's currently no way to run a program with an unusual + `argv[0]'.) + + The keyword arguments `stdin', `stdout', and `stderr' explain what to do + with the standard file descriptors. + + * `INHERIT' means that they should be left alone: the child will use a + copy of the parent's descriptor. This is the default. + + * `DISCARD' means that the descriptor should be re-opened onto + `/dev/null' (for reading or writing as appropriate). + + * `PIPE' means that the descriptor should be re-opened as (the read or + write end, as appropriate, of) a pipe, and the other end returned to + the context body. + + Simiarly, the JOBSERVER may be `INHERIT' to pass the jobserver descriptors + and environment variable down to the child, or `DISCARD' to close it. The + default is `DISCARD'. + + The CWD may be `INHERIT' to run the child with the same working directory + as the parent, or a pathname to change to an explicitly given working + directory. + + The context is returned three values, which are file descriptors for other + pipe ends for stdin, stdout, and stderr respectively, or -1 if there is no + pipe. + + The context owns the pipe descriptors, and is expected to close them + itself. (Timing of closure is significant, particularly for `stdin'.) + """ + + ## Set up. + r_in, w_in = -1, -1 + r_out, w_out = -1, -1 + r_err, w_err = -1, -1 + spew("running subprocess `%s'" % " ".join(command)) + + ## Clean up as necessary... + try: + + ## Set up stdin. + if stdin is PIPE: r_in, w_in = OS.pipe() + elif stdin is DISCARD: r_in = OS.open("/dev/null", OS.O_RDONLY) + elif stdin is not INHERIT: + raise ValueError("bad `stdin' value `%r'" % stdin) + + ## Set up stdout. + if stdout is PIPE: r_out, w_out = OS.pipe() + elif stdout is DISCARD: w_out = OS.open("/dev/null", OS.O_WRONLY) + elif stdout is not INHERIT: + raise ValueError("bad `stderr' value `%r'" % stdout) + + ## Set up stderr. + if stderr is PIPE: r_err, w_err = OS.pipe() + elif stderr is DISCARD: w_err = OS.open("/dev/null", OS.O_WRONLY) + elif stderr is not INHERIT: + raise ValueError("bad `stderr' value `%r'" % stderr) + + ## Start up the child. + kid = OS.fork() + + if kid == 0: + ## Child process. + + ## Fix up stdin. + if r_in != -1: OS.dup2(r_in, 0); OS.close(r_in) + if w_in != -1: OS.close(w_in) + + ## Fix up stdout. + if w_out != -1: OS.dup2(w_out, 1); OS.close(w_out) + if r_out != -1: OS.close(r_out) + + ## Fix up stderr. + if w_err != -1: OS.dup2(w_err, 2); OS.close(w_err) + if r_err != -1: OS.close(r_err) + + ## Change directory. + if cwd is not INHERIT: OS.chdir(cwd) + + ## Fix up the jobserver. + if jobserver is DISCARD: SCHED.close_jobserver() + + ## Run the program. + try: OS.execvp(command[0], command) + except OSError, err: + moan("failed to run `%s': %s" % err.strerror) + OS._exit(127) + + ## Close the other ends of the pipes. + if r_in != -1: OS.close(r_in); r_in = -1 + if w_out != -1: OS.close(w_out); w_out = -1 + if w_err != -1: OS.close(w_err); w_err = -1 + + ## Return control to the context body. Remember not to close its pipes. + yield w_in, r_out, r_err + w_in = r_out = r_err = -1 + + ## Collect the child process's exit status. + _, st = OS.waitpid(kid, 0) + spew("subprocess `%s' %s" % (" ".join(command), wait_outcome(st))) + if st: raise SubprocessFailure(" ".join(command), st) + + ## Tidy up. + finally: + + ## Close any left-over file descriptors. + for fd in [r_in, w_in, r_out, w_out, r_err, w_err]: + if fd != -1: OS.close(fd) + +def set_nonblocking(fd): + """Mark the descriptor FD as non-blocking.""" + FC.fcntl(fd, FC.F_SETFL, FC.fcntl(fd, FC.F_GETFL) | OS.O_NONBLOCK) + +class DribbleOut (BaseSelector): + """A simple selector to feed a string to a descriptor, in pieces.""" + def __init__(me, fd, string, *args, **kw): + super(DribbleOut, me).__init__(*args, **kw) + me._fd = fd + me._string = string + me._i = 0 + set_nonblocking(me._fd) + me.result = None + def preselect(me, rfds, wfds): + if me._fd != -1: wfds.append(me._fd) + def postselect_write(me, fd): + if fd != me._fd: return + try: n = OS.write(me._fd, me._string) + except OSError, err: + if err.errno == E.EAGAIN or err.errno == E.EWOULDBLOCK: return + elif err.errno == E.EPIPE: OS.close(me._fd); me._fd = -1; return + else: raise + if n == len(me._string): OS.close(me._fd); me._fd = -1 + else: me._string = me._string[n:] + +class DribbleIn (BaseSelector): + """A simple selector to collect all the input as a big string.""" + def __init__(me, fd, *args, **kw): + super(DribbleIn, me).__init__(*args, **kw) + me._fd = fd + me._buf = StringIO() + set_nonblocking(me._fd) + def preselect(me, rfds, wfds): + if me._fd != -1: rfds.append(me._fd) + def postselect_read(me, fd): + if fd != me._fd: return + while True: + try: buf = OS.read(me._fd, 4096) + except OSError, err: + if err.errno == E.EAGAIN or err.errno == E.EWOULDBLOCK: break + else: raise + if buf == "": OS.close(me._fd); me._fd = -1; break + else: me._buf.write(buf) + @property + def result(me): return me._buf.getvalue() + +RETURN = Tag('RETURN') +def run_program(command, + stdin = INHERIT, stdout = INHERIT, stderr = INHERIT, + *args, **kwargs): + """ + A simplifying wrapper around `subprocess'. + + The COMMAND is a list of arguments; COMMAND[0] names the program to be + invoked, as for `subprocess'. + + The keyword arguments `stdin', `stdout', and `stderr' explain what to do + with the standard file descriptors. + + * `INHERIT' means that they should be left alone: the child will use a + copy of the parent's descriptor. + + * `DISCARD' means that the descriptor should be re-opened onto + `/dev/null' (for reading or writing as appropriate). + + * `RETURN', for an output descriptor, means that all of the output + produced on that descriptor should be collected and returned as a + string. + + * A string, for stdin, means that the string should be provided on the + child's standard input. + + (The value `PIPE' is not permitted here.) + + Other arguments are passed on to `subprocess'. + + If no descriptors are marked `RETURN', then the function returns `None'; if + exactly one descriptor is so marked, then the function returns that + descriptor's output as a string; otherwise, it returns a tuple of strings + for each such descriptor, in the usual order. + """ + kw = dict(); kw.update(kwargs) + selfn = [] + + if isinstance(stdin, basestring): + kw['stdin'] = PIPE; selfn.append(lambda fds: DribbleOut(fds[0], stdin)) + elif stdin is INHERIT or stdin is DISCARD: + kw['stdin'] = stdin + else: + raise ValueError("bad `stdin' value `%r'" % stdin) + + if stdout is RETURN: + kw['stdout'] = PIPE; selfn.append(lambda fds: DribbleIn(fds[1])) + elif stdout is INHERIT or stdout is DISCARD: + kw['stdout'] = stdout + else: + raise ValueError("bad `stdout' value `%r'" % stdout) + + if stderr is RETURN: + kw['stderr'] = PIPE; selfn.append(lambda fds: DribbleIn(fds[2])) + elif stderr is INHERIT or stderr is DISCARD: + kw['stderr'] = stderr + else: + raise ValueError("bad `stderr' value `%r'" % stderr) + + with subprocess(command, *args, **kw) as fds: + sel = [fn(fds) for fn in selfn] + select_loop(sel) + rr = [] + for s in sel: + r = s.result + if r is not None: rr.append(r) + if len(rr) == 0: return None + if len(rr) == 1: return rr[0] + else: return tuple(rr) + +###-------------------------------------------------------------------------- +### Other system-ish utilities. + +@CTX.contextmanager +def safewrite(path): + """ + Context manager for writing to a file. + + A new file, named `PATH.new', is opened for writing, and the file object + provided to the context body. If the body completes normally, the file is + closed and renamed to PATH. If the body raises an exception, the file is + still closed, but not renamed into place. + """ + new = path + ".new" + with open(new, "w") as f: yield f + OS.rename(new, path) + +@CTX.contextmanager +def safewrite_root(path, mode = None, uid = None, gid = None): + """ + Context manager for writing to a file with root privileges. + + This is as for `safewrite', but the file is opened and written as root. + """ + new = path + ".new" + with subprocess(C.ROOTLY + ["tee", new], + stdin = PIPE, stdout = DISCARD) as (fd_in, _, _): + pipe = OS.fdopen(fd_in, 'w') + try: yield pipe + finally: pipe.close() + if mode is not None: run_program(C.ROOTLY + ["chmod", mode, new]) + if uid is not None: + run_program(C.ROOTLY + ["chown", + uid + (gid is not None and ":" + gid or ""), + new]) + elif gid is not None: + run_program(C.ROOTLY + ["chgrp", gid, new]) + run_program(C.ROOTLY + ["mv", new, path]) + +def mountpoint_p(dir): + """Return true if DIR is a mountpoint.""" + + ## A mountpoint can be distinguished because it is a directory whose device + ## number differs from its parent. + try: st1 = OS.stat(dir) + except OSError, err: + if err.errno == E.ENOENT: return False + else: raise + if not ST.S_ISDIR(st1.st_mode): return False + st0 = OS.stat(OS.path.join(dir, "..")) + return st0.st_dev != st1.st_dev + +def mkdir_p(dir, mode = 0777): + """ + Make a directory DIR, and any parents, as necessary. + + Unlike `OS.makedirs', this doesn't fail if DIR already exists. + """ + d = "" + for p in dir.split("/"): + d = OS.path.join(d, p) + if d == "": continue + try: OS.mkdir(d, mode) + except OSError, err: + if err.errno == E.EEXIST: pass + else: raise + +def umount(fs): + """ + Unmount the filesystem FS. + + The FS may be the block device holding the filesystem, or (more usually) + the mount point. + """ + + ## Sometimes random things can prevent unmounting. Be persistent. + for i in xrange(5): + try: run_program(C.ROOTLY + ["umount", fs], stderr = DISCARD) + except SubprocessFailure, err: + if err.rc == 32: pass + else: raise + else: return + T.sleep(0.2) + run_program(C.ROOTLY + ["umount", fs], stderr = DISCARD) + +@CTX.contextmanager +def lockfile(lock, exclp = True, waitp = True): + """ + Acquire an exclusive lock on a named file LOCK while executing the body. + + If WAITP is true, wait until the lock is available; if false, then fail + immediately if the lock can't be acquired. + """ + fd = -1 + flag = 0 + if exclp: flag |= FC.LOCK_EX + else: flag |= FC.LOCK_SH + if not waitp: flag |= FC.LOCK_NB + spew("acquiring %s lock on `%s'" % + (exclp and "exclusive" or "shared", lock)) + try: + while True: + + ## Open the file and take note of which file it is. + fd = OS.open(lock, OS.O_RDWR | OS.O_CREAT, 0666) + st0 = OS.fstat(fd) + + ## Acquire the lock, waiting if necessary. + FC.lockf(fd, flag) + + ## Check that the lock file is still the same one. It's permissible + ## for the lock holder to release the lock by unlinking or renaming the + ## lock file, in which case there might be a different lockfile there + ## now which we need to acquire instead. + ## + ## It's tempting to `optimize' this code by opening a new file + ## descriptor here so as to elide the additional call to fstat(2) + ## above. But this doesn't work: if we successfully acquire the lock, + ## we then have two file descriptors open on the lock file, so we have + ## to close one -- but, under the daft fcntl(2) rules, even closing + ## `nfd' will release the lock immediately. + try: + st1 = OS.stat(lock) + except OSError, err: + if err.errno == E.ENOENT: pass + else: raise + if st0.st_dev == st1.st_dev and st0.st_ino == st1.st_ino: break + OS.close(fd) + + ## We have the lock, so away we go. + spew("lock `%s' acquired" % lock) + yield None + spew("lock `%s' released" % lock) + + finally: + if fd != -1: OS.close(fd) + +def block_device_p(dev): + """Return true if DEV names a block device.""" + try: st = OS.stat(dev) + except OSError, err: + if err.errno == E.ENOENT: return False + else: raise + else: return ST.S_ISBLK(st.st_mode) + +###-------------------------------------------------------------------------- +### Running parallel jobs. + +## Return codes from `check' +SLEEP = Tag('SLEEP') +READY = Tag('READY') +FAILED = Tag('FAILED') +DONE = Tag('DONE') + +class BaseJob (object): + """ + Base class for jobs. + + Subclasses must implement `run' and `_mkname', and probably ought to extend + `prepare' and `check'. + """ + + ## A magic token to prevent sneaky uninterned jobs. + _MAGIC = Tag('MAGIC') + + ## A map from job names to objects. + _MAP = {} + + ## Number of tail lines of the log to print on failure. + LOGLINES = 20 + + def __init__(me, _token, *args, **kw): + """ + Initialize a job. + + Jobs are interned! Don't construct instances (of subclasses) directly: + use the `ensure' class method. + """ + assert _token is me._MAGIC + super(BaseJob, me).__init__(*args, **kw) + + ## Dependencies on other jobs. + me._deps = None + me._waiting = set() + + ## Attributes maintained by the JobServer. + me.done = False + me.started = False + me.win = None + me._token = None + me._known = False + me._st = None + me._logkid = -1 + me._logfile = None + + def prepare(me): + """ + Establish any prerequisite jobs. + + Delaying this allows command-line settings to override those chosen by + dependent jobs. + """ + pass + + @classmethod + def ensure(cls, *args, **kw): + """ + Return the unique job with the given parameters. + + If a matching job already exists, then return it. Otherwise, create the + new job, register it in the table, and notify the scheduler about it. + """ + me = cls(_token = cls._MAGIC, *args, **kw) + try: + job = cls._MAP[me.name] + except KeyError: + cls._MAP[me.name] = me + SCHED.add(me) + return me + else: + return job + + ## Naming. + @property + def name(me): + """Return the job's name, as calculated by `_mkname'.""" + try: name = me._name + except AttributeError: name = me._name = me._mkname() + return name + + ## Subclass responsibilities. + def _mkname(me): + """ + Return the job's name. + + By default, this is an unhelpful string which is distinct for every job. + Subclasses should normally override this method to return a name as an + injective function of the job parameters. + """ + return "%s.%x" % (me.__class__.__name__, id(me)) + + def check(me): + """ + Return whether the job is ready to run. + + Returns a pair STATE, REASON. The REASON is a human-readable string + explaining what's going on, or `None' if it's not worth explaining. The + STATE is one of the following. + + * `READY' -- the job can be run at any time. + + * `FAILED' -- the job can't be started. Usually, this means that some + prerequisite job failed, there was some error in the job's + parameters, or the environment is unsuitable for the job to run. + + * `DONE' -- the job has nothing to do. Usually, this means that the + thing the job acts on is already up-to-date. It's bad form to do + even minor work in `check'. + + * `SLEEP' -- the job can't be run right now. It has arranged to be + retried if conditions change. (Spurious wakeups are permitted and + must be handled correctly.) + + The default behaviour checks the set of dependencies, as built by the + `await' method, and returns `SLEEP' or `FAILED' as appropriate, or + `READY' if all the prerequisite jobs have completed successfully. + """ + for job in me._deps: + if not job.done: + job._waiting.add(me) + return SLEEP, "waiting for job `%s'" % job.name + elif not job.win and not OPT.ignerr: + return FAILED, "dependent on failed job `%s'" % job.name + return READY, None + + ## Subclass utilities. + def await(me, job): + """Make sure that JOB completes before allowing this job to start.""" + me._deps.add(job) + + def _logtail(me): + """ + Dump the last `LOGLINES' lines of the logfile. + + This is called if the job fails and was being run quietly, to provide the + user with some context for the failure. + """ + + ## Gather blocks from the end of the log until we have enough lines. + with open(me._logfile, 'r') as f: + nlines = 0 + bufs = [] + bufsz = 4096 + f.seek(0, 2); off = f.tell() + spew("start: off = %d" % off) + while nlines <= me.LOGLINES and off > 0: + off = max(0, off - bufsz) + f.seek(off, 0) + spew("try at off = %d" % off) + buf = f.read(bufsz) + nlines += buf.count("\n") + spew("now lines = %d" % nlines) + bufs.append(buf) + buf = ''.join(reversed(bufs)) + + ## We probably overshot. Skip the extra lines from the start. + i = 0 + while nlines > me.LOGLINES: i = buf.index("\n", i) + 1; nlines -= 1 + + ## If we ended up trimming the log, print an ellipsis. + if off > 0 or i > 0: print "%-*s * [...]" % (TAGWD, me.name) + + ## Print the log tail. + lines = buf[i:].split("\n") + if lines and lines[-1] == '': lines.pop() + for line in lines: print "%-*s %s" % (TAGWD, me.name, line) + +class BaseJobToken (object): + """ + A job token is the authorization for a job to be run. + + Subclasses must implement `recycle' to allow some other job to use the + token. + """ + pass + +class TrivialJobToken (BaseJobToken): + """ + A trivial reusable token, for when issuing jobs in parallel without limit. + + There only needs to be one of these. + """ + def recycle(me): + spew("no token needed; nothing to recycle") +TRIVIAL_TOKEN = TrivialJobToken() + +class JobServerToken (BaseJobToken): + """A job token storing a byte from the jobserver pipe.""" + def __init__(me, char, pipefd, *args, **kw): + super(JobServerToken, me).__init__(*args, **kw) + me._char = char + me._fd = pipefd + def recycle(me): + spew("returning token to jobserver pipe") + OS.write(me._fd, me._char) + +class PrivateJobToken (BaseJobToken): + """ + The private job token belonging to a scheduler. + + When running under a GNU Make jobserver, there is a token for each byte in + the pipe, and an additional one which represents the slot we're actually + running in. This class represents that additional token. + """ + def __init__(me, sched, *args, **kw): + super(PrivateJobToken, me).__init__(*args, **kw) + me._sched = sched + def recycle(me): + assert me._sched._privtoken is None + spew("recycling private token") + me._sched._privtoken = me + +TAGWD = 29 +LOGKEEP = 20 + +class JobScheduler (object): + """ + The main machinery for running and ordering jobs. + + This handles all of the details of job scheduling. + """ + + def __init__(me, rfd = -1, wfd = -1, npar = 1): + """ + Initialize a scheduler. + + * RFD and WFD are the read and write ends of the jobserver pipe, as + determined from the `MAKEFLAGS' environment variable, or -1. + + * NPAR is the maximum number of jobs to run in parallel, or `True' if + there is no maximum (i.e., we're in `forkbomb' mode). + """ + + ## Set the parallelism state. The `_rfd' and `_wfd' are the read and + ## write ends of the jobserver pipe, or -1 if there is no jobserver. + ## `_par' is true if we're meant to run jobs in parallel. The case _par + ## and _rfd = -1 means unconstrained parallelism. + ## + ## The jobserver pipe contains a byte for each shared job slot. A + ## scheduler reads a byte from the pipe for each job it wants to run + ## (nearly -- see `_privtoken' below), and puts the byte back when the + ## job finishes. The GNU Make jobserver protocol specification insists + ## that we preserve the value of the byte in the pipe (though doesn't + ## currently make any use of this flexibility), so we record it in a + ## `JobToken' object's `_char' attribute. + me._par = rfd != -1 or npar is True or npar != 1 + spew("par is %r" % me._par) + if rfd == -1 and npar > 1: + rfd, wfd = OS.pipe() + OS.write(wfd, (npar - 1)*'+') + OS.environ["MAKEFLAGS"] = \ + (" -j --jobserver-auth=%(rfd)d,%(wfd)d " + + "--jobserver-fds=%(rfd)d,%(wfd)d") % dict(rfd = rfd, wfd = wfd) + me._rfd = rfd; me._wfd = wfd + + ## The scheduler state. A job starts in the `_check' list. Each + ## iteration of the scheduler loop will inspect the jobs here and see + ## whether it's ready to run: if not, it gets put in the `_sleep' list, + ## where it will languish until something moves it back; if it is ready, + ## it gets moved to the `_ready' list to wait for a token from the + ## jobserver. At that point the job can be started, and it moves to the + ## `_kidmap', which associates a process-id with each running job. + ## Finally, jobs which have completed are simply forgotten. The `_njobs' + ## counter keeps track of how many jobs are outstanding, so that we can + ## stop when there are none left. + me._check = set() + me._sleep = set() + me._ready = set() + me._kidmap = {} + me._logkidmap = {} + me._njobs = 0 + + ## As well as the jobserver pipe, we implicitly have one extra job slot, + ## which is the one we took when we were started by our parent. The + ## right to do processing in this slot is represnted by the `private + ## token' here, distinguished from tokens from the jobserver pipe by + ## having `None' as its `_char' value. + me._privtoken = PrivateJobToken(me) + + def add(me, job): + """Notice a new job and arrange for it to (try to) run.""" + if job._known: return + spew("adding new job `%s'" % job.name) + job._known = True + me._check.add(job) + me._njobs += 1 + + def close_jobserver(me): + """ + Close the jobserver file descriptors. + + This should be called within child processes to prevent them from messing + with the jobserver. + """ + if me._rfd != -1: OS.close(me._rfd); me._rfd = -1 + if me._wfd != -1: OS.close(me._wfd); me._wfd = -1 + try: del OS.environ["MAKEFLAGS"] + except KeyError: pass + + def _killall(me): + """Zap all jobs which aren't yet running.""" + for jobset in [me._sleep, me._check, me._ready]: + while jobset: + job = jobset.pop() + job.done = True + job.win = False + me._njobs -= 1 + + def _retire(me, job, win, outcome): + """ + Declare that a job has stopped, and deal with the consequences. + + JOB is the completed job, which should not be on any of the job queues. + WIN is true if the job succeeded, and false otherwise. OUTCOME is a + human-readable string explaining how the job came to its end, or `None' + if no message should be reported. + """ + + global RC + + ## Return the job's token to the pool. + if job._token is not None: job._token.recycle() + job._token = None + me._njobs -= 1 + + ## Update and maybe report the job's status. + job.done = True + job.win = win + if outcome is not None and not OPT.silent: + if OPT.quiet and not job.win and job._logfile: job._logtail() + if not job.win or not OPT.quiet: + print "%-*s %c (%s)" % \ + (TAGWD, job.name, job.win and '|' or '*', outcome) + + ## If the job failed, and we care, arrange to exit nonzero. + if not win and not OPT.ignerr: RC = 2 + + ## If the job failed, and we're supposed to give up after the first + ## error, then zap all of the waiting jobs. + if not job.win and not OPT.keepon and not OPT.ignerr: me._killall() + + ## If this job has dependents then wake them up and see whether they're + ## ready to run. + for j in job._waiting: + try: me._sleep.remove(j) + except KeyError: pass + else: + spew("waking dependent job `%s'" % j.name) + me._check.add(j) + + def _reap(me, kid, st): + """ + Deal with the child with process-id KID having exited with status ST. + """ + + ## Figure out what kind of child this is. Note that it has finished. + try: job = me._kidmap[kid] + except KeyError: + try: job = me._logkidmap[kid] + except KeyError: + spew("unknown child %d exits with status 0x%04x" % (kid, st)) + return + else: + ## It's a logging child. + del me._logkidmap[kid] + job._logkid = DONE + spew("logging process for job `%s' exits with status 0x%04x" % + (job.name, st)) + else: + job._st = st + del me._kidmap[kid] + spew("main process for job `%s' exits with status 0x%04x" % + (job.name, st)) + + ## If either of the job's associated processes is still running then we + ## should stop now and give the other one a chance. + if job._st is None or job._logkid is not DONE: + spew("deferring retirement for job `%s'" % job.name) + return + spew("completing deferred retirement for job `%s'" % job.name) + + ## Update and (maybe) report the job status. + if job._st == 0: win = True; outcome = None + else: win = False; outcome = wait_outcome(job._st) + + ## Retire the job. + me._retire(job, win, outcome) + + def _reapkids(me): + """Reap all finished child processes.""" + while True: + try: kid, st = OS.waitpid(-1, OS.WNOHANG) + except OSError, err: + if err.errno == E.ECHILD: break + else: raise + if kid == 0: break + me._reap(kid, st) + + def run_job(me, job): + """Start running the JOB.""" + + job.started = True + if OPT.dryrun: return None, None + + ## Make pipes to collect the job's output and error reports. + r_out, w_out = OS.pipe() + r_err, w_err = OS.pipe() + + ## Find a log file to write. Avoid races over the log names; but this + ## means that the log descriptor needs to be handled somewhat carefully. + logdir = OS.path.join(C.STATE, "log"); mkdir_p(logdir) + logseq = 1 + while True: + logfile = OS.path.join(logdir, "%s-%s#%d" % (job.name, TODAY, logseq)) + try: + logfd = OS.open(logfile, OS.O_WRONLY | OS.O_CREAT | OS.O_EXCL, 0666) + except OSError, err: + if err.errno == E.EEXIST: logseq += 1; continue + else: raise + else: + break + job._logfile = logfile + + ## Make sure there's no pending output, or we might get two copies. (I + ## don't know how to flush all output streams in Python, but this is good + ## enough for our purposes.) + SYS.stdout.flush() + + ## Set up the logging child first. If we can't, take down the whole job. + try: job._logkid = OS.fork() + except OSError, err: OS.close(logfd); return None, err + if not job._logkid: + ## The main logging loop. + + ## Close the jobserver descriptors, and the write ends of the pipes. + me.close_jobserver() + OS.close(w_out); OS.close(w_err) + + ## Capture the job's stdout and stderr and wait for everything to + ## happen. + def log_lines(fd, marker): + def fn(line): + if not OPT.quiet: + OS.write(1, "%-*s %s %s\n" % (TAGWD, job.name, marker, line)) + OS.write(logfd, "%s %s\n" % (marker, line)) + return ReadLinesSelector(fd, fn) + select_loop([log_lines(r_out, "|"), log_lines(r_err, "*")]) + + ## We're done. (Closing the descriptors here would be like polishing + ## the floors before the building is demolished.) + OS._exit(0) + + ## Back in the main process: record the logging child. At this point we + ## no longer need the logfile descriptor. + me._logkidmap[job._logkid] = job + OS.close(logfd) + + ## Start the main job process. + try: kid = OS.fork() + except OSError, err: return None, err + if not kid: + ## The main job. + + ## Close the read ends of the pipes, and move the write ends to the + ## right places. (This will go wrong if we were started without enough + ## descriptors. Fingers crossed.) + OS.dup2(w_out, 1); OS.dup2(w_err, 2) + OS.close(r_out); OS.close(w_out) + OS.close(r_err); OS.close(w_err) + spew("running job `%s' as pid %d" % (job.name, OS.getpid())) + + ## Run the job, catching nonlocal flow. + try: + job.run() + except ExpectedError, err: + moan(str(err)) + OS._exit(2) + except Exception, err: + TB.print_exc(SYS.stderr) + OS._exit(3) + except BaseException, err: + moan("caught unexpected exception: %r" % err) + OS._exit(112) + else: + spew("job `%s' ran to completion" % job.name) + + ## Clean up old logs. + match = [] + pat = RX.compile(r"^%s-(\d{4})-(\d{2})-(\d{2})\#(\d+)$" % + RX.escape(job.name)) + for f in OS.listdir(logdir): + m = pat.match(f) + if m: match.append((f, int(m.group(1)), int(m.group(2)), + int(m.group(3)), int(m.group(4)))) + match.sort(key = lambda (_, y, m, d, q): (y, m, d, q)) + if len(match) > LOGKEEP: + for (f, _, _, _, _) in match[:-LOGKEEP]: + try: OS.unlink(OS.path.join(logdir, f)) + except OSError, err: + if err.errno == E.ENOENT: pass + else: raise + + ## All done. + OS._exit(0) + + ## Back in the main process: close both the pipes and return the child + ## process. + OS.close(r_out); OS.close(w_out) + OS.close(r_err); OS.close(w_err) + if OPT.quiet: print "%-*s | (started)" % (TAGWD, job.name) + return kid, None + + def run(me): + """Run the scheduler.""" + + spew("JobScheduler starts") + + while True: + ## The main scheduler loop. We go through three main phases: + ## + ## * Inspect the jobs in the `check' list to see whether they can + ## run. After this, the `check' list will be empty. + ## + ## * If there are running jobs, check to see whether any of them have + ## stopped, and deal with the results. Also, if there are jobs + ## ready to start and a job token has become available, then + ## retrieve the token. (Doing these at the same time is the tricky + ## part.) + ## + ## * If there is a job ready to run, and we retrieved a token, then + ## start running the job. + + ## Check the pending jobs to see if they can make progress: run each + ## job's `check' method and move it to the appropriate queue. (It's OK + ## if `check' methods add more jobs to the list, as long as things + ## settle down eventually.) + while True: + try: job = me._check.pop() + except KeyError: break + if job._deps is None: + job._deps = set() + job.prepare() + state, reason = job.check() + tail = reason is not None and ": %s" % reason or "" + if state == READY: + spew("job `%s' ready to run%s" % (job.name, tail)) + me._ready.add(job) + elif state is FAILED: + spew("job `%s' refused to run%s" % (job.name, tail)) + me._retire(job, False, "refused to run%s" % tail) + elif state is DONE: + spew("job `%s' has nothing to do%s" % (job.name, tail)) + me._retire(job, True, reason) + elif state is SLEEP: + spew("job `%s' can't run yet%s" % (job.name, tail)) + me._sleep.add(job) + else: + raise ValueError("unexpected job check from `%s': %r, %r" % + (job.name, state, reason)) + + ## If there are no jobs left, then we're done. + if not me._njobs: + spew("all jobs completed") + break + + ## Make sure we can make progress. There are no jobs on the check list + ## any more, because we just cleared it. We assume that jobs which are + ## ready to run will eventually receive a token. So we only end up in + ## trouble if there are jobs asleep, but none running or ready to run. + ##spew("#jobs = %d" % me._njobs) + ##spew("sleeping: %s" % ", ".join([j.name for j in me._sleep])) + ##spew("ready: %s" % ", ".join([j.name for j in me._ready])) + ##spew("running: %s" % ", ".join([j.name for j in me._kidmap.itervalues()])) + assert not me._sleep or me._kidmap or me._logkidmap or me._ready + + ## Wait for something to happen. + if not me._ready or (not me._par and me._privtoken is None): + ## If we have no jobs ready to run, then we must wait for an existing + ## child to exit. Hopefully, a sleeping job will be able to make + ## progress after this. + ## + ## Alternatively, if we're not supposed to be running jobs in + ## parallel and we don't have the private token, then we have no + ## choice but to wait for the running job to complete. + ## + ## There's no check here for `ECHILD'. We really shouldn't be here + ## if there are no children to wait for. (The check list must be + ## empty because we just drained it. If the ready list is empty, + ## then all of the jobs must be running or sleeping; but the + ## assertion above means that either there are no jobs at all, in + ## which case we should have stopped, or at least one is running, in + ## which case it's safe to wait for it. The other case is that we're + ## running jobs sequentially, and one is currently running, so + ## there's nothing for it but to wait for it -- and hope that it will + ## wake up one of the sleeping jobs. The remaining possibility is + ## that we've miscounted somewhere, which will cause a crash.) + if not me._ready: + spew("no new jobs ready: waiting for outstanding jobs to complete") + else: + spew("job running without parallelism: waiting for it to finish") + kid, st = OS.waitpid(-1, 0) + me._reap(kid, st) + me._reapkids() + continue + + ## We have jobs ready to run, so try to acquire a token. + if me._rfd == -1 and me._par: + ## We're running with unlimited parallelism, so we don't need a token + ## to run a job. + spew("running new job without token") + token = TRIVIAL_TOKEN + elif me._privtoken: + ## Our private token is available, so we can use that to start + ## a new job. + spew("private token available: assigning to new job") + token = me._privtoken + me._privtoken = None + else: + ## We have to read from the jobserver pipe. Unfortunately, we're not + ## allowed to set the pipe nonblocking, because make is also using it + ## and will get into a serious mess. And we must deal with `SIGCHLD' + ## arriving at any moment. We use the same approach as GNU Make. We + ## start by making a copy of the jobserver descriptor: it's this + ## descriptor we actually try to read from. We set a signal handler + ## to close this descriptor if a child exits. And we try one last + ## time to reap any children which have exited just before we try + ## reading the jobserver pipe. This way we're covered: + ## + ## * If a child exits during the main loop, before we establish the + ## descriptor copy then we'll notice when we try reaping + ## children. + ## + ## * If a child exits between the last-chance reap and the read, + ## the signal handler will close the descriptor and the `read' + ## call will fail with `EBADF'. + ## + ## * If a child exits while we're inside the `read' system call, + ## then the syscall will fail with `EINTR'. + ## + ## The only problem is that we can't do this from Python, because + ## Python signal handlers are delayed. This is what the `jobclient' + ## module is for. + ## + ## The `jobclient' function is called as + ## + ## jobclient(FD) + ## + ## It returns a tuple of three values: TOKEN, PID, STATUS. If TOKEN + ## is not `None', then reading the pipe succeeded; if TOKEN is empty, + ## then the pipe returned EOF, so we should abort; otherwise, TOKEN + ## is a singleton string holding the token character. If PID is not + ## `None', then PID is the process id of a child which exited, and + ## STATUS is its exit status. + spew("waiting for token from jobserver") + tokch, kid, st = JC.jobclient(me._rfd) + + if kid is not None: + me._reap(kid, st) + me._reapkids() + if tokch is None: + spew("no token; trying again") + continue + elif token == '': + error("jobserver pipe closed; giving up") + me._killall() + continue + spew("received token from jobserver") + token = JobServerToken(tokch, me._wfd) + + ## We have a token, so we should start up the job. + job = me._ready.pop() + job._token = token + spew("start new job `%s'" % job.name) + kid, err = me.run_job(job) + if err is not None: + me._retire(job, False, "failed to fork: %s" % err) + continue + if kid is None: me._retire(job, True, "dry run") + else: me._kidmap[kid] = job + + ## We ran out of work to do. + spew("JobScheduler done") + +###-------------------------------------------------------------------------- +### Configuration. + +R_CONFIG = RX.compile(r"^([a-zA-Z0-9_]+)='(.*)'$") + +class Config (object): + + def _conv_str(s): return s + def _conv_list(s): return s.split() + def _conv_set(s): return set(s.split()) + + _CONVERT = { + "ROOTLY": _conv_list, + "DISTS": _conv_set, + "MYARCH": _conv_set, + "NATIVE_ARCHS": _conv_set, + "FOREIGN_ARCHS": _conv_set, + "FOREIGN_GNUARCHS": _conv_list, + "ALL_ARCHS": _conv_set, + "NATIVE_CHROOTS": _conv_set, + "FOREIGN_CHROOTS": _conv_set, + "ALL_CHROOTS": _conv_set, + "BASE_PACKAGES": _conv_list, + "EXTRA_PACKAGES": _conv_list, + "CROSS_PACKAGES": _conv_list, + "CROSS_PATHS": _conv_list, + "APTCONF": _conv_list, + "LOCALPKGS": _conv_list, + "SCHROOT_COPYFILES": _conv_list, + "SCHROOT_NSSDATABASES": _conv_list + } + + _CONV_MAP = { + "*_APTCONFSRC": ("APTCONFSRC", _conv_str), + "*_DEPS": ("PKGDEPS", _conv_list), + "*_QEMUHOST": ("QEMUHOST", _conv_str), + "*_QEMUARCH": ("QEMUARCH", _conv_str), + "*_ALIASES": ("DISTALIAS", _conv_str) + } + + _conv_str = staticmethod(_conv_str) + _conv_list = staticmethod(_conv_list) + _conv_set = staticmethod(_conv_set) + + def __init__(me): + raw = r""" + """; raw = open('state/config.sh').read(); _ignore = """ @@@config@@@ + """ + me._conf = {} + for line in raw.split("\n"): + line = line.strip() + if not line or line.startswith('#'): continue + m = R_CONFIG.match(line) + if not m: raise ExpectedError("bad config line `%s'" % line) + k, v = m.group(1), m.group(2).replace("'\\''", "'") + d = me._conf + try: conv = me._CONVERT[k] + except KeyError: + i = 0 + while True: + try: i = k.index("_", i + 1) + except ValueError: conv = me._conv_str; break + try: map, conv = me._CONV_MAP["*" + k[i:]] + except KeyError: pass + else: + d = me._conf.setdefault(map, dict()) + k = k[:i] + if k.startswith("_"): k = k[1:] + break + d[k] = conv(v) + + def __getattr__(me, attr): + try: return me._conf[attr] + except KeyError, err: raise AttributeError(err.args[0]) + +with toplevel_handler(): C = Config() + +###-------------------------------------------------------------------------- +### Chroot maintenance utilities. + +CREATE = Tag("CREATE") +FORCE = Tag("FORCE") + +def check_fresh(fresh, update): + """ + Compare a refresh mode FRESH against an UPDATE time. + + Return a (STATUS, REASON) pair, suitable for returning from a job `check' + method. + + The FRESH argument may be one of the following: + + * `CREATE' is satisfied if the thing exists at all: it returns `READY' if + the thing doesn't yet exist (UPDATE is `None'), or `DONE' otherwise. + + * `FORCE' is never satisfied: it always returns `READY'. + + * an integer N is satisfied if UPDATE time is at most N seconds earlier + than the present: if returns `READY' if the UPDATE is too old, or + `DONE' otherwise. + """ + if update is None: return READY, "must create" + elif fresh is FORCE: return READY, "update forced" + elif fresh is CREATE: return DONE, "already created" + elif NOW - unzulu(update) > fresh: return READY, "too stale: updating" + else: return DONE, "already sufficiently up-to-date" + +def lockfile_path(file): + """ + Return the full path for a lockfile named FILE. + + Create the lock directory if necessary. + """ + lockdir = OS.path.join(C.STATE, "lock"); mkdir_p(lockdir) + return OS.path.join(lockdir, file) + +def chroot_src_lockfile(dist, arch): + """ + Return the lockfile for the source-chroot for DIST on ARCH. + + It is not allowed to acquire a source-chroot lock while holding any other + locks. + """ + return lockfile_path("source.%s-%s" % (dist, arch)) + +def chroot_src_lv(dist, arch): + """ + Return the logical volume name for the source-chroot for DIST on ARCH. + """ + return "%s%s-%s" % (C.LVPREFIX, dist, arch) + +def chroot_src_blkdev(dist, arch): + """ + Return the block-device name for the source-chroot for DIST on ARCH. + """ + return OS.path.join("/dev", C.VG, chroot_src_lv(dist, arch)) + +def chroot_src_mntpt(dist, arch): + """ + Return mountpoint path for setting up the source-chroot for DIST on ARCH. + + Note that this is not the mountpoint that schroot(1) uses. + """ + mnt = OS.path.join(C.STATE, "mnt", "%s-%s" % (dist, arch)) + mkdir_p(mnt) + return mnt + +def chroot_session_mntpt(session): + """Return the mountpoint for an schroot session.""" + return OS.path.join("/schroot", session) + +def crosstools_lockfile(dist, arch): + """ + Return the lockfile for the cross-build tools for DIST, hosted by ARCH. + + When locking multiple cross-build tools, you must acquire the locks in + lexicographically ascending order. + """ + return lockfile_path("cross-tools.%s-%s" % (dist, arch)) + +def switch_prefix(string, map): + """ + Replace the prefix of a STRING, according to the given MAP. + + MAP is a sequence of (OLD, NEW) pairs. For each such pair in turn, test + whether STRING starts with OLD: if so, return STRING, but with the prefix + OLD replaced by NEW. If no OLD prefix matches, then raise a `ValueError'. + """ + for old, new in map: + if string.startswith(old): return new + string[len(old):] + raise ValueError("expected `%s' to start with one of %s" % + ", ".join(["`%s'" % old for old, new in map])) + +def host_to_chroot(path): + """ + Convert a host path under `C.LOCAL' to the corresponding chroot path under + `/usr/local.schroot'. + """ + return switch_prefix(path, [(C.LOCAL + "/", "/usr/local.schroot/")]) + +def chroot_to_host(path): + """ + Convert a chroot path under `/usr/local.schroot' to the corresponding + host path under `C.LOCAL'. + """ + return switch_prefix(path, [("/usr/local.schroot/", C.LOCAL + "/")]) + +def split_dist_arch(spec): + """Split a SPEC of the form `DIST-ARCH' into the pair (DIST, ARCH).""" + dash = spec.index("-") + return spec[:dash], spec[dash + 1:] + +def elf_binary_p(arch, path): + """Return whether PATH is an ELF binary for ARCH.""" + if not OS.path.isfile(path): return False + with open(path, 'rb') as f: magic = f.read(20) + if magic[0:4] != "\x7fELF": return False + if magic[8:16] != 8*"\0": return False + if arch == "i386": + if magic[4:7] != "\x01\x01\x01": return False + if magic[18:20] != "\x03\x00": return False + elif arch == "amd64": + if magic[4:7] != "\x02\x01\x01": return False + if magic[18:20] != "\x3e\x00": return False + else: + raise ValueError("unsupported donor architecture `%s'" % arch) + return True + +def progress(msg): + """ + Print a progress message MSG. + + This is intended to be called within a job's `run' method, so it doesn't + check `OPT.quiet' or `OPT.silent'. + """ + OS.write(1, ";; %s\n" % msg) + +class NoSuchChroot (Exception): + """ + Exception indicating that a chroot does not exist. + + Specifically, it means that it doesn't even have a logical volume. + """ + def __init__(me, dist, arch): + me.dist = dist + me.arch = arch + def __str__(me): + return "chroot for `%s' on `%s' not found" % (me.dist, me.arch) + +@CTX.contextmanager +def mount_chroot_src(dist, arch): + """ + Context manager for mounting the source-chroot for DIST on ARCH. + + The context manager automatically unmounts the filesystem again when the + body exits. You must hold the appropriate source-chroot lock before + calling this routine. + """ + dev = chroot_src_blkdev(dist, arch) + if not block_device_p(dev): raise NoSuchChroot(dist, arch) + mnt = chroot_src_mntpt(dist, arch) + try: + run_program(C.ROOTLY + ["mount", dev, mnt]) + yield mnt + finally: + umount(mnt) + +@CTX.contextmanager +def chroot_session(dist, arch, sourcep = False): + """ + Context manager for running an schroot(1) session. + + Returns the (ugly, automatically generated) session name to the context + body. By default, a snapshot session is started: set SOURCEP true to start + a source-chroot session. You must hold the appropriate source-chroot lock + before starting a source-chroot session. + + The context manager automatically closes the session again when the body + exits. + """ + chroot = chroot_src_lv(dist, arch) + if sourcep: chroot = "source:" + chroot + session = run_program(["schroot", "-uroot", "-b", "-c", chroot], + stdout = RETURN).rstrip("\n") + try: + root = OS.path.join(chroot_session_mntpt(session), "fs") + yield session, root + finally: + run_program(["schroot", "-e", "-c", session]) + +def run_root(command, **kw): + """Run a COMMAND as root. Arguments are as for `run_program'.""" + return run_program(C.ROOTLY + command, **kw) + +def run_schroot_session(session, command, rootp = False, **kw): + """ + Run a COMMAND within an schroot(1) session. + + Arguments are as for `run_program'. + """ + if rootp: + return run_program(["schroot", "-uroot", "-r", + "-c", session, "--"] + command, **kw) + else: + return run_program(["schroot", "-r", + "-c", session, "--"] + command, **kw) + +def run_schroot_source(dist, arch, command, **kw): + """ + Run a COMMAND through schroot(1), in the source-chroot for DIST on ARCH. + + Arguments are as for `run_program'. You must hold the appropriate source- + chroot lock before calling this routine. + """ + return run_program(["schroot", "-uroot", + "-c", "source:%s" % chroot_src_lv(dist, arch), + "--"] + command, **kw) + +###-------------------------------------------------------------------------- +### Metadata files. + +class MetadataClass (type): + """ + Metaclass for metadata classes. + + Notice a `VARS' attribute in the class dictionary, and augment it with a + `_VARSET' attribute, constructed as a set containing the same items. (We + need them both: the set satisfies fast lookups, while the original sequence + remembers the ordering.) + """ + def __new__(me, name, supers, dict): + try: vars = dict['VARS'] + except KeyError: pass + else: dict['_VARSET'] = set(vars) + return super(MetadataClass, me).__new__(me, name, supers, dict) + +class BaseMetadata (object): + """ + Base class for metadate objects. + + Metadata bundles are simple collections of key/value pairs. Keys should + usually be Python identifiers because they're used to name attributes. + Values are strings, but shouldn't have leading or trailing whitespace, and + can't contain newlines. + + Metadata bundles are written to files. The format is simple enough: empty + lines and lines starting with `#' are ignored; otherwise, the line must + have the form + + KEY = VALUE + + where KEY does not contain `='; spaces around the `=' are optional, and + spaces around the KEY and VALUE are stripped. The order of keys is + unimportant; keys are always written in a standard order on output. + """ + __metaclass__ = MetadataClass + + def __init__(me, **kw): + """Initialize a metadata bundle from keyword arguments.""" + for k, v in kw.iteritems(): + setattr(me, k, v) + for v in me.VARS: + try: getattr(me, v) + except AttributeError: setattr(me, v, None) + + def __setattr__(me, attr, value): + """ + Try to set an attribute. + + Only attribute names listed in the `VARS' class attribute are permitted. + """ + if attr not in me._VARSET: raise AttributeError, attr + super(BaseMetadata, me).__setattr__(attr, value) + + @classmethod + def read(cls, path): + """Return a new metadata bundle read from a named PATH.""" + map = {} + with open(path) as f: + for line in f: + line = line.strip() + if line == "" or line.startswith("#"): continue + k, v = line.split("=", 1) + map[k.strip()] = v.strip() + return cls(**map) + + def _write(me, file): + """ + Write the metadata bundle to the FILE (a file-like object). + + This is intended for use by subclasses which want to override the default + I/O behaviour of the main `write' method. + """ + file.write("### -*-conf-*-\n") + for k in me.VARS: + try: v = getattr(me, k) + except AttributeError: pass + else: + if v is not None: file.write("%s = %s\n" % (k, v)) + + def write(me, path): + """ + Write the metadata bundle to a given PATH. + + The file is replaced atomically. + """ + with safewrite(path) as f: me._write(f) + + def __repr__(me): + return "#<%s: %s>" % (me.__class__.__name__, + ", ".join("%s=%r" % (k, getattr(me, k, None)) + for k in me.VARS)) + +class ChrootMetadata (BaseMetadata): + VARS = ['dist', 'arch', 'update'] + + @classmethod + def read(cls, dist, arch): + try: + with lockfile(chroot_src_lockfile(dist, arch), exclp = False): + with mount_chroot_src(dist, arch) as mnt: + return super(ChrootMetadata, cls).read(OS.path.join(mnt, "META")) + except IOError, err: + if err.errno == E.ENOENT: pass + else: raise + except NoSuchChroot: pass + return cls(dist = dist, arch = arch) + + def write(me): + with mount_chroot_src(me.dist, me.arch) as mnt: + with safewrite_root(OS.path.join(mnt, "META")) as f: + me._write(f) + +class CrossToolsMetadata (BaseMetadata): + VARS = ['dist', 'arch', 'update'] + + @classmethod + def read(cls, dist, arch): + try: + return super(CrossToolsMetadata, cls)\ + .read(OS.path.join(C.LOCAL, "cross", "%s-%s" % (dist, arch), "META")) + except IOError, err: + if err.errno == E.ENOENT: pass + else: raise + return cls(dist = dist, arch = arch) + + def write(me, dir = None): + if dir is None: + dir = OS.path.join(C.LOCAL, "cross", "%s-%s" % (me.dist, me.arch)) + with safewrite_root(OS.path.join(dir, "META")) as f: + me._write(f) + +###-------------------------------------------------------------------------- +### Constructing a chroot. + +R_DIVERT = RX.compile(r"^diversion of (.*) to .* by install-cross-tools$") + +class ChrootJob (BaseJob): + """ + Create or update a chroot. + """ + + SPECS = C.ALL_CHROOTS + + def __init__(me, spec, fresh = CREATE, *args, **kw): + super(ChrootJob, me).__init__(*args, **kw) + me._dist, me._arch = split_dist_arch(spec) + me._fresh = fresh + me._meta = ChrootMetadata.read(me._dist, me._arch) + me._tools_chroot = me._qemu_chroot = None + + def _mkname(me): return "chroot.%s-%s" % (me._dist, me._arch) + + def prepare(me): + if me._arch in C.FOREIGN_ARCHS: + me._tools_chroot = CrossToolsJob.ensure\ + ("%s-%s" % (me._dist, C.TOOLSARCH), FRESH) + me._qemu_chroot = CrossToolsJob.ensure\ + ("%s-%s" % (me._dist, C.QEMUHOST[me._arch]), FRESH) + me.await(me._tools_chroot) + me.await(me._qemu_chroot) + + def check(me): + status, reason = super(ChrootJob, me).check() + if status is not READY: return status, reason + if (me._tools_chroot is not None and me._tools_chroot.started) or \ + (me._qemu_chroot is not None and me._qemu_chroot.started): + return READY, "prerequisites run" + return check_fresh(me._fresh, me._meta.update) + + def _install_cross_tools(me): + """ + Install or refresh cross-tools in the source-chroot. + + This function version assumes that the source-chroot lock is already + held. + + Note that there isn't a job class corresponding to this function. It's + done automatically as part of source-chroot setup and update for foreign + architectures. + """ + with Cleanup() as clean: + + dist, arch = me._dist, me._arch + + mymulti = run_program(["dpkg-architecture", "-a", C.TOOLSARCH, + "-qDEB_HOST_MULTIARCH"], + stdout = RETURN).rstrip("\n") + gnuarch = run_program(["dpkg-architecture", "-A", arch, + "-qDEB_TARGET_GNU_TYPE"], + stdout = RETURN).rstrip("\n") + + crossdir = OS.path.join(C.LOCAL, "cross", + "%s-%s" % (dist, C.TOOLSARCH)) + + qarch, qhost = C.QEMUARCH[arch], C.QEMUHOST[arch] + qemudir = OS.path.join(C.LOCAL, "cross", + "%s-%s" % (dist, qhost), "QEMU") + + ## Acquire lockfiles in a canonical order to prevent deadlocks. + donors = [C.TOOLSARCH] + if qarch != C.TOOLSARCH: donors.append(qarch) + donors.sort() + for a in donors: + clean.enter(lockfile(crosstools_lockfile(dist, a), exclp = False)) + + ## Open a session. + session, root = clean.enter(chroot_session(dist, arch, sourcep = True)) + + ## Search the cross-tools tree for tools, to decide what to do with + ## each file. Make lists: + ## + ## * `want_div' is simply a set of all files in the chroot which need + ## dpkg diversions to prevent foreign versions of the tools from + ## clobbering our native versions. + ## + ## * `want_link' is a dictionary mapping paths which need symbolic + ## links into the cross-tools trees to their link destinations. + progress("scan cross-tools tree") + want_div = set() + want_link = dict() + cross_prefix = crossdir + "/" + qemu_prefix = qemudir + "/" + toolchain_prefix = OS.path.join(crossdir, "TOOLCHAIN", gnuarch) + "/" + def examine(path): + dest = switch_prefix(path, [(qemu_prefix, "/usr/bin/"), + (toolchain_prefix, "/usr/bin/"), + (cross_prefix, "/")]) + if OS.path.islink(path): src = OS.readlink(path) + else: src = host_to_chroot(path) + want_link[dest] = src + if not OS.path.isdir(path): want_div.add(dest) + examine(OS.path.join(qemudir, "qemu-%s-static" % qarch)) + examine(OS.path.join(crossdir, "lib", mymulti)) + examine(OS.path.join(crossdir, "usr/lib", mymulti)) + examine(OS.path.join(crossdir, "usr/lib/gcc-cross")) + def visit(_, dir, files): + ff = [] + for f in files: + if f == "META" or f == "QEMU" or f == "TOOLCHAIN" or \ + (dir.endswith("/lib") and (f == mymulti or f == "gcc-cross")): + continue + ff.append(f) + path = OS.path.join(dir, f) + if not OS.path.isdir(path): examine(path) + files[:] = ff + OS.path.walk(crossdir, visit, None) + OS.path.walk(OS.path.join(crossdir, "TOOLCHAIN", gnuarch), + visit, None) + + ## Build the set `have_div' of paths which already have diversions. + progress("scan chroot") + have_div = set() + with subprocess(["schroot", "-uroot", "-r", "-c", session, "--", + "dpkg-divert", "--list"], + stdout = PIPE) as (_, fd_out, _): + try: + f = OS.fdopen(fd_out) + for line in f: + m = R_DIVERT.match(line.rstrip("\n")) + if m: have_div.add(m.group(1)) + finally: + f.close() + + ## Build a dictionary `have_link' of symbolic links into the cross- + ## tools trees. Also, be sure to collect all of the relative symbolic + ## links which are in the cross-tools tree. + have_link = dict() + with subprocess(["schroot", "-uroot", "-r", "-c", session, "--", + "sh", "-e", "-c", """ + find / -xdev -lname "/usr/local.schroot/cross/*" -printf "%p %l\n" + """], stdout = PIPE) as (_, fd_out, _): + try: + f = OS.fdopen(fd_out) + for line in f: + dest, src = line.split() + have_link[dest] = src + finally: + f.close() + for path in want_link.iterkeys(): + real = root + path + if not OS.path.islink(real): continue + have_link[path] = OS.readlink(real) + + ## Add diversions for the paths which need one, but don't have one. + ## There's a hack here because the `--no-rename' option was required in + ## the same version in which it was introduced, so there's no single + ## incantation that will work across the boundary. + progress("add missing diversions") + with subprocess(["schroot", "-uroot", "-r", "-c", session, "--", + "sh", "-e", "-c", """ + a="%(arch)s" + + if dpkg-divert >/dev/null 2>&1 --no-rename --help + then no_rename=--no-rename + else no_rename= + fi + + while read path; do + dpkg-divert --package "install-cross-tools" $no_rename \ + --divert "$path.$a" --add "$path" + done + """ % dict(arch = arch)], stdin = PIPE) as (fd_in, _, _): + try: + f = OS.fdopen(fd_in, 'w') + for path in want_div: + if path not in have_div: f.write(path + "\n") + finally: + f.close() + + ## Go through each diverted tool, and, if it hasn't been moved aside, + ## then /link/ it across now. If we rename it, then the chroot will + ## stop working -- which is why we didn't allow `dpkg-divert' to do the + ## rename. We can tell a tool that hasn't been moved, because it's a + ## symlink into one of the cross trees. + progress("preserve existing foreign files") + chroot_cross_prefix = host_to_chroot(crossdir) + "/" + chroot_qemu_prefix = host_to_chroot(qemudir) + "/" + for path in want_div: + real = root + path; div = real + "." + arch; cross = crossdir + path + if OS.path.exists(div): continue + if not OS.path.exists(real): continue + if OS.path.islink(real): + realdest = OS.readlink(real) + if realdest.startswith(chroot_cross_prefix) or \ + realdest.startswith(chroot_qemu_prefix): + continue + if OS.path.islink(cross) and realdest == OS.readlink(cross): + continue + progress("preserve existing foreign file `%s'" % path) + run_root(["ln", real, div]) + + ## Update all of the symbolic links which are currently wrong: add + ## links which are missing, delete ones which are obsolete, and update + ## ones which have the wrong target. + progress("update symlinks") + for path, src in want_link.iteritems(): + real = root + path + try: old_src = have_link[path] + except KeyError: pass + else: + if src == old_src: continue + new = real + ".new" + progress("link `%s' -> `%s'" % (path, src)) + dir = OS.path.dirname(real) + if not OS.path.isdir(dir): run_root(["mkdir", "-p", dir]) + if OS.path.exists(new): run_root(["rm", "-f", new]) + run_root(["ln", "-s", src, new]) + run_root(["mv", new, real]) + for path in have_link.iterkeys(): + if path in want_link: continue + progress("remove obsolete link `%s' -> `%s'" % path) + real = root + path + run_root(["rm", "-f", real]) + + ## Remove diversions from paths which don't need them any more. Here + ## it's safe to rename, because either the tool isn't there, in which + ## case it obviously wasn't important, or it is, and `dpkg-divert' will + ## atomically replace our link with the foreign version. + progress("remove obsolete diversions") + with subprocess(["schroot", "-uroot", "-r", "-c", session, "--", + "sh", "-e", "-c", """ + a="%(arch)s" + + while read path; do + dpkg-divert --package "install-cross-tools" --rename \ + --divert "$path.$a" --remove "$path" + done + """ % dict(arch = arch)], stdin = PIPE) as (fd_in, _, _): + try: + f = OS.fdopen(fd_in, 'w') + for path in have_div: + if path not in want_div: f.write(path + "\n") + finally: + f.close() + + def _make_chroot(me): + """ + Create the source-chroot with chroot metadata META. + + This will recreate a source-chroot from scratch, destroying the existing + logical volume if necessary. + """ + with Cleanup() as clean: + + dist, arch = me._dist, me._arch + clean.enter(lockfile(chroot_src_lockfile(dist, arch))) + + mnt = chroot_src_mntpt(dist, arch) + dev = chroot_src_blkdev(dist, arch) + lv = chroot_src_lv(dist, arch) + newlv = lv + ".new" + + ## Clean up any leftover debris. + if mountpoint_p(mnt): umount(mnt) + if block_device_p(dev): + run_root(["lvremove", "-f", "%s/%s" % (C.VG, lv)]) + + ## Create the logical volume and filesystem. It's important that the + ## logical volume not have its official name until after it contains a + ## mountable filesystem. + progress("create filesystem") + run_root(["lvcreate", "--yes", C.LVSZ, "-n", newlv, C.VG]) + run_root(["mkfs", "-j", "-L%s-%s" % (dist, arch), + OS.path.join("/dev", C.VG, newlv)]) + run_root(["lvrename", C.VG, newlv, lv]) + + ## Start installing the chroot. + with mount_chroot_src(dist, arch) as mnt: + + ## Set the basic structure. + run_root(["mkdir", "-m755", OS.path.join(mnt, "fs")]) + run_root(["chmod", "750", mnt]) + + ## Install the base system. + progress("install base system") + run_root(["eatmydata", "debootstrap"] + + (arch in C.FOREIGN_ARCHS and ["--foreign"] or []) + + ["--arch=" + arch, "--variant=minbase", + "--include=" + ",".join(C.BASE_PACKAGES), + dist, OS.path.join(mnt, "fs"), C.DEBMIRROR]) + + ## If this is a cross-installation, then install the necessary `qemu' + ## and complete the installation. + if arch in C.FOREIGN_ARCHS: + qemu = OS.path.join("cross", "%s-%s" % (dist, C.QEMUHOST[arch]), + "QEMU", "qemu-%s-static" % C.QEMUARCH[arch]) + run_root(["install", OS.path.join(C.LOCAL, qemu), + OS.path.join(mnt, "fs/usr/bin")]) + run_root(["chroot", OS.path.join(mnt, "fs"), + "/debootstrap/debootstrap", "--second-stage"]) + run_root(["ln", "-sf", + OS.path.join("/usr/local.schroot", qemu), + OS.path.join(mnt, "fs/usr/bin")]) + + ## Set up `/usr/local'. + progress("install `/usr/local' symlink") + run_root(["rm", "-rf", OS.path.join(mnt, "fs/usr/local")]) + run_root(["ln", "-s", + OS.path.join("local.schroot", arch), + OS.path.join(mnt, "fs/usr/local")]) + + ## Install the `apt' configuration. + progress("configure package manager") + run_root(["rm", "-f", OS.path.join(mnt, "fs/etc/apt/sources.list")]) + for c in C.APTCONF: + run_root(["ln", "-s", + OS.path.join("/usr/local.schroot/etc/apt/apt.conf.d", c), + OS.path.join(mnt, "fs/etc/apt/apt.conf.d")]) + run_root(["ln", "-s", + "/usr/local.schroot/etc/apt/sources.%s" % dist, + OS.path.join(mnt, "fs/etc/apt/sources.list")]) + + with safewrite_root\ + (OS.path.join(mnt, "fs/etc/apt/apt.conf.d/20arch")) as f: + f.write("""\ + ### -*-conf-*- + + APT { + Architecture "%s"; + }; + """ % arch) + + ## Set up the locale and time zone from the host system. + progress("configure locales and timezone") + run_root(["cp", "/etc/locale.gen", "/etc/timezone", + OS.path.join(mnt, "fs/etc")]) + with open("/etc/timezone") as f: tz = f.readline().strip() + run_root(["ln", "-sf", + OS.path.join("/usr/share/timezone", tz), + OS.path.join(mnt, "fs/etc/localtime")]) + run_root(["cp", "/etc/default/locale", + OS.path.join(mnt, "fs/etc/default")]) + + ## Fix `/etc/mtab'. + progress("set `/etc/mtab'") + run_root(["ln", "-sf", "/proc/mounts", + OS.path.join(mnt, "fs/etc/mtab")]) + + ## Prevent daemons from starting within the chroot. + progress("inhibit daemon startup") + with safewrite_root(OS.path.join(mnt, "fs/usr/sbin/policy-rc.d"), + mode = "755") as f: + f.write("""\ + #! /bin/sh + echo >&2 "policy-rc.d: Services disabled by policy." + exit 101 + """) + + ## Hack the dynamic linker to prefer libraries in `/usr' over + ## `/usr/local'. This prevents `dpkg-shlibdeps' from becoming + ## confused. + progress("configure dynamic linker") + with safewrite_root\ + (OS.path.join(mnt, "fs/etc/ld.so.conf.d/libc.conf")) as f: + f.write("# libc default configuration") + with safewrite_root\ + (OS.path.join(mnt, "fs/etc/ld.so.conf.d/zzz-local.conf")) as f: + f.write("""\ + ### -*-conf-*- + ### Local hack to make /usr/local/ late. + /usr/local/lib + """) + + ## If this is a foreign architecture then we need to set it up. + if arch in C.FOREIGN_ARCHS: + + ## Keep the chroot's native Qemu out of our way: otherwise we'll stop + ## being able to run programs in the chroot. There's a hack here + ## because the `--no-rename' option was required in the same version + ## in which is was introduced, so there's no single incantation that + ## will work across the boundary. + progress("divert emulator") + run_schroot_source(dist, arch, ["eatmydata", "sh", "-e", "-c", """ + if dpkg-divert >/dev/null 2>&1 --no-rename --help + then no_rename=--no-rename + else no_rename= + fi + + dpkg-divert --package install-cross-tools $no_rename \ + --divert /usr/bin/%(qemu)s.%(arch)s --add /usr/bin/%(qemu)s + """ % dict(arch = arch, qemu = "qemu-%s-static" % C.QEMUARCH[arch])]) + + ## Install faster native tools. + me._install_cross_tools() + + ## Finishing touches. + progress("finishing touches") + run_schroot_source(dist, arch, ["eatmydata", "sh", "-e", "-c", """ + apt-get update + apt-get -y upgrade + apt-get -y install "$@" + ldconfig + apt-get -y autoremove + apt-get clean + """, "."] + C.EXTRA_PACKAGES, stdin = DISCARD) + + ## Mark the chroot as done. + me._meta.update = zulu() + me._meta.write() + + def _update_chroot(me): + """Refresh the source-chroot with chroot metadata META.""" + with Cleanup() as clean: + dist, arch = me._dist, me._arch + clean.enter(lockfile(chroot_src_lockfile(dist, arch))) + run_schroot_source(dist, arch, ["eatmydata", "sh", "-e", "-c", """ + apt-get update + apt-get -y dist-upgrade + apt-get -y autoremove + apt-get -y clean + """], stdin = DISCARD) + if arch in C.FOREIGN_ARCHS: me._install_cross_tools() + me._meta.update = zulu(); me._meta.write() + + def run(me): + if me._meta.update is not None: me._update_chroot() + else: me._make_chroot() + +###-------------------------------------------------------------------------- +### Extracting the cross tools. + +class CrossToolsJob (BaseJob): + """Extract cross-tools from a donor chroot.""" + + SPECS = C.NATIVE_CHROOTS + + def __init__(me, spec, fresh = CREATE, *args, **kw): + super(CrossToolsJob, me).__init__(*args, **kw) + me._dist, me._arch = split_dist_arch(spec) + me._meta = CrossToolsMetadata.read(me._dist, me._arch) + me._fresh = fresh + me._chroot = None + + def _mkname(me): return "cross-tools.%s-%s" % (me._dist, me._arch) + + def prepare(me): + st, r = check_fresh(me._fresh, me._meta.update) + if st is DONE: return + me._chroot = ChrootJob.ensure("%s-%s" % (me._dist, me._arch), FRESH) + me.await(me._chroot) + + def check(me): + status, reason = super(CrossToolsJob, me).check() + if status is not READY: return status, reason + if me._chroot is not None and me._chroot.started: + return READY, "prerequisites run" + return check_fresh(me._fresh, me._meta.update) + + def run(me): + with Cleanup() as clean: + + dist, arch = me._dist, me._arch + + mymulti = run_program(["dpkg-architecture", "-a" + arch, + "-qDEB_HOST_MULTIARCH"], + stdout = RETURN).rstrip("\n") + crossarchs = [run_program(["dpkg-architecture", "-A" + a, + "-qDEB_TARGET_GNU_TYPE"], + stdout = RETURN).rstrip("\n") + for a in C.FOREIGN_ARCHS] + + crossdir = OS.path.join(C.LOCAL, "cross", "%s-%s" % (dist, arch)) + crossold = crossdir + ".old"; crossnew = crossdir + ".new" + usrbin = OS.path.join(crossnew, "usr/bin") + + clean.enter(lockfile(crosstools_lockfile(dist, arch))) + run_program(["rm", "-rf", crossnew]) + mkdir_p(crossnew) + + ## Open a session to the donor chroot. + progress("establish snapshot") + session, root = clean.enter(chroot_session(dist, arch)) + + ## Make sure the donor tree is up-to-date, and install the extra + ## packages we need. + progress("install tools packages") + run_schroot_session(session, ["eatmydata", "sh", "-e", "-c", """ + apt-get update + apt-get -y upgrade + apt-get -y install "$@" + """, "."] + C.CROSS_PACKAGES, rootp = True, stdin = DISCARD) + + def chase(path): + dest = "" + + ## Work through the remaining components of the PATH. + while path != "": + try: sl = path.index("/") + except ValueError: step = path; path = "" + else: step, path = path[:sl], path[sl + 1:] + + ## Split off and analyse the first component. + if step == "" or step == ".": + ## A redundant `/' or `./'. Skip it. + pass + elif step == "..": + ## A `../'. Strip off the trailing component of DEST. + dest = dest[:dest.rindex("/")] + else: + ## Something else. Transfer the component name to DEST. + dest += "/" + step + + ## If DEST refers to something in the cross-tools tree then we're + ## good. + crossdest = crossnew + dest + try: st = OS.lstat(crossdest) + except OSError, err: + if err.errno == E.ENOENT: + ## No. We need to copy something from the donor tree so that + ## the name works. + + st = OS.lstat(root + dest) + if ST.S_ISDIR(st.st_mode): + OS.mkdir(crossdest) + else: + progress("copy `%s'" % dest) + run_program(["rsync", "-aHR", + "%s/.%s" % (root, dest), + crossnew]) + else: + raise + + ## If DEST refers to a symbolic link, then prepend the link target + ## to PATH so that we can be sure the link will work. + if ST.S_ISLNK(st.st_mode): + link = OS.readlink(crossdest) + if link.startswith("/"): dest = ""; link = link[1:] + else: + try: dest = dest[:dest.rindex("/")] + except ValueError: dest = "" + if path == "": path = link + else: path = "%s/%s" % (path, link) + + ## Work through the shopping list, copying the things it names into the + ## cross-tools tree. + scan = [] + for pat in C.CROSS_PATHS: + pat = pat.replace("MULTI", mymulti) + any = False + for rootpath in GLOB.iglob(root + pat): + any = True + path = rootpath[len(root):] + progress("copy `%s'" % path) + run_program(["rsync", "-aHR", "%s/.%s" % (root, path), crossnew]) + if not any: + raise RuntimeError("no matches for cross-tool pattern `%s'" % pat) + + ## Scan the new tree: chase down symbolic links, copying extra stuff + ## that we'll need; and examine ELF binaries to make sure we get the + ## necessary shared libraries. + def visit(_, dir, files): + for f in files: + path = OS.path.join(dir, f) + inside = switch_prefix(path, [(crossnew + "/", "/")]) + if OS.path.islink(path): chase(inside) + if elf_binary_p(arch, path): scan.append(inside) + OS.path.walk(crossnew, visit, None) + + ## Work through the ELF binaries in `scan', determining which shared + ## libraries they'll need. + ## + ## The rune running in the chroot session reads ELF binary names on + ## stdin, one per line, and runs `ldd' on them to discover the binary's + ## needed libraries and resolve them into pathnames. Each pathname is + ## printed to stderr as a line `+PATHNAME', followed by a final line + ## consisting only of `-' as a terminator. This is necessary so that + ## we can tell when we've finished, because newly discovered libraries + ## need to be fed back to discover their recursive dependencies. (This + ## is why the `WriteLinesSelector' interface is quite so hairy.) + with subprocess(["schroot", "-r", "-c", session, "--", + "sh", "-e", "-c", """ + while read path; do + ldd "$path" | while read a b c d; do + case $a:$b:$c:$d in + not:a:dynamic:executable) ;; + statically:linked::) ;; + /*) echo "+$a" ;; + *:=\\>:/*) echo "+$c" ;; + linux-*) ;; + *) echo >&2 "failed to find shared library \\`$a'"; exit 2 ;; + esac + done + echo - + done + """], stdin = PIPE, stdout = PIPE) as (fd_in, fd_out, _): + + ## Keep track of the number of binaries we've reported to the `ldd' + ## process for which we haven't yet seen all of their dependencies. + ## (This is wrapped in a `Struct' because of Python's daft scoping + ## rules.) + v = Struct(n = 0) + + def line_in(): + ## Provide a line in., so raise `StopIteration' to signal this. + + try: + ## See if there's something to scan. + path = scan.pop() + + except IndexError: + ## There's nothing currently waiting to be scanned. + if v.n: + ## There are still outstanding replies, so stall. + return None + else: + ## There are no outstanding replies left, and we have nothing + ## more to scan, then we must be finished. + raise StopIteration + + else: + ## The `scan' list isn't empty, so return an item from that, and + ## remember that there's one more thing we expect to see answers + ## from. + v.n += 1; return path + + def line_out(line): + ## We've received a line from the `ldd' process. + + if line == "-": + ## It's finished processing one of our binaries. Note this. + ## Maybe it's time to stop + v.n -= 1 + return + + ## Strip the leading marker (which is just there so that the + ## terminating `-' is unambiguous). + assert line.startswith("+") + lib = line[1:] + + ## If we already have this binary then we'll already have submitted + ## it. + path = crossnew + lib + try: OS.lstat(path) + except OSError, err: + if err.errno == E.ENOENT: pass + else: raise + else: return + + ## Copy it into the tools tree, together with any symbolic links + ## along the path. + chase(lib) + + ## If this is an ELF binary (and it ought to be!) then submit it + ## for further scanning. + if elf_binary_p(arch, path): + scan.append(switch_prefix(path, [(crossnew + "/", "/")])) + + ## And run this entire contraption. When this is done, we should + ## have all of the library dependencies for all of our binaries. + select_loop([WriteLinesSelector(fd_in, line_in), + ReadLinesSelector(fd_out, line_out)]) + + ## Set up the cross-compiler and emulator. Start by moving the cross + ## compilers and emulator into their specific places, so they don't end + ## up cluttering chroots for non-matching architectures. + progress("establish TOOLCHAIN and QEMU") + OS.mkdir(OS.path.join(crossnew, "TOOLCHAIN")) + qemudir = OS.path.join(crossnew, "QEMU") + OS.mkdir(qemudir) + for gnu in C.FOREIGN_GNUARCHS: + OS.mkdir(OS.path.join(crossnew, "TOOLCHAIN", gnu)) + for f in OS.listdir(usrbin): + for gnu in C.FOREIGN_GNUARCHS: + gnuprefix = gnu + "-" + if f.startswith(gnuprefix): + tooldir = OS.path.join(crossnew, "TOOLCHAIN", gnu) + OS.rename(OS.path.join(usrbin, f), OS.path.join(tooldir, f)) + OS.symlink(f, OS.path.join(tooldir, f[len(gnuprefix):])) + break + else: + if f.startswith("qemu-") and f.endswith("-static"): + OS.rename(OS.path.join(usrbin, f), OS.path.join(qemudir, f)) + + ## The GNU cross compilers try to find their additional pieces via a + ## relative path, which isn't going to end well. Add a symbolic link + ## at the right place to where the things are actually going to live. + toollib = OS.path.join(crossnew, "TOOLCHAIN", "lib") + OS.mkdir(toollib) + OS.symlink("../../usr/lib/gcc-cross", + OS.path.join(toollib, "gcc-cross")) + + ## We're done. Replace the old cross-tools with our new one. + me._meta.update = zulu() + me._meta.write(crossnew) + if OS.path.exists(crossdir): run_program(["mv", crossdir, crossold]) + OS.rename(crossnew, crossdir) + run_program(["rm", "-rf", crossold]) + +###-------------------------------------------------------------------------- +### Buliding and installing local packages. + +def pkg_metadata_lockfile(pkg): + return lockfile_path("pkg-meta.%s" % pkg) + +def pkg_srcdir_lockfile(pkg, ver): + return lockfile_path("pkg-source.%s-%s" % (pkg, ver)) + +def pkg_srcdir(pkg, ver): + return OS.path.join(C.LOCAL, "src", "%s-%s" % (pkg, ver)) + +def pkg_builddir(pkg, ver, arch): + return OS.path.join(pkg_srcdir(pkg, ver), "build.%s" % arch) + +class PackageMetadata (BaseMetadata): + VARS = ["pkg"] + list(C.ALL_ARCHS) + + @classmethod + def read(cls, pkg): + try: + return super(PackageMetadata, cls)\ + .read(OS.path.join(C.LOCAL, "src", "META.%s" % pkg)) + except IOError, err: + if err.errno == E.ENOENT: pass + else: raise + return cls(pkg = pkg) + + def write(me): + super(PackageMetadata, me)\ + .write(OS.path.join(C.LOCAL, "src", "META.%s" % me.pkg)) + +class PackageSourceJob (BaseJob): + + SPECS = C.LOCALPKGS + + def __init__(me, pkg, fresh = CREATE, *args, **kw): + super(PackageSourceJob, me).__init__(*args, **kw) + me._pkg = pkg + tar = None; ver = None + r = RX.compile("^%s-(\d.*)\.tar.(?:Z|z|gz|bz2|xz|lzma)$" % + RX.escape(pkg)) + for f in OS.listdir("pkg"): + m = r.match(f) + if not m: pass + elif tar is not None: + raise ExpectedError("multiple source tarballs of package `%s'" % pkg) + else: tar, ver = f, m.group(1) + me.version = ver + me.tarball = OS.path.join("pkg", tar) + + def _mkname(me): return "pkg-source.%s" % me._pkg + + def check(me): + status, reason = super(PackageSourceJob, me).check() + if status is not READY: return status, reason + if OS.path.isdir(pkg_srcdir(me._pkg, me.version)): + return DONE, "already unpacked" + else: + return READY, "no source tree" + + def run(me): + with Cleanup() as clean: + pkg, ver, tar = me._pkg, me.version, me.tarball + srcdir = pkg_srcdir(pkg, ver) + newdir = srcdir + ".new" + + progress("unpack `%s'" % me.tarball) + clean.enter(lockfile(pkg_srcdir_lockfile(pkg, ver))) + run_program(["rm", "-rf", newdir]) + mkdir_p(newdir) + run_program(["tar", "xf", OS.path.join(OS.getcwd(), me.tarball)], + cwd = newdir) + things = OS.listdir(newdir) + if len(things) == 1: + OS.rename(OS.path.join(newdir, things[0]), srcdir) + OS.rmdir(newdir) + else: + OS.rename(newdir, srcdir) + +class PackageBuildJob (BaseJob): + + SPECS = ["%s:%s" % (pkg, arch) + for pkg in C.LOCALPKGS + for arch in C.ALL_ARCHS] + + def __init__(me, spec, fresh = CREATE, *args, **kw): + super(PackageBuildJob, me).__init__(*args, **kw) + colon = spec.index(":") + me._pkg, me._arch = spec[:colon], spec[colon + 1:] + + def _mkname(me): return "pkg-build.%s:%s" % (me._pkg, me._arch) + + def prepare(me): + me.await(ChrootJob.ensure("%s-%s" % (C.PRIMARY_DIST, me._arch), CREATE)) + me._meta = PackageMetadata.read(me._pkg) + me._src = PackageSourceJob.ensure(me._pkg, FRESH); me.await(me._src) + me._prereq = [PackageBuildJob.ensure("%s:%s" % (prereq, me._arch), FRESH) + for prereq in C.PKGDEPS[me._pkg]] + for j in me._prereq: me.await(j) + + def check(me): + status, reason = super(PackageBuildJob, me).check() + if status is not READY: return status, reason + if me._src.started: return READY, "fresh source directory" + for j in me._prereq: + if j.started: + return READY, "dependency `%s' freshly installed" % j._pkg + if getattr(me._meta, me._arch) == me._src.version: + return DONE, "already installed" + return READY, "not yet installed" + + def run(me): + with Cleanup() as clean: + pkg, ver, arch = me._pkg, me._src.version, me._arch + + session, _ = clean.enter(chroot_session(C.PRIMARY_DIST, arch)) + builddir = OS.path.join(pkg_srcdir(pkg, ver), "build.%s" % arch) + chroot_builddir = host_to_chroot(builddir) + run_program(["rm", "-rf", builddir]) + OS.mkdir(builddir) + + progress("prepare %s chroot" % (arch)) + run_schroot_session(session, + ["eatmydata", "apt-get", "update"], + rootp = True, stdin = DISCARD) + run_schroot_session(session, + ["eatmydata", "apt-get", "-y", "upgrade"], + rootp = True, stdin = DISCARD) + run_schroot_session(session, + ["eatmydata", "apt-get", "-y", + "install", "pkg-config"], + rootp = True, stdin = DISCARD) + run_schroot_session(session, + ["mount", "-oremount,rw", "/usr/local.schroot"], + rootp = True, stdin = DISCARD) + + progress("configure `%s' %s for %s" % (pkg, ver, arch)) + run_schroot_session(session, ["sh", "-e", "-c", """ + cd "$1" && + ../configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig.hidden + """, ".", chroot_builddir]) + + progress("compile `%s' %s for %s" % (pkg, ver, arch)) + run_schroot_session(session, ["sh", "-e", "-c", """ + cd "$1" && make -j4 && make -j4 check + """, ".", chroot_builddir]) + + existing = getattr(me._meta, arch, None) + if existing is not None and existing != ver: + progress("uninstall existing `%s' %s for %s" % (pkg, existing, arch)) + run_schroot_session(session, ["sh", "-e", "-c", """ + cd "$1" && make uninstall + """, ".", OS.path.join(pkg_srcdir(pkg, existing), + "build.%s" % arch)], + rootp = True) + + progress("install `%s' %s for %s" % (pkg, existing, arch)) + run_schroot_session(session, ["sh", "-e", "-c", """ + cd "$1" && make install + mkdir -p /usr/local/lib/pkgconfig.hidden + mv /usr/local/lib/pkgconfig/*.pc /usr/local/lib/pkgconfig.hidden || : + """, ".", chroot_builddir], rootp = True) + + clean.enter(lockfile(pkg_metadata_lockfile(pkg))) + me._meta = PackageMetadata.read(pkg) + setattr(me._meta, arch, ver); me._meta.write() + + with lockfile(chroot_src_lockfile(C.PRIMARY_DIST, arch)): + run_schroot_source(C.PRIMARY_DIST, arch, ["ldconfig"]) + +###-------------------------------------------------------------------------- +### Process the configuration and options. + +OPTIONS = OP.OptionParser\ + (usage = "chroot-maint [-diknqs] [-fFRESH] [-jN] JOB[.SPEC,...] ...") +for short, long, props in [ + ("-d", "--debug", { + 'dest': 'debug', 'default': False, 'action': 'store_true', + 'help': "print lots of debugging drivel" }), + ("-f", "--fresh", { + 'dest': 'fresh', 'metavar': 'FRESH', 'default': "create", + 'help': "how fresh (`create', `force', or `N[s|m|h|d|w]')" }), + ("-i", "--ignore-errors", { + 'dest': 'ignerr', 'default': False, 'action': 'store_true', + 'help': "ignore all errors encountered while processing" }), + ("-j", "--jobs", { + 'dest': 'njobs', 'metavar': 'N', 'default': 1, 'type': 'int', + 'help': 'run up to N jobs in parallel' }), + ("-J", "--forkbomb", { + 'dest': 'njobs', 'action': 'store_true', + 'help': 'run as many jobs in parallel as possible' }), + ("-k", "--keep-going", { + 'dest': 'keepon', 'default': False, 'action': 'store_true', + 'help': "keep going even if independent jobs fail" }), + ("-n", "--dry-run", { + 'dest': 'dryrun', 'default': False, 'action': 'store_true', + 'help': "don't actually do anything" }), + ("-q", "--quiet", { + 'dest': 'quiet', 'default': False, 'action': 'store_true', + 'help': "don't print the output from successful jobs" }), + ("-s", "--silent", { + 'dest': 'silent', 'default': False, 'action': 'store_true', + 'help': "don't print progress messages" })]: + OPTIONS.add_option(short, long, **props) + +###-------------------------------------------------------------------------- +### Main program. + +R_JOBSERV = RX.compile(r'^--jobserver-(?:fds|auth)=(\d+),(\d+)$') + +JOBMAP = { "chroot": ChrootJob, + "cross-tools": CrossToolsJob, + "pkg-source": PackageSourceJob, + "pkg-build": PackageBuildJob } + +R_FRESH = RX.compile(r"^(?:create|force|(\d+)(|[smhdw]))$") + +def parse_fresh(spec): + m = R_FRESH.match(spec) + if not m: raise ExpectedError("bad freshness `%s'" % spec) + if spec == "create": fresh = CREATE + elif spec == "force": fresh = FORCE + else: + n, u = int(m.group(1)), m.group(2) + if u == "" or u == "s": fresh = n + elif u == "m": fresh = 60*n + elif u == "h": fresh = 3600*n + elif u == "d": fresh = 86400*n + elif u == "w": fresh = 604800*n + else: assert False + return fresh + +with toplevel_handler(): + OPT, args = OPTIONS.parse_args() + rfd, wfd = -1, -1 + njobs = OPT.njobs + try: mkflags = OS.environ['MAKEFLAGS'] + except KeyError: pass + else: + ff = mkflags.split() + for f in ff: + if f == "--": break + m = R_JOBSERV.match(f) + if m: rfd, wfd = int(m.group(1)), int(m.group(2)) + elif f == '-j': njobs = None + elif not f.startswith('-'): + for ch in f: + if ch == 'i': OPT.ignerr = True + elif ch == 'k': OPT.keepon = True + elif ch == 'n': OPT.dryrun = True + elif ch == 's': OPT.silent = True + if OPT.njobs < 1: + raise ExpectedError("running no more than %d jobs is silly" % OPT.njobs) + + FRESH = parse_fresh(OPT.fresh) + + SCHED = JobScheduler(rfd, wfd, njobs) + OS.environ["http_proxy"] = C.PROXY + + jobs = [] + if not args: OPTIONS.print_usage(SYS.stderr); SYS.exit(2) + for arg in args: + try: sl = arg.index("/") + except ValueError: fresh = FRESH + else: arg, fresh = arg[:sl], parse_fresh(arg[sl + 1:]) + try: dot = arg.index(".") + except ValueError: jty, pats = arg, "*" + else: jty, pats = arg[:dot], arg[dot + 1:] + try: jcls = JOBMAP[jty] + except KeyError: raise ExpectedError("unknown job type `%s'" % jty) + specs = [] + for pat in pats.split(","): + any = False + for s in jcls.SPECS: + if FM.fnmatch(s, pat): specs.append(s); any = True + if not any: raise ExpectedError("no match for `%s'" % pat) + for s in specs: + jobs.append(jcls.ensure(s, fresh)) + + SCHED.run() + +SYS.exit(RC) + +###----- That's all, folks -------------------------------------------------- diff --git a/bin/install-cross-tools b/bin/install-cross-tools index 7673229..b985551 100755 --- a/bin/install-cross-tools +++ b/bin/install-cross-tools @@ -119,7 +119,7 @@ schroot -uroot -rc$sess -- sh -ec ' t=$(readlink $s) case $t in /usr/local.schroot/cross/*) continue ;; esac echo "$s $t" - done /dev/null } | sort >/mnt/LINK.have' ## Add diversions for the paths which need one, but don't have one. There's diff --git a/bin/mkbuildchroot b/bin/mkbuildchroot index cb17ed2..a6ac135 100755 --- a/bin/mkbuildchroot +++ b/bin/mkbuildchroot @@ -78,10 +78,6 @@ chmod 750 $mnt/ ## Install the base system. want=$BASE_PACKAGES -case $qemup in - t) want="$want $FOREIGN_BASE_PACKAGES" ;; - nil) want="$want $NATIVE_BASE_PACKAGES" ;; -esac pkgs=; for p in $want; do pkgs=${pkgs:+$pkgs,}$p; done eatmydata debootstrap $dbsopts --arch=$a --variant=minbase \ --include=$pkgs $d $mnt/fs/ $DEBMIRROR @@ -163,13 +159,13 @@ case $qemup in ## Install faster native tools. $STATE/bin/install-cross-tools $d $a - - ## Install `build-essential', which had been delayed from earlier. - schroot -uroot -csource:$lv -- \ - eatmydata apt-get -y install build-essential - ;; esac +## Install extra packages now that everything should go fairly quickly. +want=$EXTRA_PACKAGES +pkgs=; for p in $want; do pkgs=${pkgs:+$pkgs,}$p; done +schroot -uroot -csource:$lv -- eatmydata apt-get -y install $pkgs + ## Set the chroot's package state up properly. schroot -uroot -csource:$lv -- eatmydata sh -e -c ' apt-get update diff --git a/bin/mkchrootconf b/bin/mkchrootconf index 642fb37..dac6fe3 100755 --- a/bin/mkchrootconf +++ b/bin/mkchrootconf @@ -97,6 +97,7 @@ type=lvm-snapshot description=Debian $dist/$arch autobuilder device=/dev/$VG/$LVPREFIX$dist-$arch lvm-snapshot-options=$SNAPOPT +lvm.suppress-fd-warnings=t mount-options=-onosuid,data=writeback,barrier=0,commit=3600,noatime location=/fs groups=root,sbuild -- 2.11.0