From: Mark Wooding Date: Sun, 8 Sep 2019 19:44:58 +0000 (+0100) Subject: Very rough work-in-progress. X-Git-Url: https://git.distorted.org.uk/~mdw/distorted-chroot/commitdiff_plain/e36b4f2551f8a3a819bf2614665c5d1d5a93ae0a Very rough work-in-progress. Doesn't work at all yet. --- e36b4f2551f8a3a819bf2614665c5d1d5a93ae0a diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2706c7e --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/etc/apt-conf.d/9* +/etc/aptsrc.site.conf +/etc/aptsrc.local.conf +/local.mk +/local.schroot/ +/state/ diff --git a/.skelrc b/.skelrc new file mode 100644 index 0000000..cbb4fbb --- /dev/null +++ b/.skelrc @@ -0,0 +1,8 @@ +;;; -*-emacs-lisp-*- + +(setq skel-alist + (append + '((author . "Mark Wooding") + (program . "distorted-chroot") + (full-title . "the distorted.org.uk chroot maintenance tools")) + skel-alist)) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..dd96989 --- /dev/null +++ b/Makefile @@ -0,0 +1,328 @@ +### -*-makefile-*- +### +### Main maintenance script for chroots +### +### (c) 2018 Mark Wooding +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of the distorted.org.uk chroot maintenance tools. +### +### distorted-chroot is free software: you can redistribute it and/or +### modify it under the terms of the GNU General Public License as +### published by the Free Software Foundation; either version 2 of the +### License, or (at your option) any later version. +### +### distorted-chroot is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +### General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with distorted-chroot. If not, write to the Free Software +### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, +### USA. + +all:: +clean:: +.PHONY: all clean +.SECONDEXPANSION: #sorry + +###-------------------------------------------------------------------------- +### Configuration. + +CONFIG_VARS = + +## Volume group from which to allocate chroot volumes and snapshots. +CONFIG_VARS += VG LVPREFIX +VG = vg-$(shell hostname) +LVPREFIX = + +## Logical volume size, as an LVM option. +CONFIG_VARS += LVSZ +LVSZ = -L8G + +## Debian mirror. +CONFIG_VARS += DEBMIRROR +DEBMIRROR = http://deb.debian.org/debian/ + +## APT sources configurations. (Also, $($d_APTSRC) for each $d in $(DISTS).) +CONFIG_VARS += APTSRC +APTSRC = etc/aptsrc.conf $(wildcard etc/aptsrc.local.conf) + +## APT configuration fragment names. These will be linked into +## `/etc/apt/apt.conf.d' in each chroot. To put a fragment f in a surprising +## place, set $($f_APTCONFSRC). +CONFIG_VARS += APTCONF +APTCONF_DIR = etc/apt-conf.d +APTCONF = $(notdir $(wildcard $(APTCONF_DIR)/[0-9]*[!~])) + +## Proxy setting. +CONFIG_VARS += PROXY +PROXY := $(shell \ + eval $$(apt-config $(foreach a,$(APTCONF), -c$(APTCONF_DIR)/$a) \ + shell proxy Acquire::http::proxy); \ + case $${proxy+t} in (t) echo "$$proxy" ;; (*) echo nil ;; esac) + +## Debian distributions to support. +CONFIG_VARS += PRIMARY_DIST DISTS +PRIMARY_DIST = stretch +DISTS = $(PRIMARY_DIST) buster bullseye sid + +## Host's native architecture(s). +CONFIG_VARS += MYARCH NATIVE_ARCHS +MYARCH = $(shell dpkg --print-architecture) +OTHERARCHS = $(shell dpkg --print-foreign-architectures) +NATIVE_ARCHS = $(MYARCH) $(OTHERARCHS) + +## Foreign (emulated) architectures to support. +CONFIG_VARS += FOREIGN_ARCHS +FOREIGN_ARCHS = + +## Master lists of chroots to build and maintain. +NATIVE_CHROOTS = $(foreach a,$(NATIVE_ARCHS), \ + $(foreach d,$(or $($a_DISTS) $(DISTS)), \ + $d-$a)) +FOREIGN_CHROOTS = $(foreach a,$(FOREIGN_ARCHS), \ + $(foreach d,$(or $($a_DISTS) $(DISTS)), \ + $d-$a)) + +## Which host architecture to use for foreign architectures. It turns out +## that it's best to use a Qemu with the same host bitness as the target +## architecture; otherwise it has to do a difficult job of converting +## arguments and results between kernel ABI flavours. +32BIT_QEMUHOST = $(or $(filter i386,$(NATIVE_ARCHS)),$(TOOLSARCH)) +64BIT_QEMUHOST = $(or $(filter amd64,$(NATIVE_ARCHS)),$(TOOLSARCH)) + +CONFIG_VARS += $(foreach a,$(FOREIGN_ARCHS), $a_QEMUHOST) +armel_QEMUHOST = $(32BIT_QEMUHOST) +armhf_QEMUHOST = $(32BIT_QEMUHOST) +arm64_QEMUHOST = $(64BIT_QEMUHOST) +i386_QEMUHOST = $(32BIT_QEMUHOST) +amd64_QEMUHOST = $(64BIT_QEMUHOST) + +## Qemu architecture names. These tell us which Qemu binary to use for a +## particular Debian architecture. +CONFIG_VARS += $(foreach a,$(FOREIGN_ARCHS), $a_QEMUARCH) +armel_QEMUARCH = arm +armhf_QEMUARCH = arm +arm64_QEMUARCH = aarch64 +i386_QEMUARCH = i386 +amd64_QEMUARCH = x86_64 + +## Alias mapping for chroots. +CONFIG_VARS += $(foreach d,$(DISTS), $d_ALIASES) +stretch_ALIASES = oldstable +buster_ALIASES = stable +bullseye_ALIASE = testing +sid_ALIASES = unstable + +## Which host architecture to use for commonly used tools in foreign chroots. +CONFIG_VARS += TOOLSARCH +TOOLSARCH = $(MYARCH) + +## A directory in which we can maintain private files and state. +CONFIG_VARS += STATE +STATE = state + +## A directory which will be spliced into chroots as `/usr/local.schroot/'. +## This will be our primary point of contact with the chroot. +CONFIG_VARS += LOCAL +LOCAL = local.schroot + +## How to run a command as a privileged user. +ROOTLY = sudo + +## Files to be copied into a chroot from the host. +SCHROOT_COPYFILES = /etc/resolv.conf +SCHROOT_NSSDATABASES = passwd shadow group gshadow + +## Local configuration hook. +-include local.mk + +## All chroot names. +CONFIG_VARS += ALL_CHROOTS +ALL_CHROOTS = $(NATIVE_CHROOTS) $(FOREIGN_CHROOTS) + +###-------------------------------------------------------------------------- +### Utilities. + +## $(call chroot-dist,D-A) -> D +## $(call chroot-arch,D-A) -> A +## +## Parse chroot descriptions. +chroot-dist = $(patsubst %/,%,$(dir $(subst -,/,$1))) +chroot-arch = $(notdir $(subst -,/,$1)) + +## $(call native-chroot-p,D-A) -> D | +## +## Answer whether D-A is a native chroot. +native-chroot-p = $(filter $(call chroot-arch,$1), $(NATIVE_ARCHS)) + +## $(call chroot-qemuhost,D-A) -> AA +## +## Answer the apporopriate Qemu host architecture for foreign chroot D-A. +chroot-qemuhost = $($(call chroot-arch,$1)_QEMUHOST) + +## $(call chroot-deps,PRE,D-A) -> PRE/DD-AA ... | +## +## Answer a list of additional dependencies for the chroot D-A: specifically, +## if D-A is foreign then include PRE/DD-AA entries for the tools +## architecture, and Qemu host architecture (if that's different). +chroot-deps = $(if $(call native-chroot-p,$2),, \ + $(addprefix $1/$(call chroot-dist,$2)-,\ + $(sort $(call chroot-toolsarch,$2) \ + $(call chroot-qemuhost,$2)))) + +## Substituting placeholders in files. +SUBSTITUTIONS := local=$(shell realpath $(LOCAL)) +SUBSTITUTIONS += primary-dist=$(PRIMARY_DIST) +subst-file = { \ + echo "$1 GENERATED by distorted-chroot: do not edit"; \ + substs=""; \ + for v in $(SUBSTITUTIONS); do \ + var=$${v%%=*} value=$${v\#*=}; \ + substs="$$substs s@$$var@$$valueg;"; \ + done; \ + sed "$$substs"; \ +} + +## Silent-rules machinery. +V = 0 +V_AT = $(V_AT_$V) +V_AT_0 = @ +v_print = $(call v_print_$V,$1,$2) +v_print_0 = printf " %-8s %s\n" "$1" "$2"; +v_tag = $(V_AT)$(call v_print,$1,$@) +v_log = $(call v_log_$V,$1) +v_log_ = $(call v_log_1,$1) +v_log_0 = >$1.log 2>&1 +v_log_1 = 2>&1 | tee $1.log +CLEANFILES += *.log + +###-------------------------------------------------------------------------- +### APT configuration. + +## In a chroot, `/etc/apt/sources.list' links to +## `/usr/local.schroot/etc/apt/sources.$d' for the appropriate distribution. +APT_SOURCES = $(foreach d,$(DISTS), $(LOCAL)/etc/apt/sources.$d) +CLEANFILES += $(APT_SOURCES) +all:: $(APT_SOURCES) + +$(foreach d,$(DISTS), $(STATE)/etc/apt/aptsrc.$d): $(STATE)/etc/apt/aptsrc.%: + $(V_AT)mkdir -p $(dir $@) + $(call v_tag,GEN){ \ + echo "### -*-conf-*- GENERATED by distorted-chroot: do not edit"; \ + echo "subscribe"; \ + echo " debian : $*"; \ + } >$@.new && mv $@.new $@ + +$(APT_SOURCES): $(LOCAL)/etc/apt/sources.%: \ + $$(APTSRC) $$($$*_APTSRC) $$(STATE)/etc/apt/aptsrc.$$* + $(V_AT)mkdir -p $(dir $@) + $(call v_tag,GEN)bin/mkaptsrc \ + $(APTSRC) $($*_APTSRC) $(STATE)/etc/apt/aptsrc.$* \ + >$@.new && mv $@.new $@ + +## In a chroot, a link `/etc/apt/apt.conf.d/FOO' is created for each file in +## `/usr/local.schroot/etc/apt/apt.conf.d/FOO'. +APT_CONFIGS = $(addprefix $(LOCAL)/etc/apt/apt.conf.d/,$(APTCONF)) +all:: $(APT_CONFIGS) +$(APT_CONFIGS): $(LOCAL)/etc/apt/apt.conf.d/%: \ + $$(or $$($$*_APTCONFSRC) $$(APTCONF_DIR)/$$*) + $(V_AT)mkdir -p $(dir $@) + $(call v_tag,COPY)cp $< $@.new && mv $@.new $@ + +###-------------------------------------------------------------------------- +### `schroot' and `sbuild' configuration. + +all:: schroot-config +schroot-config:: +.PHONY: schroot-config + +%print-varlist = { \ + echo "\#\#\# -*-sh-*- GENERATED by distorted-chroot: do not edit"; \ + $(foreach v,$1, echo $v=\''$(subst ','\'\\\'\'',$($v))'\';) \ +} #' +schroot-config_HASH := \ + $(shell $(call %print-varlist,$(CONFIG_VARS)) | \ + sha256sum | cut -c1-32) +schroot-config_FILE = $(STATE)/config.sh-$(schroot-config_HASH) +$(schroot-config_FILE): + $(V_AT)mkdir -p $(STATE) + $(V_AT)rm -f $(STATE)/config.sh-* + $(call v_tag,STAMP)$(call %print-varlist,$(CONFIG_VARS)) \ + >$@.new && mv $@.new $@ + +schroot-config:: $(STATE)/config.sh +$(STATE)/config.sh: $(schroot-config_FILE) + $(call v_tag,SYMLINK)ln -sf $(notdir $<) $@ + +schroot-config:: $(STATE)/etc/schroot/sbuild.schroot +$(STATE)/etc/schroot/sbuild.schroot: $(STATE)/config.sh bin/mkchrootconf + $(V_AT)mkdir -p $(dir $@) + $(call v_tag,GEN)bin/mkchrootconf >$@.new && mv $@.new $@ + +schroot-config:: $(STATE)/etc/schroot/sbuild.profile/copyfiles +$(STATE)/etc/schroot/sbuild.profile/copyfiles: $(schroot-config_STAMP) + $(V_AT)mkdir -p $(dir $@) + $(call v_tag,GEN){ \ + echo "### -*-conf-*- GENERATED by distorted-chroot: do not edit"; \ + for i in $(SCHROOT_COPYFILES); do echo "$$i"; done; \ + } >$@.new && mv $@.new $@ + +schroot-config:: $(STATE)/etc/schroot/sbuild.profile/nssdatabases +$(STATE)/etc/schroot/sbuild.profile/nssdatabases: $(schroot-config_STAMP) + $(V_AT)mkdir -p $(dir $@) + $(call v_tag,GEN){ \ + echo "### -*-conf-*- GENERATED by distorted-chroot: do not edit"; \ + for i in $(SCHROOT_NSSDATABASES); do echo "$$i"; done; \ + } >$@.new && mv $@.new $@ + +schroot-config:: $(STATE)/etc/schroot/sbuild.profile/fstab +$(STATE)/etc/schroot/sbuild.profile/fstab: \ + etc/sbuild.fstab.in $(schroot-config_STAMP) + $(V_AT)mkdir -p $(dir $@) + $(call v_tag,SUBST)$(call subst-file,### -*-conf-*-) \ + <$< >$@.new && mv $@.new $@ + +schroot-config:: $(STATE)/etc/sbuild.conf +$(STATE)/etc/sbuild.conf: etc/sbuild.conf.in $(schroot-config_STAMP) + $(V_AT)mkdir -p $(dir $@) + $(call v_tag,SUBST)$(call subst-file,### -*-perl-*-) \ + <$< >$@.new && mv $@.new $@ + +###-------------------------------------------------------------------------- +### Constructing chroots. + +BUILD_CHROOTS = $(addprefix chroot/,$(ALL_CHROOTS)) +CHROOT_STAMPS = $(addprefix $(STATE)/stamp/chroot.,$(ALL_CHROOTS)) +all:: setup-chroots +setup-chroots: $(BUILD_CHROOTS) +$(BUILD_CHROOTS): chroot/%: $(STATE)/stamp/chroot.% +.PHONY: setup-chroots $(BUILD_CHROOTS) + +$(CHROOT_STAMPS): $(STATE)/stamp/chroot.%: $(STATE)/config.sh bin/mkbuildchroot + $(call v_tag,CHROOT)bin/mkbuildchroot \ + $(call chroot-dist,$*) $(call chroot-arch,$*) \ + $(call v_log,setup-chroot.$*) + +###-------------------------------------------------------------------------- +### Installing basic custom software. + + + +###-------------------------------------------------------------------------- +### Other maintenance targets. + +show:; : $x + +clean:: + rm -f $(CLEANFILES) + rm -rf $(STATE) + +realclean:: clean + rm -rf $(LOCAL) + +###----- That's all, folks -------------------------------------------------- diff --git a/README b/README new file mode 100644 index 0000000..feb4ab7 --- /dev/null +++ b/README @@ -0,0 +1,5 @@ +Installation for ~distorted-chroot~ + + * Mount a filesystem on ~/var/lib/sbuild/build/~. I usually make a + logical volume named ~build-scratch~ for this, and make a symlink + ~/build~ pointing to ~/var/lib/sbuild/build~. diff --git a/bin/mkaptsrc b/bin/mkaptsrc new file mode 100755 index 0000000..102d598 --- /dev/null +++ b/bin/mkaptsrc @@ -0,0 +1,644 @@ +#! /usr/bin/perl +### +### Construct an APT `sources.list' file +### +### (c) 2012 Mark Wooding +### + +###----- Licensing notice --------------------------------------------------- +### +### This program is free software; you can redistribute it and/or modify +### it under the terms of the GNU General Public License as published by +### the Free Software Foundation; either version 2 of the License, or +### (at your option) any later version. +### +### This program is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +### GNU General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with this program; if not, write to the Free Software Foundation, +### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +use Data::Dumper; +use File::FnMatch qw(:fnmatch); +use Getopt::Long qw(:config gnu_compat bundling no_ignore_case); +use Text::ParseWords; + +###-------------------------------------------------------------------------- +### Miscellaneous utilities. + +(our $QUIS = $0) =~ s:.*/::; +our $VERSION = "1.0.0"; + +sub fail ($) { + my ($msg) = @_; + ## Report a fatal error MSG and exit. + + print STDERR "$QUIS: $msg\n"; + exit 1; +} + +sub literalp ($) { + my ($s) = @_; + ## Answer whether the string S is free of metacharacters. + + return $s !~ /[\\?*[]/; +} + +###-------------------------------------------------------------------------- +### Configuration sets. + +sub cset_new () { + ## Return a new configuration set. + + return { "%meta" => { allstarp => 1, ixlist => [], ixmap => {} } }; +} + +sub cset_indices ($$) { + my ($cset, $what) = @_; + ## Return the list of literal indices in the configuration set CSET. If an + ## explicit `indices' tag is defined, then use its value (split at + ## whitespace). If there are explicit literal indices, return them (in the + ## correct order). If all indices are `*', return a single `default' item. + ## Otherwise report an error. + + if (defined (my $it = $cset->{indices}{"*"})) { + return shellwords $it; + } else { + my $meta = $cset->{"%meta"}; + $meta->{allstarp} and return "default"; + return @{$meta->{ixlist}} if @{$meta->{ixlist}}; + fail "no literal indices for `$what'"; + } +} + +sub cset_store ($$$$) { + my ($cset, $tag, $ix, $value) = @_; + ## Store VALUE in the configuration set CSET as the value for TAG with + ## index pattern IX. + + my $meta = $cset->{"%meta"}; + $ix eq "*" or $meta->{allstarp} = 0; + if (!$meta->{ixmap}{$ix} && literalp $ix) { + $meta->{ixmap}{$ix} = 1; + push @{$meta->{ixlist}}, $ix; + } + $cset->{$tag}{$ix} = $value; +} + +sub cset_expand (\@$$); +%PAINT = (); + +sub cset_lookup (\@$$;$) { + my ($cset, $tag, $ix, $mustp) = @_; + ## Look up TAG in the CSETs using index IX. Return the value corresponding + ## to the most specific match to IX in the earliest configuration set in + ## the list. If no set contains a matching value at all, then the + ## behaviour depends on MUSTP: if true, report an error; if false, return + ## undef. + + if ($PAINT{$tag}) { fail "recursive expansion of `\%${tag}[$ix]'"; } + my $val = undef; + CSET: for my $cs (@$cset) { + defined (my $tset = $cs->{$tag}) or next CSET; + if (defined (my $it = $tset->{$ix})) { $val = $it; last CSET; }; + my $pat = undef; + PAT: while (my ($p, $v) = each %$tset) { + fnmatch $p, $ix or next PAT; + unless (defined($pat) && fnmatch($p, $pat)) { $val = $v; $pat = $p; } + } + last CSET if defined $val; + } + if (defined $val) { + local $PAINT{$tag} = 1; + my $exp = cset_expand @$cset, $ix, $val; + return $exp; + } elsif ($mustp) { fail "variable `$tag\[$ix]' undefined"; } + else { return undef; } +} + +sub cset_fetch (\%\@$$@) { + my ($a, $cset, $mustp, $ix, @tag) = @_; + ## Populate the hash A with values retrieved from the CSETs. Each TAG is + ## looked up with index IX, and if a value is found, it is stored in A with + ## key TAG. If MUSTP is true, then an error is reported unless a value is + ## found for every TAG. + + for my $tag (@tag) { + my $v = cset_lookup @$cset, $tag, $ix, $mustp; + $a->{$tag} = $v if defined $v; + } +} + +sub cset_expand (\@$$) { + my ($cset, $ix, $s) = @_; + ## Expand placeholders %TAG or %{TAG} in the string S, relative to the + ## CSETs and the index IX. + + $s =~ s{ + % (?: (?P\w+) + | \{ (?P\w+) \} ) + }{ + cset_lookup(@$cset, $+{NAME}, $ix, 1) + }xeg; + return $s; +} + +###-------------------------------------------------------------------------- +### Parsing. + +our %DEFAULT = %{+cset_new}; # Default assignments. +our %CSET = (); # Map of distro configuration sets. +our @SUB = (); # List of subscriptions. + +sub parse ($) { + my ($fn) = @_; + ## Parse the file named by FN and add definitions to the tables %DEFAULT, + ## %CSET and @SUB. + + ## Open the file and prepare to read. + open my $fh, "<", $fn or fail "open `$fn': $!"; + my $ln = 0; + + ## Report a syntax error, citing the offending file and line. + my $syntax = sub { fail "$fn:$ln: $_[0]" }; + + ## Report an error about an indented line with no stanza header. + my $nomode = sub { $syntax->("missing stanza header") }; + my $mode = $nomode; + + ## Parse an assignment LINE and store it in CSET. + my $assign = sub { + my ($cset, $line) = @_; + $line =~ m{ + ^ \s* + (?P \w+) + (?: \[ (?P [^\]]+) \] )? + \s* = \s* + (?P | \S | \S.*\S) + \s* $ + }x or $syntax->("invalid assignment"); + cset_store $cset, $+{TAG}, $+{IX} // "*", $+{VALUE}; + }; + + ## Parse a subscription LINE and store it in @SUB. + my $subscribe = sub { + my ($line) = @_; + my @w = shellwords $line; + my @dist = (); + while (my $w = shift @w) { last if $w eq ":"; push @dist, $w; } + @w and @dist or $syntax->("empty distribution or release list"); + push @SUB, [\@dist, \@w]; + }; + + for (;;) { + + ## Read a line. If it's empty or a comment then ignore it. + defined (my $line = readline $fh) + or last; + $ln++; + next if $line =~ /^\s*($|\#)/; + chomp $line; + + ## If the line begins with whitespace then process it according to the + ## prevailing mode. + if ($line =~ /^\s/) { + $mode->($line); + next; + } + + ## Split the header line into tokens and determine an action. + my @w = shellwords $line; + $mode = $nomode; + if ($w[0] eq "distribution") { + @w == 2 or $syntax->("usage: distribution NAME"); + my $cset = $CSET{$w[1]} //= cset_new; + $mode = sub { $assign->($cset, @_) }; + } elsif ($w[0] eq "default") { + @w == 1 or $syntax->("usage: default"); + $mode = sub { $assign->(\%DEFAULT, @_) }; + } elsif ($w[0] eq "subscribe") { + @w == 1 or $syntax->("usage: subscribe"); + $mode = $subscribe; + } else { + $syntax->("unknown toplevel directive `$w[0]'"); + } + } + + ## Done. Make sure we read everything. + close $fh or die "read `$fn': $!"; +} + +###-------------------------------------------------------------------------- +### Main program. + +our $USAGE = "usage: $QUIS FILE|DIR ..."; +sub version { print "$QUIS, version $VERSION\n"; } +sub help { + print < sub { version; help; exit; }, + 'version|v' => sub { version; exit; }) + and @ARGV >= 1 + or do { print STDERR $USAGE, "\n"; exit 1; }; + +## Read the input files. +for my $fn (@ARGV) { + if (-d $fn) { + opendir my $dh, $fn or fail "opendir `$fn': $!"; + my @f = (); + FILE: while (my $f = readdir $dh) { + $f =~ /^[-\w.]+$/ or next FILE; + $f = "$fn/$f"; + -f $f or next FILE; + push @f, $f; + } + closedir $dh; + for my $f (sort @f) { parse $f; } + } else { + parse $fn; + } +} + +## Start writing output. +print <[0]}; + my @rel = @{$pair->[1]}; + + ## Write a stanza for each distribution. + for my $dist (@dist) { + + ## Find the configuration set for the distribution. + defined (my $cset = $CSET{$dist}) + or fail "unknown distribution `$dist'"; + my @ix = cset_indices $cset, $dist; + + ## Print a banner to break up the monotony. + my %a = (); + cset_fetch %a, @{[$cset, \%DEFAULT]}, 0, "default", qw(banner); + print "###", "-" x 74, "\n"; + print "### ", $a{banner}, "\n" if exists $a{banner}; + + ## Write a paragraph for each release. + for my $rel (@rel) { + + ## Write a header. + print "\n## $rel\n"; + + ## Prepare a list of configuration sections to provide variables for + ## expansion. + my @cset = ({ RELEASE => { "*" => $rel } }, $cset, \%DEFAULT); + + ## Work through each index. + IX: for my $ix (@ix) { + + ## Fetch properties from the configuration set. + %a = (options => undef, + release => $rel, + releases => "*", + types => "deb deb-src"); + cset_fetch %a, @cset, 1, $ix, qw(uri components); + cset_fetch %a, @cset, 0, $ix, qw(types options release releases); + + ## Check that this release matches the index. + my $matchp = 0; + for my $rpat (shellwords $a{releases}) { + $matchp = 1, last if fnmatch $rpat, $rel; + } + next IX unless $matchp; + + ## Build an output line. + my $out = ""; + if (defined (my $opt = $a{options})) { $out .= "[ $opt ] "; } + $out .= "$a{uri} $a{release} $a{components}"; + + ## Canonify whitespace. + $out =~ s/^\s+//; $out =~ s/\s+$//; $out =~ s/\s+/ /; + + ## Write out the necessary + print "$_ $out\n" for shellwords $a{types}; + } + } + print "\n"; + } +} + +## Write a trailer. +print "###----- That's all, folks ", "-" x 50, "\n"; +print "### GENERATED by $QUIS: NO NOT EDIT!\n"; + +###-------------------------------------------------------------------------- +### Documentation. + +=head1 NAME + +mkaptsrc - generate APT `sources.list' files + +=head1 SYNOPSIS + +B I|I... + +=head1 DESCRIPTION + +The B progrem generates an APT F file from a +collection of configuration files. It allows a site to use a single master +file defining all (or most) of the available archives, while allowing each +individiual host to describe succinctly which archives it actually wants to +track. + +The command line arguments are a list of one or more filenames and/or +directories. The program reads the files one by one, in order; a directory +stands for all of the regular files it contains whose names consist only of +alphanumeric characters, dots C<.>, underscores C<_>, and hyphens C<->, in +ascending lexicographic order. (Nested subdirectories are ignored.) Later +files can override settings from earlier ones. + +=head2 Command-line syntax + +The following command-line options are recognized. + +=over + +=item B<-h>, B<--help> + +Print help about the program to standard output, and exit. + +=item B<-v>, B<--version> + +Print B's version number to standard output, and exit. + +=back + +=head2 Configuration syntax + +The configuration files are split into stanze. Each stanza begins with an +unindented header line, followed by zero or more indented body lines. Blank +lines (containing only whitespace) and comments (whose first non-whitespace +character is C<#>) are ignored E and in particular are not considered +when determining the boundaries of stanze. It is not possible to split a +stanza between two files. + +A I consists of a line + +=over + +B I + +=back + +followed by a number of indented assignments + +=over + +I = I + +=back + +or + +=over + +IB<[>IB<]> = I + +=back + +Here, I is a name for this distribution; this name is entirely internal +to the configuration and has no external meaning. Several stanze may use the +same I: the effect is the same as a single big stanza containing all of +the assignments in order. + +Each assignment line sets the value of a I for the distribution; if the +I has already been assigned a value then the old value is forgotten. +The optional I may be used to assign different values to the same tag +according to different I, distinguished by glob patterns: see the +description below. Omitting the I is equivalent to using the wildcard +pattern C<*>. + +A I consists of a line + +=over + +B + +=back + +followed by assignments as for a distribution stanza. Again, there may be +many default stanze, and the effect is the same as a single big default +stanza containing all of the assignments in order. During output, tags are +looked up first in the relevant distribution, and if there no matching +assignments then the B assignments are searched. + +A I consists of a line + +=over + +B + +=back + +followed by indented subscription lines + +=over + +I [I ...] B<:> I [I ...] + +=back + +Such a line is equivalent to a sequence of lines + +=over + +I B<:> I [I ...] + +=back + +one for each I, in order. + +It is permitted for several lines to name the same I, though currently +the behaviour is not good: they are treated entirely independently. The +author is not sure what the correct behaviour ought to be. + +=head2 Tag lookup and value expansion + +The output of B is largely constructed by looking up tags and using +their values. A tag is always looked up in a particular I and +with reference to a particular I. Contexts are named with an +I. The resulting value is the last assignment in the distribution's +stanze whose tag is equal to the tag being looked up, and whose pattern is +either absent or matches the context index. If there is no matching +assignment, then the default assignments are checked, and again the last +match is used. If there is no default assignment either, then the lookup +fails; this might or might not be an error. + +Once the value has been found, it is I before use. Any +placeholders of the form B<%>I or B<%{>IB<}> (the latter may be +used to distinguish the I name from any immediately following text) are +replaced by the (expanded) value of the I, using the same distribution +and context as the original lookup. It is a fatal error for a lookup of a +tag to fail during expansion. Recursive expansion is forbidden. + +There are some special tags given values by B. Their names are +written in all upper-case. + +=head2 Output + +The output is always written to stdout. It begins with a header comment +(which you can't modify), including a warning that the file is generated and +shouldn't be edited. + +The output is split into sections, one for each I in the subcription +stanze. Each section begins with a comment banner, whose text is the result +of looking up the tag B in the distribution, using the context index +B; if the lookup fails then no banner text is added. + +The distribution section is split into paragraphs, one for each I +listed in the subscription line, and headed with a comment naming the +I. The contents of the paragraph are determined by assignments in +the distribution stanza for I. + +The set of context indices for the paragraph is determined, as follows. + +=over + +=item * + +The tag B is looked up in the distribution I. This lookup is +special in three ways: firstly, lookup will I fall back to the +B assignments; secondly, only assignments with no pattern (or, +equivalently, with pattern C<*>) are examined; and, thirdly, the result is +I subject to expansion. If a value is found, then the context indices +are precisely the space-separated words of the value. + +=item * + +If there assignments in the distribution I whose patterns are +I E i.e., contain no metacharacters C<*>, C, C<[>, or +C<\\> E then the context indices are precisely these literal patterns, +in the order in which they first appeared. + +=item * + +If all of the assignments for the distribution I have no pattern (or, +equivalently, have pattern C<*>), then there is exactly one context index +B. + +=item * + +Otherwise the situation is a fatal error. You should resolve this unlikely +situation by setting an explicit B value. + +=back + +The contexts are now processed in turn. Each lookup described below happens +in the distribution I, with respect to the context being processed. +Furthermore, the special tag B is given the value I. + +The tag B is looked up, and split into a space-separated sequence +of glob patterns. If the I doesn't match any of these patterns then +the context is ignored. (If the lookup fails, the context is always used, +as if the value had been C<*>.) + +Finally, a sequence of lines is written, of the form + +=over + +I S I B<]>> I I I + +=back + +one for each word in the value of B, defaulting to B B. +Other pieces correspond to the values of tags to be looked up: I +defaults to the name provided in the B stanza; if I is +omitted then there will be no S I B<]>> piece; it is a fatal +error if other lookups fail. + +=head1 EXAMPLES + +The package repository for the official Linux Spotify client can be described +as follows. + + distribution spotify + banner = Spotify client for Linux. + uri = http://repository.spotify.com/ + components = non-free + types = deb + + subscribe + spotify : stable + +This produces the output + + ###------------------------------------------------------------ + ### Spotify client for Linux. + + ## stable + deb http://repository.spotify.com/ stable non-free + +As a more complex example, I describe the official Debian package archive as +follows. + + default + debmirror = http://mirror.distorted.org.uk + debsecurity = http://security.debian.org + + distribution debian + banner = Debian GNU/Linux. + uri[base] = %debmirror/debian/ + uri[security-local] = %debmirror/debian-security/ + uri[security-upstream] = %debsecurity/debian-security/ + release[security-*] = %RELEASE/updates + releases[security-*] = oldstable stable testing + components = main non-free contrib + components[security-*] = main + + subscribe + debian : stable testing unstable + +This arranges to use my local mirror for both the main archive and for +security updates, but I to use the upstream archive for security +updates which I might not have mirrored yet. Setting B +copes with the fact that there are no separate security releases for the +B release. + +On machines which are far away from my mirror, I override these settings by +writing + + distribution debian + debmirror = http://ftp.uk.debian.org + indices = base security-upstream + +in a host-local file (which has the effect of disabling the B +context implicitly defined in the base stanza. + +=head1 BUGS + +Redefinition of subscriptions currently isn't well behaved. + +=head1 SEE ALSO + +L + +=head1 AUTHOR + +Mark Wooding + +=cut + +###----- That's all, folks -------------------------------------------------- diff --git a/bin/mkbuildchroot b/bin/mkbuildchroot new file mode 100755 index 0000000..40f925b --- /dev/null +++ b/bin/mkbuildchroot @@ -0,0 +1,154 @@ +#! /bin/sh -e +### +### Construct a fresh build-chroot base +### +### (c) 2018 Mark Wooding +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of the distorted.org.uk chroot maintenance tools. +### +### distorted-chroot is free software: you can redistribute it and/or +### modify it under the terms of the GNU General Public License as +### published by the Free Software Foundation; either version 2 of the +### License, or (at your option) any later version. +### +### distorted-chroot is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +### General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with distorted-chroot. If not, write to the Free Software +### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, +### USA. + +. state/config.sh # @@@config@@@ + +case $PROXY in nil) ;; *) http_proxy=$PROXY; export PROXY ;; esac + +badp=nil forcep=nil +while getopts "f" opt; do + case $opt in + f) forcep=t ;; + *) badp=t ;; + esac +done +shift $(( $OPTIND - 1 )) +case $# in 2) ;; *) badp=t ;; esac +case $badp in t) echo >&2 "usage: $0 [-f] DIST ARCH"; exit 2 ;; esac +d=$1 a=$2 + +case $d-$a in *-*-*) echo >&2 "$0: bad chroot name \`$arg'"; exit 2 ;; esac +if [ ! -d /dev/$VG/ ]; then echo >&2 "$0: no volume group \`$VG'"; exit 2; fi + +qemup=nil dbsopts= +for fa in $FOREIGN_ARCHS; do + case $fa in + "$a") + qemup=t qbsopts=--foreign + eval qhost=\$${a}_QEMUARCH qhost=\$${a}_QEMUHOST + break + ;; + esac +done + +lv=$LVPREFIX$d-$a +mnt=mnt/$lv +mkdir -p $mnt +if mountpoint -q $mnt; then umount $mnt; fi +if [ -b /dev/$VG/$lv ]; then + case $forcep in + nil) echo >&2 "$0: volume \`$lv' already exists"; exit 2 ;; + t) lvremove -f $VG/$lv ;; + esac +fi +lvcreate --yes $LVSZ -n$lv $VG +mkfs -j -L$d-$a /dev/$VG/$lv +mount -orelatime,data=writeback,commit=3600,barrier=0 /dev/$VG/$lv $mnt/ +mkdir -m755 $mnt/fs/ +chmod 750 $mnt/ +pkgs=ccache,eatmydata,fakeroot,libfile-fcntllock-perl,locales,tzdata +eatmydata debootstrap $dbsopts --arch=$a --variant=minbase \ + --include=$pkgs $d $mnt/fs/ $DEBMIRROR + +case $qemup in + t) + install $LOCAL/cross/$d-$qhost/QEMU/qemu-$qarch-static \ + $mnt/fs/usr/bin/ + chroot $mnt/fs/ /debootstrap/debootstrap --second-stage + ln -sf /usr/local.schroot/cross/$d-$qhost/QEMU/qemu-$qarch-static \ + $mnt/fs/usr/bin/ + ;; +esac + +cd $mnt/fs/usr/ +rm -rf local/; ln -s local.schroot/$a local + +cd $mnt/fs/etc/apt/ +rm -rf apt.conf sources.list +ln -s /usr/local.schroot/config/apt/conf.d/10sbuild apt.conf.d/ +ln -s /usr/local.schroot/config/apt/conf.d/90local apt.conf.d/ +ln -s /usr/local.schroot/config/apt/sources.$d sources.list + +cat >apt.conf.d/20arch <policy-rc.d <&2 "policy-rc.d: Services disabled by policy." +exit 101 +EOF +chmod +x policy-rc.d + +cd $mnt/fs/etc/ld.so.conf.d/ +cat >libc.conf <zzz-local.conf </dev/null 2>&1 --no-rename --help + then no_rename=--no-rename + else no_rename= + fi + + dpkg-divert --package install-cross-tools \$no_rename \ + --divert /usr/bin/$qemu.$a --add /usr/bin/$qemu" + $STATE/bin/install-cross-tools $d $a + ;; +esac + +schroot -uroot -csource:$LVPREFIX$d-$a -- eatmydata sh -e -c ' + apt-get update + apt-get -y upgrade + locale-gen + ldconfig + apt-get -y autoremove + apt-get clean' + +###----- That's all, folks -------------------------------------------------- diff --git a/bin/mkchrootconf b/bin/mkchrootconf new file mode 100755 index 0000000..9aff2a6 --- /dev/null +++ b/bin/mkchrootconf @@ -0,0 +1,110 @@ +#! /bin/sh -e +### +### Write `schroot' configuration for build chroots +### +### (c) 2018 Mark Wooding +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of the distorted.org.uk chroot maintenance tools. +### +### distorted-chroot is free software: you can redistribute it and/or +### modify it under the terms of the GNU General Public License as +### published by the Free Software Foundation; either version 2 of the +### License, or (at your option) any later version. +### +### distorted-chroot is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +### General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with distorted-chroot. If not, write to the Free Software +### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, +### USA. + +. state/config.sh # @@@config@@@ + +## Parse the command-line. +badp=nil +while getopts "" opt; do badp=t; done +shift $(( $OPTIND - 1 )) +case $# in 0) ;; *) badp=t ;; esac +case $badp in t) echo >&2 "usage: $0"; exit 2 ;; esac + +for arg in $ALL_CHROOTS; do + case $arg in *-*-*) echo >&2 "$0: bad chroot name \`$arg'"; exit 2 ;; esac +done +if [ ! -d /dev/$VG/ ]; then echo >&2 "$0: no volume group \`$VG'"; exit 2; fi + +## Start producing output. +cat < "/private", +}; + +## Actually resolve alternatives properly. +$resolve_alternatives = 1; + +## Don't do anything clever if the build dependencies fail. The clever thing +## doesn't work on older distributions. Also, it takes extra time, and +## that's not desirable. +$bd_uninstallable_explainer = "apt"; + +###----- That's all, folks -------------------------------------------------- + +1; diff --git a/etc/sbuild.fstab.in b/etc/sbuild.fstab.in new file mode 100644 index 0000000..e486ad6 --- /dev/null +++ b/etc/sbuild.fstab.in @@ -0,0 +1,23 @@ +### -*-conf-*- +### Filesystem mounts for chroots. +### +### FILSYS MOUNTPT FSTYPE OPTIONS DUMP FSCK + +## Standard mountpoints. +/proc /proc none rw,bind 0 0 +/sys /sys none rw,bind 0 0 +/dev/pts /dev/pts none rw,bind 0 0 +tmpfs /dev/shm tmpfs defaults 0 0 + +## Splice the host's `/tmp' and `/home' for manual debugging convenience. +/tmp /tmp none rw,bind 0 0 +/home /home none rw,bind 0 0 + +## Splice in the support file tree. This needs to be done in two steps: for +## stupid reasons, a plain bind-mount entry leaves the mount read-write. +@local@ /usr/local.schroot none rw,bind 0 0 +@local@ /usr/local.schroot none remount,ro,bind 0 0 + +## Mount a large scratch space for the build, so we don't use up space on an +## LVM snapshot of the chroot itself. +/var/lib/sbuild/build /build none rw,bind 0 0