[profile] / bin / mdw-sbuild-server

#! /bin/sh -e
###
### Build a Debian package on supported architectures
###
### (c) 2016 Mark Wooding
###

###----- Licensing notice ---------------------------------------------------
###
### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation; either version 2 of the License, or
### (at your option) any later version.
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
### You should have received a copy of the GNU General Public License
### along with this program; if not, write to the Free Software Foundation,
### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

###--------------------------------------------------------------------------
### Configuration.

unset buildroot default_targets parallel
for i in \
  "/etc/mdw-sbuild.conf" \
  "${XDG_CONFIG_HOME-$HOME/.config}/mdw-sbuild.conf"
do
  if [ -f "$i" ]; then . "$i"; fi
done
: ${buildroot=$HOME/build}
: ${default_targets="wheezy-amd64 wheezy-i386"}
: ${DEB_BUILD_OPTIONS=parallel=4}; export DEB_BUILD_OPTIONS

###--------------------------------------------------------------------------
### Some utilities.

prog=${0##*/}

fail () { echo >&2 "$prog: $*"; exit 1; }
usage () { echo "usage: $prog [-aiknT] [-t TARGET] [-A DBPARGS] COMMAND [ARGUMENTS ...]"; }
fail_usage () { usage >&2; exit 1; }

want_1 () {
  what=$1 pat=$2 type=$3; shift 3
  for i in "$@"; do
    [ $type "$i" ] || fail "$what not found: \`$i'"
  done
  case $# in
    1) ;;
    *) fail "expected exactly one $what matching \`$pat', but found $#" ;;
  esac
  echo "$1"
}

run () {
  case $notreally in
    t) echo "+ $*" ;;
    nil) nice "$@" ;;
  esac
}

decor () {
  tag=$1 marker=$2
  while IFS= read -r line; do
    printf "%-21s %c %s\n" "$tag" "$marker" "$line"
  done
}

###--------------------------------------------------------------------------
### Parse options.

bogusp=nil archp=nil indepp=nil keepon=nil notreally=nil
unset targets dbpargs

while getopts "haint:A:T" opt; do
  case $opt in
    h)
      usage
      cat <<EOF

Options:
	-h		Show this help text.
	-a		Build only architecture-dependent packages.
	-i		Build only architecture-neutral packages.
	-k		Keep going even if one fails.
	-n		Don't actually do the build.
	-t TARGET	Build in TARGET build environment.
	-A ARGS		Pass ARGS to \`dpkg-buildpackage'.
	-T		Don't run the tests.

Commands available:

	dir PROJECT/VERSION
		Return a freshly-made directory for the source code to
		go in.

	build BUILDDIR
		Build the package placed in BUILDDIR, which should contain
		exactly one \`.dsc' file, and whatever source archive files
		are necessary.
EOF
      exit
      ;;
    a) archp=t ;;
    i) indepp=t ;;
    k) keepon=t ;;
    n) notreally=t ;;
    t) targets="${targets+$targets }$OPTARG" ;;
    A) dbpargs="${dbpargs+$dbpargs }$OPTARG" ;;
    T)
      case " $DEB_BUILD_OPTIONS " in
	*" nocheck "*) ;;
	*) DEB_BUILD_OPTIONS=${DEB_BUILD_OPTIONS+"$DEB_BUILD_OPTIONS "} nocheck ;;
      esac
      ;;
    *) bogusp=nil ;;
  esac
done
shift $(( $OPTIND - 1 ))

case $bogusp in t) fail_usage ;; esac
case $archp,$indepp in nil,nil) archp=t indepp=t ;; esac
case ${targets+t} in t) ;; *) targets=$default_targets ;; esac

###--------------------------------------------------------------------------
### Main work.

case "$#,$1" in
  0,*) fail_usage ;;
  *,*,*) fail "bad command name \`$1'" ;;

  2,dir)
    ## dirname PROJECT/VERSION

    ## Try to create a fresh build directory.
    dist=$2
    case "$dist" in */*/*) fail "bad distribution name \`$dist'" ;; esac
    proj=${dist%/*} ver=${dist#*/}
    cd "$buildroot"
    mkdir -p "$proj"
    cd "$proj"
    i=0
    winp=nil
    while [ $i -lt 50 ]; do
      i=$(( $i + 1 ))

      ## Find a sequence number different from all of the existing builds of
      ## this version.
      nn=1
      for j in "$ver#"*; do
	case "$j" in "$ver#*") break ;; esac
	n=${j##*\#}
	if [ $nn -le $n ]; then nn=$(( $n + 1 )); fi
      done

      ## Try to make the build directory.  This might not work if we're
      ## racing with another process, but that's why we're trying in a loop.
      if mkdir "$ver#$nn" >/dev/null 2>&1; then
	winp=t
	cd "$ver#$nn"
	break
      fi

      ## Make sure it actually failed because a directory appeared, rather
      ## than for some other reason.
      [ -e "$ver#$nn" ] || \
	fail "unexpectedly couldn't create \`$buildroot/$dist#$nn'"
    done

    ## Make sure we actually succeeded.
    case $winp in t) ;; *) fail "failed to create build directory" ;; esac

    ## Make an empty directory for dependency packages.
    mkdir -p pkgs/

    ## Done.
    echo "$buildroot/$dist#$nn"
    ;;

  *,dir)
    echo >&2 "usage: $prog dir PROJECT/VERSION"; exit 1 ;;

  2,build)
    ## build BUILDDIR

    ## Track down the build directory.
    builddir=$2
    cd "$builddir"
    dsc=$(want_1 "file" "*.dsc" -f *.dsc)

    ## Figure out which targets need building.  If the `.dsc' file isn't
    ## telling, assume it needs building everywhere and let sbuild(1) sort
    ## out the mess.
    os=$(dpkg-architecture -qDEB_HOST_ARCH_OS)
    unset first rest; anyp=nil depp=nil allp=nil
    wantarchs=$(sed -n '/^[Aa]rchitecture:/ s/^[^:]*: *//p' "$dsc")
    : ${wantarchs:=any}
    unset buildarchs buildarchs_seen=:

    ## Work through the available targets assigning builds to them.  This is
    ## actually a little tricky.
    for t in $targets; do

      ## Dissect the target name.
      suite=${t%%-*} archs=${t#*-}
      case $archs in
	*/*) target=${archs%/*} host=${archs#*/} ;;
	*) target=$archs host=$archs; t=$suite-$target/$host ;;
      esac
      case $buildarchs_seen in
	*:$target:*)
	  ;;
	*)
	  buildarchs=${buildarchs+$buildarchs }$target
	  buildarchs_seen=$buildarchs_seen$target:
	  ;;
      esac

      ## Work through the architectures which we can build.
      for arch in $wantarchs; do
	case $arch in
	  all)
	    ## Package suitable for all architectures.

	    ## If we don't want to build architecture-neutral packages then
	    ## there's nothing to do.
	    case $indepp in nil) continue ;; esac

	    ## Pick this up if nobody has volunteered.  However, we should be
	    ## ready to let some other architecture build this if it's going
	    ## to build some architecture-dependent package too.
	    case $anyp in nil) first=$t anyp=t allp=t ;; esac
	    ;;
	  *)
	    ## There's at least one architecture-specific package.

	    ## If we don't want to build architecture-specific packages then
	    ## there's nothing to do.
	    case $archp in nil) continue ;; esac

	    ## If we can't build it then we shouldn't try.
	    if ! dpkg-architecture -a"$os-$target" -i"$arch"; then
	      continue
	    fi

	    ## Decide whether we should take responsibility for the
	    ## architecture-neutral packages.  If nobody's claimed them yet,
	    ## or the previous claimant wasn't building architecture-specific
	    ## packages, we should take over.
	    case $depp in
	      nil) first=$t depp=t anyp=t ;;
	      t) rest="${rest+$rest }$t" ;;
	    esac
	    ;;
	esac
      done
    done

    ## If we never found a match then we can't do anything.
    case $anyp in nil) echo "$prog: no packages to build"; exit 0 ;; esac

    ## Figure out the right options to use.
    case $indepp in
      t) firstopt="--arch-all" ;;
      nil) firstopt="--no-arch-all" ;;
    esac
    case $archp in
      t) ;;
      nil) firstopt="$firstopt --no-arch-any" ;;
    esac

    ## Sort out the additional packages.  This is rather annoying, because
    ## sbuild(1) does this in a really stupid way.
    rm -rf pkgs.*
    for a in $buildarchs; do
      mkdir pkgs.$a
      for f in $(dpkg-scanpackages -a$a pkgs/ |
		    sed -n '/^Filename: /s///p')
      do
	ln $f pkgs.$a/
      done
    done

    ## Build the builds sequentially.  Tests can conflict with each other,
    ## e.g., over port numbers.
    rc=0 buildopt=$firstopt
    for t in $first $rest; do
      host=${t##*/} full=${t%/*}
      suite=${full%%-*} target=${full#*-}

      ## And we're ready to go.
      exec 3>&1
      thisrc=$(
	{ { { { set +e
		run sbuild --extra-package=pkgs.$target \
		    --dist=$suite --build=$host --host=$target \
		    --chroot=$suite-$host --verbose $buildopt $dsc \
		    ${dbpargs+--debbuildopts="$dbpargs"} \
		    3>&- 4>&- 5>&-
		echo $? >&5
	      } |
		decor "$full" "|" >&4; } 2>&1 |
	      decor "$full" "*" >&4; } 4>&1 |
	    cat -u >&3; } 5>&1 </dev/null)
      exec 3>&-
      case $thisrc in 0) ;;
	*)
	  echo failed rc=$thisrc >$stat; rc=$thisrc
	  case $keepon in nil) break ;; esac
	  ;;
      esac
      buildopt=--no-arch-all
    done
    exit $rc
    ;;
  build,*)
    echo >&2 "usage: $prog build BUILDDIR"; exit 1 ;;

  *)
    fail "unknown command \`$1'"
    ;;
esac

###----- That's all, folks --------------------------------------------------
Commit	Line	Data
b94830d9 MW	1	#! /bin/sh -e
	2	###
	3	### Build a Debian package on supported architectures
	4	###
	5	### (c) 2016 Mark Wooding
	6	###
	7
	8	###----- Licensing notice ---------------------------------------------------
	9	###
	10	### This program is free software; you can redistribute it and/or modify
	11	### it under the terms of the GNU General Public License as published by
	12	### the Free Software Foundation; either version 2 of the License, or
	13	### (at your option) any later version.
	14	###
	15	### This program is distributed in the hope that it will be useful,
	16	### but WITHOUT ANY WARRANTY; without even the implied warranty of
	17	### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	18	### GNU General Public License for more details.
	19	###
	20	### You should have received a copy of the GNU General Public License
	21	### along with this program; if not, write to the Free Software Foundation,
	22	### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	23
	24	###--------------------------------------------------------------------------
	25	### Configuration.
	26
	27	unset buildroot default_targets parallel
	28	for i in \
	29	"/etc/mdw-sbuild.conf" \
	30	"${XDG_CONFIG_HOME-$HOME/.config}/mdw-sbuild.conf"
	31	do
	32	if [ -f "$i" ]; then . "$i"; fi
	33	done
	34	: ${buildroot=$HOME/build}
	35	: ${default_targets="wheezy-amd64 wheezy-i386"}
b94830d9 MW	36	: ${DEB_BUILD_OPTIONS=parallel=4}; export DEB_BUILD_OPTIONS
	37
	38	###--------------------------------------------------------------------------
	39	### Some utilities.
	40
	41	prog=${0##*/}
	42
	43	fail () { echo >&2 "$prog: $*"; exit 1; }
ff55a023	44	usage () { echo "usage: $prog [-aiknT] [-t TARGET] [-A DBPARGS] COMMAND [ARGUMENTS ...]"; }
b94830d9 MW	45	fail_usage () { usage >&2; exit 1; }
	46
	47	want_1 () {
	48	what=$1 pat=$2 type=$3; shift 3
	49	for i in "$@"; do
	50	[ $type "$i" ] \|\| fail "$what not found: \`$i'"
	51	done
	52	case $# in
	53	1) ;;
	54	*) fail "expected exactly one $what matching \`$pat', but found $#" ;;
	55	esac
	56	echo "$1"
	57	}
	58
ba9719eb MW	59	run () {
	60	case $notreally in
	61	t) echo "+ $*" ;;
5b950572	62	nil) nice "$@" ;;
ba9719eb MW	63	esac
	64	}
	65
	66	decor () {
	67	tag=$1 marker=$2
	68	while IFS= read -r line; do
	69	printf "%-21s %c %s\n" "$tag" "$marker" "$line"
	70	done
	71	}
	72
b94830d9 MW	73	###--------------------------------------------------------------------------
	74	### Parse options.
	75
ba9719eb	76	bogusp=nil archp=nil indepp=nil keepon=nil notreally=nil
5e9e7146	77	unset targets dbpargs
b94830d9	78
ff55a023	79	while getopts "haint:A:T" opt; do
b94830d9 MW	80	case $opt in
	81	h)
	82	usage
	83	cat <<EOF
	84
	85	Options:
	86	-h Show this help text.
	87	-a Build only architecture-dependent packages.
	88	-i Build only architecture-neutral packages.
ba9719eb	89	-k Keep going even if one fails.
b94830d9 MW	90	-n Don't actually do the build.
b94830d9 MW	91	-t TARGET Build in TARGET build environment.
5e9e7146	92	-A ARGS Pass ARGS to \`dpkg-buildpackage'.
ff55a023	93	-T Don't run the tests.
b94830d9 MW	94
	95	Commands available:
	96
	97	dir PROJECT/VERSION
	98	Return a freshly-made directory for the source code to
	99	go in.
	100
	101	build BUILDDIR
	102	Build the package placed in BUILDDIR, which should contain
	103	exactly one \`.dsc' file, and whatever source archive files
	104	are necessary.
	105	EOF
	106	exit
	107	;;
	108	a) archp=t ;;
	109	i) indepp=t ;;
ba9719eb MW	110	k) keepon=t ;;
ba9719eb MW	111	n) notreally=t ;;
b94830d9	112	t) targets="${targets+$targets }$OPTARG" ;;
5e9e7146	113	A) dbpargs="${dbpargs+$dbpargs }$OPTARG" ;;
ff55a023 MW	114	T)
	115	case " $DEB_BUILD_OPTIONS " in
	116	" nocheck ") ;;
	117	*) DEB_BUILD_OPTIONS=${DEB_BUILD_OPTIONS+"$DEB_BUILD_OPTIONS "} nocheck ;;
	118	esac
	119	;;
b94830d9 MW	120	*) bogusp=nil ;;
	121	esac
	122	done
	123	shift $(( $OPTIND - 1 ))
	124
	125	case $bogusp in t) fail_usage ;; esac
	126	case $archp,$indepp in nil,nil) archp=t indepp=t ;; esac
	127	case ${targets+t} in t) ;; *) targets=$default_targets ;; esac
	128
	129	###--------------------------------------------------------------------------
	130	### Main work.
	131
	132	case "$#,$1" in
	133	0,*) fail_usage ;;
	134	,,*) fail "bad command name \`$1'" ;;
	135
	136	2,dir)
	137	## dirname PROJECT/VERSION
	138
	139	## Try to create a fresh build directory.
	140	dist=$2
	141	case "$dist" in //*) fail "bad distribution name \`$dist'" ;; esac
	142	proj=${dist%/} ver=${dist#/}
	143	cd "$buildroot"
	144	mkdir -p "$proj"
	145	cd "$proj"
	146	i=0
	147	winp=nil
	148	while [ $i -lt 50 ]; do
	149	i=$(( $i + 1 ))
	150
	151	## Find a sequence number different from all of the existing builds of
	152	## this version.
	153	nn=1
	154	for j in "$ver#"*; do
	155	case "$j" in "$ver#*") break ;; esac
	156	n=${j##*\#}
	157	if [ $nn -le $n ]; then nn=$(( $n + 1 )); fi
	158	done
	159
	160	## Try to make the build directory. This might not work if we're
	161	## racing with another process, but that's why we're trying in a loop.
	162	if mkdir "$ver#$nn" >/dev/null 2>&1; then
	163	winp=t
6cf97414	164	cd "$ver#$nn"
b94830d9 MW	165	break
	166	fi
	167
	168	## Make sure it actually failed because a directory appeared, rather
	169	## than for some other reason.
	170	[ -e "$ver#$nn" ] \|\| \
	171	fail "unexpectedly couldn't create \`$buildroot/$dist#$nn'"
	172	done
	173
	174	## Make sure we actually succeeded.
	175	case $winp in t) ;; *) fail "failed to create build directory" ;; esac
	176
6cf97414 MW	177	## Make an empty directory for dependency packages.
	178	mkdir -p pkgs/
	179
b94830d9 MW	180	## Done.
	181	echo "$buildroot/$dist#$nn"
	182	;;
	183
	184	*,dir)
	185	echo >&2 "usage: $prog dir PROJECT/VERSION"; exit 1 ;;
	186
	187	2,build)
	188	## build BUILDDIR
	189
	190	## Track down the build directory.
	191	builddir=$2
	192	cd "$builddir"
	193	dsc=$(want_1 "file" ".dsc" -f .dsc)
	194
	195	## Figure out which targets need building. If the `.dsc' file isn't
	196	## telling, assume it needs building everywhere and let sbuild(1) sort
	197	## out the mess.
	198	os=$(dpkg-architecture -qDEB_HOST_ARCH_OS)
	199	unset first rest; anyp=nil depp=nil allp=nil
	200	wantarchs=$(sed -n '/^[Aa]rchitecture:/ s/^[^:]: //p' "$dsc")
	201	: ${wantarchs:=any}
be0b4ef6	202	unset buildarchs buildarchs_seen=:
b94830d9 MW	203
	204	## Work through the available targets assigning builds to them. This is
	205	## actually a little tricky.
	206	for t in $targets; do
	207
92b05c85 MW	208	## Dissect the target name.
	209	suite=${t%%-} archs=${t#-}
	210	case $archs in
	211	/) target=${archs%/} host=${archs#/} ;;
	212	*) target=$archs host=$archs; t=$suite-$target/$host ;;
	213	esac
be0b4ef6 MW	214	case $buildarchs_seen in
	215	:$target:)
	216	;;
	217	*)
	218	buildarchs=${buildarchs+$buildarchs }$target
	219	buildarchs_seen=$buildarchs_seen$target:
	220	;;
	221	esac
92b05c85	222
b94830d9 MW	223	## Work through the architectures which we can build.
	224	for arch in $wantarchs; do
	225	case $arch in
	226	all)
	227	## Package suitable for all architectures.
	228
	229	## If we don't want to build architecture-neutral packages then
	230	## there's nothing to do.
	231	case $indepp in nil) continue ;; esac
	232
	233	## Pick this up if nobody has volunteered. However, we should be
	234	## ready to let some other architecture build this if it's going
	235	## to build some architecture-dependent package too.
	236	case $anyp in nil) first=$t anyp=t allp=t ;; esac
	237	;;
	238	*)
	239	## There's at least one architecture-specific package.
	240
28f48d70	241	## If we don't want to build architecture-specific packages then
b94830d9 MW	242	## there's nothing to do.
	243	case $archp in nil) continue ;; esac
	244
	245	## If we can't build it then we shouldn't try.
92b05c85	246	if ! dpkg-architecture -a"$os-$target" -i"$arch"; then
b94830d9 MW	247	continue
	248	fi
	249
	250	## Decide whether we should take responsibility for the
	251	## architecture-neutral packages. If nobody's claimed them yet,
	252	## or the previous claimant wasn't building architecture-specific
	253	## packages, we should take over.
	254	case $depp in
	255	nil) first=$t depp=t anyp=t ;;
	256	t) rest="${rest+$rest }$t" ;;
	257	esac
	258	;;
	259	esac
	260	done
	261	done
	262
	263	## If we never found a match then we can't do anything.
	264	case $anyp in nil) echo "$prog: no packages to build"; exit 0 ;; esac
	265
	266	## Figure out the right options to use.
	267	case $indepp in
	268	t) firstopt="--arch-all" ;;
	269	nil) firstopt="--no-arch-all" ;;
	270	esac
	271	case $archp in
	272	t) ;;
ea91eab4	273	nil) firstopt="$firstopt --no-arch-any" ;;
b94830d9 MW	274	esac
b94830d9 MW	275
6cf97414 MW	276	## Sort out the additional packages. This is rather annoying, because
	277	## sbuild(1) does this in a really stupid way.
	278	rm -rf pkgs.*
	279	for a in $buildarchs; do
	280	mkdir pkgs.$a
	281	for f in $(dpkg-scanpackages -a$a pkgs/ \|
	282	sed -n '/^Filename: /s///p')
	283	do
	284	ln $f pkgs.$a/
	285	done
	286	done
	287
ba9719eb MW	288	## Build the builds sequentially. Tests can conflict with each other,
	289	## e.g., over port numbers.
	290	rc=0 buildopt=$firstopt
	291	for t in $first $rest; do
	292	host=${t##/} full=${t%/}
	293	suite=${full%%-} target=${full#-}
	294
	295	## And we're ready to go.
	296	exec 3>&1
	297	thisrc=$(
	298	{ { { { set +e
bac824a6	299	run sbuild --extra-package=pkgs.$target \
ba9719eb MW	300	--dist=$suite --build=$host --host=$target \
ba9719eb MW	301	--chroot=$suite-$host --verbose $buildopt $dsc \
5e9e7146	302	${dbpargs+--debbuildopts="$dbpargs"} \
ba9719eb MW	303	3>&- 4>&- 5>&-
	304	echo $? >&5
	305	} \|
	306	decor "$full" "\|" >&4; } 2>&1 \|
	307	decor "$full" "*" >&4; } 4>&1 \|
	308	cat -u >&3; } 5>&1 </dev/null)
	309	exec 3>&-
	310	case $thisrc in 0) ;;
	311	*)
	312	echo failed rc=$thisrc >$stat; rc=$thisrc
	313	case $keepon in nil) break ;; esac
	314	;;
	315	esac
	316	buildopt=--no-arch-all
	317	done
b94830d9 MW	318	exit $rc
	319	;;
	320	build,*)
	321	echo >&2 "usage: $prog build BUILDDIR"; exit 1 ;;
	322
	323	*)
	324	fail "unknown command \`$1'"
	325	;;
	326	esac
	327
	328	###----- That's all, folks --------------------------------------------------