[profile] / bin / mdw-sbuild-server

#! /bin/sh -e
###
### Build a Debian package on supported architectures
###
### (c) 2016 Mark Wooding
###

###----- Licensing notice ---------------------------------------------------
###
### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation; either version 2 of the License, or
### (at your option) any later version.
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
### You should have received a copy of the GNU General Public License
### along with this program; if not, write to the Free Software Foundation,
### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

###--------------------------------------------------------------------------
### Configuration.

unset buildroot default_targets parallel
for i in \
  "/etc/mdw-sbuild.conf" \
  "${XDG_CONFIG_HOME-$HOME/.config}/mdw-sbuild.conf"
do
  if [ -f "$i" ]; then . "$i"; fi
done
: ${buildroot=$HOME/build}
: ${default_targets="wheezy-amd64 wheezy-i386"}
: ${DEB_BUILD_OPTIONS=parallel=4}; export DEB_BUILD_OPTIONS

###--------------------------------------------------------------------------
### Some utilities.

prog=${0##*/}

fail () { echo >&2 "$prog: $*"; exit 1; }
usage () { echo "usage: $prog [-ain] [-t TARGET] COMMAND [ARGUMENTS ...]"; }
fail_usage () { usage >&2; exit 1; }

want_1 () {
  what=$1 pat=$2 type=$3; shift 3
  for i in "$@"; do
    [ $type "$i" ] || fail "$what not found: \`$i'"
  done
  case $# in
    1) ;;
    *) fail "expected exactly one $what matching \`$pat', but found $#" ;;
  esac
  echo "$1"
}

run () {
  case $notreally in
    t) echo "+ $*" ;;
    nil) "$@" ;;
  esac
}

decor () {
  tag=$1 marker=$2
  while IFS= read -r line; do
    printf "%-21s %c %s\n" "$tag" "$marker" "$line"
  done
}

###--------------------------------------------------------------------------
### Parse options.

bogusp=nil archp=nil indepp=nil keepon=nil notreally=nil
unset targets

while getopts "haint:" opt; do
  case $opt in
    h)
      usage
      cat <<EOF

Options:
	-h		Show this help text.
	-a		Build only architecture-dependent packages.
	-i		Build only architecture-neutral packages.
	-k		Keep going even if one fails.
	-n		Don't actually do the build.
	-t TARGET	Build in TARGET build environment.

Commands available:

	dir PROJECT/VERSION
		Return a freshly-made directory for the source code to
		go in.

	build BUILDDIR
		Build the package placed in BUILDDIR, which should contain
		exactly one \`.dsc' file, and whatever source archive files
		are necessary.
EOF
      exit
      ;;
    a) archp=t ;;
    i) indepp=t ;;
    k) keepon=t ;;
    n) notreally=t ;;
    t) targets="${targets+$targets }$OPTARG" ;;
    *) bogusp=nil ;;
  esac
done
shift $(( $OPTIND - 1 ))

case $bogusp in t) fail_usage ;; esac
case $archp,$indepp in nil,nil) archp=t indepp=t ;; esac
case ${targets+t} in t) ;; *) targets=$default_targets ;; esac

###--------------------------------------------------------------------------
### Main work.

case "$#,$1" in
  0,*) fail_usage ;;
  *,*,*) fail "bad command name \`$1'" ;;

  2,dir)
    ## dirname PROJECT/VERSION

    ## Try to create a fresh build directory.
    dist=$2
    case "$dist" in */*/*) fail "bad distribution name \`$dist'" ;; esac
    proj=${dist%/*} ver=${dist#*/}
    cd "$buildroot"
    mkdir -p "$proj"
    cd "$proj"
    i=0
    winp=nil
    while [ $i -lt 50 ]; do
      i=$(( $i + 1 ))

      ## Find a sequence number different from all of the existing builds of
      ## this version.
      nn=1
      for j in "$ver#"*; do
	case "$j" in "$ver#*") break ;; esac
	n=${j##*\#}
	if [ $nn -le $n ]; then nn=$(( $n + 1 )); fi
      done

      ## Try to make the build directory.  This might not work if we're
      ## racing with another process, but that's why we're trying in a loop.
      if mkdir "$ver#$nn" >/dev/null 2>&1; then
	winp=t
	cd "$ver#$nn"
	break
      fi

      ## Make sure it actually failed because a directory appeared, rather
      ## than for some other reason.
      [ -e "$ver#$nn" ] || \
	fail "unexpectedly couldn't create \`$buildroot/$dist#$nn'"
    done

    ## Make sure we actually succeeded.
    case $winp in t) ;; *) fail "failed to create build directory" ;; esac

    ## Make an empty directory for dependency packages.
    mkdir -p pkgs/

    ## Done.
    echo "$buildroot/$dist#$nn"
    ;;

  *,dir)
    echo >&2 "usage: $prog dir PROJECT/VERSION"; exit 1 ;;

  2,build)
    ## build BUILDDIR

    ## Track down the build directory.
    builddir=$2
    cd "$builddir"
    dsc=$(want_1 "file" "*.dsc" -f *.dsc)

    ## Figure out which targets need building.  If the `.dsc' file isn't
    ## telling, assume it needs building everywhere and let sbuild(1) sort
    ## out the mess.
    os=$(dpkg-architecture -qDEB_HOST_ARCH_OS)
    unset first rest; anyp=nil depp=nil allp=nil
    wantarchs=$(sed -n '/^[Aa]rchitecture:/ s/^[^:]*: *//p' "$dsc")
    : ${wantarchs:=any}
    unset buildarchs buildarchs_seen=:

    ## Work through the available targets assigning builds to them.  This is
    ## actually a little tricky.
    for t in $targets; do

      ## Dissect the target name.
      suite=${t%%-*} archs=${t#*-}
      case $archs in
	*/*) target=${archs%/*} host=${archs#*/} ;;
	*) target=$archs host=$archs; t=$suite-$target/$host ;;
      esac
      case $buildarchs_seen in
	*:$target:*)
	  ;;
	*)
	  buildarchs=${buildarchs+$buildarchs }$target
	  buildarchs_seen=$buildarchs_seen$target:
	  ;;
      esac

      ## Work through the architectures which we can build.
      for arch in $wantarchs; do
	case $arch in
	  all)
	    ## Package suitable for all architectures.

	    ## If we don't want to build architecture-neutral packages then
	    ## there's nothing to do.
	    case $indepp in nil) continue ;; esac

	    ## Pick this up if nobody has volunteered.  However, we should be
	    ## ready to let some other architecture build this if it's going
	    ## to build some architecture-dependent package too.
	    case $anyp in nil) first=$t anyp=t allp=t ;; esac
	    ;;
	  *)
	    ## There's at least one architecture-specific package.

	    ## If we don't want to build architecture-specific packages then
	    ## there's nothing to do.
	    case $archp in nil) continue ;; esac

	    ## If we can't build it then we shouldn't try.
	    if ! dpkg-architecture -a"$os-$target" -i"$arch"; then
	      continue
	    fi

	    ## Decide whether we should take responsibility for the
	    ## architecture-neutral packages.  If nobody's claimed them yet,
	    ## or the previous claimant wasn't building architecture-specific
	    ## packages, we should take over.
	    case $depp in
	      nil) first=$t depp=t anyp=t ;;
	      t) rest="${rest+$rest }$t" ;;
	    esac
	    ;;
	esac
      done
    done

    ## If we never found a match then we can't do anything.
    case $anyp in nil) echo "$prog: no packages to build"; exit 0 ;; esac

    ## Figure out the right options to use.
    case $indepp in
      t) firstopt="--arch-all" ;;
      nil) firstopt="--no-arch-all" ;;
    esac
    case $archp in
      t) ;;
      nil) firstopt="$firstopt --no-arch-any" ;;
    esac

    ## Sort out the additional packages.  This is rather annoying, because
    ## sbuild(1) does this in a really stupid way.
    rm -rf pkgs.*
    for a in $buildarchs; do
      mkdir pkgs.$a
      for f in $(dpkg-scanpackages -a$a pkgs/ |
		    sed -n '/^Filename: /s///p')
      do
	ln $f pkgs.$a/
      done
    done

    ## Build the builds sequentially.  Tests can conflict with each other,
    ## e.g., over port numbers.
    rc=0 buildopt=$firstopt
    for t in $first $rest; do
      host=${t##*/} full=${t%/*}
      suite=${full%%-*} target=${full#*-}

      ## And we're ready to go.
      exec 3>&1
      thisrc=$(
	{ { { { set +e
		run sbuild --extra-package=$pkgs.$target \
		    --dist=$suite --build=$host --host=$target \
		    --chroot=$suite-$host --verbose $buildopt $dsc \
		    3>&- 4>&- 5>&-
		echo $? >&5
	      } |
		decor "$full" "|" >&4; } 2>&1 |
	      decor "$full" "*" >&4; } 4>&1 |
	    cat -u >&3; } 5>&1 </dev/null)
      exec 3>&-
      case $thisrc in 0) ;;
	*)
	  echo failed rc=$thisrc >$stat; rc=$thisrc
	  case $keepon in nil) break ;; esac
	  ;;
      esac
      buildopt=--no-arch-all
    done
    exit $rc
    ;;
  build,*)
    echo >&2 "usage: $prog build BUILDDIR"; exit 1 ;;

  *)
    fail "unknown command \`$1'"
    ;;
esac

###----- That's all, folks --------------------------------------------------
Commit	Line	Data
b94830d9 MW	1	#! /bin/sh -e
	2	###
	3	### Build a Debian package on supported architectures
	4	###
	5	### (c) 2016 Mark Wooding
	6	###
	7
	8	###----- Licensing notice ---------------------------------------------------
	9	###
	10	### This program is free software; you can redistribute it and/or modify
	11	### it under the terms of the GNU General Public License as published by
	12	### the Free Software Foundation; either version 2 of the License, or
	13	### (at your option) any later version.
	14	###
	15	### This program is distributed in the hope that it will be useful,
	16	### but WITHOUT ANY WARRANTY; without even the implied warranty of
	17	### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	18	### GNU General Public License for more details.
	19	###
	20	### You should have received a copy of the GNU General Public License
	21	### along with this program; if not, write to the Free Software Foundation,
	22	### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	23
	24	###--------------------------------------------------------------------------
	25	### Configuration.
	26
	27	unset buildroot default_targets parallel
	28	for i in \
	29	"/etc/mdw-sbuild.conf" \
	30	"${XDG_CONFIG_HOME-$HOME/.config}/mdw-sbuild.conf"
	31	do
	32	if [ -f "$i" ]; then . "$i"; fi
	33	done
	34	: ${buildroot=$HOME/build}
	35	: ${default_targets="wheezy-amd64 wheezy-i386"}
b94830d9 MW	36	: ${DEB_BUILD_OPTIONS=parallel=4}; export DEB_BUILD_OPTIONS
	37
	38	###--------------------------------------------------------------------------
	39	### Some utilities.
	40
	41	prog=${0##*/}
	42
	43	fail () { echo >&2 "$prog: $*"; exit 1; }
	44	usage () { echo "usage: $prog [-ain] [-t TARGET] COMMAND [ARGUMENTS ...]"; }
	45	fail_usage () { usage >&2; exit 1; }
	46
	47	want_1 () {
	48	what=$1 pat=$2 type=$3; shift 3
	49	for i in "$@"; do
	50	[ $type "$i" ] \|\| fail "$what not found: \`$i'"
	51	done
	52	case $# in
	53	1) ;;
	54	*) fail "expected exactly one $what matching \`$pat', but found $#" ;;
	55	esac
	56	echo "$1"
	57	}
	58
ba9719eb MW	59	run () {
	60	case $notreally in
	61	t) echo "+ $*" ;;
	62	nil) "$@" ;;
	63	esac
	64	}
	65
	66	decor () {
	67	tag=$1 marker=$2
	68	while IFS= read -r line; do
	69	printf "%-21s %c %s\n" "$tag" "$marker" "$line"
	70	done
	71	}
	72
b94830d9 MW	73	###--------------------------------------------------------------------------
	74	### Parse options.
	75
ba9719eb	76	bogusp=nil archp=nil indepp=nil keepon=nil notreally=nil
b94830d9 MW	77	unset targets
	78
	79	while getopts "haint:" opt; do
	80	case $opt in
	81	h)
	82	usage
	83	cat <<EOF
	84
	85	Options:
	86	-h Show this help text.
	87	-a Build only architecture-dependent packages.
	88	-i Build only architecture-neutral packages.
ba9719eb	89	-k Keep going even if one fails.
b94830d9 MW	90	-n Don't actually do the build.
	91	-t TARGET Build in TARGET build environment.
	92
	93	Commands available:
	94
	95	dir PROJECT/VERSION
	96	Return a freshly-made directory for the source code to
	97	go in.
	98
	99	build BUILDDIR
	100	Build the package placed in BUILDDIR, which should contain
	101	exactly one \`.dsc' file, and whatever source archive files
	102	are necessary.
	103	EOF
	104	exit
	105	;;
	106	a) archp=t ;;
	107	i) indepp=t ;;
ba9719eb MW	108	k) keepon=t ;;
ba9719eb MW	109	n) notreally=t ;;
b94830d9 MW	110	t) targets="${targets+$targets }$OPTARG" ;;
	111	*) bogusp=nil ;;
	112	esac
	113	done
	114	shift $(( $OPTIND - 1 ))
	115
	116	case $bogusp in t) fail_usage ;; esac
	117	case $archp,$indepp in nil,nil) archp=t indepp=t ;; esac
	118	case ${targets+t} in t) ;; *) targets=$default_targets ;; esac
	119
	120	###--------------------------------------------------------------------------
	121	### Main work.
	122
	123	case "$#,$1" in
	124	0,*) fail_usage ;;
	125	,,*) fail "bad command name \`$1'" ;;
	126
	127	2,dir)
	128	## dirname PROJECT/VERSION
	129
	130	## Try to create a fresh build directory.
	131	dist=$2
	132	case "$dist" in //*) fail "bad distribution name \`$dist'" ;; esac
	133	proj=${dist%/} ver=${dist#/}
	134	cd "$buildroot"
	135	mkdir -p "$proj"
	136	cd "$proj"
	137	i=0
	138	winp=nil
	139	while [ $i -lt 50 ]; do
	140	i=$(( $i + 1 ))
	141
	142	## Find a sequence number different from all of the existing builds of
	143	## this version.
	144	nn=1
	145	for j in "$ver#"*; do
	146	case "$j" in "$ver#*") break ;; esac
	147	n=${j##*\#}
	148	if [ $nn -le $n ]; then nn=$(( $n + 1 )); fi
	149	done
	150
	151	## Try to make the build directory. This might not work if we're
	152	## racing with another process, but that's why we're trying in a loop.
	153	if mkdir "$ver#$nn" >/dev/null 2>&1; then
	154	winp=t
6cf97414	155	cd "$ver#$nn"
b94830d9 MW	156	break
	157	fi
	158
	159	## Make sure it actually failed because a directory appeared, rather
	160	## than for some other reason.
	161	[ -e "$ver#$nn" ] \|\| \
	162	fail "unexpectedly couldn't create \`$buildroot/$dist#$nn'"
	163	done
	164
	165	## Make sure we actually succeeded.
	166	case $winp in t) ;; *) fail "failed to create build directory" ;; esac
	167
6cf97414 MW	168	## Make an empty directory for dependency packages.
	169	mkdir -p pkgs/
	170
b94830d9 MW	171	## Done.
	172	echo "$buildroot/$dist#$nn"
	173	;;
	174
	175	*,dir)
	176	echo >&2 "usage: $prog dir PROJECT/VERSION"; exit 1 ;;
	177
	178	2,build)
	179	## build BUILDDIR
	180
	181	## Track down the build directory.
	182	builddir=$2
	183	cd "$builddir"
	184	dsc=$(want_1 "file" ".dsc" -f .dsc)
	185
	186	## Figure out which targets need building. If the `.dsc' file isn't
	187	## telling, assume it needs building everywhere and let sbuild(1) sort
	188	## out the mess.
	189	os=$(dpkg-architecture -qDEB_HOST_ARCH_OS)
	190	unset first rest; anyp=nil depp=nil allp=nil
	191	wantarchs=$(sed -n '/^[Aa]rchitecture:/ s/^[^:]: //p' "$dsc")
	192	: ${wantarchs:=any}
be0b4ef6	193	unset buildarchs buildarchs_seen=:
b94830d9 MW	194
	195	## Work through the available targets assigning builds to them. This is
	196	## actually a little tricky.
	197	for t in $targets; do
	198
92b05c85 MW	199	## Dissect the target name.
	200	suite=${t%%-} archs=${t#-}
	201	case $archs in
	202	/) target=${archs%/} host=${archs#/} ;;
	203	*) target=$archs host=$archs; t=$suite-$target/$host ;;
	204	esac
be0b4ef6 MW	205	case $buildarchs_seen in
	206	:$target:)
	207	;;
	208	*)
	209	buildarchs=${buildarchs+$buildarchs }$target
	210	buildarchs_seen=$buildarchs_seen$target:
	211	;;
	212	esac
92b05c85	213
b94830d9 MW	214	## Work through the architectures which we can build.
	215	for arch in $wantarchs; do
	216	case $arch in
	217	all)
	218	## Package suitable for all architectures.
	219
	220	## If we don't want to build architecture-neutral packages then
	221	## there's nothing to do.
	222	case $indepp in nil) continue ;; esac
	223
	224	## Pick this up if nobody has volunteered. However, we should be
	225	## ready to let some other architecture build this if it's going
	226	## to build some architecture-dependent package too.
	227	case $anyp in nil) first=$t anyp=t allp=t ;; esac
	228	;;
	229	*)
	230	## There's at least one architecture-specific package.
	231
28f48d70	232	## If we don't want to build architecture-specific packages then
b94830d9 MW	233	## there's nothing to do.
	234	case $archp in nil) continue ;; esac
	235
	236	## If we can't build it then we shouldn't try.
92b05c85	237	if ! dpkg-architecture -a"$os-$target" -i"$arch"; then
b94830d9 MW	238	continue
	239	fi
	240
	241	## Decide whether we should take responsibility for the
	242	## architecture-neutral packages. If nobody's claimed them yet,
	243	## or the previous claimant wasn't building architecture-specific
	244	## packages, we should take over.
	245	case $depp in
	246	nil) first=$t depp=t anyp=t ;;
	247	t) rest="${rest+$rest }$t" ;;
	248	esac
	249	;;
	250	esac
	251	done
	252	done
	253
	254	## If we never found a match then we can't do anything.
	255	case $anyp in nil) echo "$prog: no packages to build"; exit 0 ;; esac
	256
	257	## Figure out the right options to use.
	258	case $indepp in
	259	t) firstopt="--arch-all" ;;
	260	nil) firstopt="--no-arch-all" ;;
	261	esac
	262	case $archp in
	263	t) ;;
ea91eab4	264	nil) firstopt="$firstopt --no-arch-any" ;;
b94830d9 MW	265	esac
b94830d9 MW	266
6cf97414 MW	267	## Sort out the additional packages. This is rather annoying, because
	268	## sbuild(1) does this in a really stupid way.
	269	rm -rf pkgs.*
	270	for a in $buildarchs; do
	271	mkdir pkgs.$a
	272	for f in $(dpkg-scanpackages -a$a pkgs/ \|
	273	sed -n '/^Filename: /s///p')
	274	do
	275	ln $f pkgs.$a/
	276	done
	277	done
	278
ba9719eb MW	279	## Build the builds sequentially. Tests can conflict with each other,
	280	## e.g., over port numbers.
	281	rc=0 buildopt=$firstopt
	282	for t in $first $rest; do
	283	host=${t##/} full=${t%/}
	284	suite=${full%%-} target=${full#-}
	285
	286	## And we're ready to go.
	287	exec 3>&1
	288	thisrc=$(
	289	{ { { { set +e
	290	run sbuild --extra-package=$pkgs.$target \
	291	--dist=$suite --build=$host --host=$target \
	292	--chroot=$suite-$host --verbose $buildopt $dsc \
	293	3>&- 4>&- 5>&-
	294	echo $? >&5
	295	} \|
	296	decor "$full" "\|" >&4; } 2>&1 \|
	297	decor "$full" "*" >&4; } 4>&1 \|
	298	cat -u >&3; } 5>&1 </dev/null)
	299	exec 3>&-
	300	case $thisrc in 0) ;;
	301	*)
	302	echo failed rc=$thisrc >$stat; rc=$thisrc
	303	case $keepon in nil) break ;; esac
	304	;;
	305	esac
	306	buildopt=--no-arch-all
	307	done
b94830d9 MW	308	exit $rc
	309	;;
	310	build,*)
	311	echo >&2 "usage: $prog build BUILDDIR"; exit 1 ;;
	312
	313	*)
	314	fail "unknown command \`$1'"
	315	;;
	316	esac
	317
	318	###----- That's all, folks --------------------------------------------------