[profile] / bin / mdw-sbuild-server

#! /bin/sh -e
###
### Build a Debian package on supported architectures
###
### (c) 2016 Mark Wooding
###

###----- Licensing notice ---------------------------------------------------
###
### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation; either version 2 of the License, or
### (at your option) any later version.
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
### You should have received a copy of the GNU General Public License
### along with this program; if not, write to the Free Software Foundation,
### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

###--------------------------------------------------------------------------
### Configuration.

unset buildroot default_targets parallel
for i in \
  "/etc/mdw-sbuild.conf" \
  "${XDG_CONFIG_HOME-$HOME/.config}/mdw-sbuild.conf"
do
  if [ -f "$i" ]; then . "$i"; fi
done
: ${buildroot=$HOME/build}
: ${default_targets="wheezy-amd64 wheezy-i386"}
: ${parallel=-j3}
: ${DEB_BUILD_OPTIONS=parallel=4}; export DEB_BUILD_OPTIONS

###--------------------------------------------------------------------------
### Some utilities.

prog=${0##*/}

fail () { echo >&2 "$prog: $*"; exit 1; }
usage () { echo "usage: $prog [-ain] [-t TARGET] COMMAND [ARGUMENTS ...]"; }
fail_usage () { usage >&2; exit 1; }

want_1 () {
  what=$1 pat=$2 type=$3; shift 3
  for i in "$@"; do
    [ $type "$i" ] || fail "$what not found: \`$i'"
  done
  case $# in
    1) ;;
    *) fail "expected exactly one $what matching \`$pat', but found $#" ;;
  esac
  echo "$1"
}

###--------------------------------------------------------------------------
### Parse options.

bogusp=nil archp=nil indepp=nil makeopts=""
unset targets

while getopts "haint:" opt; do
  case $opt in
    h)
      usage
      cat <<EOF

Options:
	-h		Show this help text.
	-a		Build only architecture-dependent packages.
	-i		Build only architecture-neutral packages.
	-n		Don't actually do the build.
	-t TARGET	Build in TARGET build environment.

Commands available:

	dir PROJECT/VERSION
		Return a freshly-made directory for the source code to
		go in.

	build BUILDDIR
		Build the package placed in BUILDDIR, which should contain
		exactly one \`.dsc' file, and whatever source archive files
		are necessary.
EOF
      exit
      ;;
    a) archp=t ;;
    i) indepp=t ;;
    n) makeopts="${makeopts+$makeopts }-n" ;;
    t) targets="${targets+$targets }$OPTARG" ;;
    *) bogusp=nil ;;
  esac
done
shift $(( $OPTIND - 1 ))

case $bogusp in t) fail_usage ;; esac
case $archp,$indepp in nil,nil) archp=t indepp=t ;; esac
case ${targets+t} in t) ;; *) targets=$default_targets ;; esac

###--------------------------------------------------------------------------
### Main work.

case "$#,$1" in
  0,*) fail_usage ;;
  *,*,*) fail "bad command name \`$1'" ;;

  2,dir)
    ## dirname PROJECT/VERSION

    ## Try to create a fresh build directory.
    dist=$2
    case "$dist" in */*/*) fail "bad distribution name \`$dist'" ;; esac
    proj=${dist%/*} ver=${dist#*/}
    cd "$buildroot"
    mkdir -p "$proj"
    cd "$proj"
    i=0
    winp=nil
    while [ $i -lt 50 ]; do
      i=$(( $i + 1 ))

      ## Find a sequence number different from all of the existing builds of
      ## this version.
      nn=1
      for j in "$ver#"*; do
	case "$j" in "$ver#*") break ;; esac
	n=${j##*\#}
	if [ $nn -le $n ]; then nn=$(( $n + 1 )); fi
      done

      ## Try to make the build directory.  This might not work if we're
      ## racing with another process, but that's why we're trying in a loop.
      if mkdir "$ver#$nn" >/dev/null 2>&1; then
	winp=t
	cd "$ver#$nn"
	break
      fi

      ## Make sure it actually failed because a directory appeared, rather
      ## than for some other reason.
      [ -e "$ver#$nn" ] || \
	fail "unexpectedly couldn't create \`$buildroot/$dist#$nn'"
    done

    ## Make sure we actually succeeded.
    case $winp in t) ;; *) fail "failed to create build directory" ;; esac

    ## Make an empty directory for dependency packages.
    mkdir -p pkgs/

    ## Done.
    echo "$buildroot/$dist#$nn"
    ;;

  *,dir)
    echo >&2 "usage: $prog dir PROJECT/VERSION"; exit 1 ;;

  2,build)
    ## build BUILDDIR

    ## Track down the build directory.
    builddir=$2
    cd "$builddir"
    dsc=$(want_1 "file" "*.dsc" -f *.dsc)

    ## Figure out which targets need building.  If the `.dsc' file isn't
    ## telling, assume it needs building everywhere and let sbuild(1) sort
    ## out the mess.
    os=$(dpkg-architecture -qDEB_HOST_ARCH_OS)
    unset first rest; anyp=nil depp=nil allp=nil
    wantarchs=$(sed -n '/^[Aa]rchitecture:/ s/^[^:]*: *//p' "$dsc")
    : ${wantarchs:=any}
    unset buildarchs buildarchs_seen=:

    ## Work through the available targets assigning builds to them.  This is
    ## actually a little tricky.
    for t in $targets; do

      ## Dissect the target name.
      suite=${t%%-*} archs=${t#*-}
      case $archs in
	*/*) target=${archs%/*} host=${archs#*/} ;;
	*) target=$archs host=$archs; t=$suite-$target/$host ;;
      esac
      case $buildarchs_seen in
	*:$target:*)
	  ;;
	*)
	  buildarchs=${buildarchs+$buildarchs }$target
	  buildarchs_seen=$buildarchs_seen$target:
	  ;;
      esac

      ## Work through the architectures which we can build.
      for arch in $wantarchs; do
	case $arch in
	  all)
	    ## Package suitable for all architectures.

	    ## If we don't want to build architecture-neutral packages then
	    ## there's nothing to do.
	    case $indepp in nil) continue ;; esac

	    ## Pick this up if nobody has volunteered.  However, we should be
	    ## ready to let some other architecture build this if it's going
	    ## to build some architecture-dependent package too.
	    case $anyp in nil) first=$t anyp=t allp=t ;; esac
	    ;;
	  *)
	    ## There's at least one architecture-specific package.

	    ## If we don't want to build architecture-specific packages then
	    ## there's nothing to do.
	    case $archp in nil) continue ;; esac

	    ## If we can't build it then we shouldn't try.
	    if ! dpkg-architecture -a"$os-$target" -i"$arch"; then
	      continue
	    fi

	    ## Decide whether we should take responsibility for the
	    ## architecture-neutral packages.  If nobody's claimed them yet,
	    ## or the previous claimant wasn't building architecture-specific
	    ## packages, we should take over.
	    case $depp in
	      nil) first=$t depp=t anyp=t ;;
	      t) rest="${rest+$rest }$t" ;;
	    esac
	    ;;
	esac
      done
    done

    ## If we never found a match then we can't do anything.
    case $anyp in nil) echo "$prog: no packages to build"; exit 0 ;; esac

    ## Figure out the right options to use.
    case $indepp in
      t) firstopt="--arch-all" ;;
      nil) firstopt="--no-arch-all" ;;
    esac
    case $archp in
      t) ;;
      nil) firstopt="$firstopt --no-arch-any" ;;
    esac

    ## Sort out the additional packages.  This is rather annoying, because
    ## sbuild(1) does this in a really stupid way.
    rm -rf pkgs.*
    for a in $buildarchs; do
      mkdir pkgs.$a
      for f in $(dpkg-scanpackages -a$a pkgs/ |
		    sed -n '/^Filename: /s///p')
      do
	ln $f pkgs.$a/
      done
    done

    ## Build a cheesy makefile to run these in parallel.
    cat >build.mk <<EOF
### -*-makefile-*-
DSC = $dsc
FIRST = $first
REST = $rest
sbuild-wrap = \\
	t=\$@; \\
	host=\$\${t\#\#*/} full=\$\${t%/*}; \\
	suite=\$\${full%%-*} target=\$\${full\#*-}; \\
	{ echo started >build-status.\$\$full; \\
	  sbuild \\
		--extra-package=pkgs.\$\$target/ \\
		--dist=\$\$suite --build=\$\$host --host=\$\$target \\
		--chroot=\$\$suite-\$\$host --verbose \$1 \$(DSC); \\
	  rc=\$\$?; case \$\$rc in \\
	    0) echo ok >build-status.\$\$full ;; \\
	    *) echo failed rc=\$\$rc >build-status.\$\$full ;; \\
	  esac; } | \\
	while IFS= read -r line; do \\
	  printf "%-21s | %s\n" "\$\$full" "\$\$line"; \\
	done; \\
	read st _ <build-status.\$\$full && \\
	case \$\$st in ok) exit 0 ;; *) exit 1 ;; esac
all: \$(FIRST) \$(REST)
\$(FIRST):; \$(call sbuild-wrap,$firstopt)
\$(REST):; \$(call sbuild-wrap,--no-arch-all)
EOF

    ## Make some marker files to say things are in progress.
    for i in $first $rest; do echo "starting" >build-status.${i%/*}; done

    ## And we're ready to go.
    mkfifo pipeout
    cat pipeout& catpid=$!
    set +e; make -fbuild.mk $parallel $makeopts -k all >pipeout
    rc=$?; set -e
    wait $!
    rm build.mk pipeout build-status.*
    find . -maxdepth 1 -type l -exec rm {} \;
    exit $rc
    ;;
  build,*)
    echo >&2 "usage: $prog build BUILDDIR"; exit 1 ;;

  *)
    fail "unknown command \`$1'"
    ;;
esac

###----- That's all, folks --------------------------------------------------
Commit	Line	Data
b94830d9 MW	1	#! /bin/sh -e
	2	###
	3	### Build a Debian package on supported architectures
	4	###
	5	### (c) 2016 Mark Wooding
	6	###
	7
	8	###----- Licensing notice ---------------------------------------------------
	9	###
	10	### This program is free software; you can redistribute it and/or modify
	11	### it under the terms of the GNU General Public License as published by
	12	### the Free Software Foundation; either version 2 of the License, or
	13	### (at your option) any later version.
	14	###
	15	### This program is distributed in the hope that it will be useful,
	16	### but WITHOUT ANY WARRANTY; without even the implied warranty of
	17	### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	18	### GNU General Public License for more details.
	19	###
	20	### You should have received a copy of the GNU General Public License
	21	### along with this program; if not, write to the Free Software Foundation,
	22	### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	23
	24	###--------------------------------------------------------------------------
	25	### Configuration.
	26
	27	unset buildroot default_targets parallel
	28	for i in \
	29	"/etc/mdw-sbuild.conf" \
	30	"${XDG_CONFIG_HOME-$HOME/.config}/mdw-sbuild.conf"
	31	do
	32	if [ -f "$i" ]; then . "$i"; fi
	33	done
	34	: ${buildroot=$HOME/build}
	35	: ${default_targets="wheezy-amd64 wheezy-i386"}
	36	: ${parallel=-j3}
	37	: ${DEB_BUILD_OPTIONS=parallel=4}; export DEB_BUILD_OPTIONS
	38
	39	###--------------------------------------------------------------------------
	40	### Some utilities.
	41
	42	prog=${0##*/}
	43
	44	fail () { echo >&2 "$prog: $*"; exit 1; }
	45	usage () { echo "usage: $prog [-ain] [-t TARGET] COMMAND [ARGUMENTS ...]"; }
	46	fail_usage () { usage >&2; exit 1; }
	47
	48	want_1 () {
	49	what=$1 pat=$2 type=$3; shift 3
	50	for i in "$@"; do
	51	[ $type "$i" ] \|\| fail "$what not found: \`$i'"
	52	done
	53	case $# in
	54	1) ;;
	55	*) fail "expected exactly one $what matching \`$pat', but found $#" ;;
	56	esac
	57	echo "$1"
	58	}
	59
	60	###--------------------------------------------------------------------------
	61	### Parse options.
	62
	63	bogusp=nil archp=nil indepp=nil makeopts=""
	64	unset targets
65
66	while getopts "haint:" opt; do
67	case $opt in
68	h)
69	usage
70	cat <<EOF
71
72	Options:
73	-h Show this help text.
74	-a Build only architecture-dependent packages.
75	-i Build only architecture-neutral packages.
76	-n Don't actually do the build.
77	-t TARGET Build in TARGET build environment.
78
79	Commands available:
80
81	dir PROJECT/VERSION
82	Return a freshly-made directory for the source code to
83	go in.
84
85	build BUILDDIR
86	Build the package placed in BUILDDIR, which should contain
87	exactly one \`.dsc' file, and whatever source archive files
88	are necessary.
89	EOF
90	exit
91	;;
92	a) archp=t ;;
93	i) indepp=t ;;
94	n) makeopts="${makeopts+$makeopts }-n" ;;
95	t) targets="${targets+$targets }$OPTARG" ;;
96	*) bogusp=nil ;;
97	esac
98	done
99	shift $(( $OPTIND - 1 ))
100
101	case $bogusp in t) fail_usage ;; esac
102	case $archp,$indepp in nil,nil) archp=t indepp=t ;; esac
103	case ${targets+t} in t) ;; *) targets=$default_targets ;; esac
104
105	###--------------------------------------------------------------------------
106	### Main work.
107
108	case "$#,$1" in
109	0,*) fail_usage ;;
110	,,*) fail "bad command name \`$1'" ;;
111
112	2,dir)
113	## dirname PROJECT/VERSION
114
115	## Try to create a fresh build directory.
116	dist=$2
117	case "$dist" in //*) fail "bad distribution name \`$dist'" ;; esac
118	proj=${dist%/} ver=${dist#/}
119	cd "$buildroot"
120	mkdir -p "$proj"
121	cd "$proj"
122	i=0
123	winp=nil
124	while [ $i -lt 50 ]; do
125	i=$(( $i + 1 ))
126
127	## Find a sequence number different from all of the existing builds of
128	## this version.
129	nn=1
130	for j in "$ver#"*; do
131	case "$j" in "$ver#*") break ;; esac
132	n=${j##*\#}
133	if [ $nn -le $n ]; then nn=$(( $n + 1 )); fi
134	done
135
136	## Try to make the build directory. This might not work if we're
137	## racing with another process, but that's why we're trying in a loop.
138	if mkdir "$ver#$nn" >/dev/null 2>&1; then
139	winp=t
6cf97414	140	cd "$ver#$nn"
b94830d9 MW	141	break
	142	fi
	143
	144	## Make sure it actually failed because a directory appeared, rather
	145	## than for some other reason.
	146	[ -e "$ver#$nn" ] \|\| \
	147	fail "unexpectedly couldn't create \`$buildroot/$dist#$nn'"
	148	done
	149
	150	## Make sure we actually succeeded.
	151	case $winp in t) ;; *) fail "failed to create build directory" ;; esac
	152
6cf97414 MW	153	## Make an empty directory for dependency packages.
	154	mkdir -p pkgs/
	155
b94830d9 MW	156	## Done.
	157	echo "$buildroot/$dist#$nn"
	158	;;
	159
	160	*,dir)
	161	echo >&2 "usage: $prog dir PROJECT/VERSION"; exit 1 ;;
	162
	163	2,build)
	164	## build BUILDDIR
	165
	166	## Track down the build directory.
	167	builddir=$2
	168	cd "$builddir"
	169	dsc=$(want_1 "file" ".dsc" -f .dsc)
	170
	171	## Figure out which targets need building. If the `.dsc' file isn't
	172	## telling, assume it needs building everywhere and let sbuild(1) sort
	173	## out the mess.
	174	os=$(dpkg-architecture -qDEB_HOST_ARCH_OS)
	175	unset first rest; anyp=nil depp=nil allp=nil
	176	wantarchs=$(sed -n '/^[Aa]rchitecture:/ s/^[^:]: //p' "$dsc")
	177	: ${wantarchs:=any}
be0b4ef6	178	unset buildarchs buildarchs_seen=:
b94830d9 MW	179
	180	## Work through the available targets assigning builds to them. This is
	181	## actually a little tricky.
	182	for t in $targets; do
	183
92b05c85 MW	184	## Dissect the target name.
	185	suite=${t%%-} archs=${t#-}
	186	case $archs in
	187	/) target=${archs%/} host=${archs#/} ;;
	188	*) target=$archs host=$archs; t=$suite-$target/$host ;;
	189	esac
be0b4ef6 MW	190	case $buildarchs_seen in
	191	:$target:)
	192	;;
	193	*)
	194	buildarchs=${buildarchs+$buildarchs }$target
	195	buildarchs_seen=$buildarchs_seen$target:
	196	;;
	197	esac
92b05c85	198
b94830d9 MW	199	## Work through the architectures which we can build.
	200	for arch in $wantarchs; do
	201	case $arch in
	202	all)
	203	## Package suitable for all architectures.
	204
	205	## If we don't want to build architecture-neutral packages then
	206	## there's nothing to do.
	207	case $indepp in nil) continue ;; esac
	208
	209	## Pick this up if nobody has volunteered. However, we should be
	210	## ready to let some other architecture build this if it's going
	211	## to build some architecture-dependent package too.
	212	case $anyp in nil) first=$t anyp=t allp=t ;; esac
	213	;;
	214	*)
	215	## There's at least one architecture-specific package.
	216
28f48d70	217	## If we don't want to build architecture-specific packages then
b94830d9 MW	218	## there's nothing to do.
	219	case $archp in nil) continue ;; esac
	220
	221	## If we can't build it then we shouldn't try.
92b05c85	222	if ! dpkg-architecture -a"$os-$target" -i"$arch"; then
b94830d9 MW	223	continue
	224	fi
	225
	226	## Decide whether we should take responsibility for the
	227	## architecture-neutral packages. If nobody's claimed them yet,
	228	## or the previous claimant wasn't building architecture-specific
	229	## packages, we should take over.
	230	case $depp in
	231	nil) first=$t depp=t anyp=t ;;
	232	t) rest="${rest+$rest }$t" ;;
	233	esac
	234	;;
	235	esac
	236	done
	237	done
	238
	239	## If we never found a match then we can't do anything.
	240	case $anyp in nil) echo "$prog: no packages to build"; exit 0 ;; esac
	241
	242	## Figure out the right options to use.
	243	case $indepp in
	244	t) firstopt="--arch-all" ;;
	245	nil) firstopt="--no-arch-all" ;;
	246	esac
	247	case $archp in
	248	t) ;;
ea91eab4	249	nil) firstopt="$firstopt --no-arch-any" ;;
b94830d9 MW	250	esac
b94830d9 MW	251
6cf97414 MW	252	## Sort out the additional packages. This is rather annoying, because
	253	## sbuild(1) does this in a really stupid way.
	254	rm -rf pkgs.*
	255	for a in $buildarchs; do
	256	mkdir pkgs.$a
	257	for f in $(dpkg-scanpackages -a$a pkgs/ \|
	258	sed -n '/^Filename: /s///p')
	259	do
	260	ln $f pkgs.$a/
	261	done
	262	done
	263
b94830d9 MW	264	## Build a cheesy makefile to run these in parallel.
	265	cat >build.mk <<EOF
	266	### --makefile--
	267	DSC = $dsc
	268	FIRST = $first
	269	REST = $rest
01c036af MW	270	sbuild-wrap = \\
01c036af MW	271	t=\$@; \\
92b05c85 MW	272	host=\$\${t\#\#/} full=\$\${t%/}; \\
	273	suite=\$\${full%%-} target=\$\${full\#-}; \\
	274	{ echo started >build-status.\$\$full; \\
01c036af	275	sbuild \\
6cf97414	276	--extra-package=pkgs.\$\$target/ \\
92b05c85 MW	277	--dist=\$\$suite --build=\$\$host --host=\$\$target \\
92b05c85 MW	278	--chroot=\$\$suite-\$\$host --verbose \$1 \$(DSC); \\
01c036af	279	rc=\$\$?; case \$\$rc in \\
92b05c85 MW	280	0) echo ok >build-status.\$\$full ;; \\
92b05c85 MW	281	*) echo failed rc=\$\$rc >build-status.\$\$full ;; \\
01c036af MW	282	esac; } \| \\
01c036af MW	283	while IFS= read -r line; do \\
b1f23788	284	printf "%-21s \| %s\n" "\$\$full" "\$\$line"; \\
01c036af	285	done; \\
92b05c85	286	read st _ <build-status.\$\$full && \\
01c036af	287	case \$\$st in ok) exit 0 ;; *) exit 1 ;; esac
b94830d9	288	all: \$(FIRST) \$(REST)
01c036af MW	289	\$(FIRST):; \$(call sbuild-wrap,$firstopt)
01c036af MW	290	\$(REST):; \$(call sbuild-wrap,--no-arch-all)
b94830d9 MW	291	EOF
b94830d9 MW	292
01c036af	293	## Make some marker files to say things are in progress.
92b05c85	294	for i in $first $rest; do echo "starting" >build-status.${i%/*}; done
01c036af	295
b94830d9 MW	296	## And we're ready to go.
	297	mkfifo pipeout
	298	cat pipeout& catpid=$!
	299	set +e; make -fbuild.mk $parallel $makeopts -k all >pipeout
	300	rc=$?; set -e
	301	wait $!
01c036af	302	rm build.mk pipeout build-status.*
b94830d9 MW	303	find . -maxdepth 1 -type l -exec rm {} \;
	304	exit $rc
	305	;;
	306	build,*)
	307	echo >&2 "usage: $prog build BUILDDIR"; exit 1 ;;
	308
	309	*)
	310	fail "unknown command \`$1'"
	311	;;
	312	esac
	313
	314	###----- That's all, folks --------------------------------------------------