From 0fe058e9a36482f8d15abb83471439d8d9f8e538 Mon Sep 17 00:00:00 2001 From: Mark Wooding Date: Fri, 5 Aug 2011 11:28:09 +0100 Subject: [PATCH] Refactor the Debian mirror configuration. * Factor the architecture-exclusion logic out into a separate program bin/debian-excludes. * Factor the common ftpsync configuration options into etc/ftpsync-common.conf, and source this from the archive-specific configuration scripts. --- bin/debian-excludes | 52 +++++++++++ etc/ftpsync-common.conf | 168 ++++++++++++++++++++++++++++++++++++ etc/ftpsync-debian-security.conf | 161 +++------------------------------- etc/ftpsync-debian.conf | 181 +-------------------------------------- 4 files changed, 236 insertions(+), 326 deletions(-) create mode 100755 bin/debian-excludes create mode 100644 etc/ftpsync-common.conf diff --git a/bin/debian-excludes b/bin/debian-excludes new file mode 100755 index 0000000..bac4992 --- /dev/null +++ b/bin/debian-excludes @@ -0,0 +1,52 @@ +#! /bin/sh + +set -e +case $# in + 0 | 1 | 2 | 3) + echo >&2 "Usage: $0 HOST PATH ARCH:ARCH:... DIST..." + exit 1 + ;; + *) + RSYNC_HOST=$1 RSYNC_PATH=$2 WANT_ARCH=$3; shift 3 + ;; +esac + +## Check the available distributions for architectures. +for dist in "$@"; do + rsync --list-only $RSYNC_HOST::$RSYNC_PATH/dists/$dist/main/ +done | { + + ## Gather up excluded architectures as we go. + excludes="" + + while read mode size date time name; do + + ## Check directories of binary packages. If it's an architecture we + ## don't want to reject, then continue on. + case "$name" in + binary-all) + continue + ;; + binary-*) + arch=${name#binary-} + case ":$WANT_ARCH:" in *:"$arch":*) continue ;; esac + ;; + *) + continue + ;; + esac + + ## Pick out the architecture name. Check whether we've seen it before. + arch=${name#binary-} + case " $excludes " in + *" $arch "*) + ;; + *) + excludes="${excludes+$excludes }$arch" + ;; + esac + done + + ## Done. Print out the finished list. + echo $excludes +} diff --git a/etc/ftpsync-common.conf b/etc/ftpsync-common.conf new file mode 100644 index 0000000..61712f4 --- /dev/null +++ b/etc/ftpsync-common.conf @@ -0,0 +1,168 @@ +### -*-sh-*- +### +### Common ftpsync configuration for Debian mirrors. +### +### This file is sourced by configuration scripts for specfic mirrors. + +###-------------------------------------------------------------------------- +### Settings expected from the calling script. + +## Mirrorname. This is used for things like the trace file and should always +## be the full hostname of the mirror. +: ${MIRRORNAME=`hostname -f`} + +## Destination of the mirrored files. Should be an empty directory. +## CAREFUL, this directory will contain the mirror. Everything else that +## might have happened to be in there WILL BE GONE after the mirror sync! +: ${LABEL="debian"} +: ${TO="/mnt/mirror/$LABEL/"} + +## The upstream name of the rsync share. +: ${RSYNC_PATH="$LABEL"} + +## The host we mirror from +: ${RSYNC_HOST=ftp.de.debian.org} + +## In case we need a user to access the rsync share at our upstream host +: ${RSYNC_USER=} + +## If we need a user we also need a password +: ${RSYNC_PASSWORD=} + +## Architectures we want to carry. +: ${WANT_ARCH="i386:amd64"} + +## Distributions to inspect for available architectures. +: ${SEARCH_DISTS="oldstable stable testing unstable experimental"} + +###-------------------------------------------------------------------------- +### Common settings. + +## In which directory should logfiles end up +## Note that BASEDIR defaults to $HOME, but can be set before calling the +## ftpsync script to any value you want (for example using pam_env) +#LOGDIR="${BASEDIR}/log" + +## Name of our own logfile. +## Note that ${NAME} is set by the ftpsync script depending on the way it +## is called. See README for a description of the multi-archive capability +## and better always include ${NAME} in this path. +#LOG="${LOGDIR}/${NAME}.log" + +## The script can send logs (or error messages) to a mail address. +## If this is unset it will default to the local root user unless it is run +## on a .debian.org machine where it will default to the mirroradm people. +#MAILTO="root" + +## If you do want a mail about every single sync, set this to false +## Everything else will only send mails if a mirror sync fails +#ERRORSONLY="true" + +## If you want the logs to also include output of rsync, set this to true. +## Careful, the logs can get pretty big, especially if it is the first mirror +## run +#FULLLOGS="false" + +## If you do want to exclude files from the mirror run, put --exclude +## statements here. See rsync(1) for the exact syntax, these are passed to +## rsync as written here. DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH +## THIS, IT WILL NOT WORK! +#EXCLUDE="" + +## If you do want to exclude an architecture, this is for you. +## Use as space seperated list. +## Possible values are: +## alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, kfreebsd-amd64, +## kfreebsd-i386, m68k, mipsel, mips, powerpc, s390, sh, sparc and source +## eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc" +## An unset value will mirror all architectures (default!) +#ARCH_EXCLUDE="" + +## Actually, no: we want to /include/ only some architectures. So we need to +## find out which ones are available so that we can exclude the ones we don't +## want. Crazy, no? +ARCH_EXCLUDE=$( + bin/debian-excludes $RSYNC_HOST $RSYNC_PATH $WANT_ARCH $SEARCH_DISTS + echo source +) + +## Do we have leaf mirror to signal we are done and they should sync? +## If so set it to true and make sure you configure runmirrors.mirrors +## and runmirrors.conf for your need. +#HUB=false + +## We do create three logfiles for every run. To save space we rotate them, +## this defines how many we keep +#LOGROTATE=14 + +## Our own lockfile (only one sync should run at any time) +#LOCK="${TO}/Archive-Update-in-Progress-${MIRRORNAME}" + +## Timeout for the lockfile, in case we have bash older than v4 (and no +## /proc) +# LOCKTIMEOUT=${LOCKTIMEOUT:-3600} + +## The following file is used to make sure we will end up with a correctly +## synced mirror even if we get multiple pushes in a short timeframe +#UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" + +## The trace file is used by a mirror check tool to see when we last had a +## successful mirror sync. Make sure that it always ends up in project/trace +## and always shows the full hostname. This is *relative* to ${TO} +#TRACE="project/trace/${MIRRORNAME}" + +## We sync our mirror using rsync (everything else would be insane), so +## we need a few options set. +## The rsync program +: ${RSYNC="rsync"} + +## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! +## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! +## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! +## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! + +## limit I/O bandwidth. Value is KBytes per second, unset or 0 means +## unlimited +#RSYNC_BW="" + +## Default rsync options every rsync invocation sees. +#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}" + +## Options the first pass gets. We do not want the Packages/Source indices +## here, and we also do not want to delete any files yet. +#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*" + +## Options the second pass gets. Now we want the Packages/Source indices too +## and we also want to delete files. We also want to delete files that are +## excluded. +#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded" + +## You may establish the connection via a web proxy by setting the +## environment variable RSYNC_PROXY to a hostname:port pair pointing to your +## web proxy. Note that your web proxy's configuration must support proxy +## connections to port 873. +# RSYNC_PROXY= + +## The following three options are used in case we want to "callback" the +## host we got pushed from. +#CALLBACKUSER="archvsync" +#CALLBACKHOST="none" +#CALLBACKKEY="none" + +## Hook scripts can be run at various places during the sync. +## Leave them blank if you don't want any +## Hook1: After lock is acquired, before first rsync +## Hook2: After first rsync, if successful +## Hook3: After second rsync, if successful +## Hook4: Right before leaf mirror triggering +## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true) +## +## Note that Hook3 and Hook4 are likely to be called directly after each +## other. Difference is: Hook3 is called *every* time the second rsync was +## successful, but even if the mirroring needs to re-run thanks to a second +## push. Hook4 is only effective if we are done with mirroring. +#HOOK1= +#HOOK2= +#HOOK3= +#HOOK4= +#HOOK5= diff --git a/etc/ftpsync-debian-security.conf b/etc/ftpsync-debian-security.conf index df490f1..a6b47fb 100644 --- a/etc/ftpsync-debian-security.conf +++ b/etc/ftpsync-debian-security.conf @@ -1,148 +1,13 @@ -######################################################################## -######################################################################## -## This is a sample configuration file for the ftpsync mirror script. ## -## Most of the values are commented out and just shown here for ## -## completeness, together with their default value. ## -######################################################################## -######################################################################## - -## Mirrorname. This is used for things like the trace file and should always -## be the full hostname of the mirror. -#MIRRORNAME=`hostname -f` - -## Destination of the mirrored files. Should be an empty directory. -## CAREFUL, this directory will contain the mirror. Everything else -## that might have happened to be in there WILL BE GONE after the mirror sync! -#TO="/org/ftp.debian.org/ftp/" - -## The upstream name of the rsync share. -#RSYNC_PATH="ftp" - -## The host we mirror from -#RSYNC_HOST=some.mirror.debian.org - -## In case we need a user to access the rsync share at our upstream host -#RSYNC_USER= - -## If we need a user we also need a password -#RSYNC_PASSWORD= - -## In which directory should logfiles end up -## Note that BASEDIR defaults to $HOME, but can be set before calling the -## ftpsync script to any value you want (for example using pam_env) -#LOGDIR="${BASEDIR}/log" - -## Name of our own logfile. -## Note that ${NAME} is set by the ftpsync script depending on the way it -## is called. See README for a description of the multi-archive capability -## and better always include ${NAME} in this path. -#LOG="${LOGDIR}/${NAME}.log" - -## The script can send logs (or error messages) to a mail address. -## If this is unset it will default to the local root user unless it is run -## on a .debian.org machine where it will default to the mirroradm people. -#MAILTO="root" - -## If you do want a mail about every single sync, set this to false -## Everything else will only send mails if a mirror sync fails -#ERRORSONLY="true" - -## If you want the logs to also include output of rsync, set this to true. -## Careful, the logs can get pretty big, especially if it is the first mirror -## run -#FULLLOGS="false" - -## If you do want to exclude files from the mirror run, put --exclude statements here. -## See rsync(1) for the exact syntax, these are passed to rsync as written here. -## DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH THIS, IT WILL NOT WORK! -#EXCLUDE="" - -## If you do want to exclude an architecture, this is for you. -## Use as space seperated list. -## Possible values are: -## alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, kfreebsd-amd64, -## kfreebsd-i386, m68k, mipsel, mips, powerpc, s390, sh, sparc and source -## eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc" -## An unset value will mirror all architectures (default!) -#ARCH_EXCLUDE="" - -## Do we have leaf mirror to signal we are done and they should sync? -## If so set it to true and make sure you configure runmirrors.mirrors -## and runmirrors.conf for your need. -#HUB=false - -## We do create three logfiles for every run. To save space we rotate them, this -## defines how many we keep -#LOGROTATE=14 - -## Our own lockfile (only one sync should run at any time) -#LOCK="${TO}/Archive-Update-in-Progress-${MIRRORNAME}" - -# Timeout for the lockfile, in case we have bash older than v4 (and no /proc) -# LOCKTIMEOUT=${LOCKTIMEOUT:-3600} - -## The following file is used to make sure we will end up with a correctly -## synced mirror even if we get multiple pushes in a short timeframe -#UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" - -## The trace file is used by a mirror check tool to see when we last -## had a successful mirror sync. Make sure that it always ends up in -## project/trace and always shows the full hostname. -## This is *relative* to ${TO} -#TRACE="project/trace/${MIRRORNAME}" - -## We sync our mirror using rsync (everything else would be insane), so -## we need a few options set. -## The rsync program -#RSYNC=rsync - -## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! -## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! -## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! -## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! - -## limit I/O bandwidth. Value is KBytes per second, unset or 0 means unlimited -#RSYNC_BW="" - -## Default rsync options every rsync invocation sees. -#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}" - -## Options the first pass gets. We do not want the Packages/Source indices -## here, and we also do not want to delete any files yet. -#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*" - -## Options the second pass gets. Now we want the Packages/Source indices too -## and we also want to delete files. We also want to delete files that are -## excluded. -#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded" - -## You may establish the connection via a web proxy by setting the environment -## variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note -## that your web proxy's configuration must support proxy connections to port 873. -# RSYNC_PROXY= - - -## The following three options are used in case we want to "callback" the host -## we got pushed from. -#CALLBACKUSER="archvsync" -#CALLBACKHOST="none" -#CALLBACKKEY="none" - - -## Hook scripts can be run at various places during the sync. -## Leave them blank if you don't want any -## Hook1: After lock is acquired, before first rsync -## Hook2: After first rsync, if successful -## Hook3: After second rsync, if successful -## Hook4: Right before leaf mirror triggering -## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true) -## -## Note that Hook3 and Hook4 are likely to be called directly after each other. -## Difference is: Hook3 is called *every* time the second rsync was successful, -## but even if the mirroring needs to re-run thanks to a second push. -## Hook4 is only effective if we are done with mirroring. -#HOOK1= -#HOOK2= -#HOOK3= -#HOOK4= -#HOOK5= +### -*-sh-*- +### +### ftpsync configuration for the Debian security archive. + +LABEL=debian-security +RSYNC_HOST=security.debian.org +SEARCH_DISTS=$( + for i in oldstable stable testing unstable experimental; do + echo $i/updates + done +) + +. etc/ftpsync-common.conf diff --git a/etc/ftpsync-debian.conf b/etc/ftpsync-debian.conf index 15d5c83..3417c94 100644 --- a/etc/ftpsync-debian.conf +++ b/etc/ftpsync-debian.conf @@ -2,183 +2,8 @@ ### ### ftpsync configuration for the Debian main archive. -## Mirrorname. This is used for things like the trace file and should always -## be the full hostname of the mirror. -#MIRRORNAME=`hostname -f` - -## Destination of the mirrored files. Should be an empty directory. -## CAREFUL, this directory will contain the mirror. Everything else that -## might have happened to be in there WILL BE GONE after the mirror sync! -TO="/mnt/ftp/pub/mirrors/debian/" - -## The upstream name of the rsync share. -RSYNC_PATH="debian" - -## The host we mirror from +LABEL=debian RSYNC_HOST=ftp.de.debian.org +SEARCH_DISTS="oldstable stable testing unstable experimental" -## In case we need a user to access the rsync share at our upstream host -#RSYNC_USER= - -## If we need a user we also need a password -#RSYNC_PASSWORD= - -## In which directory should logfiles end up -## Note that BASEDIR defaults to $HOME, but can be set before calling the -## ftpsync script to any value you want (for example using pam_env) -#LOGDIR="${BASEDIR}/log" - -## Name of our own logfile. -## Note that ${NAME} is set by the ftpsync script depending on the way it -## is called. See README for a description of the multi-archive capability -## and better always include ${NAME} in this path. -#LOG="${LOGDIR}/${NAME}.log" - -## The script can send logs (or error messages) to a mail address. -## If this is unset it will default to the local root user unless it is run -## on a .debian.org machine where it will default to the mirroradm people. -#MAILTO="root" - -## If you do want a mail about every single sync, set this to false -## Everything else will only send mails if a mirror sync fails -#ERRORSONLY="true" - -## If you want the logs to also include output of rsync, set this to true. -## Careful, the logs can get pretty big, especially if it is the first mirror -## run -#FULLLOGS="false" - -## If you do want to exclude files from the mirror run, put --exclude -## statements here. See rsync(1) for the exact syntax, these are passed to -## rsync as written here. DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH -## THIS, IT WILL NOT WORK! -#EXCLUDE="" - -## If you do want to exclude an architecture, this is for you. -## Use as space seperated list. -## Possible values are: -## alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, kfreebsd-amd64, -## kfreebsd-i386, m68k, mipsel, mips, powerpc, s390, sh, sparc and source -## eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc" -## An unset value will mirror all architectures (default!) -#ARCH_EXCLUDE="" - -## Actually, no: we want to /include/ only some architectures. So we need to -## find out which ones are available so that we can exclude the ones we don't -## want. Crazy, no? -ARCH_EXCLUDE=$( - - ## Check the available distributions for architectures. - for dist in oldstable stable testing unstable experimental; do - rsync --list-only $RSYNC_HOST::$RSYNC_PATH/dists/$dist/main/ - done | { - - ## Gather up excluded architectures as we go. - excludes="" - - while read mode size date time name; do - - ## Check directories of binary packages. If it's an architecture we - ## don't want to reject, then continue on. - case "$name" in - (binary-all | binary-i386 | binary-amd64) continue ;; - (binary-*) ;; - (*) continue ;; - esac - - ## Pick out the architecture name. Check whether we've seen it before. - arch=${name#binary-} - case " $excludes " in - (*" $arch "*) - ;; - (*) - excludes="${excludes+$excludes }$arch" - ;; - esac - done - - ## Done. Print out the finished list. - echo $excludes - } -) - -## Do we have leaf mirror to signal we are done and they should sync? -## If so set it to true and make sure you configure runmirrors.mirrors -## and runmirrors.conf for your need. -#HUB=false - -## We do create three logfiles for every run. To save space we rotate them, -## this defines how many we keep -#LOGROTATE=14 - -## Our own lockfile (only one sync should run at any time) -#LOCK="${TO}/Archive-Update-in-Progress-${MIRRORNAME}" - -## Timeout for the lockfile, in case we have bash older than v4 (and no -## /proc) -# LOCKTIMEOUT=${LOCKTIMEOUT:-3600} - -## The following file is used to make sure we will end up with a correctly -## synced mirror even if we get multiple pushes in a short timeframe -#UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" - -## The trace file is used by a mirror check tool to see when we last had a -## successful mirror sync. Make sure that it always ends up in project/trace -## and always shows the full hostname. This is *relative* to ${TO} -#TRACE="project/trace/${MIRRORNAME}" - -## We sync our mirror using rsync (everything else would be insane), so -## we need a few options set. -## The rsync program -RSYNC="timeout 6h rsync" - -## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! -## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! -## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! -## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! - -## limit I/O bandwidth. Value is KBytes per second, unset or 0 means -## unlimited -#RSYNC_BW="" - -## Default rsync options every rsync invocation sees. -#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}" - -## Options the first pass gets. We do not want the Packages/Source indices -## here, and we also do not want to delete any files yet. -#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*" - -## Options the second pass gets. Now we want the Packages/Source indices too -## and we also want to delete files. We also want to delete files that are -## excluded. -#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded" - -## You may establish the connection via a web proxy by setting the -## environment variable RSYNC_PROXY to a hostname:port pair pointing to your -## web proxy. Note that your web proxy's configuration must support proxy -## connections to port 873. -# RSYNC_PROXY= - -## The following three options are used in case we want to "callback" the -## host we got pushed from. -#CALLBACKUSER="archvsync" -#CALLBACKHOST="none" -#CALLBACKKEY="none" - -## Hook scripts can be run at various places during the sync. -## Leave them blank if you don't want any -## Hook1: After lock is acquired, before first rsync -## Hook2: After first rsync, if successful -## Hook3: After second rsync, if successful -## Hook4: Right before leaf mirror triggering -## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true) -## -## Note that Hook3 and Hook4 are likely to be called directly after each -## other. Difference is: Hook3 is called *every* time the second rsync was -## successful, but even if the mirroring needs to re-run thanks to a second -## push. Hook4 is only effective if we are done with mirroring. -#HOOK1= -#HOOK2= -#HOOK3= -#HOOK4= -#HOOK5= +. etc/ftpsync-common.conf -- 2.11.0