--- /dev/null
+#! /bin/sh
+
+set -e
+case $# in
+ 0 | 1 | 2 | 3)
+ echo >&2 "Usage: $0 HOST PATH ARCH:ARCH:... DIST..."
+ exit 1
+ ;;
+ *)
+ RSYNC_HOST=$1 RSYNC_PATH=$2 WANT_ARCH=$3; shift 3
+ ;;
+esac
+
+## Check the available distributions for architectures.
+for dist in "$@"; do
+ rsync --list-only $RSYNC_HOST::$RSYNC_PATH/dists/$dist/main/
+done | {
+
+ ## Gather up excluded architectures as we go.
+ excludes=""
+
+ while read mode size date time name; do
+
+ ## Check directories of binary packages. If it's an architecture we
+ ## don't want to reject, then continue on.
+ case "$name" in
+ binary-all)
+ continue
+ ;;
+ binary-*)
+ arch=${name#binary-}
+ case ":$WANT_ARCH:" in *:"$arch":*) continue ;; esac
+ ;;
+ *)
+ continue
+ ;;
+ esac
+
+ ## Pick out the architecture name. Check whether we've seen it before.
+ arch=${name#binary-}
+ case " $excludes " in
+ *" $arch "*)
+ ;;
+ *)
+ excludes="${excludes+$excludes }$arch"
+ ;;
+ esac
+ done
+
+ ## Done. Print out the finished list.
+ echo $excludes
+}
--- /dev/null
+### -*-sh-*-
+###
+### Common ftpsync configuration for Debian mirrors.
+###
+### This file is sourced by configuration scripts for specfic mirrors.
+
+###--------------------------------------------------------------------------
+### Settings expected from the calling script.
+
+## Mirrorname. This is used for things like the trace file and should always
+## be the full hostname of the mirror.
+: ${MIRRORNAME=`hostname -f`}
+
+## Destination of the mirrored files. Should be an empty directory.
+## CAREFUL, this directory will contain the mirror. Everything else that
+## might have happened to be in there WILL BE GONE after the mirror sync!
+: ${LABEL="debian"}
+: ${TO="/mnt/mirror/$LABEL/"}
+
+## The upstream name of the rsync share.
+: ${RSYNC_PATH="$LABEL"}
+
+## The host we mirror from
+: ${RSYNC_HOST=ftp.de.debian.org}
+
+## In case we need a user to access the rsync share at our upstream host
+: ${RSYNC_USER=}
+
+## If we need a user we also need a password
+: ${RSYNC_PASSWORD=}
+
+## Architectures we want to carry.
+: ${WANT_ARCH="i386:amd64"}
+
+## Distributions to inspect for available architectures.
+: ${SEARCH_DISTS="oldstable stable testing unstable experimental"}
+
+###--------------------------------------------------------------------------
+### Common settings.
+
+## In which directory should logfiles end up
+## Note that BASEDIR defaults to $HOME, but can be set before calling the
+## ftpsync script to any value you want (for example using pam_env)
+#LOGDIR="${BASEDIR}/log"
+
+## Name of our own logfile.
+## Note that ${NAME} is set by the ftpsync script depending on the way it
+## is called. See README for a description of the multi-archive capability
+## and better always include ${NAME} in this path.
+#LOG="${LOGDIR}/${NAME}.log"
+
+## The script can send logs (or error messages) to a mail address.
+## If this is unset it will default to the local root user unless it is run
+## on a .debian.org machine where it will default to the mirroradm people.
+#MAILTO="root"
+
+## If you do want a mail about every single sync, set this to false
+## Everything else will only send mails if a mirror sync fails
+#ERRORSONLY="true"
+
+## If you want the logs to also include output of rsync, set this to true.
+## Careful, the logs can get pretty big, especially if it is the first mirror
+## run
+#FULLLOGS="false"
+
+## If you do want to exclude files from the mirror run, put --exclude
+## statements here. See rsync(1) for the exact syntax, these are passed to
+## rsync as written here. DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH
+## THIS, IT WILL NOT WORK!
+#EXCLUDE=""
+
+## If you do want to exclude an architecture, this is for you.
+## Use as space seperated list.
+## Possible values are:
+## alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, kfreebsd-amd64,
+## kfreebsd-i386, m68k, mipsel, mips, powerpc, s390, sh, sparc and source
+## eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc"
+## An unset value will mirror all architectures (default!)
+#ARCH_EXCLUDE=""
+
+## Actually, no: we want to /include/ only some architectures. So we need to
+## find out which ones are available so that we can exclude the ones we don't
+## want. Crazy, no?
+ARCH_EXCLUDE=$(
+ bin/debian-excludes $RSYNC_HOST $RSYNC_PATH $WANT_ARCH $SEARCH_DISTS
+ echo source
+)
+
+## Do we have leaf mirror to signal we are done and they should sync?
+## If so set it to true and make sure you configure runmirrors.mirrors
+## and runmirrors.conf for your need.
+#HUB=false
+
+## We do create three logfiles for every run. To save space we rotate them,
+## this defines how many we keep
+#LOGROTATE=14
+
+## Our own lockfile (only one sync should run at any time)
+#LOCK="${TO}/Archive-Update-in-Progress-${MIRRORNAME}"
+
+## Timeout for the lockfile, in case we have bash older than v4 (and no
+## /proc)
+# LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
+
+## The following file is used to make sure we will end up with a correctly
+## synced mirror even if we get multiple pushes in a short timeframe
+#UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
+
+## The trace file is used by a mirror check tool to see when we last had a
+## successful mirror sync. Make sure that it always ends up in project/trace
+## and always shows the full hostname. This is *relative* to ${TO}
+#TRACE="project/trace/${MIRRORNAME}"
+
+## We sync our mirror using rsync (everything else would be insane), so
+## we need a few options set.
+## The rsync program
+: ${RSYNC="rsync"}
+
+## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
+## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
+## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
+## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
+
+## limit I/O bandwidth. Value is KBytes per second, unset or 0 means
+## unlimited
+#RSYNC_BW=""
+
+## Default rsync options every rsync invocation sees.
+#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}"
+
+## Options the first pass gets. We do not want the Packages/Source indices
+## here, and we also do not want to delete any files yet.
+#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"
+
+## Options the second pass gets. Now we want the Packages/Source indices too
+## and we also want to delete files. We also want to delete files that are
+## excluded.
+#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"
+
+## You may establish the connection via a web proxy by setting the
+## environment variable RSYNC_PROXY to a hostname:port pair pointing to your
+## web proxy. Note that your web proxy's configuration must support proxy
+## connections to port 873.
+# RSYNC_PROXY=
+
+## The following three options are used in case we want to "callback" the
+## host we got pushed from.
+#CALLBACKUSER="archvsync"
+#CALLBACKHOST="none"
+#CALLBACKKEY="none"
+
+## Hook scripts can be run at various places during the sync.
+## Leave them blank if you don't want any
+## Hook1: After lock is acquired, before first rsync
+## Hook2: After first rsync, if successful
+## Hook3: After second rsync, if successful
+## Hook4: Right before leaf mirror triggering
+## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true)
+##
+## Note that Hook3 and Hook4 are likely to be called directly after each
+## other. Difference is: Hook3 is called *every* time the second rsync was
+## successful, but even if the mirroring needs to re-run thanks to a second
+## push. Hook4 is only effective if we are done with mirroring.
+#HOOK1=
+#HOOK2=
+#HOOK3=
+#HOOK4=
+#HOOK5=
-########################################################################
-########################################################################
-## This is a sample configuration file for the ftpsync mirror script. ##
-## Most of the values are commented out and just shown here for ##
-## completeness, together with their default value. ##
-########################################################################
-########################################################################
-
-## Mirrorname. This is used for things like the trace file and should always
-## be the full hostname of the mirror.
-#MIRRORNAME=`hostname -f`
-
-## Destination of the mirrored files. Should be an empty directory.
-## CAREFUL, this directory will contain the mirror. Everything else
-## that might have happened to be in there WILL BE GONE after the mirror sync!
-#TO="/org/ftp.debian.org/ftp/"
-
-## The upstream name of the rsync share.
-#RSYNC_PATH="ftp"
-
-## The host we mirror from
-#RSYNC_HOST=some.mirror.debian.org
-
-## In case we need a user to access the rsync share at our upstream host
-#RSYNC_USER=
-
-## If we need a user we also need a password
-#RSYNC_PASSWORD=
-
-## In which directory should logfiles end up
-## Note that BASEDIR defaults to $HOME, but can be set before calling the
-## ftpsync script to any value you want (for example using pam_env)
-#LOGDIR="${BASEDIR}/log"
-
-## Name of our own logfile.
-## Note that ${NAME} is set by the ftpsync script depending on the way it
-## is called. See README for a description of the multi-archive capability
-## and better always include ${NAME} in this path.
-#LOG="${LOGDIR}/${NAME}.log"
-
-## The script can send logs (or error messages) to a mail address.
-## If this is unset it will default to the local root user unless it is run
-## on a .debian.org machine where it will default to the mirroradm people.
-#MAILTO="root"
-
-## If you do want a mail about every single sync, set this to false
-## Everything else will only send mails if a mirror sync fails
-#ERRORSONLY="true"
-
-## If you want the logs to also include output of rsync, set this to true.
-## Careful, the logs can get pretty big, especially if it is the first mirror
-## run
-#FULLLOGS="false"
-
-## If you do want to exclude files from the mirror run, put --exclude statements here.
-## See rsync(1) for the exact syntax, these are passed to rsync as written here.
-## DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH THIS, IT WILL NOT WORK!
-#EXCLUDE=""
-
-## If you do want to exclude an architecture, this is for you.
-## Use as space seperated list.
-## Possible values are:
-## alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, kfreebsd-amd64,
-## kfreebsd-i386, m68k, mipsel, mips, powerpc, s390, sh, sparc and source
-## eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc"
-## An unset value will mirror all architectures (default!)
-#ARCH_EXCLUDE=""
-
-## Do we have leaf mirror to signal we are done and they should sync?
-## If so set it to true and make sure you configure runmirrors.mirrors
-## and runmirrors.conf for your need.
-#HUB=false
-
-## We do create three logfiles for every run. To save space we rotate them, this
-## defines how many we keep
-#LOGROTATE=14
-
-## Our own lockfile (only one sync should run at any time)
-#LOCK="${TO}/Archive-Update-in-Progress-${MIRRORNAME}"
-
-# Timeout for the lockfile, in case we have bash older than v4 (and no /proc)
-# LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
-
-## The following file is used to make sure we will end up with a correctly
-## synced mirror even if we get multiple pushes in a short timeframe
-#UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
-
-## The trace file is used by a mirror check tool to see when we last
-## had a successful mirror sync. Make sure that it always ends up in
-## project/trace and always shows the full hostname.
-## This is *relative* to ${TO}
-#TRACE="project/trace/${MIRRORNAME}"
-
-## We sync our mirror using rsync (everything else would be insane), so
-## we need a few options set.
-## The rsync program
-#RSYNC=rsync
-
-## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
-## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
-## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
-## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
-
-## limit I/O bandwidth. Value is KBytes per second, unset or 0 means unlimited
-#RSYNC_BW=""
-
-## Default rsync options every rsync invocation sees.
-#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}"
-
-## Options the first pass gets. We do not want the Packages/Source indices
-## here, and we also do not want to delete any files yet.
-#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"
-
-## Options the second pass gets. Now we want the Packages/Source indices too
-## and we also want to delete files. We also want to delete files that are
-## excluded.
-#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"
-
-## You may establish the connection via a web proxy by setting the environment
-## variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note
-## that your web proxy's configuration must support proxy connections to port 873.
-# RSYNC_PROXY=
-
-
-## The following three options are used in case we want to "callback" the host
-## we got pushed from.
-#CALLBACKUSER="archvsync"
-#CALLBACKHOST="none"
-#CALLBACKKEY="none"
-
-
-## Hook scripts can be run at various places during the sync.
-## Leave them blank if you don't want any
-## Hook1: After lock is acquired, before first rsync
-## Hook2: After first rsync, if successful
-## Hook3: After second rsync, if successful
-## Hook4: Right before leaf mirror triggering
-## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true)
-##
-## Note that Hook3 and Hook4 are likely to be called directly after each other.
-## Difference is: Hook3 is called *every* time the second rsync was successful,
-## but even if the mirroring needs to re-run thanks to a second push.
-## Hook4 is only effective if we are done with mirroring.
-#HOOK1=
-#HOOK2=
-#HOOK3=
-#HOOK4=
-#HOOK5=
+### -*-sh-*-
+###
+### ftpsync configuration for the Debian security archive.
+
+LABEL=debian-security
+RSYNC_HOST=security.debian.org
+SEARCH_DISTS=$(
+ for i in oldstable stable testing unstable experimental; do
+ echo $i/updates
+ done
+)
+
+. etc/ftpsync-common.conf
###
### ftpsync configuration for the Debian main archive.
-## Mirrorname. This is used for things like the trace file and should always
-## be the full hostname of the mirror.
-#MIRRORNAME=`hostname -f`
-
-## Destination of the mirrored files. Should be an empty directory.
-## CAREFUL, this directory will contain the mirror. Everything else that
-## might have happened to be in there WILL BE GONE after the mirror sync!
-TO="/mnt/ftp/pub/mirrors/debian/"
-
-## The upstream name of the rsync share.
-RSYNC_PATH="debian"
-
-## The host we mirror from
+LABEL=debian
RSYNC_HOST=ftp.de.debian.org
+SEARCH_DISTS="oldstable stable testing unstable experimental"
-## In case we need a user to access the rsync share at our upstream host
-#RSYNC_USER=
-
-## If we need a user we also need a password
-#RSYNC_PASSWORD=
-
-## In which directory should logfiles end up
-## Note that BASEDIR defaults to $HOME, but can be set before calling the
-## ftpsync script to any value you want (for example using pam_env)
-#LOGDIR="${BASEDIR}/log"
-
-## Name of our own logfile.
-## Note that ${NAME} is set by the ftpsync script depending on the way it
-## is called. See README for a description of the multi-archive capability
-## and better always include ${NAME} in this path.
-#LOG="${LOGDIR}/${NAME}.log"
-
-## The script can send logs (or error messages) to a mail address.
-## If this is unset it will default to the local root user unless it is run
-## on a .debian.org machine where it will default to the mirroradm people.
-#MAILTO="root"
-
-## If you do want a mail about every single sync, set this to false
-## Everything else will only send mails if a mirror sync fails
-#ERRORSONLY="true"
-
-## If you want the logs to also include output of rsync, set this to true.
-## Careful, the logs can get pretty big, especially if it is the first mirror
-## run
-#FULLLOGS="false"
-
-## If you do want to exclude files from the mirror run, put --exclude
-## statements here. See rsync(1) for the exact syntax, these are passed to
-## rsync as written here. DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH
-## THIS, IT WILL NOT WORK!
-#EXCLUDE=""
-
-## If you do want to exclude an architecture, this is for you.
-## Use as space seperated list.
-## Possible values are:
-## alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, kfreebsd-amd64,
-## kfreebsd-i386, m68k, mipsel, mips, powerpc, s390, sh, sparc and source
-## eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc"
-## An unset value will mirror all architectures (default!)
-#ARCH_EXCLUDE=""
-
-## Actually, no: we want to /include/ only some architectures. So we need to
-## find out which ones are available so that we can exclude the ones we don't
-## want. Crazy, no?
-ARCH_EXCLUDE=$(
-
- ## Check the available distributions for architectures.
- for dist in oldstable stable testing unstable experimental; do
- rsync --list-only $RSYNC_HOST::$RSYNC_PATH/dists/$dist/main/
- done | {
-
- ## Gather up excluded architectures as we go.
- excludes=""
-
- while read mode size date time name; do
-
- ## Check directories of binary packages. If it's an architecture we
- ## don't want to reject, then continue on.
- case "$name" in
- (binary-all | binary-i386 | binary-amd64) continue ;;
- (binary-*) ;;
- (*) continue ;;
- esac
-
- ## Pick out the architecture name. Check whether we've seen it before.
- arch=${name#binary-}
- case " $excludes " in
- (*" $arch "*)
- ;;
- (*)
- excludes="${excludes+$excludes }$arch"
- ;;
- esac
- done
-
- ## Done. Print out the finished list.
- echo $excludes
- }
-)
-
-## Do we have leaf mirror to signal we are done and they should sync?
-## If so set it to true and make sure you configure runmirrors.mirrors
-## and runmirrors.conf for your need.
-#HUB=false
-
-## We do create three logfiles for every run. To save space we rotate them,
-## this defines how many we keep
-#LOGROTATE=14
-
-## Our own lockfile (only one sync should run at any time)
-#LOCK="${TO}/Archive-Update-in-Progress-${MIRRORNAME}"
-
-## Timeout for the lockfile, in case we have bash older than v4 (and no
-## /proc)
-# LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
-
-## The following file is used to make sure we will end up with a correctly
-## synced mirror even if we get multiple pushes in a short timeframe
-#UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
-
-## The trace file is used by a mirror check tool to see when we last had a
-## successful mirror sync. Make sure that it always ends up in project/trace
-## and always shows the full hostname. This is *relative* to ${TO}
-#TRACE="project/trace/${MIRRORNAME}"
-
-## We sync our mirror using rsync (everything else would be insane), so
-## we need a few options set.
-## The rsync program
-RSYNC="timeout 6h rsync"
-
-## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
-## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
-## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
-## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
-
-## limit I/O bandwidth. Value is KBytes per second, unset or 0 means
-## unlimited
-#RSYNC_BW=""
-
-## Default rsync options every rsync invocation sees.
-#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}"
-
-## Options the first pass gets. We do not want the Packages/Source indices
-## here, and we also do not want to delete any files yet.
-#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"
-
-## Options the second pass gets. Now we want the Packages/Source indices too
-## and we also want to delete files. We also want to delete files that are
-## excluded.
-#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"
-
-## You may establish the connection via a web proxy by setting the
-## environment variable RSYNC_PROXY to a hostname:port pair pointing to your
-## web proxy. Note that your web proxy's configuration must support proxy
-## connections to port 873.
-# RSYNC_PROXY=
-
-## The following three options are used in case we want to "callback" the
-## host we got pushed from.
-#CALLBACKUSER="archvsync"
-#CALLBACKHOST="none"
-#CALLBACKKEY="none"
-
-## Hook scripts can be run at various places during the sync.
-## Leave them blank if you don't want any
-## Hook1: After lock is acquired, before first rsync
-## Hook2: After first rsync, if successful
-## Hook3: After second rsync, if successful
-## Hook4: Right before leaf mirror triggering
-## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true)
-##
-## Note that Hook3 and Hook4 are likely to be called directly after each
-## other. Difference is: Hook3 is called *every* time the second rsync was
-## successful, but even if the mirroring needs to re-run thanks to a second
-## push. Hook4 is only effective if we are done with mirroring.
-#HOOK1=
-#HOOK2=
-#HOOK3=
-#HOOK4=
-#HOOK5=
+. etc/ftpsync-common.conf
~mdw / mirror-admin / commitdiff