# Based losely on a number of existing scripts, written by an
# unknown number of different people over the years.
#
-# Copyright (C) 2008,2009,2010,2011 Joerg Jaspert <joerg@debian.org>
+# Copyright (C) 2008-2012 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
# by Joerg Jaspert and the Debian mirroradm group.
# This is used to track which mirror is using which script version.
-VERSION="80387"
+VERSION="20130605"
# Source our common functions
. "${BASEDIR}/etc/common"
LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"}
# timeout for the lockfile, in case we have bash older than v4 (and no /proc)
LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
+# sleeping time when an AUIP file is found but is not ours
+UIPSLEEP=${UIPSLEEP:-1200}
+# retries whenever an upstream (or possibly stale) AUIP file is found
+UIPRETRIES=${UIPRETRIES:-3}
# Do we need another rsync run?
UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
# Trace file for mirror stats and checks (make sure we get full hostname)
TRACE=${TRACE:-"project/trace/${MIRRORNAME}"}
+# The trace file can have different format/contents. Here you can select
+# what it will be.
+# Possible values are
+# "full" - all information
+# "terse" - basic, timestamp only (date -u)
+# "touch" - just touch the file in existance
+# "none" - no tracefile at all
+#
+# Default and required value for Debian mirrors is full.
+EXTENDEDTRACE=${EXTENDEDTRACE:-"full"}
# rsync program
RSYNC=${RSYNC:-rsync}
RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"}
# limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited
RSYNC_BW=${RSYNC_BW:-0}
+RSYNC_PROTOCOL=$(rsync_protocol)
+
+# Set the delete method to --delete-delay if protocol version is 30 or
+# greater (meaning rsync 3.0.0 or greater is used). Use --delete-after
+# otherwise.
+if [ 30 -le $RSYNC_PROTOCOL ]; then
+ RSYNC_DELETE_METHOD=delay
+else
+ RSYNC_DELETE_METHOD=after
+fi
+
# Default rsync options for *every* rsync call
RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
# Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files
-RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"}
+RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --exclude=i18n/* --exclude=ls-lR*"}
# Options for the second pass, where we do want everything, including deletion of old and now unused files
-RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
+RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-excluded"}
# Which rsync share to use on our upstream mirror?
RSYNC_PATH=${RSYNC_PATH:-"debian"}
+# Extra rsync options as defined by the admin locally. Won't be set
+# to any default by ftpsync. Those will be added to EACH AND EVERY rsync call.
+RSYNC_EXTRA=${RSYNC_EXTRA:-""}
+
# Now add the bwlimit option. As default is 0 we always add it, rsync interprets
# 0 as unlimited, so this is safe.
-RSYNC_OPTIONS="--bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}"
+RSYNC_OPTIONS="${RSYNC_EXTRA} --bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}"
+
+# Finally, make sure RSYNC_OPTIONS2 has either --delete-after or --deleter-delay
+RSYNC_OPTION_REGEX="--delete-(after|delay)"
+if ! [[ ${RSYNC_OPTIONS2} =~ ${RSYNC_OPTION_REGEX} ]]; then
+ RSYNC_OPTIONS2+=" --delete-${RSYNC_DELETE_METHOD}"
+fi
+unset RSYNC_OPTION_REGEX
# We have no default host to sync from, but will error out if its unset
RSYNC_HOST=${RSYNC_HOST:-""}
SYNCALL=${SYNCALL:-"true"}
# Do we have a mhop sync?
SYNCMHOP=${SYNCMHOP:-"false"}
-# Do we callback?
+# Do we callback? (May get changed later)
SYNCCALLBACK=${SYNCCALLBACK:-"false"}
# If we call back we need some more options defined in the config file.
CALLBACKUSER=${CALLBACKUSER:-"archvsync"}
EXCLUDE=${EXCLUDE:-""}
# The temp directory used by rsync --delay-updates is not
-# world-readable remotely. Always exclude it to avoid errors.
-EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
+# world-readable remotely. Always exclude it to avoid errors.
+EXCLUDE="${EXCLUDE} --exclude=.~tmp~/"
SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""}
ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
# Exclude architectures defined in $ARCH_EXCLUDE
for ARCH in ${ARCH_EXCLUDE}; do
- EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes"
+ EXCLUDE="${EXCLUDE} --exclude=binary-${ARCH}/ --exclude=installer-${ARCH}/ --exclude=Contents-${ARCH}.gz --exclude=Contents-udeb-${ARCH}.gz --exclude=Contents-${ARCH}.diff/ --exclude=arch-${ARCH}.files --exclude=arch-${ARCH}.list.gz --exclude=*_${ARCH}.deb --exclude=*_${ARCH}.udeb --exclude=*_${ARCH}.changes"
if [ "${ARCH}" = "source" ]; then
if [ -z ${SOURCE_EXCLUDE} ]; then
- SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.tar.xz --exclude *.diff.bz2 --exclude *.dsc "
+ SOURCE_EXCLUDE=" --exclude=source/ --exclude=*.tar.gz --exclude=*.diff.gz --exclude=*.tar.bz2 --exclude=*.tar.xz --exclude=*.diff.bz2 --exclude=*.dsc "
fi
fi
done
if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then
# We have a recent enough bash version, lets do it the easy way,
# the lock will contain the right pid, thanks to $BASHPID
- if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
+ if ! $(kill -0 $(< ${LOCK}) 2>/dev/null); then
# Process does either not exist or is not owned by us.
echo "$$" > "${LOCK}"
else
- echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
+ echo "Unable to start rsync, lock file still exists, PID $(< ${LOCK})"
exit 1
fi
else
export RSYNC_PASSWORD
export RSYNC_PROXY
+UPDATE_RETRIES=0
+
while [ -e "${UPDATEREQUIRED}" ]; do
log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
# if we want stage2 *or* all
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
- log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+ upstream_uip=false
+ for aupfile in "${TO}/Archive-Update-in-Progress-"*; do
+ case "$aupfile" in
+ "${TO}/Archive-Update-in-Progress-*")
+ error "Lock file is missing, this should not happen"
+ ;;
+ "${LOCK}")
+ :
+ ;;
+ *)
+ if [ -f "$aupfile" ]; then
+ # Remove the file, it will be synced again if
+ # upstream is still not done
+ rm -f "$aupfile"
+ else
+ log "AUIP file '$aupfile' is not really a file, weird"
+ fi
+ upstream_uip=true
+ ;;
+ esac
+ done
- set +e
- # We are lucky, it worked. Now do step 2 and sync again, this time including
- # the packages/releases files
- ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
- ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
- result=$?
- set -e
+ if [ "xtruex" = "x${upstream_uip}x" ]; then
+ log "Upstream archive update in progress, skipping stage2"
+ if [ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]; then
+ log "Retrying update in ${UIPSLEEP}"
+ touch "${UPDATEREQUIRED}"
+ UPDATE_RETRIES=$(($UPDATE_RETRIES+1))
+ sleep "${UIPSLEEP}"
+ result=0
+ else
+ error "Update has been retried ${UPDATEREQUIRED} times, aborting"
+ log "Perhaps upstream is still updating or there's a stale AUIP file"
+ result=1
+ fi
+ else
+ log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+
+ set +e
+ # We are lucky, it worked. Now do step 2 and sync again, this time including
+ # the packages/releases files
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
+ result=$?
+ set -e
- log "Back from rsync with returncode ${result}"
+ log "Back from rsync with returncode ${result}"
+ fi
else
# Fake a good resultcode
result=0
# We only update our tracefile when we had a stage2 or an all sync.
# Otherwise we would update it after stage1 already, which is wrong.
+
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
- if [ -d "$(dirname "${TO}/${TRACE}")" ]; then
- LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
- echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
- echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
- fi
+ case ${EXTENDEDTRACE} in
+ none)
+ log "No trace file wanted. Not creating one"
+ ;;
+ touch)
+ log "Just touching the trace file"
+ touch "${TO}/${TRACE}"
+ ;;
+ terse|full)
+ log "Creating a ${EXTENDEDTRACE} trace file"
+ if [ -d "$(dirname "${TO}/${TRACE}")" ]; then
+ LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}.new"
+ echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}.new"
+ echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}.new"
+ if [ "xfullx" = "x${EXTENDEDTRACE}x" ]; then
+ GLOBALARCHLIST="source amd64 armel armhf hurd-i386 i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc"
+
+ AEXCLUDE="^${ARCH_EXCLUDE// /\$|^}$"
+ ARCHLIST=""
+ for ARCH in ${GLOBALARCHLIST}; do
+ if ! [[ ${ARCH} =~ ${AEXCLUDE} ]]; then
+ ARCHLIST="${ARCHLIST} ${ARCH}"
+ fi
+ done
+ out="GUESSED:{${ARCHLIST}}"
+ echo "Architectures: ${out}" >> "${TO}/${TRACE}.new"
+ echo "Upstream-mirror: ${RSYNC_HOST}" >> "${TO}/${TRACE}.new"
+ fi # full trace
+ mv "${TO}/${TRACE}.new" "${TO}/${TRACE}"
+ fi
+ ;;
+ *)
+ error "Unsupported EXTENDEDTRACE value configured in ${BASEDIR}/etc/${NAME}.conf, please fix"
+ ;;
+ esac
fi
+
HOOK=(
HOOKNR=4
HOOKSCR=${HOOK4}
# Remove the Archive-Update-in-Progress file before we push our downstreams.
rm -f "${LOCK}"
+# Check if there is a newer version of ftpsync. If so inform the admin, but not
+# more than once every third day.
+if [ -r "${TO}/project/ftpsync/LATEST.VERSION" ]; then
+ LATEST=$(< "${TO}/project/ftpsync/LATEST.VERSION")
+ if ! [[ ${LATEST} =~ [0-9]+ ]]; then
+ LATEST=0
+ fi
+ if [ ${LATEST} -gt ${VERSION} ]; then
+ if [ -n "${MAILTO}" ]; then
+ difference=0
+ if [ -f "${LOGDIR}/ftpsync.newversion" ]; then
+ stamptime=$(< "${LOGDIR}/ftpsync.newversion")
+ unixtime=$(date +%s)
+ difference=$(( $unixtime - $stamptime ))
+ fi
+ if [ ${difference} -ge 259200 ]; then
+ # Only warn every third day
+ mail -e -s "[$(hostname -s)] Update for ftpsync available" ${MAILTO} <<EOF
+Hello admin,
+
+i found that there is a new version of me available.
+Me lonely ftpsync is currently version: ${VERSION}
+New release of myself is available as: ${LATEST}
+
+Me, myself and I - and the Debian mirroradmins - would be very grateful
+if you could update me. You can find the latest version on your mirror,
+check $(hostname -s):${TO}/project/ftpsync/ftpsync-${LATEST}.tar.gz
+
+You can ensure the validity of that file by using sha512sum or md5sum
+against the available checksum files secured with a signature from the
+Debian FTPMaster signing key.
+
+EOF
+
+ date +%s > "${LOGDIR}/ftpsync.newversion"
+ fi
+ fi
+ else
+ # Remove a possible stampfile
+ rm -f "${LOGDIR}/ftpsync.newversion"
+ fi
+fi
+
if [ x${HUB} = "xtrue" ]; then
# Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
~mdw / mirror-admin / blobdiff