--- /dev/null
+#!/usr/bin/perl
+# Uses symlinks to merge the files contained in a set of vcs
+# checkouts to into a single directory. Keeps track of when files are
+# removed from the merged directories and removes the symlinks.
+#
+# Only merges files that match the specified pattern.
+#
+# Note that the directories given to merge should be paths that will work
+# for symlink targets from the destination directory (so either full paths,
+# or they should be right inside the destination directory).
+#
+# Note that other files in the destination directory will be left as-is.
+#
+# Copyright 2006 by Joey Hess, licensed under the GPL.
+
+if (! @ARGV) {
+ die "usage: dircombine include-pattern dest dir1 [dir2 ...]\n";
+}
+
+my $pattern=shift;
+my $dest=shift;
+
+foreach my $dir (@ARGV) {
+ my %known;
+
+ # Link in each thing from the dir.
+ opendir(DIR, $dir) || die "opendir: $!";
+ while ($_=readdir(DIR)) {
+ next if $_ eq '.' || $_ eq '..' || $_ eq 'known' || $_ eq '.svn' || $_ eq '.git' || $_ eq '.gitignore' || $_ eq '_darcs';
+ next unless /$pattern/;
+
+ $known{$_}=1;
+
+ if (! -l "$dest/$_" && -e "$dest/$_") {
+ print STDERR "$_ in $dir is also in $dest\n";
+ }
+ elsif (! -l "$dest/$_") {
+ system("ln", "-svf", "$dir/$_", $dest);
+ }
+ }
+ closedir(DIR);
+
+ # Remove anything that was previously linked in but is not in the
+ # dir anymore.
+ if (-e "$dir/known") {
+ open(KNOWN, "$dir/known") || die "open $dir/known: $!";
+ while (<KNOWN>) {
+ chomp;
+ if (! $known{$_}) {
+ system("rm", "-vf", "$dest/$_");
+ }
+ }
+ close KNOWN;
+ }
+
+ # Save state for next time.
+ open(KNOWN, ">$dir/known") || die "write $dir/known: $!";
+ foreach my $file (sort keys %known) {
+ print KNOWN "$file\n";
+ }
+ close KNOWN;
+}
--- /dev/null
+#! /bin/bash
+# No, we can not deal with sh alone.
+
+set -e
+set -u
+# ERR traps should be inherited from functions too. (And command
+# substitutions and subshells and whatnot, but for us the function is
+# the important part here)
+set -E
+
+# ftpsync script for Debian
+# Based losely on a number of existing scripts, written by an
+# unknown number of different people over the years.
+#
+# Copyright (C) 2008,2009,2010,2011 Joerg Jaspert <joerg@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# In case the admin somehow wants to have this script located someplace else,
+# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
+# How the admin sets this isn't our place to deal with. One could use a wrapper
+# for that. Or pam_env. Or whatever fits in the local setup. :)
+BASEDIR=${BASEDIR:-"${HOME}"}
+
+# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
+# by Joerg Jaspert and the Debian mirroradm group.
+# This is used to track which mirror is using which script version.
+VERSION="80387"
+
+# Source our common functions
+. "${BASEDIR}/etc/common"
+
+########################################################################
+########################################################################
+## functions ##
+########################################################################
+########################################################################
+# We want to be able to get told what kind of sync we should do. This
+# might be anything, from the archive to sync, the stage to do, etc. A
+# list of currently understood and valid options is below. Multiple
+# options are seperated by space. All the words have to have the word
+# sync: in front or nothing will get used!
+#
+# Option Behaviour
+# stage1 Only do stage1 sync
+# stage2 Only do stage2 sync
+# all Do a complete sync
+# mhop Do a mhop sync, usually additionally to stage1
+# archive:foo Sync archive foo (if config for foo is available)
+# callback Call back when done (needs proper ssh setup for this to
+# work). It will always use the "command" callback:$HOSTNAME
+# where $HOSTNAME is the one defined below/in config and
+# will happen before slave mirrors are triggered.
+#
+# So to get us to sync all of the archive behind bpo and call back when
+# we are done, a trigger command of
+# "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the
+# trick.
+check_commandline() {
+ while [ $# -gt 0 ]; do
+ case "$1" in
+ sync:stage1)
+ SYNCSTAGE1="true"
+ SYNCALL="false"
+ ;;
+ sync:stage2)
+ SYNCSTAGE2="true"
+ SYNCALL="false"
+ ;;
+ sync:callback)
+ SYNCCALLBACK="true"
+ ;;
+ sync:archive:*)
+ ARCHIVE=${1##sync:archive:}
+ # We do not like / or . in the remotely supplied archive name.
+ ARCHIVE=${ARCHIVE//\/}
+ ARCHIVE=${ARCHIVE//.}
+ ;;
+ sync:all)
+ SYNCALL="true"
+ ;;
+ sync:mhop)
+ SYNCMHOP="true"
+ ;;
+ *)
+ echo "Unknown option ${1} ignored"
+ ;;
+ esac
+ shift # Check next set of parameters.
+ done
+}
+
+# All the stuff we want to do when we exit, no matter where
+cleanup() {
+ trap - ERR TERM HUP INT QUIT EXIT
+ # all done. Mail the log, exit.
+ log "Mirrorsync done";
+
+ # Lets get a statistical value
+ SPEED="unknown"
+ if [ -f "${LOGDIR}/rsync-${NAME}.log" ]; then
+ SPEED=$(
+ SPEEDLINE=$(egrep '[0-9.]+ bytes/sec' "${LOGDIR}/rsync-${NAME}.log")
+ set "nothing" ${SPEEDLINE}
+ echo ${8:-""}
+ )
+ if [ -n "${SPEED}" ]; then
+ SPEED=${SPEED%%.*}
+ SPEED=$(( $SPEED / 1024 ))
+ fi
+ fi
+ log "Rsync transfer speed: ${SPEED} KB/s"
+
+ if [ -n "${MAILTO}" ]; then
+ # In case rsync had something on stderr
+ if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
+ mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
+ fi
+ if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
+ # And the normal log
+ MAILFILES="${LOG}"
+ if [ "x${FULLLOGS}x" = "xtruex" ]; then
+ # Someone wants full logs including rsync
+ MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
+ fi
+ cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
+ fi
+ fi
+
+ savelog "${LOGDIR}/rsync-${NAME}.log"
+ savelog "${LOGDIR}/rsync-${NAME}.error"
+ savelog "$LOG" > /dev/null
+
+ rm -f "${LOCK}"
+}
+
+# Check rsyncs return value
+check_rsync() {
+ ret=$1
+ msg=$2
+
+ # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
+ # and us re-running. If it's not, uplink is broken anyways.
+ case "${ret}" in
+ 0) return 0;;
+ 24) return 0;;
+ 23) return 2;;
+ 30) return 2;;
+ *)
+ error "ERROR: ${msg}"
+ return 1
+ ;;
+ esac
+}
+
+########################################################################
+########################################################################
+
+
+# As what are we called?
+NAME="$(basename $0)"
+# The original command line arguments need to be saved!
+if [ $# -gt 0 ]; then
+ ORIGINAL_COMMAND=$*
+else
+ ORIGINAL_COMMAND=""
+fi
+
+SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""}
+# Now, check if we got told about stuff via ssh
+if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then
+ # We deliberately add "nothing" and ignore it right again, to avoid
+ # people from outside putting some set options in the first place,
+ # making us parse them...
+ set "nothing" "${SSH_ORIGINAL_COMMAND}"
+ shift
+ # Yes, unqouted $* here. Or the function will only see it as one
+ # parameter, which doesnt help the case in it.
+ check_commandline $*
+fi
+
+# Now, we can locally override all the above variables by just putting
+# them into the .ssh/authorized_keys file forced command.
+if [ -n "${ORIGINAL_COMMAND}" ]; then
+ set ${ORIGINAL_COMMAND}
+ check_commandline $*
+fi
+
+# If we have been told to do stuff for a different archive than default,
+# set the name accordingly.
+ARCHIVE=${ARCHIVE:-""}
+if [ -n "${ARCHIVE}" ]; then
+ NAME="${NAME}-${ARCHIVE}"
+fi
+
+# Now source the config for the archive we run on.
+# (Yes, people can also overwrite the options above in the config file
+# if they want to)
+if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then
+ . "${BASEDIR}/etc/${NAME}.conf"
+else
+ echo "Nono, you can't tell us about random archives. Bad boy!"
+ exit 1
+fi
+
+########################################################################
+# Config options go here. Feel free to overwrite them in the config #
+# file if you need to. #
+# On debian.org machines the defaults should be ok. #
+# #
+# The following extra variables can be defined in the config file: #
+# #
+# ARCH_EXCLUDE #
+# can be used to exclude a complete architecture from #
+# mirrorring. Use as space seperated list. #
+# Possible values are: #
+# alpha, amd64, arm, armel, armhf, hppa, hurd-i386, i386, ia64, mips #
+# mipsel, powerpc, s390, s390x, sparc, kfreebsd-i386, kfreebsd-amd64 #
+# and source. #
+# eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 s390x sparc" #
+# #
+# An unset value will mirror all architectures #
+########################################################################
+
+########################################################################
+# There should be nothing to edit here, use the config file #
+########################################################################
+MIRRORNAME=${MIRRORNAME:-$(hostname -f)}
+# Where to put logfiles in
+LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
+# Our own logfile
+LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
+
+# Where should we put all the mirrored files?
+TO=${TO:-"/srv/mirrors/debian/"}
+
+# used by log() and error()
+PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
+
+# Where to send mails about mirroring to?
+if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
+else
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+fi
+# Want errors only or every log?
+ERRORSONLY=${ERRORSONLY:-"true"}
+# Want full logs, ie. including the rsync one?
+FULLLOGS=${FULLLOGS:-"false"}
+
+# How many logfiles to keep
+LOGROTATE=${LOGROTATE:-14}
+
+# Our lockfile
+LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"}
+# timeout for the lockfile, in case we have bash older than v4 (and no /proc)
+LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
+# Do we need another rsync run?
+UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
+# Trace file for mirror stats and checks (make sure we get full hostname)
+TRACE=${TRACE:-"project/trace/${MIRRORNAME}"}
+
+# rsync program
+RSYNC=${RSYNC:-rsync}
+# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
+# excluded files
+RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"}
+# limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited
+RSYNC_BW=${RSYNC_BW:-0}
+# Default rsync options for *every* rsync call
+RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
+# Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files
+RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"}
+# Options for the second pass, where we do want everything, including deletion of old and now unused files
+RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
+# Which rsync share to use on our upstream mirror?
+RSYNC_PATH=${RSYNC_PATH:-"debian"}
+
+# Now add the bwlimit option. As default is 0 we always add it, rsync interprets
+# 0 as unlimited, so this is safe.
+RSYNC_OPTIONS="--bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}"
+
+# We have no default host to sync from, but will error out if its unset
+RSYNC_HOST=${RSYNC_HOST:-""}
+# Error out if we have no host to sync from
+if [ -z "${RSYNC_HOST}" ]; then
+ error "Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf"
+fi
+
+# our username for the rsync share
+RSYNC_USER=${RSYNC_USER:-""}
+# the password
+RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}
+
+# a possible proxy
+RSYNC_PROXY=${RSYNC_PROXY:-""}
+
+# Do we sync stage1?
+SYNCSTAGE1=${SYNCSTAGE1:-"false"}
+# Do we sync stage2?
+SYNCSTAGE2=${SYNCSTAGE2:-"false"}
+# Do we sync all?
+SYNCALL=${SYNCALL:-"true"}
+# Do we have a mhop sync?
+SYNCMHOP=${SYNCMHOP:-"false"}
+# Do we callback?
+SYNCCALLBACK=${SYNCCALLBACK:-"false"}
+# If we call back we need some more options defined in the config file.
+CALLBACKUSER=${CALLBACKUSER:-"archvsync"}
+CALLBACKHOST=${CALLBACKHOST:-"none"}
+CALLBACKKEY=${CALLBACKKEY:-"none"}
+
+# General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that!
+EXCLUDE=${EXCLUDE:-""}
+
+# The temp directory used by rsync --delay-updates is not
+# world-readable remotely. Always exclude it to avoid errors.
+EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
+
+SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""}
+ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
+# Exclude architectures defined in $ARCH_EXCLUDE
+for ARCH in ${ARCH_EXCLUDE}; do
+ EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes"
+ if [ "${ARCH}" = "source" ]; then
+ if [ -z ${SOURCE_EXCLUDE} ]; then
+ SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.tar.xz -exclude *.diff.bz2 --exclude *.dsc "
+ fi
+ fi
+done
+
+# Hooks
+HOOK1=${HOOK1:-""}
+HOOK2=${HOOK2:-""}
+HOOK3=${HOOK3:-""}
+HOOK4=${HOOK4:-""}
+HOOK5=${HOOK5:-""}
+
+# Are we a hub?
+HUB=${HUB:-"false"}
+
+########################################################################
+# Really nothing to see below here. Only code follows. #
+########################################################################
+########################################################################
+
+# Some sane defaults
+cd "${BASEDIR}"
+umask 022
+
+# If we are here for the first time, create the
+# destination and the trace directory
+mkdir -p "${TO}/project/trace"
+
+# Used to make sure we will have the archive fully and completly synced before
+# we stop, even if we get multiple pushes while this script is running.
+# Otherwise we can end up with a half-synced archive:
+# - get a push
+# - sync, while locked
+# - get another push. Of course no extra sync run then happens, we are locked.
+# - done. Archive not correctly synced, we don't have all the changes from the second push.
+touch "${UPDATEREQUIRED}"
+
+# Check to see if another sync is in progress
+if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
+ if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then
+ # We have a recent enough bash version, lets do it the easy way,
+ # the lock will contain the right pid, thanks to $BASHPID
+ if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
+ # Process does either not exist or is not owned by us.
+ echo "$$" > "${LOCK}"
+ else
+ echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
+ exit 1
+ fi
+ else
+ # Old bash, means we dont have the right pid in our lockfile
+ # So take a different way - guess if it is still there by comparing its age.
+ # Not optimal, but hey.
+ stamptime=$(date --reference="${LOCK}" +%s)
+ unixtime=$(date +%s)
+ difference=$(( $unixtime - $stamptime ))
+ if [ ${difference} -ge ${LOCKTIMEOUT} ]; then
+ # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock
+ echo "$$" > "${LOCK}"
+ else
+ echo "Unable to start rsync, lock file younger than one hour"
+ exit 1
+ fi
+ fi
+fi
+
+# When we exit normally we call cleanup on our own. Otherwise we want it called by
+# this trap. (We can not trap on EXIT, because that is called when the main script
+# exits. Which also happens when we background the mainroutine, ie. while we still
+# run!)
+trap cleanup ERR TERM HUP INT QUIT
+
+# Start log by redirecting stdout and stderr there and closing stdin
+exec >"$LOG" 2>&1 <&-
+log "Mirrorsync start"
+
+# Look who pushed us and note that in the log.
+SSH_CONNECTION=${SSH_CONNECTION:-""}
+PUSHFROM="${SSH_CONNECTION%%\ *}"
+if [ -n "${PUSHFROM}" ]; then
+ log "We got pushed from ${PUSHFROM}"
+fi
+
+if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
+ if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then
+ SYNCCALLBACK="false"
+ error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
+ fi
+fi
+
+HOOK=(
+ HOOKNR=1
+ HOOKSCR=${HOOK1}
+)
+hook $HOOK
+
+# Now, we might want to sync from anonymous too.
+# This is that deep in this script so hook1 could, if wanted, change things!
+if [ -z ${RSYNC_USER} ]; then
+ RSYNCPTH="${RSYNC_HOST}"
+else
+ RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
+fi
+
+# Now do the actual mirroring, and run as long as we have an updaterequired file.
+export RSYNC_PASSWORD
+export RSYNC_PROXY
+
+while [ -e "${UPDATEREQUIRED}" ]; do
+ log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
+
+ # if we want stage1 *or* all
+ if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ while [ -e "${UPDATEREQUIRED}" ]; do
+ rm -f "${UPDATEREQUIRED}"
+ log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+
+ set +e
+ # Step one, sync everything except Packages/Releases
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
+ result=$?
+ set -e
+
+ log "Back from rsync with returncode ${result}"
+ done
+ else
+ # Fake a good resultcode
+ result=0
+ fi # Sync stage 1?
+ rm -f "${UPDATEREQUIRED}"
+
+ set +e
+ check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
+ GO=$?
+ set -e
+ if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
+ log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
+ elif [ ${GO} -ne 0 ]; then
+ exit 3
+ fi
+
+ HOOK=(
+ HOOKNR=2
+ HOOKSCR=${HOOK2}
+ )
+ hook $HOOK
+
+ # if we want stage2 *or* all
+ if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+
+ set +e
+ # We are lucky, it worked. Now do step 2 and sync again, this time including
+ # the packages/releases files
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
+ result=$?
+ set -e
+
+ log "Back from rsync with returncode ${result}"
+ else
+ # Fake a good resultcode
+ result=0
+ fi # Sync stage 2?
+
+ set +e
+ check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
+ GO=$?
+ set -e
+ if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
+ log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
+ elif [ ${GO} -ne 0 ]; then
+ exit 4
+ fi
+
+ HOOK=(
+ HOOKNR=3
+ HOOKSCR=${HOOK3}
+ )
+ hook $HOOK
+done
+
+# We only update our tracefile when we had a stage2 or an all sync.
+# Otherwise we would update it after stage1 already, which is wrong.
+if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ if [ -d "$(dirname "${TO}/${TRACE}")" ]; then
+ LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
+ echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
+ echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
+ fi
+fi
+
+HOOK=(
+ HOOKNR=4
+ HOOKSCR=${HOOK4}
+)
+hook $HOOK
+
+if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
+ set +e
+ callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
+ set -e
+fi
+
+# Remove the Archive-Update-in-Progress file before we push our downstreams.
+rm -f "${LOCK}"
+
+if [ x${HUB} = "xtrue" ]; then
+ # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
+ if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
+ RUNMIRRORARGS=""
+ if [ -n "${ARCHIVE}" ]; then
+ # We tell runmirrors about the archive we are running on.
+ RUNMIRRORARGS="-a ${ARCHIVE}"
+ fi
+ # We also tell runmirrors that we are running it from within ftpsync, so it can change
+ # the way it works with mhop based on that.
+ RUNMIRRORARGS="${RUNMIRRORARGS} -f"
+
+ if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
+ # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
+ # not get to this point.
+ # So if that happens, tell runmirrors we are doing mhop
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
+ elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
+ elif [ "xtruex" = "x${SYNCALL}x" ]; then
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
+ fi
+ log "Trigger slave mirrors using ${RUNMIRRORARGS}"
+ ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
+ log "Trigger slave done"
+
+ HOOK=(
+ HOOKNR=5
+ HOOKSCR=${HOOK5}
+ )
+ hook $HOOK
+ fi
+fi
+
+# All done, lets call cleanup
+cleanup
--- /dev/null
+#! /bin/bash
+
+set -e
+set -u
+
+# psuhpdo script for Debian
+#
+# Copyright (C) 2008 Joerg Jaspert <joerg@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# In case the admin somehow wants to have this script located someplace else,
+# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
+BASEDIR=${BASEDIR:-"${HOME}"}
+
+NAME="`basename $0`"
+
+
+# Read our config file
+. "${BASEDIR}/etc/${NAME}.conf"
+
+# Source our common functions
+. "${BASEDIR}/etc/common"
+
+# Set sane defaults if the configfile didn't do that for us.
+# The directory for our logfiles
+LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
+# Our own logfile
+LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
+# How many logfiles to keep
+LOGROTATE=${LOGROTATE:-14}
+# Our mirrorfile
+MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"}
+# used by log()
+PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
+# extra ssh options we might want hostwide
+SSH_OPTS=${SSH_OPTS:-""}
+# Which ssh key to use?
+KEYFILE=${KEYFILE:-".ssh/pushpackages"}
+# which path to "mirror"
+MIRRORPATH=${MIRRORPATH:-"/org/packages.debian.org/mirror/"}
+# where to send mails to
+if [ "x$(hostname -s)x" != "x${MIRRORNAME%%.debian.org}x" ]; then
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
+else
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+fi
+
+if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
+ error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG}
+ exit 5
+fi
+
+# Some sane defaults
+cd ${BASEDIR}
+umask 022
+
+# Make sure we have our log and lock directories
+mkdir -p "${LOGDIR}"
+
+trap 'log "Pdopush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT
+
+log "Pushing pdo mirrors" >> ${LOG}
+
+# From here on we do *NOT* want to exit on errors. We don't want to
+# stop pushing mirrors just because we can't reach one of them.
+set +e
+
+
+# Now read our mirrorfile and push the mirrors defined in there.
+# We use grep to easily sort out all lines having a # in front of them or are empty.
+egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
+while read MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
+ # Process the two options that can be left blank in the config
+ if [ -z ${MPROTO} ]; then
+ MPROTO=2
+ fi
+ if [ -z ${MKEYFILE} ]; then
+ MKEYFILE="${BASEDIR}/${KEYFILE}"
+ fi
+ # Now, people can do stupid things and leave out the protocol, but
+ # define a keyfile...
+ if [ ${MPROTO} -ne 1 ] && [ ${MPROTO} -ne 2 ]; then
+ error "Need a correct ssh protocol version for ${MLNAME}, skipping" >> ${LOG}
+ continue
+ fi
+
+ # And finally, push the mirror
+ log "Pushing ${MLNAME}" >> ${LOG}
+ # This needs a limited ssh key on the other side, something like
+ # no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="rsync --server -vlogDtpr . /srv/mirrors/packages.debian.org/",from="87.106.64.223,2001:8d8:80:11::35d,powell.debian.org" ssh-rsa.....
+ rsync -e "ssh -i ${MKEYFILE} -${MPROTO} ${SSH_OPTS}" -av --stats "${MIRRORPATH}" ${MUSER}@${MHOSTNAME}:/does/not/matter >"${LOGDIR}/${MLNAME}.log"
+ log "Pushing ${MLNAME} done" >> ${LOG}
+ savelog ${LOGDIR}${MLNAME}.log
+
+ set +e
+done
+
+exit 0
--- /dev/null
+#! /bin/bash
+
+set -e
+set -u
+
+# runmirrors script for Debian
+# Based losely on existing scripts, written by an unknown number of
+# different people over the years.
+#
+# Copyright (C) 2008, 2009 Joerg Jaspert <joerg@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# In case the admin somehow wants to have this script located someplace else,
+# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
+BASEDIR=${BASEDIR:-"${HOME}"}
+
+NAME="$(basename $0)"
+
+HELP="$0, (C) 2008, 2009 by Joerg Jaspert <joerg@debian.org>\n
+Usage:\n\n
+
+1.) a single parameter with NO leading -.\n
+\t This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n
+\t have us look for ${NAME}-security.{conf,mirror} files.\n\n
+
+2.) using getopt style parameters:\n
+\t -a [NAME] - Same as 1.) above, used for the config files. Default empty.\n
+\t -k [TYPE] - Type of push. all, stage2, mhop. Default mhop.\n
+\t -f - Run from within the mirrorscript ftpsync. Don't use from commandline!\n
+\t -h - Print this help and exit
+"
+# If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle
+# it will not start with a -. If we find oldstyle we assume its only one, the config
+# name we run on.
+if [ $# -gt 0 ]; then
+ if [ "x${1:0:1}x" != "x-x" ]; then
+ # Yes, does not start with a -, so use it for the config name.
+ CONF=${1:-""}
+ if [ -n "${CONF}" ]; then
+ NAME="${NAME}-${CONF}"
+ fi
+ else
+ # Yeah well, new style, starting with - for getopts
+ while getopts ':a:k:fh' OPTION ; do
+ case $OPTION in
+ a) CONF="${OPTARG}"
+ if [ -n "${CONF}" ]; then
+ NAME="${NAME}-${CONF}"
+ fi
+ ;;
+ k) PUSHKIND="${OPTARG}"
+ ;;
+ f) FROMFTPSYNC="true"
+ ;;
+ h) echo -e $HELP
+ exit 0
+ ;;
+
+ *) echo "Invalid usage"
+ echo -e $HELP
+ exit 1
+ ;;
+ esac
+ done
+ fi
+fi
+# Make sure the values are always defined, even if there was no commandline option
+# for them
+# Default config is empty
+CONF=${CONF:-""}
+
+# Set the default to all, if we didnt get told about it. Currently
+# valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1,
+# stage2 - staged push, second phase. Default is mhop.
+PUSHKIND=${PUSHKIND:-"mhop"}
+
+# If we are pushed from within ftpsync. Default false.
+FROMFTPSYNC=${FROMFTPSYNC:-"false"}
+
+########################################################################
+# Read our config file
+. "${BASEDIR}/etc/${NAME}.conf"
+
+# Source our common functions
+. "${BASEDIR}/etc/common"
+
+# Set sane defaults if the configfile didn't do that for us.
+# The directory for our logfiles
+LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
+# Our own logfile
+LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
+# Our lockfile directory
+LOCKDIR=${LOCKDIR:-"${BASEDIR}/locks"}
+# How many logfiles to keep
+LOGROTATE=${LOGROTATE:-14}
+# Our mirrorfile
+MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"}
+# used by log()
+PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
+# extra ssh options we might want hostwide
+SSH_OPTS=${SSH_OPTS:-"-o StrictHostKeyChecking=no"}
+# Whats our archive name? We will also tell our leafs about it
+PUSHARCHIVE=${PUSHARCHIVE:-"${CONF}"}
+# How long to wait for mirrors to do stage1 if we have multi-stage syncing
+PUSHDELAY=${PUSHDELAY:-600}
+# Which ssh key to use?
+KEYFILE=${KEYFILE:-".ssh/pushmirror"}
+# where to send mails to
+if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
+else
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+fi
+
+if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
+ error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> "${LOG}"
+ exit 5
+fi
+
+# Hooks
+HOOK1=${HOOK1:-""}
+HOOK2=${HOOK2:-""}
+HOOK3=${HOOK3:-""}
+
+########################################################################
+
+# Some sane defaults
+cd "${BASEDIR}"
+umask 022
+
+# Make sure we have our log and lock directories
+mkdir -p "${LOGDIR}"
+mkdir -p "${LOCKDIR}"
+
+trap 'log "Mirrorpush done" >> "${LOG}"; savelog "${LOG}" > /dev/null' EXIT
+
+log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> "${LOG}"
+
+HOOK=(
+ HOOKNR=1
+ HOOKSCR=${HOOK1}
+)
+hook $HOOK
+
+# From here on we do *NOT* want to exit on errors. We don't want to
+# stop pushing mirrors just because we can't reach one of them.
+set +e
+
+# Built up our list of 2-stage mirrors.
+PUSHLOCKS=""
+PUSHLOCKS=$(get2stage)
+
+# In case we have it - remove. It is used to synchronize multi-stage mirroring
+rm -f "${LOCKDIR}/all_stage1"
+
+# Now read our mirrorfile and push the mirrors defined in there.
+# We use grep to easily sort out all lines having a # in front of them or are empty.
+egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
+while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
+ if [ "x${MTYPE}x" = "xDELAYx" ]; then
+ # We should wait a bit.
+ if [ -z ${MLNAME} ]; then
+ MLNAME=600
+ fi
+ log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}"
+ sleep ${MLNAME}
+ continue
+ fi
+
+ # If we are told we have a mhop sync to do and are called from within ftpsync,
+ # we will only look at staged/mhop entries and ignore the rest.
+ if [ "x${PUSHKIND}x" = "xmhopx" ] && [ "x${FROMFTPSYNC}x" = "xtruex" ]; then
+ if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then
+ continue
+ fi
+ fi
+
+ # Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken
+ # as a set of options to give to ssh, we pass it without doing anything with it.
+ # If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use,
+ # and also means we look if there is one value more after a space. That value would then
+ # be the ssh keyfile we use with -i. That gives us full flexibility for all
+ # ssh options but doesn't destroy backwards compatibility.
+ # If it is empty we assume proto 2 and the default keyfile.
+ #
+ # There is one bug in here. We will give out the master keyfile, even if there is a
+ # "-i /bla/bla" in the options. ssh stuffs them together and presents two keys to the
+ # target server. In the case both keys do some actions- the first one presented wins.
+ # And this might not be what one wants.
+ #
+ # The only sane way to go around this, i think, is by dropping backward compability.
+ # Which I don't really like.
+ if [ -n "${MSSHOPT}" ]; then
+ # So its not empty, lets check if it starts with a - and as such is a "new-style"
+ # ssh options set.
+ if [ "x${MSSHOPT:0:1}x" = "x-x" ]; then
+ # Yes we start with a -
+ SSHOPT="${MSSHOPT}"
+ MPROTO="99"
+ MKEYFILE="${BASEDIR}/${KEYFILE}"
+ elif [ ${MSSHOPT:0:1} -eq 1 ] || [ ${MSSHOPT:0:1} -eq 2 ]; then
+ # We do seem to have oldstyle options here.
+ MPROTO=${MSSHOPT:0:1}
+ MKEYFILE=${MSSHOPT:1}
+ SSHOPT=""
+ else
+ error "I don't know what is configured for mirror ${MLNAME}"
+ continue
+ fi
+ else
+ MPROTO=2
+ MKEYFILE="${BASEDIR}/${KEYFILE}"
+ SSHOPT=""
+ fi
+
+ # Built our array
+ SIGNAL_OPTS=(
+ MIRROR="${MLNAME}"
+ HOSTNAME="${MHOSTNAME}"
+ USERNAME="${MUSER}"
+ SSHPROTO="${MPROTO}"
+ SSHKEY="${MKEYFILE}"
+ SSHOPTS="${SSHOPT/ /#}"
+ PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1"
+ PUSHTYPE="${MTYPE}"
+ PUSHARCHIVE=${PUSHARCHIVE}
+ PUSHKIND=${PUSHKIND}
+ FROMFTPSYNC=${FROMFTPSYNC}
+ )
+
+ # And finally, push the mirror
+ log "Trigger ${MLNAME}" >> "${LOG}"
+ signal "${SIGNAL_OPTS}" &
+ log "Trigger for ${MLNAME} done" >> "${LOG}"
+
+ HOOK=(
+ HOOKNR=2
+ HOOKSCR=${HOOK2}
+ )
+ hook $HOOK
+ set +e
+done
+
+# If we are run from within ftpsync *and* have an mhop push to send on, we have
+# to wait until the push is gone through and they all returned, or we will exit
+# much too early.
+# As the signal routine touches $LOCKDIR/all_stage1 when all are done, its
+# easy enough just to wait for that to appear. Of course we do the same game
+# with PUSHDELAY to not wait forever.
+if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then
+ tries=0
+ # We do not wait forever
+ while [ ${tries} -lt ${PUSHDELAY} ]; do
+ if [ -f "${LOCKDIR}/all_stage1" ]; then
+ break
+ fi
+ tries=$((tries + 5))
+ sleep 5
+ done
+
+ if [ ${tries} -ge ${PUSHDELAY} ]; then
+ error "Failed to wait for our mirrors when sending mhop push down." >> "${LOG}"
+ fi
+fi
+
+HOOK=(
+ HOOKNR=3
+ HOOKSCR=${HOOK3}
+)
+hook $HOOK
+
+exit 0
--- /dev/null
+#!/usr/bin/perl -wT
+
+# Copyright (c) 2006 Anthony Towns <ajt@debian.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+use strict;
+use Fcntl ':flock';
+use File::Find;
+use POSIX qw(strftime);
+
+# configuration:
+
+my $local_dir = "/srv/ftp.debian.org/mirror";
+my $rsync_host = undef; #"merkel.debian.org";
+my $rsync_dir = undef; #"debian";
+
+my $dest = "/srv/ftp.debian.org/rsync/typical";
+my $max_del = 1000;
+
+$ENV{"PATH"} = "/bin:/usr/bin";
+
+# program
+
+my $hostname = `/bin/hostname -f`;
+die "bad hostname" unless $hostname =~ m/^([a-zA-Z0-9._-]+)/;
+$hostname = $1;
+
+my $lockfile = "./Archive-Update-in-Progress-$hostname";
+
+unless (open LKFILE, "> $dest/$lockfile" and flock(LKFILE, LOCK_EX)) {
+ print "$hostname is unable to start sync, lock file exists\n";
+ exit(1);
+}
+
+if (defined $rsync_host && defined $rsync_dir) {
+ system("rsync --links --hard-links --times --verbose --recursive"
+ ." --delay-updates --files-from :indices/files/typical.files"
+ ." rsync://$rsync_host/$rsync_dir/ $dest/");
+} else {
+ open FILELIST, "< $local_dir/indices/files/typical.files"
+ or die "typical.files index not found";
+ while (<FILELIST>) {
+ chomp;
+ m/^(.*)$/; $_ = $1;
+ my @l = lstat("$local_dir/$_");
+ next unless (@l);
+
+ if (-l _) {
+ my $lpath = readlink("$local_dir/$_");
+ $lpath =~ m/^(.*)$/; $lpath = $1;
+ if (-l "$dest/$_") {
+ next if ($lpath eq readlink("$dest/$_"));
+ }
+
+ unless (mk_dirname_as_dirs($dest, $_)) {
+ print "E: couldn't create path for $_\n";
+ next;
+ }
+
+ if (-d "$dest/$_") {
+ rename "$dest/$_", "$dest/$_.remove" or print "E: couldn't rename old dir $_ out of the way\n";
+ } elsif (-e "$dest/$_") {
+ unlink("$dest/$_") or print "E: couldn't unlink $_\n";
+ }
+ symlink($lpath, "$dest/$_") or print "E: couldn't create $_ as symlink to $lpath\n";
+ next;
+ }
+
+ next if (-d _);
+
+ unless (mk_dirname_as_dirs($dest, $_)) {
+ print "E: couldn't create path for $_\n";
+ next;
+ }
+
+ my @d = lstat("$dest/$_");
+ if (@d) {
+ if (-d _) {
+ rename("$dest/$_", "$dest/$_.remove") or print "E: couldn't rename old dir $_ out of the way\n";
+ } else {
+ next if (@l and @d and $l[0] == $d[0] and $l[1] == $d[1]);
+ #next if (@l and @d and $l[7] == $d[7]);
+ print "I: updating $_\n";
+ unlink("$dest/$_");
+ }
+ }
+
+ link("$local_dir/$_", "$dest/$_") or print "E: couldn't link $_\n";
+ }
+ close(FILELIST);
+}
+
+print "Files synced, now deleting any unnecessary files\n";
+
+my %expected_files = ();
+open FILES, "< $dest/indices/files/typical.files"
+ or die "typical.files index not found";
+while (<FILES>) {
+ chomp;
+ $expected_files{$_} = 1;
+}
+close(FILES);
+
+chdir($dest);
+
+my $del_count = 0;
+my $last = '';
+finddepth({wanted => \&wanted, no_chdir => 1}, ".");
+
+open TRACE, "> $dest/project/trace/$hostname" or die "couldn't open trace";
+print TRACE strftime("%a %b %e %H:%M:%S UTC %Y", gmtime) . "\n";
+close TRACE;
+
+close LKFILE;
+unlink("$dest/$lockfile");
+exit(0);
+
+sub wanted {
+ my ($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_);
+ if (-d _) {
+ if (substr($last, 0, length($_) + 1) ne "$_/") {
+ print "Deleting empty directory: $_\n";
+ $_ = m/^(.*)$/;
+ my $f = $1;
+ rmdir($f);
+ } else {
+ $last = $_;
+ }
+ } elsif ($_ =~ m|^\./project/trace/| or $_ eq $lockfile) {
+ $last = $_;
+ } elsif (defined $expected_files{$_}) {
+ $last = $_;
+ } elsif ($del_count < $max_del) {
+ $del_count++;
+ print "Deleting file: $_\n";
+ $_ = m/^(.*)$/;
+ my $f = $1;
+ unlink($f);
+ }
+}
+
+sub mk_dirname_as_dirs {
+ my ($base, $file) = @_;
+ while ($file =~ m,^/*([^/]+)/+([^/].*)$,) {
+ $file = $2;
+ $base = "$base/$1";
+ my @blah = lstat($base);
+ if (!@blah) {
+ mkdir($base, 0777);
+ } elsif (-l _ or ! -d _) {
+ print "SHOULD BE A DIRECTORY: $base\n";
+ unlink($base);
+ mkdir($base, 0777);
+ }
+ }
+ 1;
+}
+
+
--- /dev/null
+#!/bin/bash
+
+set -e
+
+unset LC_CTYPE
+LANG=C
+HOST=`hostname -f`
+
+cd ${HOME}/archvsync
+git pull
+
+cd ${HOME}
+~/archvsync/bin/dircombine . . archvsync/ >/dev/null 2>&1
--- /dev/null
+#! /bin/bash
+# No, we can not deal with sh alone.
+
+set -e
+set -u
+# ERR traps should be inherited from functions too. (And command
+# substitutions and subshells and whatnot, but for us the function is
+# the important part here)
+set -E
+
+# websync script for Debian
+# Based losely on the old websync written by an
+# unknown number of different people over the years and ftpsync.
+#
+# Copyright (C) 2008,2009 Joerg Jaspert <joerg@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# In case the admin somehow wants to have this script located someplace else,
+# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
+# How the admin sets this isn't our place to deal with. One could use a wrapper
+# for that. Or pam_env. Or whatever fits in the local setup. :)
+BASEDIR=${BASEDIR:-"${HOME}"}
+
+# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
+# by Joerg Jaspert and the Debian mirroradm group.
+# This is used to track which mirror is using which script version.
+VERSION="0815"
+
+# Source our common functions
+. "${BASEDIR}/etc/common"
+
+########################################################################
+########################################################################
+## functions ##
+########################################################################
+########################################################################
+# All the stuff we want to do when we exit, no matter where
+cleanup() {
+ trap - ERR TERM HUP INT QUIT EXIT
+ # all done. Mail the log, exit.
+ log "Mirrorsync done";
+ if [ -n "${MAILTO}" ]; then
+ # In case rsync had something on stderr
+ if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
+ mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
+ fi
+ if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
+ # And the normal log
+ MAILFILES="${LOG}"
+ if [ "x${FULLLOGS}x" = "xtruex" ]; then
+ # Someone wants full logs including rsync
+ MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
+ fi
+ cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
+ fi
+ fi
+
+ savelog "${LOGDIR}/rsync-${NAME}.log"
+ savelog "${LOGDIR}/rsync-${NAME}.error"
+ savelog "$LOG" > /dev/null
+
+ rm -f "${LOCK}"
+}
+
+
+# Check rsyncs return value
+check_rsync() {
+
+ ret=$1
+ msg=$2
+
+ # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
+ # and us re-running. If it's not, uplink is broken anyways.
+ case "${ret}" in
+ 0) return 0;;
+ 24) return 0;;
+ 23) return 2;;
+ 30) return 2;;
+ *)
+ error "ERROR: ${msg}"
+ return 1
+ ;;
+ esac
+}
+
+########################################################################
+########################################################################
+
+# As what are we called?
+NAME="`basename $0`"
+
+# Now source the config.
+. "${BASEDIR}/etc/${NAME}.conf"
+
+########################################################################
+# Config options go here. Feel free to overwrite them in the config #
+# file if you need to. #
+# On debian.org machines the defaults should be ok. #
+########################################################################
+
+########################################################################
+# There should be nothing to edit here, use the config file #
+########################################################################
+MIRRORNAME=${MIRRORNAME:-`hostname -f`}
+# Where to put logfiles in
+LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
+# Our own logfile
+LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
+
+# Where should we put all the mirrored files?
+TO=${TO:-"/org/www.debian.org/www"}
+
+# used by log() and error()
+PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
+
+# Where to send mails about mirroring to?
+if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
+else
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+fi
+# Want errors only or every log?
+ERRORSONLY=${ERRORSONLY:-"true"}
+# Want full logs, ie. including the rsync one?
+FULLLOGS=${FULLLOGS:-"false"}
+
+# How many logfiles to keep
+LOGROTATE=${LOGROTATE:-14}
+
+# Our lockfile
+LOCK=${LOCK:-"${TO}/Website-Update-in-Progress-${MIRRORNAME}"}
+# Do we need another rsync run?
+UPDATEREQUIRED="${TO}/Website-Update-Required-${MIRRORNAME}"
+# Trace file for mirror stats and checks (make sure we get full hostname)
+TRACE=${TRACE:-".project/trace/${MIRRORNAME}"}
+
+# rsync program
+RSYNC=${RSYNC:-rsync}
+# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
+# excluded files
+RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Website-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Website-Update-Required-${MIRRORNAME}"}
+# Default rsync options for *every* rsync call
+RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
+RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
+# Which rsync share to use on our upstream mirror?
+RSYNC_PATH=${RSYNC_PATH:-"web.debian.org"}
+
+# our username for the rsync share
+RSYNC_USER=${RSYNC_USER:-""}
+# the password
+RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}
+
+# a possible proxy
+RSYNC_PROXY=${RSYNC_PROXY:-""}
+
+# General excludes.
+EXCLUDE=${EXCLUDE:-"--exclude ${HOSTNAME}"}
+
+# The temp directory used by rsync --delay-updates is not
+# world-readable remotely. Always exclude it to avoid errors.
+EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
+
+# And site specific excludes, by default its the sponsor stuff that should be local to all (except templates)
+SITE_FILTER=${SITE_FILTER:-"--include sponsor.deb.* --exclude sponsor_img.* --exclude sponsor.html --exclude sponsor.*.html --filter=protect_sponsor_img.* --filter=protect_sponsor.html --filter=protect_sponsor.*.html"}
+
+# Hooks
+HOOK1=${HOOK1:-""}
+HOOK2=${HOOK2:-""}
+HOOK3=${HOOK3:-""}
+HOOK4=${HOOK4:-""}
+
+# Are we a hub?
+HUB=${HUB:-"false"}
+
+# Some sane defaults
+cd "${BASEDIR}"
+umask 022
+
+# If we are here for the first time, create the
+# destination and the trace directory
+mkdir -p "${TO}/.project/trace"
+
+# Used to make sure we will have the archive fully and completly synced before
+# we stop, even if we get multiple pushes while this script is running.
+# Otherwise we can end up with a half-synced archive:
+# - get a push
+# - sync, while locked
+# - get another push. Of course no extra sync run then happens, we are locked.
+# - done. Archive not correctly synced, we don't have all the changes from the second push.
+touch "${UPDATEREQUIRED}"
+
+# Check to see if another sync is in progress
+if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
+ if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
+ # Process does either not exist or is not owned by us.
+ echo "$$" > "${LOCK}"
+ else
+ echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
+ exit 1
+ fi
+fi
+
+trap cleanup EXIT ERR TERM HUP INT QUIT
+
+# Start log by redirecting everything there.
+exec >"$LOG" 2>&1 </dev/null
+
+# Look who pushed us and note that in the log.
+log "Mirrorsync start"
+PUSHFROM="${SSH_CONNECTION%%\ *}"
+if [ -n "${PUSHFROM}" ]; then
+ log "We got pushed from ${PUSHFROM}"
+fi
+log "Acquired main lock"
+
+HOOK=(
+ HOOKNR=1
+ HOOKSCR=${HOOK1}
+)
+hook $HOOK
+
+# Now, we might want to sync from anonymous too.
+# This is that deep in this script so hook1 could, if wanted, change things!
+if [ -z ${RSYNC_USER} ]; then
+ RSYNCPTH="${RSYNC_HOST}"
+else
+ RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
+fi
+
+# Now do the actual mirroring, and run as long as we have an updaterequired file.
+export RSYNC_PASSWORD
+export RSYNC_PROXY
+
+while [ -e "${UPDATEREQUIRED}" ]; do
+ log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
+
+ rm -f "${UPDATEREQUIRED}"
+ log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+
+ set +e
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
+ result=$?
+ set -e
+
+ log "Back from rsync with returncode ${result}"
+
+ set +e
+ check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}"
+ GO=$?
+ set -e
+
+ if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
+ log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
+ elif [ ${GO} -ne 0 ]; then
+ exit 3
+ fi
+
+ HOOK=(
+ HOOKNR=2
+ HOOKSCR=${HOOK2}
+ )
+ hook $HOOK
+
+done
+
+mkdir -p "${TO}/.project/trace"
+LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
+echo "Used websync version: ${VERSION}" >> "${TO}/${TRACE}"
+echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
+
+HOOK=(
+ HOOKNR=3
+ HOOKSCR=${HOOK3}
+)
+hook $HOOK
+
+if [ x${HUB} = "xtrue" ]; then
+ log "Trigger slave mirrors"
+ ${BASEDIR}/bin/runmirrors "websync"
+ log "Trigger slave done"
+
+ HOOK=(
+ HOOKNR=4
+ HOOKSCR=${HOOK4}
+ )
+ hook $HOOK
+fi
+
+# All done, rest is done by cleanup hook.
--- /dev/null
+# -*- mode:sh -*-
+# Little common functions
+
+# push a mirror attached to us.
+# Arguments (using an array named SIGNAL_OPTS):
+#
+# $MIRROR - Name for the mirror, also basename for the logfile
+# $HOSTNAME - Hostname to push to
+# $USERNAME - Username there
+# $SSHPROTO - Protocol version, either 1 or 2.
+# $SSHKEY - the ssh private key file to use for this push
+# $SSHOPTS - any other option ssh accepts, passed blindly, be careful
+# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged
+# $PUSHTYPE - what kind of push should be done?
+# all - normal, just push once with ssh backgrounded and finish
+# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear,
+# then push stage2
+# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!)
+# $PUSHCB - do we want a callback?
+# $PUSHKIND - whats going on? are we doing mhop push or already stage2?
+# $FROMFTPSYNC - set to true if we run from within ftpsync.
+#
+# This function assumes that the variable LOG is set to a directory where
+# logfiles can be written to.
+# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings
+# (list of "lock"files) to wait for if you want pushtype=staged
+#
+# Pushes might be done in background (for type all).
+signal () {
+ ARGS="SIGNAL_OPTS[*]"
+ local ${!ARGS}
+
+ MIRROR=${MIRROR:-""}
+ HOSTNAME=${HOSTNAME:-""}
+ USERNAME=${USERNAME:-""}
+ SSHPROTO=${SSHPROTO:-""}
+ SSHKEY=${SSHKEY:-""}
+ SSHOPTS=${SSHOPTS:-""}
+ PUSHLOCKOWN=${PUSHLOCKOWN:-""}
+ PUSHTYPE=${PUSHTYPE:-"all"}
+ PUSHARCHIVE=${PUSHARCHIVE:-""}
+ PUSHCB=${PUSHCB:-""}
+ PUSHKIND=${PUSHKIND:-"all"}
+ FROMFTPSYNC=${FROMFTPSYNC:-"false"}
+
+ # And now get # back to space...
+ SSHOPTS=${SSHOPTS/\#/ }
+
+ # Defaults we always want, no matter what
+ SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
+
+ # If there are userdefined ssh options, add them.
+ if [ -n "${SSH_OPTS}" ]; then
+ SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}"
+ fi
+
+ # Does this machine need a special key?
+ if [ -n "${SSHKEY}" ]; then
+ SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}"
+ fi
+
+ # Does this machine have an extra own set of ssh options?
+ if [ -n "${SSHOPTS}" ]; then
+ SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}"
+ fi
+
+ # Set the protocol version
+ if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ] && [ ${SSHPROTO} -ne 99 ]; then
+ # Idiots, we only want 1 or 2. Cant decide? Lets force 2.
+ SSHPROTO=2
+ fi
+ if [ -n "${SSHPROTO}" ] && [ ${SSHPROTO} -ne 99 ]; then
+ SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}"
+ fi
+
+ date -u >> "${LOGDIR}/${MIRROR}.log"
+
+ PUSHARGS=""
+ # PUSHARCHIVE empty or not, we always add the sync:archive: command to transfer.
+ # Otherwise, if nothing else is added, ssh -f would not work ("no command to execute")
+ # But ftpsync does treat "sync:archive:" as the main archive, so this works nicely.
+ PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}"
+
+ # We have a callback wish, tell downstreams
+ if [ -n "${PUSHCB}" ]; then
+ PUSHARGS="${PUSHARGS} sync:callback"
+ fi
+ # If we are running an mhop push AND our downstream is one to receive it, tell it.
+ if [ "xmhopx" = "x${PUSHKIND}x" ] && [ "xmhopx" = "x${PUSHTYPE}x" ]; then
+ PUSHARGS="${PUSHARGS} sync:mhop"
+ fi
+
+ if [ "xallx" = "x${PUSHTYPE}x" ]; then
+ # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings
+ echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log"
+ PUSHARGS1="sync:all"
+ ssh -f $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log"
+ elif [ "xstagedx" = "x${PUSHTYPE}x" ] || [ "xmhopx" = "x${PUSHTYPE}x" ]; then
+ # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings.
+ echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log"
+
+ # Only send stage1 if we havent already send it. When called with stage2, we already did.
+ if [ "xstage2x" != "x${PUSHKIND}x" ]; then
+ # Step1: Do a push to only sync stage1, do not background
+ PUSHARGS1="sync:stage1"
+ ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1
+ touch "${PUSHLOCKOWN}"
+
+ # Step2: Wait for all the other "lock"files to appear.
+ tries=0
+ # We do not wait forever
+ while [ ${tries} -lt ${PUSHDELAY} ]; do
+ total=0
+ found=0
+ for file in ${PUSHLOCKS}; do
+ total=$((total + 1))
+ if [ -f ${file} ]; then
+ found=$((found + 1))
+ fi
+ done
+ if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then
+ touch "${LOCKDIR}/all_stage1"
+ break
+ fi
+ tries=$((tries + 5))
+ sleep 5
+ done
+ # In case we did not have all PUSHLOCKS and still continued, note it
+ # This is a little racy, especially if the other parts decide to do this
+ # at the same time, but it wont hurt more than a mail too much, so I don't care much
+ if [ ${tries} -ge ${PUSHDELAY} ]; then
+ echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log"
+ for file in ${PUSHLOCKS}; do
+ if [ ! -f ${file} ]; then
+ echo "${file}" >> "${LOGDIR}/${MIRROR}.log"
+ error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing"
+ fi
+ done
+ fi
+ rm -f "${PUSHLOCKOWN}"
+ fi
+
+ # Step3: It either timed out or we have all the "lock"files, do the rest
+ # If we are doing mhop AND are called from ftpsync - we now exit.
+ # That way we notify our uplink that we and all our clients are done with their
+ # stage1. It can then finish its own, and if all our upstreams downlinks are done,
+ # it will send us stage2.
+ # If we are not doing mhop or are not called from ftpsync, we start stage2
+ if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then
+ return
+ else
+ PUSHARGS2="sync:stage2"
+ echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log"
+ ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1
+ fi
+ else
+ # Can't decide? Then you get nothing.
+ return
+ fi
+}
+
+# callback, used by ftpsync
+callback () {
+ # Defaults we always want, no matter what
+ SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
+ ssh $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME}
+}
+
+# log something (basically echo it together with a timestamp)
+#
+# Set $PROGRAM to a string to have it added to the output.
+log () {
+ if [ -z "${PROGRAM}" ]; then
+ echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@"
+ else
+ echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@"
+ fi
+}
+
+# log the message using log() but then also send a mail
+# to the address configured in MAILTO (if non-empty)
+error () {
+ log "$@"
+ if [ -z "${MAILTO}" ]; then
+ echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO}
+ fi
+}
+
+# run a hook
+# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR
+# the script to run.
+hook () {
+ ARGS='HOOK[@]'
+ local "${!ARGS}"
+ if [ -n "${HOOKSCR}" ]; then
+ log "Running hook $HOOKNR: ${HOOKSCR}"
+ set +e
+ "${HOOKSCR}"
+ result=$?
+ set -e
+ log "Back from hook $HOOKNR, got returncode ${result}"
+ return $result
+ else
+ return 0
+ fi
+}
+
+# Return the list of 2-stage mirrors.
+get2stage() {
+ egrep '^(staged|mhop)' "${MIRRORS}" | {
+ while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
+ PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}"
+ done
+ echo "$PUSHLOCKS"
+ }
+}
+
+# Rotate logfiles
+savelog() {
+ torotate="$1"
+ count=${2:-${LOGROTATE}}
+ while [ ${count} -gt 0 ]; do
+ prev=$(( count - 1 ))
+ if [ -e "${torotate}.${prev}" ]; then
+ mv "${torotate}.${prev}" "${torotate}.${count}"
+ fi
+ count=$prev
+ done
+ mv "${torotate}" "${torotate}.0"
+}
~mdw / mirror-admin / commitdiff