New ftpsync upstream.

[mirror-admin] / bin / runmirrors

#! /bin/bash

set -e
set -u

# runmirrors script for Debian
# Based losely on existing scripts, written by an unknown number of
# different people over the years.
#
# Copyright (C) 2008, 2009 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# In case the admin somehow wants to have this script located someplace else,
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
BASEDIR=${BASEDIR:-"${HOME}"}

NAME="$(basename $0)"

HELP="$0, (C) 2008, 2009 by Joerg Jaspert <joerg@debian.org>\n
Usage:\n\n

1.) a single parameter with NO leading -.\n
\t  This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n
\t  have us look for ${NAME}-security.{conf,mirror} files.\n\n

2.) using getopt style parameters:\n
\t -a [NAME]   - Same as 1.) above, used for the config files. Default empty.\n
\t -k [TYPE]   - Type of push. all, stage2, mhop. Default mhop.\n
\t -f          - Run from within the mirrorscript ftpsync. Don't use from commandline!\n
\t -h          - Print this help and exit
"
# If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle
# it will not start with a -. If we find oldstyle we assume its only one, the config
# name we run on.
if [ $# -gt 0 ]; then
    if [ "x${1:0:1}x" != "x-x" ]; then
        # Yes, does not start with a -, so use it for the config name.
        CONF=${1:-""}
        if [ -n "${CONF}" ]; then
            NAME="${NAME}-${CONF}"
        fi
    else
        # Yeah well, new style, starting with - for getopts
        while getopts ':a:k:fh' OPTION ; do
            case $OPTION in
                a)	CONF="${OPTARG}"
                    if [ -n "${CONF}" ]; then
                        NAME="${NAME}-${CONF}"
                    fi
                    ;;
                k)	PUSHKIND="${OPTARG}"
                    ;;
                f)	FROMFTPSYNC="true"
                    ;;
                h)	echo -e $HELP
                    exit 0
                    ;;

                *)  echo "Invalid usage"
                    echo -e $HELP 
                    exit 1
                    ;;
            esac
        done
    fi
fi
# Make sure the values are always defined, even if there was no commandline option
# for them
# Default config is empty
CONF=${CONF:-""}

# Set the default to all, if we didnt get told about it. Currently
# valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1,
# stage2 - staged push, second phase. Default is mhop.
PUSHKIND=${PUSHKIND:-"mhop"}

# If we are pushed from within ftpsync. Default false.
FROMFTPSYNC=${FROMFTPSYNC:-"false"}

########################################################################
# Read our config file
. "${BASEDIR}/etc/${NAME}.conf"

# Source our common functions
. "${BASEDIR}/etc/common"

# Set sane defaults if the configfile didn't do that for us.
# The directory for our logfiles
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
# Our own logfile
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
# Our lockfile directory
LOCKDIR=${LOCKDIR:-"${BASEDIR}/locks"}
# How many logfiles to keep
LOGROTATE=${LOGROTATE:-14}
# Our mirrorfile
MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"}
# used by log()
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# extra ssh options we might want hostwide
SSH_OPTS=${SSH_OPTS:-"-o StrictHostKeyChecking=no"}
# Whats our archive name? We will also tell our leafs about it
PUSHARCHIVE=${PUSHARCHIVE:-"${CONF}"}
# How long to wait for mirrors to do stage1 if we have multi-stage syncing
PUSHDELAY=${PUSHDELAY:-600}
# Which ssh key to use?
KEYFILE=${KEYFILE:-".ssh/pushmirror"}
# where to send mails to
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
    # We are not on a debian.org host
    MAILTO=${MAILTO:-"root"}
else
    # Yay, on a .debian.org host
    MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi

if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
    error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> "${LOG}"
    exit 5
fi

# Hooks
HOOK1=${HOOK1:-""}
HOOK2=${HOOK2:-""}
HOOK3=${HOOK3:-""}

########################################################################

# Some sane defaults
cd "${BASEDIR}"
umask 022

# Make sure we have our log and lock directories
mkdir -p "${LOGDIR}"
mkdir -p "${LOCKDIR}"

trap 'log "Mirrorpush done" >> "${LOG}"; savelog "${LOG}" > /dev/null' EXIT

log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> "${LOG}"

HOOK=(
    HOOKNR=1
    HOOKSCR=${HOOK1}
)
hook $HOOK

# From here on we do *NOT* want to exit on errors. We don't want to
# stop pushing mirrors just because we can't reach one of them.
set +e

# Built up our list of 2-stage mirrors.
PUSHLOCKS=""
PUSHLOCKS=$(get2stage)

# In case we have it - remove. It is used to synchronize multi-stage mirroring
rm -f "${LOCKDIR}/all_stage1"

# Now read our mirrorfile and push the mirrors defined in there.
# We use grep to easily sort out all lines having a # in front of them or are empty.
egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
    if [ "x${MTYPE}x" = "xDELAYx" ]; then
        # We should wait a bit.
        if [ -z ${MLNAME} ]; then
            MLNAME=600
        fi
        log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}"
        sleep ${MLNAME}
        continue
    fi

    # If we are told we have a mhop sync to do and are called from within ftpsync,
    # we will only look at staged/mhop entries and ignore the rest.
    if [ "x${PUSHKIND}x" = "xmhopx" ] && [ "x${FROMFTPSYNC}x" = "xtruex" ]; then
        if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then
            continue
        fi
    fi

    # Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken
    # as a set of options to give to ssh, we pass it without doing anything with it.
    # If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use,
    # and also means we look if there is one value more after a space. That value would then
    # be the ssh keyfile we use with -i. That gives us full flexibility for all
    # ssh options but doesn't destroy backwards compatibility.
    # If it is empty we assume proto 2 and the default keyfile.
    #
    # There is one bug in here. We will give out the master keyfile, even if there is a
    # "-i /bla/bla" in the options. ssh stuffs them together and presents two keys to the
    # target server. In the case both keys do some actions- the first one presented wins.
    # And this might not be what one wants.
    #
    # The only sane way to go around this, i think, is by dropping backward compability.
    # Which I don't really like.
    if [ -n "${MSSHOPT}" ]; then
        # So its not empty, lets check if it starts with a - and as such is a "new-style"
        # ssh options set.
        if [ "x${MSSHOPT:0:1}x" = "x-x" ]; then
            # Yes we start with a -
            SSHOPT="${MSSHOPT}"
            MPROTO="99"
            MKEYFILE="${BASEDIR}/${KEYFILE}"
        elif [ ${MSSHOPT:0:1} -eq 1 ] || [ ${MSSHOPT:0:1} -eq 2 ]; then
            # We do seem to have oldstyle options here.
            MPROTO=${MSSHOPT:0:1}
            MKEYFILE=${MSSHOPT:2}
            SSHOPT=""
        else
            error "I don't know what is configured for mirror ${MLNAME}"
            continue
        fi
    else
        MPROTO=2
        MKEYFILE="${BASEDIR}/${KEYFILE}"
        SSHOPT=""
    fi

    # Built our array
    SIGNAL_OPTS=(
        MIRROR="${MLNAME}"
        HOSTNAME="${MHOSTNAME}"
        USERNAME="${MUSER}"
        SSHPROTO="${MPROTO}"
        SSHKEY="${MKEYFILE}"
        SSHOPTS="${SSHOPT/ /#}"
        PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1"
        PUSHTYPE="${MTYPE}"
        PUSHARCHIVE=${PUSHARCHIVE}
        PUSHKIND=${PUSHKIND}
        FROMFTPSYNC=${FROMFTPSYNC}
    )

    # And finally, push the mirror
    log "Trigger ${MLNAME}" >> "${LOG}"
    signal "${SIGNAL_OPTS}" &
    log "Trigger for ${MLNAME} done" >> "${LOG}"

    HOOK=(
        HOOKNR=2
        HOOKSCR=${HOOK2}
    )
    hook $HOOK
    set +e
done

# If we are run from within ftpsync *and* have an mhop push to send on, we have
# to wait until the push is gone through and they all returned, or we will exit
# much too early.
# As the signal routine touches $LOCKDIR/all_stage1 when all are done, its
# easy enough just to wait for that to appear. Of course we do the same game
# with PUSHDELAY to not wait forever.
if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then
    tries=0
    # We do not wait forever
    while [ ${tries} -lt ${PUSHDELAY} ]; do
        if [ -f "${LOCKDIR}/all_stage1" ]; then
            break
        fi
        tries=$((tries + 5))
        sleep 5
    done

    if [ ${tries} -ge ${PUSHDELAY} ]; then
        error "Failed to wait for our mirrors when sending mhop push down." >> "${LOG}"
    fi
fi

HOOK=(
    HOOKNR=3
    HOOKSCR=${HOOK3}
)
hook $HOOK

exit 0