mdw@git.distorted.org.uk Git - mirror-admin/blob - bin/websync

   1 #! /bin/bash
   2 # No, we can not deal with sh alone.
   3
   4 set -e
   5 set -u
   6 # ERR traps should be inherited from functions too. (And command
   7 # substitutions and subshells and whatnot, but for us the function is
   8 # the important part here)
   9 set -E
  10
  11 # websync script for Debian
  12 # Based losely on the old websync written by an
  13 # unknown number of different people over the years and ftpsync.
  14 #
  15 # Copyright (C) 2008,2009 Joerg Jaspert <joerg@debian.org>
  16 #
  17 # This program is free software; you can redistribute it and/or
  18 # modify it under the terms of the GNU General Public License as
  19 # published by the Free Software Foundation; version 2.
  20 #
  21 # This program is distributed in the hope that it will be useful, but
  22 # WITHOUT ANY WARRANTY; without even the implied warranty of
  23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  24 # General Public License for more details.
  25 #
  26 # You should have received a copy of the GNU General Public License
  27 # along with this program; if not, write to the Free Software
  28 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  29
  30 # In case the admin somehow wants to have this script located someplace else,
  31 # he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
  32 # How the admin sets this isn't our place to deal with. One could use a wrapper
  33 # for that. Or pam_env. Or whatever fits in the local setup. :)
  34 BASEDIR=${BASEDIR:-"${HOME}"}
  35
  36 # Script version. DO NOT CHANGE, *unless* you change the master copy maintained
  37 # by Joerg Jaspert and the Debian mirroradm group.
  38 # This is used to track which mirror is using which script version.
  39 VERSION="0815"
  40
  41 # Source our common functions
  42 . "${BASEDIR}/etc/common"
  43
  44 ########################################################################
  45 ########################################################################
  46 ## functions                                                          ##
  47 ########################################################################
  48 ########################################################################
  49 # All the stuff we want to do when we exit, no matter where
  50 cleanup() {
  51     trap - ERR TERM HUP INT QUIT EXIT
  52     # all done. Mail the log, exit.
  53     log "Mirrorsync done";
  54     if [ -n "${MAILTO}" ]; then
  55         # In case rsync had something on stderr
  56         if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
  57             mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
  58         fi
  59         if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
  60             # And the normal log
  61             MAILFILES="${LOG}"
  62             if [ "x${FULLLOGS}x" = "xtruex" ]; then
  63                 # Someone wants full logs including rsync
  64                 MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
  65             fi
  66             cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
  67         fi
  68     fi
  69
  70     savelog "${LOGDIR}/rsync-${NAME}.log"
  71     savelog "${LOGDIR}/rsync-${NAME}.error"
  72     savelog "$LOG" > /dev/null
  73
  74     rm -f "${LOCK}"
  75 }
  76
  77
  78 # Check rsyncs return value
  79 check_rsync() {
  80
  81     ret=$1
  82     msg=$2
  83
  84     # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
  85     # and us re-running. If it's not, uplink is broken anyways.
  86     case "${ret}" in
  87         0) return 0;;
  88         24) return 0;;
  89         23) return 2;;
  90         30) return 2;;
  91         *)
  92             error "ERROR: ${msg}"
  93             return 1
  94             ;;
  95     esac
  96 }
  97
  98 ########################################################################
  99 ########################################################################
 100
 101 # As what are we called?
 102 NAME="`basename $0`"
 103
 104 # Now source the config.
 105 . "${BASEDIR}/etc/${NAME}.conf"
 106
 107 ########################################################################
 108 # Config options go here. Feel free to overwrite them in the config    #
 109 # file if you need to.                                                 #
 110 # On debian.org machines the defaults should be ok.                    #
 111 ########################################################################
 112
 113 ########################################################################
 114 # There should be nothing to edit here, use the config file            #
 115 ########################################################################
 116 MIRRORNAME=${MIRRORNAME:-`hostname -f`}
 117 # Where to put logfiles in
 118 LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
 119 # Our own logfile
 120 LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
 121
 122 # Where should we put all the mirrored files?
 123 TO=${TO:-"/org/www.debian.org/www"}
 124
 125 # used by log() and error()
 126 PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
 127
 128 # Where to send mails about mirroring to?
 129 if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
 130     # We are not on a debian.org host
 131     MAILTO=${MAILTO:-"root"}
 132 else
 133     # Yay, on a .debian.org host
 134     MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
 135 fi
 136 # Want errors only or every log?
 137 ERRORSONLY=${ERRORSONLY:-"true"}
 138 # Want full logs, ie. including the rsync one?
 139 FULLLOGS=${FULLLOGS:-"false"}
 140
 141 # How many logfiles to keep
 142 LOGROTATE=${LOGROTATE:-14}
 143
 144 # Our lockfile
 145 LOCK=${LOCK:-"${TO}/Website-Update-in-Progress-${MIRRORNAME}"}
 146 # Do we need another rsync run?
 147 UPDATEREQUIRED="${TO}/Website-Update-Required-${MIRRORNAME}"
 148 # Trace file for mirror stats and checks (make sure we get full hostname)
 149 TRACE=${TRACE:-".project/trace/${MIRRORNAME}"}
 150
 151 # rsync program
 152 RSYNC=${RSYNC:-rsync}
 153 # Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
 154 # excluded files
 155 RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Website-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Website-Update-Required-${MIRRORNAME}"}
 156 # Default rsync options for *every* rsync call
 157 RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
 158 RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
 159 # Which rsync share to use on our upstream mirror?
 160 RSYNC_PATH=${RSYNC_PATH:-"web.debian.org"}
 161
 162 # our username for the rsync share
 163 RSYNC_USER=${RSYNC_USER:-""}
 164 # the password
 165 RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}
 166
 167 # a possible proxy
 168 RSYNC_PROXY=${RSYNC_PROXY:-""}
 169
 170 # General excludes.
 171 EXCLUDE=${EXCLUDE:-"--exclude ${HOSTNAME}"}
 172
 173 # The temp directory used by rsync --delay-updates is not
 174 # world-readable remotely. Always exclude it to avoid errors.
 175 EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
 176
 177 # And site specific excludes, by default its the sponsor stuff that should be local to all (except templates)
 178 SITE_FILTER=${SITE_FILTER:-"--include sponsor.deb.* --exclude sponsor_img.* --exclude sponsor.html --exclude sponsor.*.html --filter=protect_sponsor_img.* --filter=protect_sponsor.html --filter=protect_sponsor.*.html"}
 179
 180 # Hooks
 181 HOOK1=${HOOK1:-""}
 182 HOOK2=${HOOK2:-""}
 183 HOOK3=${HOOK3:-""}
 184 HOOK4=${HOOK4:-""}
 185
 186 # Are we a hub?
 187 HUB=${HUB:-"false"}
 188
 189 # Some sane defaults
 190 cd "${BASEDIR}"
 191 umask 022
 192
 193 # If we are here for the first time, create the
 194 # destination and the trace directory
 195 mkdir -p "${TO}/.project/trace"
 196
 197 # Used to make sure we will have the archive fully and completly synced before
 198 # we stop, even if we get multiple pushes while this script is running.
 199 # Otherwise we can end up with a half-synced archive:
 200 # - get a push
 201 # - sync, while locked
 202 # - get another push. Of course no extra sync run then happens, we are locked.
 203 # - done. Archive not correctly synced, we don't have all the changes from the second push.
 204 touch "${UPDATEREQUIRED}"
 205
 206 # Check to see if another sync is in progress
 207 if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
 208     if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
 209         # Process does either not exist or is not owned by us.
 210         echo "$$" > "${LOCK}"
 211     else
 212         echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
 213         exit 1
 214     fi
 215 fi
 216
 217 trap cleanup EXIT ERR TERM HUP INT QUIT
 218
 219 # Start log by redirecting everything there.
 220 exec >"$LOG" 2>&1 </dev/null
 221
 222 # Look who pushed us and note that in the log.
 223 log "Mirrorsync start"
 224 PUSHFROM="${SSH_CONNECTION%%\ *}"
 225 if [ -n "${PUSHFROM}" ]; then
 226     log "We got pushed from ${PUSHFROM}"
 227 fi
 228 log "Acquired main lock"
 229
 230 HOOK=(
 231     HOOKNR=1
 232     HOOKSCR=${HOOK1}
 233 )
 234 hook $HOOK
 235
 236 # Now, we might want to sync from anonymous too.
 237 # This is that deep in this script so hook1 could, if wanted, change things!
 238 if [ -z ${RSYNC_USER} ]; then
 239     RSYNCPTH="${RSYNC_HOST}"
 240 else
 241     RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
 242 fi
 243
 244 # Now do the actual mirroring, and run as long as we have an updaterequired file.
 245 export RSYNC_PASSWORD
 246 export RSYNC_PROXY
 247
 248 while [ -e "${UPDATEREQUIRED}" ]; do
 249     log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
 250
 251     rm -f "${UPDATEREQUIRED}"
 252     log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER}  ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
 253
 254     set +e
 255     ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \
 256         ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
 257     result=$?
 258     set -e
 259
 260     log "Back from rsync with returncode ${result}"
 261
 262     set +e
 263     check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}"
 264     GO=$?
 265     set -e
 266
 267     if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
 268         log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
 269     elif [ ${GO} -ne 0 ]; then
 270         exit 3
 271     fi
 272
 273     HOOK=(
 274         HOOKNR=2
 275         HOOKSCR=${HOOK2}
 276     )
 277     hook $HOOK
 278
 279 done
 280
 281 mkdir -p "${TO}/.project/trace"
 282 LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
 283 echo "Used websync version: ${VERSION}" >> "${TO}/${TRACE}"
 284 echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
 285
 286 HOOK=(
 287     HOOKNR=3
 288     HOOKSCR=${HOOK3}
 289 )
 290 hook $HOOK
 291
 292 if [ x${HUB} = "xtrue" ]; then
 293     log "Trigger slave mirrors"
 294     ${BASEDIR}/bin/runmirrors "websync"
 295     log "Trigger slave done"
 296
 297     HOOK=(
 298         HOOKNR=4
 299         HOOKSCR=${HOOK4}
 300     )
 301     hook $HOOK
 302 fi
 303
 304 # All done, rest is done by cleanup hook.