| 1 | #! /bin/bash |
| 2 | # No, we can not deal with sh alone. |
| 3 | |
| 4 | set -e |
| 5 | set -u |
| 6 | # ERR traps should be inherited from functions too. (And command |
| 7 | # substitutions and subshells and whatnot, but for us the function is |
| 8 | # the important part here) |
| 9 | set -E |
| 10 | |
| 11 | # websync script for Debian |
| 12 | # Based losely on the old websync written by an |
| 13 | # unknown number of different people over the years and ftpsync. |
| 14 | # |
| 15 | # Copyright (C) 2008,2009 Joerg Jaspert <joerg@debian.org> |
| 16 | # |
| 17 | # This program is free software; you can redistribute it and/or |
| 18 | # modify it under the terms of the GNU General Public License as |
| 19 | # published by the Free Software Foundation; version 2. |
| 20 | # |
| 21 | # This program is distributed in the hope that it will be useful, but |
| 22 | # WITHOUT ANY WARRANTY; without even the implied warranty of |
| 23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 24 | # General Public License for more details. |
| 25 | # |
| 26 | # You should have received a copy of the GNU General Public License |
| 27 | # along with this program; if not, write to the Free Software |
| 28 | # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
| 29 | |
| 30 | # In case the admin somehow wants to have this script located someplace else, |
| 31 | # he can set BASEDIR, and we will take that. If it is unset we take ${HOME} |
| 32 | # How the admin sets this isn't our place to deal with. One could use a wrapper |
| 33 | # for that. Or pam_env. Or whatever fits in the local setup. :) |
| 34 | BASEDIR=${BASEDIR:-"${HOME}"} |
| 35 | |
| 36 | # Script version. DO NOT CHANGE, *unless* you change the master copy maintained |
| 37 | # by Joerg Jaspert and the Debian mirroradm group. |
| 38 | # This is used to track which mirror is using which script version. |
| 39 | VERSION="0815" |
| 40 | |
| 41 | # Source our common functions |
| 42 | . "${BASEDIR}/etc/common" |
| 43 | |
| 44 | ######################################################################## |
| 45 | ######################################################################## |
| 46 | ## functions ## |
| 47 | ######################################################################## |
| 48 | ######################################################################## |
| 49 | # All the stuff we want to do when we exit, no matter where |
| 50 | cleanup() { |
| 51 | trap - ERR TERM HUP INT QUIT EXIT |
| 52 | # all done. Mail the log, exit. |
| 53 | log "Mirrorsync done"; |
| 54 | if [ -n "${MAILTO}" ]; then |
| 55 | # In case rsync had something on stderr |
| 56 | if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then |
| 57 | mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" |
| 58 | fi |
| 59 | if [ "x${ERRORSONLY}x" = "xfalsex" ]; then |
| 60 | # And the normal log |
| 61 | MAILFILES="${LOG}" |
| 62 | if [ "x${FULLLOGS}x" = "xtruex" ]; then |
| 63 | # Someone wants full logs including rsync |
| 64 | MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" |
| 65 | fi |
| 66 | cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} |
| 67 | fi |
| 68 | fi |
| 69 | |
| 70 | savelog "${LOGDIR}/rsync-${NAME}.log" |
| 71 | savelog "${LOGDIR}/rsync-${NAME}.error" |
| 72 | savelog "$LOG" > /dev/null |
| 73 | |
| 74 | rm -f "${LOCK}" |
| 75 | } |
| 76 | |
| 77 | |
| 78 | # Check rsyncs return value |
| 79 | check_rsync() { |
| 80 | |
| 81 | ret=$1 |
| 82 | msg=$2 |
| 83 | |
| 84 | # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED |
| 85 | # and us re-running. If it's not, uplink is broken anyways. |
| 86 | case "${ret}" in |
| 87 | 0) return 0;; |
| 88 | 24) return 0;; |
| 89 | 23) return 2;; |
| 90 | 30) return 2;; |
| 91 | *) |
| 92 | error "ERROR: ${msg}" |
| 93 | return 1 |
| 94 | ;; |
| 95 | esac |
| 96 | } |
| 97 | |
| 98 | ######################################################################## |
| 99 | ######################################################################## |
| 100 | |
| 101 | # As what are we called? |
| 102 | NAME="`basename $0`" |
| 103 | |
| 104 | # Now source the config. |
| 105 | . "${BASEDIR}/etc/${NAME}.conf" |
| 106 | |
| 107 | ######################################################################## |
| 108 | # Config options go here. Feel free to overwrite them in the config # |
| 109 | # file if you need to. # |
| 110 | # On debian.org machines the defaults should be ok. # |
| 111 | ######################################################################## |
| 112 | |
| 113 | ######################################################################## |
| 114 | # There should be nothing to edit here, use the config file # |
| 115 | ######################################################################## |
| 116 | MIRRORNAME=${MIRRORNAME:-`hostname -f`} |
| 117 | # Where to put logfiles in |
| 118 | LOGDIR=${LOGDIR:-"${BASEDIR}/log"} |
| 119 | # Our own logfile |
| 120 | LOG=${LOG:-"${LOGDIR}/${NAME}.log"} |
| 121 | |
| 122 | # Where should we put all the mirrored files? |
| 123 | TO=${TO:-"/org/www.debian.org/www"} |
| 124 | |
| 125 | # used by log() and error() |
| 126 | PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} |
| 127 | |
| 128 | # Where to send mails about mirroring to? |
| 129 | if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then |
| 130 | # We are not on a debian.org host |
| 131 | MAILTO=${MAILTO:-"root"} |
| 132 | else |
| 133 | # Yay, on a .debian.org host |
| 134 | MAILTO=${MAILTO:-"mirrorlogs@debian.org"} |
| 135 | fi |
| 136 | # Want errors only or every log? |
| 137 | ERRORSONLY=${ERRORSONLY:-"true"} |
| 138 | # Want full logs, ie. including the rsync one? |
| 139 | FULLLOGS=${FULLLOGS:-"false"} |
| 140 | |
| 141 | # How many logfiles to keep |
| 142 | LOGROTATE=${LOGROTATE:-14} |
| 143 | |
| 144 | # Our lockfile |
| 145 | LOCK=${LOCK:-"${TO}/Website-Update-in-Progress-${MIRRORNAME}"} |
| 146 | # Do we need another rsync run? |
| 147 | UPDATEREQUIRED="${TO}/Website-Update-Required-${MIRRORNAME}" |
| 148 | # Trace file for mirror stats and checks (make sure we get full hostname) |
| 149 | TRACE=${TRACE:-".project/trace/${MIRRORNAME}"} |
| 150 | |
| 151 | # rsync program |
| 152 | RSYNC=${RSYNC:-rsync} |
| 153 | # Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete |
| 154 | # excluded files |
| 155 | RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Website-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Website-Update-Required-${MIRRORNAME}"} |
| 156 | # Default rsync options for *every* rsync call |
| 157 | RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"} |
| 158 | RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"} |
| 159 | # Which rsync share to use on our upstream mirror? |
| 160 | RSYNC_PATH=${RSYNC_PATH:-"web.debian.org"} |
| 161 | |
| 162 | # our username for the rsync share |
| 163 | RSYNC_USER=${RSYNC_USER:-""} |
| 164 | # the password |
| 165 | RSYNC_PASSWORD=${RSYNC_PASSWORD:-""} |
| 166 | |
| 167 | # a possible proxy |
| 168 | RSYNC_PROXY=${RSYNC_PROXY:-""} |
| 169 | |
| 170 | # General excludes. |
| 171 | EXCLUDE=${EXCLUDE:-"--exclude ${HOSTNAME}"} |
| 172 | |
| 173 | # The temp directory used by rsync --delay-updates is not |
| 174 | # world-readable remotely. Always exclude it to avoid errors. |
| 175 | EXCLUDE="${EXCLUDE} --exclude .~tmp~/" |
| 176 | |
| 177 | # And site specific excludes, by default its the sponsor stuff that should be local to all (except templates) |
| 178 | SITE_FILTER=${SITE_FILTER:-"--include sponsor.deb.* --exclude sponsor_img.* --exclude sponsor.html --exclude sponsor.*.html --filter=protect_sponsor_img.* --filter=protect_sponsor.html --filter=protect_sponsor.*.html"} |
| 179 | |
| 180 | # Hooks |
| 181 | HOOK1=${HOOK1:-""} |
| 182 | HOOK2=${HOOK2:-""} |
| 183 | HOOK3=${HOOK3:-""} |
| 184 | HOOK4=${HOOK4:-""} |
| 185 | |
| 186 | # Are we a hub? |
| 187 | HUB=${HUB:-"false"} |
| 188 | |
| 189 | # Some sane defaults |
| 190 | cd "${BASEDIR}" |
| 191 | umask 022 |
| 192 | |
| 193 | # If we are here for the first time, create the |
| 194 | # destination and the trace directory |
| 195 | mkdir -p "${TO}/.project/trace" |
| 196 | |
| 197 | # Used to make sure we will have the archive fully and completly synced before |
| 198 | # we stop, even if we get multiple pushes while this script is running. |
| 199 | # Otherwise we can end up with a half-synced archive: |
| 200 | # - get a push |
| 201 | # - sync, while locked |
| 202 | # - get another push. Of course no extra sync run then happens, we are locked. |
| 203 | # - done. Archive not correctly synced, we don't have all the changes from the second push. |
| 204 | touch "${UPDATEREQUIRED}" |
| 205 | |
| 206 | # Check to see if another sync is in progress |
| 207 | if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then |
| 208 | if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then |
| 209 | # Process does either not exist or is not owned by us. |
| 210 | echo "$$" > "${LOCK}" |
| 211 | else |
| 212 | echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})" |
| 213 | exit 1 |
| 214 | fi |
| 215 | fi |
| 216 | |
| 217 | trap cleanup EXIT ERR TERM HUP INT QUIT |
| 218 | |
| 219 | # Start log by redirecting everything there. |
| 220 | exec >"$LOG" 2>&1 </dev/null |
| 221 | |
| 222 | # Look who pushed us and note that in the log. |
| 223 | log "Mirrorsync start" |
| 224 | PUSHFROM="${SSH_CONNECTION%%\ *}" |
| 225 | if [ -n "${PUSHFROM}" ]; then |
| 226 | log "We got pushed from ${PUSHFROM}" |
| 227 | fi |
| 228 | log "Acquired main lock" |
| 229 | |
| 230 | HOOK=( |
| 231 | HOOKNR=1 |
| 232 | HOOKSCR=${HOOK1} |
| 233 | ) |
| 234 | hook $HOOK |
| 235 | |
| 236 | # Now, we might want to sync from anonymous too. |
| 237 | # This is that deep in this script so hook1 could, if wanted, change things! |
| 238 | if [ -z ${RSYNC_USER} ]; then |
| 239 | RSYNCPTH="${RSYNC_HOST}" |
| 240 | else |
| 241 | RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" |
| 242 | fi |
| 243 | |
| 244 | # Now do the actual mirroring, and run as long as we have an updaterequired file. |
| 245 | export RSYNC_PASSWORD |
| 246 | export RSYNC_PROXY |
| 247 | |
| 248 | while [ -e "${UPDATEREQUIRED}" ]; do |
| 249 | log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" |
| 250 | |
| 251 | rm -f "${UPDATEREQUIRED}" |
| 252 | log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" |
| 253 | |
| 254 | set +e |
| 255 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \ |
| 256 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" |
| 257 | result=$? |
| 258 | set -e |
| 259 | |
| 260 | log "Back from rsync with returncode ${result}" |
| 261 | |
| 262 | set +e |
| 263 | check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}" |
| 264 | GO=$? |
| 265 | set -e |
| 266 | |
| 267 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then |
| 268 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" |
| 269 | elif [ ${GO} -ne 0 ]; then |
| 270 | exit 3 |
| 271 | fi |
| 272 | |
| 273 | HOOK=( |
| 274 | HOOKNR=2 |
| 275 | HOOKSCR=${HOOK2} |
| 276 | ) |
| 277 | hook $HOOK |
| 278 | |
| 279 | done |
| 280 | |
| 281 | mkdir -p "${TO}/.project/trace" |
| 282 | LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" |
| 283 | echo "Used websync version: ${VERSION}" >> "${TO}/${TRACE}" |
| 284 | echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" |
| 285 | |
| 286 | HOOK=( |
| 287 | HOOKNR=3 |
| 288 | HOOKSCR=${HOOK3} |
| 289 | ) |
| 290 | hook $HOOK |
| 291 | |
| 292 | if [ x${HUB} = "xtrue" ]; then |
| 293 | log "Trigger slave mirrors" |
| 294 | ${BASEDIR}/bin/runmirrors "websync" |
| 295 | log "Trigger slave done" |
| 296 | |
| 297 | HOOK=( |
| 298 | HOOKNR=4 |
| 299 | HOOKSCR=${HOOK4} |
| 300 | ) |
| 301 | hook $HOOK |
| 302 | fi |
| 303 | |
| 304 | # All done, rest is done by cleanup hook. |