| 1 | #! /bin/bash |
| 2 | # No, we can not deal with sh alone. |
| 3 | |
| 4 | set -e |
| 5 | set -u |
| 6 | # ERR traps should be inherited from functions too. (And command |
| 7 | # substitutions and subshells and whatnot, but for us the function is |
| 8 | # the important part here) |
| 9 | set -E |
| 10 | |
| 11 | # ftpsync script for Debian |
| 12 | # Based losely on a number of existing scripts, written by an |
| 13 | # unknown number of different people over the years. |
| 14 | # |
| 15 | # Copyright (C) 2008,2009,2010,2011 Joerg Jaspert <joerg@debian.org> |
| 16 | # |
| 17 | # This program is free software; you can redistribute it and/or |
| 18 | # modify it under the terms of the GNU General Public License as |
| 19 | # published by the Free Software Foundation; version 2. |
| 20 | # |
| 21 | # This program is distributed in the hope that it will be useful, but |
| 22 | # WITHOUT ANY WARRANTY; without even the implied warranty of |
| 23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 24 | # General Public License for more details. |
| 25 | # |
| 26 | # You should have received a copy of the GNU General Public License |
| 27 | # along with this program; if not, write to the Free Software |
| 28 | # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
| 29 | |
| 30 | # In case the admin somehow wants to have this script located someplace else, |
| 31 | # he can set BASEDIR, and we will take that. If it is unset we take ${HOME} |
| 32 | # How the admin sets this isn't our place to deal with. One could use a wrapper |
| 33 | # for that. Or pam_env. Or whatever fits in the local setup. :) |
| 34 | BASEDIR=${BASEDIR:-"${HOME}"} |
| 35 | |
| 36 | # Script version. DO NOT CHANGE, *unless* you change the master copy maintained |
| 37 | # by Joerg Jaspert and the Debian mirroradm group. |
| 38 | # This is used to track which mirror is using which script version. |
| 39 | VERSION="80387" |
| 40 | |
| 41 | # Source our common functions |
| 42 | . "${BASEDIR}/etc/common" |
| 43 | |
| 44 | ######################################################################## |
| 45 | ######################################################################## |
| 46 | ## functions ## |
| 47 | ######################################################################## |
| 48 | ######################################################################## |
| 49 | # We want to be able to get told what kind of sync we should do. This |
| 50 | # might be anything, from the archive to sync, the stage to do, etc. A |
| 51 | # list of currently understood and valid options is below. Multiple |
| 52 | # options are seperated by space. All the words have to have the word |
| 53 | # sync: in front or nothing will get used! |
| 54 | # |
| 55 | # Option Behaviour |
| 56 | # stage1 Only do stage1 sync |
| 57 | # stage2 Only do stage2 sync |
| 58 | # all Do a complete sync |
| 59 | # mhop Do a mhop sync, usually additionally to stage1 |
| 60 | # archive:foo Sync archive foo (if config for foo is available) |
| 61 | # callback Call back when done (needs proper ssh setup for this to |
| 62 | # work). It will always use the "command" callback:$HOSTNAME |
| 63 | # where $HOSTNAME is the one defined below/in config and |
| 64 | # will happen before slave mirrors are triggered. |
| 65 | # |
| 66 | # So to get us to sync all of the archive behind bpo and call back when |
| 67 | # we are done, a trigger command of |
| 68 | # "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the |
| 69 | # trick. |
| 70 | check_commandline() { |
| 71 | while [ $# -gt 0 ]; do |
| 72 | case "$1" in |
| 73 | sync:stage1) |
| 74 | SYNCSTAGE1="true" |
| 75 | SYNCALL="false" |
| 76 | ;; |
| 77 | sync:stage2) |
| 78 | SYNCSTAGE2="true" |
| 79 | SYNCALL="false" |
| 80 | ;; |
| 81 | sync:callback) |
| 82 | SYNCCALLBACK="true" |
| 83 | ;; |
| 84 | sync:archive:*) |
| 85 | ARCHIVE=${1##sync:archive:} |
| 86 | # We do not like / or . in the remotely supplied archive name. |
| 87 | ARCHIVE=${ARCHIVE//\/} |
| 88 | ARCHIVE=${ARCHIVE//.} |
| 89 | ;; |
| 90 | sync:all) |
| 91 | SYNCALL="true" |
| 92 | ;; |
| 93 | sync:mhop) |
| 94 | SYNCMHOP="true" |
| 95 | ;; |
| 96 | *) |
| 97 | echo "Unknown option ${1} ignored" |
| 98 | ;; |
| 99 | esac |
| 100 | shift # Check next set of parameters. |
| 101 | done |
| 102 | } |
| 103 | |
| 104 | # All the stuff we want to do when we exit, no matter where |
| 105 | cleanup() { |
| 106 | trap - ERR TERM HUP INT QUIT EXIT |
| 107 | # all done. Mail the log, exit. |
| 108 | log "Mirrorsync done"; |
| 109 | |
| 110 | # Lets get a statistical value |
| 111 | SPEED="unknown" |
| 112 | if [ -f "${LOGDIR}/rsync-${NAME}.log" ]; then |
| 113 | SPEED=$( |
| 114 | SPEEDLINE=$(egrep '[0-9.]+ bytes/sec' "${LOGDIR}/rsync-${NAME}.log") |
| 115 | set "nothing" ${SPEEDLINE} |
| 116 | echo ${8:-""} |
| 117 | ) |
| 118 | if [ -n "${SPEED}" ]; then |
| 119 | SPEED=${SPEED%%.*} |
| 120 | SPEED=$(( $SPEED / 1024 )) |
| 121 | fi |
| 122 | fi |
| 123 | log "Rsync transfer speed: ${SPEED} KB/s" |
| 124 | |
| 125 | if [ -n "${MAILTO}" ]; then |
| 126 | # In case rsync had something on stderr |
| 127 | if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then |
| 128 | mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" |
| 129 | fi |
| 130 | if [ "x${ERRORSONLY}x" = "xfalsex" ]; then |
| 131 | # And the normal log |
| 132 | MAILFILES="${LOG}" |
| 133 | if [ "x${FULLLOGS}x" = "xtruex" ]; then |
| 134 | # Someone wants full logs including rsync |
| 135 | MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" |
| 136 | fi |
| 137 | cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} |
| 138 | fi |
| 139 | fi |
| 140 | |
| 141 | savelog "${LOGDIR}/rsync-${NAME}.log" |
| 142 | savelog "${LOGDIR}/rsync-${NAME}.error" |
| 143 | savelog "$LOG" > /dev/null |
| 144 | |
| 145 | rm -f "${LOCK}" |
| 146 | } |
| 147 | |
| 148 | # Check rsyncs return value |
| 149 | check_rsync() { |
| 150 | ret=$1 |
| 151 | msg=$2 |
| 152 | |
| 153 | # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED |
| 154 | # and us re-running. If it's not, uplink is broken anyways. |
| 155 | case "${ret}" in |
| 156 | 0) return 0;; |
| 157 | 24) return 0;; |
| 158 | 23) return 2;; |
| 159 | 30) return 2;; |
| 160 | *) |
| 161 | error "ERROR: ${msg}" |
| 162 | return 1 |
| 163 | ;; |
| 164 | esac |
| 165 | } |
| 166 | |
| 167 | ######################################################################## |
| 168 | ######################################################################## |
| 169 | |
| 170 | |
| 171 | # As what are we called? |
| 172 | NAME="$(basename $0)" |
| 173 | # The original command line arguments need to be saved! |
| 174 | if [ $# -gt 0 ]; then |
| 175 | ORIGINAL_COMMAND=$* |
| 176 | else |
| 177 | ORIGINAL_COMMAND="" |
| 178 | fi |
| 179 | |
| 180 | SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} |
| 181 | # Now, check if we got told about stuff via ssh |
| 182 | if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then |
| 183 | # We deliberately add "nothing" and ignore it right again, to avoid |
| 184 | # people from outside putting some set options in the first place, |
| 185 | # making us parse them... |
| 186 | set "nothing" "${SSH_ORIGINAL_COMMAND}" |
| 187 | shift |
| 188 | # Yes, unqouted $* here. Or the function will only see it as one |
| 189 | # parameter, which doesnt help the case in it. |
| 190 | check_commandline $* |
| 191 | fi |
| 192 | |
| 193 | # Now, we can locally override all the above variables by just putting |
| 194 | # them into the .ssh/authorized_keys file forced command. |
| 195 | if [ -n "${ORIGINAL_COMMAND}" ]; then |
| 196 | set ${ORIGINAL_COMMAND} |
| 197 | check_commandline $* |
| 198 | fi |
| 199 | |
| 200 | # If we have been told to do stuff for a different archive than default, |
| 201 | # set the name accordingly. |
| 202 | ARCHIVE=${ARCHIVE:-""} |
| 203 | if [ -n "${ARCHIVE}" ]; then |
| 204 | NAME="${NAME}-${ARCHIVE}" |
| 205 | fi |
| 206 | |
| 207 | # Now source the config for the archive we run on. |
| 208 | # (Yes, people can also overwrite the options above in the config file |
| 209 | # if they want to) |
| 210 | if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then |
| 211 | . "${BASEDIR}/etc/${NAME}.conf" |
| 212 | else |
| 213 | echo "Nono, you can't tell us about random archives. Bad boy!" |
| 214 | exit 1 |
| 215 | fi |
| 216 | |
| 217 | ######################################################################## |
| 218 | # Config options go here. Feel free to overwrite them in the config # |
| 219 | # file if you need to. # |
| 220 | # On debian.org machines the defaults should be ok. # |
| 221 | # # |
| 222 | # The following extra variables can be defined in the config file: # |
| 223 | # # |
| 224 | # ARCH_EXCLUDE # |
| 225 | # can be used to exclude a complete architecture from # |
| 226 | # mirrorring. Use as space seperated list. # |
| 227 | # Possible values are: # |
| 228 | # alpha, amd64, arm, armel, armhf, hppa, hurd-i386, i386, ia64, mips # |
| 229 | # mipsel, powerpc, s390, s390x, sparc, kfreebsd-i386, kfreebsd-amd64 # |
| 230 | # and source. # |
| 231 | # eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 s390x sparc" # |
| 232 | # # |
| 233 | # An unset value will mirror all architectures # |
| 234 | ######################################################################## |
| 235 | |
| 236 | ######################################################################## |
| 237 | # There should be nothing to edit here, use the config file # |
| 238 | ######################################################################## |
| 239 | MIRRORNAME=${MIRRORNAME:-$(hostname -f)} |
| 240 | # Where to put logfiles in |
| 241 | LOGDIR=${LOGDIR:-"${BASEDIR}/log"} |
| 242 | # Our own logfile |
| 243 | LOG=${LOG:-"${LOGDIR}/${NAME}.log"} |
| 244 | |
| 245 | # Where should we put all the mirrored files? |
| 246 | TO=${TO:-"/srv/mirrors/debian/"} |
| 247 | |
| 248 | # used by log() and error() |
| 249 | PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} |
| 250 | |
| 251 | # Where to send mails about mirroring to? |
| 252 | if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then |
| 253 | # We are not on a debian.org host |
| 254 | MAILTO=${MAILTO:-"root"} |
| 255 | else |
| 256 | # Yay, on a .debian.org host |
| 257 | MAILTO=${MAILTO:-"mirrorlogs@debian.org"} |
| 258 | fi |
| 259 | # Want errors only or every log? |
| 260 | ERRORSONLY=${ERRORSONLY:-"true"} |
| 261 | # Want full logs, ie. including the rsync one? |
| 262 | FULLLOGS=${FULLLOGS:-"false"} |
| 263 | |
| 264 | # How many logfiles to keep |
| 265 | LOGROTATE=${LOGROTATE:-14} |
| 266 | |
| 267 | # Our lockfile |
| 268 | LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"} |
| 269 | # timeout for the lockfile, in case we have bash older than v4 (and no /proc) |
| 270 | LOCKTIMEOUT=${LOCKTIMEOUT:-3600} |
| 271 | # Do we need another rsync run? |
| 272 | UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" |
| 273 | # Trace file for mirror stats and checks (make sure we get full hostname) |
| 274 | TRACE=${TRACE:-"project/trace/${MIRRORNAME}"} |
| 275 | |
| 276 | # rsync program |
| 277 | RSYNC=${RSYNC:-rsync} |
| 278 | # Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete |
| 279 | # excluded files |
| 280 | RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"} |
| 281 | # limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited |
| 282 | RSYNC_BW=${RSYNC_BW:-0} |
| 283 | # Default rsync options for *every* rsync call |
| 284 | RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"} |
| 285 | # Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files |
| 286 | RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"} |
| 287 | # Options for the second pass, where we do want everything, including deletion of old and now unused files |
| 288 | RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"} |
| 289 | # Which rsync share to use on our upstream mirror? |
| 290 | RSYNC_PATH=${RSYNC_PATH:-"debian"} |
| 291 | |
| 292 | # Now add the bwlimit option. As default is 0 we always add it, rsync interprets |
| 293 | # 0 as unlimited, so this is safe. |
| 294 | RSYNC_OPTIONS="--bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}" |
| 295 | |
| 296 | # We have no default host to sync from, but will error out if its unset |
| 297 | RSYNC_HOST=${RSYNC_HOST:-""} |
| 298 | # Error out if we have no host to sync from |
| 299 | if [ -z "${RSYNC_HOST}" ]; then |
| 300 | error "Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf" |
| 301 | fi |
| 302 | |
| 303 | # our username for the rsync share |
| 304 | RSYNC_USER=${RSYNC_USER:-""} |
| 305 | # the password |
| 306 | RSYNC_PASSWORD=${RSYNC_PASSWORD:-""} |
| 307 | |
| 308 | # a possible proxy |
| 309 | RSYNC_PROXY=${RSYNC_PROXY:-""} |
| 310 | |
| 311 | # Do we sync stage1? |
| 312 | SYNCSTAGE1=${SYNCSTAGE1:-"false"} |
| 313 | # Do we sync stage2? |
| 314 | SYNCSTAGE2=${SYNCSTAGE2:-"false"} |
| 315 | # Do we sync all? |
| 316 | SYNCALL=${SYNCALL:-"true"} |
| 317 | # Do we have a mhop sync? |
| 318 | SYNCMHOP=${SYNCMHOP:-"false"} |
| 319 | # Do we callback? |
| 320 | SYNCCALLBACK=${SYNCCALLBACK:-"false"} |
| 321 | # If we call back we need some more options defined in the config file. |
| 322 | CALLBACKUSER=${CALLBACKUSER:-"archvsync"} |
| 323 | CALLBACKHOST=${CALLBACKHOST:-"none"} |
| 324 | CALLBACKKEY=${CALLBACKKEY:-"none"} |
| 325 | |
| 326 | # General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that! |
| 327 | EXCLUDE=${EXCLUDE:-""} |
| 328 | |
| 329 | # The temp directory used by rsync --delay-updates is not |
| 330 | # world-readable remotely. Always exclude it to avoid errors. |
| 331 | EXCLUDE="${EXCLUDE} --exclude .~tmp~/" |
| 332 | |
| 333 | SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""} |
| 334 | ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} |
| 335 | # Exclude architectures defined in $ARCH_EXCLUDE |
| 336 | for ARCH in ${ARCH_EXCLUDE}; do |
| 337 | EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes" |
| 338 | if [ "${ARCH}" = "source" ]; then |
| 339 | if [ -z ${SOURCE_EXCLUDE} ]; then |
| 340 | SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.tar.xz -exclude *.diff.bz2 --exclude *.dsc " |
| 341 | fi |
| 342 | fi |
| 343 | done |
| 344 | |
| 345 | # Hooks |
| 346 | HOOK1=${HOOK1:-""} |
| 347 | HOOK2=${HOOK2:-""} |
| 348 | HOOK3=${HOOK3:-""} |
| 349 | HOOK4=${HOOK4:-""} |
| 350 | HOOK5=${HOOK5:-""} |
| 351 | |
| 352 | # Are we a hub? |
| 353 | HUB=${HUB:-"false"} |
| 354 | |
| 355 | ######################################################################## |
| 356 | # Really nothing to see below here. Only code follows. # |
| 357 | ######################################################################## |
| 358 | ######################################################################## |
| 359 | |
| 360 | # Some sane defaults |
| 361 | cd "${BASEDIR}" |
| 362 | umask 022 |
| 363 | |
| 364 | # If we are here for the first time, create the |
| 365 | # destination and the trace directory |
| 366 | mkdir -p "${TO}/project/trace" |
| 367 | |
| 368 | # Used to make sure we will have the archive fully and completly synced before |
| 369 | # we stop, even if we get multiple pushes while this script is running. |
| 370 | # Otherwise we can end up with a half-synced archive: |
| 371 | # - get a push |
| 372 | # - sync, while locked |
| 373 | # - get another push. Of course no extra sync run then happens, we are locked. |
| 374 | # - done. Archive not correctly synced, we don't have all the changes from the second push. |
| 375 | touch "${UPDATEREQUIRED}" |
| 376 | |
| 377 | # Check to see if another sync is in progress |
| 378 | if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then |
| 379 | if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then |
| 380 | # We have a recent enough bash version, lets do it the easy way, |
| 381 | # the lock will contain the right pid, thanks to $BASHPID |
| 382 | if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then |
| 383 | # Process does either not exist or is not owned by us. |
| 384 | echo "$$" > "${LOCK}" |
| 385 | else |
| 386 | echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})" |
| 387 | exit 1 |
| 388 | fi |
| 389 | else |
| 390 | # Old bash, means we dont have the right pid in our lockfile |
| 391 | # So take a different way - guess if it is still there by comparing its age. |
| 392 | # Not optimal, but hey. |
| 393 | stamptime=$(date --reference="${LOCK}" +%s) |
| 394 | unixtime=$(date +%s) |
| 395 | difference=$(( $unixtime - $stamptime )) |
| 396 | if [ ${difference} -ge ${LOCKTIMEOUT} ]; then |
| 397 | # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock |
| 398 | echo "$$" > "${LOCK}" |
| 399 | else |
| 400 | echo "Unable to start rsync, lock file younger than one hour" |
| 401 | exit 1 |
| 402 | fi |
| 403 | fi |
| 404 | fi |
| 405 | |
| 406 | # When we exit normally we call cleanup on our own. Otherwise we want it called by |
| 407 | # this trap. (We can not trap on EXIT, because that is called when the main script |
| 408 | # exits. Which also happens when we background the mainroutine, ie. while we still |
| 409 | # run!) |
| 410 | trap cleanup ERR TERM HUP INT QUIT |
| 411 | |
| 412 | # Start log by redirecting stdout and stderr there and closing stdin |
| 413 | exec >"$LOG" 2>&1 <&- |
| 414 | log "Mirrorsync start" |
| 415 | |
| 416 | # Look who pushed us and note that in the log. |
| 417 | SSH_CONNECTION=${SSH_CONNECTION:-""} |
| 418 | PUSHFROM="${SSH_CONNECTION%%\ *}" |
| 419 | if [ -n "${PUSHFROM}" ]; then |
| 420 | log "We got pushed from ${PUSHFROM}" |
| 421 | fi |
| 422 | |
| 423 | if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then |
| 424 | if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then |
| 425 | SYNCCALLBACK="false" |
| 426 | error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" |
| 427 | fi |
| 428 | fi |
| 429 | |
| 430 | HOOK=( |
| 431 | HOOKNR=1 |
| 432 | HOOKSCR=${HOOK1} |
| 433 | ) |
| 434 | hook $HOOK |
| 435 | |
| 436 | # Now, we might want to sync from anonymous too. |
| 437 | # This is that deep in this script so hook1 could, if wanted, change things! |
| 438 | if [ -z ${RSYNC_USER} ]; then |
| 439 | RSYNCPTH="${RSYNC_HOST}" |
| 440 | else |
| 441 | RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" |
| 442 | fi |
| 443 | |
| 444 | # Now do the actual mirroring, and run as long as we have an updaterequired file. |
| 445 | export RSYNC_PASSWORD |
| 446 | export RSYNC_PROXY |
| 447 | |
| 448 | while [ -e "${UPDATEREQUIRED}" ]; do |
| 449 | log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" |
| 450 | |
| 451 | # if we want stage1 *or* all |
| 452 | if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then |
| 453 | while [ -e "${UPDATEREQUIRED}" ]; do |
| 454 | rm -f "${UPDATEREQUIRED}" |
| 455 | log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" |
| 456 | |
| 457 | set +e |
| 458 | # Step one, sync everything except Packages/Releases |
| 459 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ |
| 460 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" |
| 461 | result=$? |
| 462 | set -e |
| 463 | |
| 464 | log "Back from rsync with returncode ${result}" |
| 465 | done |
| 466 | else |
| 467 | # Fake a good resultcode |
| 468 | result=0 |
| 469 | fi # Sync stage 1? |
| 470 | rm -f "${UPDATEREQUIRED}" |
| 471 | |
| 472 | set +e |
| 473 | check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" |
| 474 | GO=$? |
| 475 | set -e |
| 476 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then |
| 477 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" |
| 478 | elif [ ${GO} -ne 0 ]; then |
| 479 | exit 3 |
| 480 | fi |
| 481 | |
| 482 | HOOK=( |
| 483 | HOOKNR=2 |
| 484 | HOOKSCR=${HOOK2} |
| 485 | ) |
| 486 | hook $HOOK |
| 487 | |
| 488 | # if we want stage2 *or* all |
| 489 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then |
| 490 | log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" |
| 491 | |
| 492 | set +e |
| 493 | # We are lucky, it worked. Now do step 2 and sync again, this time including |
| 494 | # the packages/releases files |
| 495 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ |
| 496 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error" |
| 497 | result=$? |
| 498 | set -e |
| 499 | |
| 500 | log "Back from rsync with returncode ${result}" |
| 501 | else |
| 502 | # Fake a good resultcode |
| 503 | result=0 |
| 504 | fi # Sync stage 2? |
| 505 | |
| 506 | set +e |
| 507 | check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" |
| 508 | GO=$? |
| 509 | set -e |
| 510 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then |
| 511 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" |
| 512 | elif [ ${GO} -ne 0 ]; then |
| 513 | exit 4 |
| 514 | fi |
| 515 | |
| 516 | HOOK=( |
| 517 | HOOKNR=3 |
| 518 | HOOKSCR=${HOOK3} |
| 519 | ) |
| 520 | hook $HOOK |
| 521 | done |
| 522 | |
| 523 | # We only update our tracefile when we had a stage2 or an all sync. |
| 524 | # Otherwise we would update it after stage1 already, which is wrong. |
| 525 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then |
| 526 | if [ -d "$(dirname "${TO}/${TRACE}")" ]; then |
| 527 | LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" |
| 528 | echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" |
| 529 | echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" |
| 530 | fi |
| 531 | fi |
| 532 | |
| 533 | HOOK=( |
| 534 | HOOKNR=4 |
| 535 | HOOKSCR=${HOOK4} |
| 536 | ) |
| 537 | hook $HOOK |
| 538 | |
| 539 | if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then |
| 540 | set +e |
| 541 | callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" |
| 542 | set -e |
| 543 | fi |
| 544 | |
| 545 | # Remove the Archive-Update-in-Progress file before we push our downstreams. |
| 546 | rm -f "${LOCK}" |
| 547 | |
| 548 | if [ x${HUB} = "xtrue" ]; then |
| 549 | # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop |
| 550 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then |
| 551 | RUNMIRRORARGS="" |
| 552 | if [ -n "${ARCHIVE}" ]; then |
| 553 | # We tell runmirrors about the archive we are running on. |
| 554 | RUNMIRRORARGS="-a ${ARCHIVE}" |
| 555 | fi |
| 556 | # We also tell runmirrors that we are running it from within ftpsync, so it can change |
| 557 | # the way it works with mhop based on that. |
| 558 | RUNMIRRORARGS="${RUNMIRRORARGS} -f" |
| 559 | |
| 560 | if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then |
| 561 | # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will |
| 562 | # not get to this point. |
| 563 | # So if that happens, tell runmirrors we are doing mhop |
| 564 | RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" |
| 565 | elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then |
| 566 | RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" |
| 567 | elif [ "xtruex" = "x${SYNCALL}x" ]; then |
| 568 | RUNMIRRORARGS="${RUNMIRRORARGS} -k all" |
| 569 | fi |
| 570 | log "Trigger slave mirrors using ${RUNMIRRORARGS}" |
| 571 | ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} |
| 572 | log "Trigger slave done" |
| 573 | |
| 574 | HOOK=( |
| 575 | HOOKNR=5 |
| 576 | HOOKSCR=${HOOK5} |
| 577 | ) |
| 578 | hook $HOOK |
| 579 | fi |
| 580 | fi |
| 581 | |
| 582 | # All done, lets call cleanup |
| 583 | cleanup |