| 1 | #! /bin/bash |
| 2 | # No, we can not deal with sh alone. |
| 3 | |
| 4 | set -e |
| 5 | set -u |
| 6 | # ERR traps should be inherited from functions too. (And command |
| 7 | # substitutions and subshells and whatnot, but for us the function is |
| 8 | # the important part here) |
| 9 | set -E |
| 10 | |
| 11 | # ftpsync script for Debian |
| 12 | # Based losely on a number of existing scripts, written by an |
| 13 | # unknown number of different people over the years. |
| 14 | # |
| 15 | # Copyright (C) 2008-2012 Joerg Jaspert <joerg@debian.org> |
| 16 | # |
| 17 | # This program is free software; you can redistribute it and/or |
| 18 | # modify it under the terms of the GNU General Public License as |
| 19 | # published by the Free Software Foundation; version 2. |
| 20 | # |
| 21 | # This program is distributed in the hope that it will be useful, but |
| 22 | # WITHOUT ANY WARRANTY; without even the implied warranty of |
| 23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 24 | # General Public License for more details. |
| 25 | # |
| 26 | # You should have received a copy of the GNU General Public License |
| 27 | # along with this program; if not, write to the Free Software |
| 28 | # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
| 29 | |
| 30 | # In case the admin somehow wants to have this script located someplace else, |
| 31 | # he can set BASEDIR, and we will take that. If it is unset we take ${HOME} |
| 32 | # How the admin sets this isn't our place to deal with. One could use a wrapper |
| 33 | # for that. Or pam_env. Or whatever fits in the local setup. :) |
| 34 | BASEDIR=${BASEDIR:-"${HOME}"} |
| 35 | |
| 36 | # Script version. DO NOT CHANGE, *unless* you change the master copy maintained |
| 37 | # by Joerg Jaspert and the Debian mirroradm group. |
| 38 | # This is used to track which mirror is using which script version. |
| 39 | VERSION="20130605" |
| 40 | |
| 41 | # Source our common functions |
| 42 | . "${BASEDIR}/etc/common" |
| 43 | |
| 44 | ######################################################################## |
| 45 | ######################################################################## |
| 46 | ## functions ## |
| 47 | ######################################################################## |
| 48 | ######################################################################## |
| 49 | # We want to be able to get told what kind of sync we should do. This |
| 50 | # might be anything, from the archive to sync, the stage to do, etc. A |
| 51 | # list of currently understood and valid options is below. Multiple |
| 52 | # options are seperated by space. All the words have to have the word |
| 53 | # sync: in front or nothing will get used! |
| 54 | # |
| 55 | # Option Behaviour |
| 56 | # stage1 Only do stage1 sync |
| 57 | # stage2 Only do stage2 sync |
| 58 | # all Do a complete sync |
| 59 | # mhop Do a mhop sync, usually additionally to stage1 |
| 60 | # archive:foo Sync archive foo (if config for foo is available) |
| 61 | # callback Call back when done (needs proper ssh setup for this to |
| 62 | # work). It will always use the "command" callback:$HOSTNAME |
| 63 | # where $HOSTNAME is the one defined below/in config and |
| 64 | # will happen before slave mirrors are triggered. |
| 65 | # |
| 66 | # So to get us to sync all of the archive behind bpo and call back when |
| 67 | # we are done, a trigger command of |
| 68 | # "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the |
| 69 | # trick. |
| 70 | check_commandline() { |
| 71 | while [ $# -gt 0 ]; do |
| 72 | case "$1" in |
| 73 | sync:stage1) |
| 74 | SYNCSTAGE1="true" |
| 75 | SYNCALL="false" |
| 76 | ;; |
| 77 | sync:stage2) |
| 78 | SYNCSTAGE2="true" |
| 79 | SYNCALL="false" |
| 80 | ;; |
| 81 | sync:callback) |
| 82 | SYNCCALLBACK="true" |
| 83 | ;; |
| 84 | sync:archive:*) |
| 85 | ARCHIVE=${1##sync:archive:} |
| 86 | # We do not like / or . in the remotely supplied archive name. |
| 87 | ARCHIVE=${ARCHIVE//\/} |
| 88 | ARCHIVE=${ARCHIVE//.} |
| 89 | ;; |
| 90 | sync:all) |
| 91 | SYNCALL="true" |
| 92 | ;; |
| 93 | sync:mhop) |
| 94 | SYNCMHOP="true" |
| 95 | ;; |
| 96 | *) |
| 97 | echo "Unknown option ${1} ignored" |
| 98 | ;; |
| 99 | esac |
| 100 | shift # Check next set of parameters. |
| 101 | done |
| 102 | } |
| 103 | |
| 104 | # All the stuff we want to do when we exit, no matter where |
| 105 | cleanup() { |
| 106 | trap - ERR TERM HUP INT QUIT EXIT |
| 107 | # all done. Mail the log, exit. |
| 108 | log "Mirrorsync done"; |
| 109 | |
| 110 | # Lets get a statistical value |
| 111 | SPEED="unknown" |
| 112 | if [ -f "${LOGDIR}/rsync-${NAME}.log" ]; then |
| 113 | SPEED=$( |
| 114 | SPEEDLINE=$(egrep '[0-9.]+ bytes/sec' "${LOGDIR}/rsync-${NAME}.log") |
| 115 | set "nothing" ${SPEEDLINE} |
| 116 | echo ${8:-""} |
| 117 | ) |
| 118 | if [ -n "${SPEED}" ]; then |
| 119 | SPEED=${SPEED%%.*} |
| 120 | SPEED=$(( $SPEED / 1024 )) |
| 121 | fi |
| 122 | fi |
| 123 | log "Rsync transfer speed: ${SPEED} KB/s" |
| 124 | |
| 125 | if [ -n "${MAILTO}" ]; then |
| 126 | # In case rsync had something on stderr |
| 127 | if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then |
| 128 | mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" |
| 129 | fi |
| 130 | if [ "x${ERRORSONLY}x" = "xfalsex" ]; then |
| 131 | # And the normal log |
| 132 | MAILFILES="${LOG}" |
| 133 | if [ "x${FULLLOGS}x" = "xtruex" ]; then |
| 134 | # Someone wants full logs including rsync |
| 135 | MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" |
| 136 | fi |
| 137 | cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} |
| 138 | fi |
| 139 | fi |
| 140 | |
| 141 | savelog "${LOGDIR}/rsync-${NAME}.log" |
| 142 | savelog "${LOGDIR}/rsync-${NAME}.error" |
| 143 | savelog "$LOG" > /dev/null |
| 144 | |
| 145 | rm -f "${LOCK}" |
| 146 | } |
| 147 | |
| 148 | # Check rsyncs return value |
| 149 | check_rsync() { |
| 150 | ret=$1 |
| 151 | msg=$2 |
| 152 | |
| 153 | # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED |
| 154 | # and us re-running. If it's not, uplink is broken anyways. |
| 155 | case "${ret}" in |
| 156 | 0) return 0;; |
| 157 | 24) return 0;; |
| 158 | 23) return 2;; |
| 159 | 30) return 2;; |
| 160 | *) |
| 161 | error "ERROR: ${msg}" |
| 162 | return 1 |
| 163 | ;; |
| 164 | esac |
| 165 | } |
| 166 | |
| 167 | ######################################################################## |
| 168 | ######################################################################## |
| 169 | |
| 170 | |
| 171 | # As what are we called? |
| 172 | NAME="$(basename $0)" |
| 173 | # The original command line arguments need to be saved! |
| 174 | if [ $# -gt 0 ]; then |
| 175 | ORIGINAL_COMMAND=$* |
| 176 | else |
| 177 | ORIGINAL_COMMAND="" |
| 178 | fi |
| 179 | |
| 180 | SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} |
| 181 | # Now, check if we got told about stuff via ssh |
| 182 | if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then |
| 183 | # We deliberately add "nothing" and ignore it right again, to avoid |
| 184 | # people from outside putting some set options in the first place, |
| 185 | # making us parse them... |
| 186 | set "nothing" "${SSH_ORIGINAL_COMMAND}" |
| 187 | shift |
| 188 | # Yes, unqouted $* here. Or the function will only see it as one |
| 189 | # parameter, which doesnt help the case in it. |
| 190 | check_commandline $* |
| 191 | fi |
| 192 | |
| 193 | # Now, we can locally override all the above variables by just putting |
| 194 | # them into the .ssh/authorized_keys file forced command. |
| 195 | if [ -n "${ORIGINAL_COMMAND}" ]; then |
| 196 | set ${ORIGINAL_COMMAND} |
| 197 | check_commandline $* |
| 198 | fi |
| 199 | |
| 200 | # If we have been told to do stuff for a different archive than default, |
| 201 | # set the name accordingly. |
| 202 | ARCHIVE=${ARCHIVE:-""} |
| 203 | if [ -n "${ARCHIVE}" ]; then |
| 204 | NAME="${NAME}-${ARCHIVE}" |
| 205 | fi |
| 206 | |
| 207 | # Now source the config for the archive we run on. |
| 208 | # (Yes, people can also overwrite the options above in the config file |
| 209 | # if they want to) |
| 210 | if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then |
| 211 | . "${BASEDIR}/etc/${NAME}.conf" |
| 212 | else |
| 213 | echo "Nono, you can't tell us about random archives. Bad boy!" |
| 214 | exit 1 |
| 215 | fi |
| 216 | |
| 217 | ######################################################################## |
| 218 | # Config options go here. Feel free to overwrite them in the config # |
| 219 | # file if you need to. # |
| 220 | # On debian.org machines the defaults should be ok. # |
| 221 | # # |
| 222 | # The following extra variables can be defined in the config file: # |
| 223 | # # |
| 224 | # ARCH_EXCLUDE # |
| 225 | # can be used to exclude a complete architecture from # |
| 226 | # mirrorring. Use as space seperated list. # |
| 227 | # Possible values are: # |
| 228 | # alpha, amd64, arm, armel, armhf, hppa, hurd-i386, i386, ia64, mips # |
| 229 | # mipsel, powerpc, s390, s390x, sparc, kfreebsd-i386, kfreebsd-amd64 # |
| 230 | # and source. # |
| 231 | # eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 s390x sparc" # |
| 232 | # # |
| 233 | # An unset value will mirror all architectures # |
| 234 | ######################################################################## |
| 235 | |
| 236 | ######################################################################## |
| 237 | # There should be nothing to edit here, use the config file # |
| 238 | ######################################################################## |
| 239 | MIRRORNAME=${MIRRORNAME:-$(hostname -f)} |
| 240 | # Where to put logfiles in |
| 241 | LOGDIR=${LOGDIR:-"${BASEDIR}/log"} |
| 242 | # Our own logfile |
| 243 | LOG=${LOG:-"${LOGDIR}/${NAME}.log"} |
| 244 | |
| 245 | # Where should we put all the mirrored files? |
| 246 | TO=${TO:-"/srv/mirrors/debian/"} |
| 247 | |
| 248 | # used by log() and error() |
| 249 | PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} |
| 250 | |
| 251 | # Where to send mails about mirroring to? |
| 252 | if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then |
| 253 | # We are not on a debian.org host |
| 254 | MAILTO=${MAILTO:-"root"} |
| 255 | else |
| 256 | # Yay, on a .debian.org host |
| 257 | MAILTO=${MAILTO:-"mirrorlogs@debian.org"} |
| 258 | fi |
| 259 | # Want errors only or every log? |
| 260 | ERRORSONLY=${ERRORSONLY:-"true"} |
| 261 | # Want full logs, ie. including the rsync one? |
| 262 | FULLLOGS=${FULLLOGS:-"false"} |
| 263 | |
| 264 | # How many logfiles to keep |
| 265 | LOGROTATE=${LOGROTATE:-14} |
| 266 | |
| 267 | # Our lockfile |
| 268 | LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"} |
| 269 | # timeout for the lockfile, in case we have bash older than v4 (and no /proc) |
| 270 | LOCKTIMEOUT=${LOCKTIMEOUT:-3600} |
| 271 | # sleeping time when an AUIP file is found but is not ours |
| 272 | UIPSLEEP=${UIPSLEEP:-1200} |
| 273 | # retries whenever an upstream (or possibly stale) AUIP file is found |
| 274 | UIPRETRIES=${UIPRETRIES:-3} |
| 275 | # Do we need another rsync run? |
| 276 | UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" |
| 277 | # Trace file for mirror stats and checks (make sure we get full hostname) |
| 278 | TRACE=${TRACE:-"project/trace/${MIRRORNAME}"} |
| 279 | # The trace file can have different format/contents. Here you can select |
| 280 | # what it will be. |
| 281 | # Possible values are |
| 282 | # "full" - all information |
| 283 | # "terse" - basic, timestamp only (date -u) |
| 284 | # "touch" - just touch the file in existance |
| 285 | # "none" - no tracefile at all |
| 286 | # |
| 287 | # Default and required value for Debian mirrors is full. |
| 288 | EXTENDEDTRACE=${EXTENDEDTRACE:-"full"} |
| 289 | |
| 290 | # rsync program |
| 291 | RSYNC=${RSYNC:-rsync} |
| 292 | # Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete |
| 293 | # excluded files |
| 294 | RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"} |
| 295 | # limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited |
| 296 | RSYNC_BW=${RSYNC_BW:-0} |
| 297 | RSYNC_PROTOCOL=$(rsync_protocol) |
| 298 | |
| 299 | # Set the delete method to --delete-delay if protocol version is 30 or |
| 300 | # greater (meaning rsync 3.0.0 or greater is used). Use --delete-after |
| 301 | # otherwise. |
| 302 | if [ 30 -le $RSYNC_PROTOCOL ]; then |
| 303 | RSYNC_DELETE_METHOD=delay |
| 304 | else |
| 305 | RSYNC_DELETE_METHOD=after |
| 306 | fi |
| 307 | |
| 308 | # Default rsync options for *every* rsync call |
| 309 | RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"} |
| 310 | # Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files |
| 311 | RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --exclude=i18n/* --exclude=ls-lR*"} |
| 312 | # Options for the second pass, where we do want everything, including deletion of old and now unused files |
| 313 | RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-excluded"} |
| 314 | # Which rsync share to use on our upstream mirror? |
| 315 | RSYNC_PATH=${RSYNC_PATH:-"debian"} |
| 316 | |
| 317 | # Extra rsync options as defined by the admin locally. Won't be set |
| 318 | # to any default by ftpsync. Those will be added to EACH AND EVERY rsync call. |
| 319 | RSYNC_EXTRA=${RSYNC_EXTRA:-""} |
| 320 | |
| 321 | # Now add the bwlimit option. As default is 0 we always add it, rsync interprets |
| 322 | # 0 as unlimited, so this is safe. |
| 323 | RSYNC_OPTIONS="${RSYNC_EXTRA} --bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}" |
| 324 | |
| 325 | # Finally, make sure RSYNC_OPTIONS2 has either --delete-after or --deleter-delay |
| 326 | RSYNC_OPTION_REGEX="--delete-(after|delay)" |
| 327 | if ! [[ ${RSYNC_OPTIONS2} =~ ${RSYNC_OPTION_REGEX} ]]; then |
| 328 | RSYNC_OPTIONS2+=" --delete-${RSYNC_DELETE_METHOD}" |
| 329 | fi |
| 330 | unset RSYNC_OPTION_REGEX |
| 331 | |
| 332 | # We have no default host to sync from, but will error out if its unset |
| 333 | RSYNC_HOST=${RSYNC_HOST:-""} |
| 334 | # Error out if we have no host to sync from |
| 335 | if [ -z "${RSYNC_HOST}" ]; then |
| 336 | error "Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf" |
| 337 | fi |
| 338 | |
| 339 | # our username for the rsync share |
| 340 | RSYNC_USER=${RSYNC_USER:-""} |
| 341 | # the password |
| 342 | RSYNC_PASSWORD=${RSYNC_PASSWORD:-""} |
| 343 | |
| 344 | # a possible proxy |
| 345 | RSYNC_PROXY=${RSYNC_PROXY:-""} |
| 346 | |
| 347 | # Do we sync stage1? |
| 348 | SYNCSTAGE1=${SYNCSTAGE1:-"false"} |
| 349 | # Do we sync stage2? |
| 350 | SYNCSTAGE2=${SYNCSTAGE2:-"false"} |
| 351 | # Do we sync all? |
| 352 | SYNCALL=${SYNCALL:-"true"} |
| 353 | # Do we have a mhop sync? |
| 354 | SYNCMHOP=${SYNCMHOP:-"false"} |
| 355 | # Do we callback? (May get changed later) |
| 356 | SYNCCALLBACK=${SYNCCALLBACK:-"false"} |
| 357 | # If we call back we need some more options defined in the config file. |
| 358 | CALLBACKUSER=${CALLBACKUSER:-"archvsync"} |
| 359 | CALLBACKHOST=${CALLBACKHOST:-"none"} |
| 360 | CALLBACKKEY=${CALLBACKKEY:-"none"} |
| 361 | |
| 362 | # General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that! |
| 363 | EXCLUDE=${EXCLUDE:-""} |
| 364 | |
| 365 | # The temp directory used by rsync --delay-updates is not |
| 366 | # world-readable remotely. Always exclude it to avoid errors. |
| 367 | EXCLUDE="${EXCLUDE} --exclude=.~tmp~/" |
| 368 | |
| 369 | SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""} |
| 370 | ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} |
| 371 | # Exclude architectures defined in $ARCH_EXCLUDE |
| 372 | for ARCH in ${ARCH_EXCLUDE}; do |
| 373 | EXCLUDE="${EXCLUDE} --exclude=binary-${ARCH}/ --exclude=installer-${ARCH}/ --exclude=Contents-${ARCH}.gz --exclude=Contents-udeb-${ARCH}.gz --exclude=Contents-${ARCH}.diff/ --exclude=arch-${ARCH}.files --exclude=arch-${ARCH}.list.gz --exclude=*_${ARCH}.deb --exclude=*_${ARCH}.udeb --exclude=*_${ARCH}.changes" |
| 374 | if [ "${ARCH}" = "source" ]; then |
| 375 | if [ -z ${SOURCE_EXCLUDE} ]; then |
| 376 | SOURCE_EXCLUDE=" --exclude=source/ --exclude=*.tar.gz --exclude=*.diff.gz --exclude=*.tar.bz2 --exclude=*.tar.xz --exclude=*.diff.bz2 --exclude=*.dsc " |
| 377 | fi |
| 378 | fi |
| 379 | done |
| 380 | |
| 381 | # Hooks |
| 382 | HOOK1=${HOOK1:-""} |
| 383 | HOOK2=${HOOK2:-""} |
| 384 | HOOK3=${HOOK3:-""} |
| 385 | HOOK4=${HOOK4:-""} |
| 386 | HOOK5=${HOOK5:-""} |
| 387 | |
| 388 | # Are we a hub? |
| 389 | HUB=${HUB:-"false"} |
| 390 | |
| 391 | ######################################################################## |
| 392 | # Really nothing to see below here. Only code follows. # |
| 393 | ######################################################################## |
| 394 | ######################################################################## |
| 395 | |
| 396 | # Some sane defaults |
| 397 | cd "${BASEDIR}" |
| 398 | umask 022 |
| 399 | |
| 400 | # If we are here for the first time, create the |
| 401 | # destination and the trace directory |
| 402 | mkdir -p "${TO}/project/trace" |
| 403 | |
| 404 | # Used to make sure we will have the archive fully and completly synced before |
| 405 | # we stop, even if we get multiple pushes while this script is running. |
| 406 | # Otherwise we can end up with a half-synced archive: |
| 407 | # - get a push |
| 408 | # - sync, while locked |
| 409 | # - get another push. Of course no extra sync run then happens, we are locked. |
| 410 | # - done. Archive not correctly synced, we don't have all the changes from the second push. |
| 411 | touch "${UPDATEREQUIRED}" |
| 412 | |
| 413 | # Check to see if another sync is in progress |
| 414 | if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then |
| 415 | if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then |
| 416 | # We have a recent enough bash version, lets do it the easy way, |
| 417 | # the lock will contain the right pid, thanks to $BASHPID |
| 418 | if ! $(kill -0 $(< ${LOCK}) 2>/dev/null); then |
| 419 | # Process does either not exist or is not owned by us. |
| 420 | echo "$$" > "${LOCK}" |
| 421 | else |
| 422 | echo "Unable to start rsync, lock file still exists, PID $(< ${LOCK})" |
| 423 | exit 1 |
| 424 | fi |
| 425 | else |
| 426 | # Old bash, means we dont have the right pid in our lockfile |
| 427 | # So take a different way - guess if it is still there by comparing its age. |
| 428 | # Not optimal, but hey. |
| 429 | stamptime=$(date --reference="${LOCK}" +%s) |
| 430 | unixtime=$(date +%s) |
| 431 | difference=$(( $unixtime - $stamptime )) |
| 432 | if [ ${difference} -ge ${LOCKTIMEOUT} ]; then |
| 433 | # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock |
| 434 | echo "$$" > "${LOCK}" |
| 435 | else |
| 436 | echo "Unable to start rsync, lock file younger than one hour" |
| 437 | exit 1 |
| 438 | fi |
| 439 | fi |
| 440 | fi |
| 441 | |
| 442 | # When we exit normally we call cleanup on our own. Otherwise we want it called by |
| 443 | # this trap. (We can not trap on EXIT, because that is called when the main script |
| 444 | # exits. Which also happens when we background the mainroutine, ie. while we still |
| 445 | # run!) |
| 446 | trap cleanup ERR TERM HUP INT QUIT |
| 447 | |
| 448 | # Start log by redirecting stdout and stderr there and closing stdin |
| 449 | exec >"$LOG" 2>&1 <&- |
| 450 | log "Mirrorsync start" |
| 451 | |
| 452 | # Look who pushed us and note that in the log. |
| 453 | SSH_CONNECTION=${SSH_CONNECTION:-""} |
| 454 | PUSHFROM="${SSH_CONNECTION%%\ *}" |
| 455 | if [ -n "${PUSHFROM}" ]; then |
| 456 | log "We got pushed from ${PUSHFROM}" |
| 457 | fi |
| 458 | |
| 459 | if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then |
| 460 | if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then |
| 461 | SYNCCALLBACK="false" |
| 462 | error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" |
| 463 | fi |
| 464 | fi |
| 465 | |
| 466 | HOOK=( |
| 467 | HOOKNR=1 |
| 468 | HOOKSCR=${HOOK1} |
| 469 | ) |
| 470 | hook $HOOK |
| 471 | |
| 472 | # Now, we might want to sync from anonymous too. |
| 473 | # This is that deep in this script so hook1 could, if wanted, change things! |
| 474 | if [ -z ${RSYNC_USER} ]; then |
| 475 | RSYNCPTH="${RSYNC_HOST}" |
| 476 | else |
| 477 | RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" |
| 478 | fi |
| 479 | |
| 480 | # Now do the actual mirroring, and run as long as we have an updaterequired file. |
| 481 | export RSYNC_PASSWORD |
| 482 | export RSYNC_PROXY |
| 483 | |
| 484 | UPDATE_RETRIES=0 |
| 485 | |
| 486 | while [ -e "${UPDATEREQUIRED}" ]; do |
| 487 | log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" |
| 488 | |
| 489 | # if we want stage1 *or* all |
| 490 | if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then |
| 491 | while [ -e "${UPDATEREQUIRED}" ]; do |
| 492 | rm -f "${UPDATEREQUIRED}" |
| 493 | log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" |
| 494 | |
| 495 | set +e |
| 496 | # Step one, sync everything except Packages/Releases |
| 497 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ |
| 498 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" |
| 499 | result=$? |
| 500 | set -e |
| 501 | |
| 502 | log "Back from rsync with returncode ${result}" |
| 503 | done |
| 504 | else |
| 505 | # Fake a good resultcode |
| 506 | result=0 |
| 507 | fi # Sync stage 1? |
| 508 | rm -f "${UPDATEREQUIRED}" |
| 509 | |
| 510 | set +e |
| 511 | check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" |
| 512 | GO=$? |
| 513 | set -e |
| 514 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then |
| 515 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" |
| 516 | elif [ ${GO} -ne 0 ]; then |
| 517 | exit 3 |
| 518 | fi |
| 519 | |
| 520 | HOOK=( |
| 521 | HOOKNR=2 |
| 522 | HOOKSCR=${HOOK2} |
| 523 | ) |
| 524 | hook $HOOK |
| 525 | |
| 526 | # if we want stage2 *or* all |
| 527 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then |
| 528 | upstream_uip=false |
| 529 | for aupfile in "${TO}/Archive-Update-in-Progress-"*; do |
| 530 | case "$aupfile" in |
| 531 | "${TO}/Archive-Update-in-Progress-*") |
| 532 | error "Lock file is missing, this should not happen" |
| 533 | ;; |
| 534 | "${LOCK}") |
| 535 | : |
| 536 | ;; |
| 537 | *) |
| 538 | if [ -f "$aupfile" ]; then |
| 539 | # Remove the file, it will be synced again if |
| 540 | # upstream is still not done |
| 541 | rm -f "$aupfile" |
| 542 | else |
| 543 | log "AUIP file '$aupfile' is not really a file, weird" |
| 544 | fi |
| 545 | upstream_uip=true |
| 546 | ;; |
| 547 | esac |
| 548 | done |
| 549 | |
| 550 | if [ "xtruex" = "x${upstream_uip}x" ]; then |
| 551 | log "Upstream archive update in progress, skipping stage2" |
| 552 | if [ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]; then |
| 553 | log "Retrying update in ${UIPSLEEP}" |
| 554 | touch "${UPDATEREQUIRED}" |
| 555 | UPDATE_RETRIES=$(($UPDATE_RETRIES+1)) |
| 556 | sleep "${UIPSLEEP}" |
| 557 | result=0 |
| 558 | else |
| 559 | error "Update has been retried ${UPDATEREQUIRED} times, aborting" |
| 560 | log "Perhaps upstream is still updating or there's a stale AUIP file" |
| 561 | result=1 |
| 562 | fi |
| 563 | else |
| 564 | log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" |
| 565 | |
| 566 | set +e |
| 567 | # We are lucky, it worked. Now do step 2 and sync again, this time including |
| 568 | # the packages/releases files |
| 569 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ |
| 570 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error" |
| 571 | result=$? |
| 572 | set -e |
| 573 | |
| 574 | log "Back from rsync with returncode ${result}" |
| 575 | fi |
| 576 | else |
| 577 | # Fake a good resultcode |
| 578 | result=0 |
| 579 | fi # Sync stage 2? |
| 580 | |
| 581 | set +e |
| 582 | check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" |
| 583 | GO=$? |
| 584 | set -e |
| 585 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then |
| 586 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" |
| 587 | elif [ ${GO} -ne 0 ]; then |
| 588 | exit 4 |
| 589 | fi |
| 590 | |
| 591 | HOOK=( |
| 592 | HOOKNR=3 |
| 593 | HOOKSCR=${HOOK3} |
| 594 | ) |
| 595 | hook $HOOK |
| 596 | done |
| 597 | |
| 598 | # We only update our tracefile when we had a stage2 or an all sync. |
| 599 | # Otherwise we would update it after stage1 already, which is wrong. |
| 600 | |
| 601 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then |
| 602 | case ${EXTENDEDTRACE} in |
| 603 | none) |
| 604 | log "No trace file wanted. Not creating one" |
| 605 | ;; |
| 606 | touch) |
| 607 | log "Just touching the trace file" |
| 608 | touch "${TO}/${TRACE}" |
| 609 | ;; |
| 610 | terse|full) |
| 611 | log "Creating a ${EXTENDEDTRACE} trace file" |
| 612 | if [ -d "$(dirname "${TO}/${TRACE}")" ]; then |
| 613 | LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}.new" |
| 614 | echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}.new" |
| 615 | echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}.new" |
| 616 | if [ "xfullx" = "x${EXTENDEDTRACE}x" ]; then |
| 617 | GLOBALARCHLIST="source amd64 armel armhf hurd-i386 i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc" |
| 618 | |
| 619 | AEXCLUDE="^${ARCH_EXCLUDE// /\$|^}$" |
| 620 | ARCHLIST="" |
| 621 | for ARCH in ${GLOBALARCHLIST}; do |
| 622 | if ! [[ ${ARCH} =~ ${AEXCLUDE} ]]; then |
| 623 | ARCHLIST="${ARCHLIST} ${ARCH}" |
| 624 | fi |
| 625 | done |
| 626 | out="GUESSED:{${ARCHLIST}}" |
| 627 | echo "Architectures: ${out}" >> "${TO}/${TRACE}.new" |
| 628 | echo "Upstream-mirror: ${RSYNC_HOST}" >> "${TO}/${TRACE}.new" |
| 629 | fi # full trace |
| 630 | mv "${TO}/${TRACE}.new" "${TO}/${TRACE}" |
| 631 | fi |
| 632 | ;; |
| 633 | *) |
| 634 | error "Unsupported EXTENDEDTRACE value configured in ${BASEDIR}/etc/${NAME}.conf, please fix" |
| 635 | ;; |
| 636 | esac |
| 637 | fi |
| 638 | |
| 639 | |
| 640 | HOOK=( |
| 641 | HOOKNR=4 |
| 642 | HOOKSCR=${HOOK4} |
| 643 | ) |
| 644 | hook $HOOK |
| 645 | |
| 646 | if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then |
| 647 | set +e |
| 648 | callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" |
| 649 | set -e |
| 650 | fi |
| 651 | |
| 652 | # Remove the Archive-Update-in-Progress file before we push our downstreams. |
| 653 | rm -f "${LOCK}" |
| 654 | |
| 655 | # Check if there is a newer version of ftpsync. If so inform the admin, but not |
| 656 | # more than once every third day. |
| 657 | if [ -r "${TO}/project/ftpsync/LATEST.VERSION" ]; then |
| 658 | LATEST=$(< "${TO}/project/ftpsync/LATEST.VERSION") |
| 659 | if ! [[ ${LATEST} =~ [0-9]+ ]]; then |
| 660 | LATEST=0 |
| 661 | fi |
| 662 | if [ ${LATEST} -gt ${VERSION} ]; then |
| 663 | if [ -n "${MAILTO}" ]; then |
| 664 | difference=0 |
| 665 | if [ -f "${LOGDIR}/ftpsync.newversion" ]; then |
| 666 | stamptime=$(< "${LOGDIR}/ftpsync.newversion") |
| 667 | unixtime=$(date +%s) |
| 668 | difference=$(( $unixtime - $stamptime )) |
| 669 | fi |
| 670 | if [ ${difference} -ge 259200 ]; then |
| 671 | # Only warn every third day |
| 672 | mail -e -s "[$(hostname -s)] Update for ftpsync available" ${MAILTO} <<EOF |
| 673 | Hello admin, |
| 674 | |
| 675 | i found that there is a new version of me available. |
| 676 | Me lonely ftpsync is currently version: ${VERSION} |
| 677 | New release of myself is available as: ${LATEST} |
| 678 | |
| 679 | Me, myself and I - and the Debian mirroradmins - would be very grateful |
| 680 | if you could update me. You can find the latest version on your mirror, |
| 681 | check $(hostname -s):${TO}/project/ftpsync/ftpsync-${LATEST}.tar.gz |
| 682 | |
| 683 | You can ensure the validity of that file by using sha512sum or md5sum |
| 684 | against the available checksum files secured with a signature from the |
| 685 | Debian FTPMaster signing key. |
| 686 | |
| 687 | EOF |
| 688 | |
| 689 | date +%s > "${LOGDIR}/ftpsync.newversion" |
| 690 | fi |
| 691 | fi |
| 692 | else |
| 693 | # Remove a possible stampfile |
| 694 | rm -f "${LOGDIR}/ftpsync.newversion" |
| 695 | fi |
| 696 | fi |
| 697 | |
| 698 | if [ x${HUB} = "xtrue" ]; then |
| 699 | # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop |
| 700 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then |
| 701 | RUNMIRRORARGS="" |
| 702 | if [ -n "${ARCHIVE}" ]; then |
| 703 | # We tell runmirrors about the archive we are running on. |
| 704 | RUNMIRRORARGS="-a ${ARCHIVE}" |
| 705 | fi |
| 706 | # We also tell runmirrors that we are running it from within ftpsync, so it can change |
| 707 | # the way it works with mhop based on that. |
| 708 | RUNMIRRORARGS="${RUNMIRRORARGS} -f" |
| 709 | |
| 710 | if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then |
| 711 | # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will |
| 712 | # not get to this point. |
| 713 | # So if that happens, tell runmirrors we are doing mhop |
| 714 | RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" |
| 715 | elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then |
| 716 | RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" |
| 717 | elif [ "xtruex" = "x${SYNCALL}x" ]; then |
| 718 | RUNMIRRORARGS="${RUNMIRRORARGS} -k all" |
| 719 | fi |
| 720 | log "Trigger slave mirrors using ${RUNMIRRORARGS}" |
| 721 | ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} |
| 722 | log "Trigger slave done" |
| 723 | |
| 724 | HOOK=( |
| 725 | HOOKNR=5 |
| 726 | HOOKSCR=${HOOK5} |
| 727 | ) |
| 728 | hook $HOOK |
| 729 | fi |
| 730 | fi |
| 731 | |
| 732 | # All done, lets call cleanup |
| 733 | cleanup |