2 # No, we can not deal with sh alone.
6 # ERR traps should be inherited from functions too. (And command
7 # substitutions and subshells and whatnot, but for us the function is
8 # the important part here)
11 # ftpsync script for Debian
12 # Based losely on a number of existing scripts, written by an
13 # unknown number of different people over the years.
15 # Copyright (C) 2008,2009,2010,2011 Joerg Jaspert <joerg@debian.org>
17 # This program is free software; you can redistribute it and/or
18 # modify it under the terms of the GNU General Public License as
19 # published by the Free Software Foundation; version 2.
21 # This program is distributed in the hope that it will be useful, but
22 # WITHOUT ANY WARRANTY; without even the implied warranty of
23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24 # General Public License for more details.
26 # You should have received a copy of the GNU General Public License
27 # along with this program; if not, write to the Free Software
28 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
30 # In case the admin somehow wants to have this script located someplace else,
31 # he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
32 # How the admin sets this isn't our place to deal with. One could use a wrapper
33 # for that. Or pam_env. Or whatever fits in the local setup. :)
34 BASEDIR
=${BASEDIR:-"${HOME}"}
36 # Script version. DO NOT CHANGE, *unless* you change the master copy maintained
37 # by Joerg Jaspert and the Debian mirroradm group.
38 # This is used to track which mirror is using which script version.
41 # Source our common functions
42 . "${BASEDIR}/etc
/common
"
44 ########################################################################
45 ########################################################################
47 ########################################################################
48 ########################################################################
49 # We want to be able to get told what kind of sync we should do. This
50 # might be anything, from the archive to sync, the stage to do, etc. A
51 # list of currently understood and valid options is below. Multiple
52 # options are seperated by space. All the words have to have the word
53 # sync: in front or nothing will get used!
56 # stage1 Only do stage1 sync
57 # stage2 Only do stage2 sync
58 # all Do a complete sync
59 # mhop Do a mhop sync, usually additionally to stage1
60 # archive:foo Sync archive foo (if config for foo is available)
61 # callback Call back when done (needs proper ssh setup for this to
62 # work). It will always use the "command" callback:$HOSTNAME
63 # where $HOSTNAME is the one defined below/in config and
64 # will happen before slave mirrors are triggered.
66 # So to get us to sync all of the archive behind bpo and call back when
67 # we are done, a trigger command of
68 # "ssh $USER@
$HOST sync
:all sync
:archive
:bpo sync
:callback
" will do the
71 while [ $# -gt 0 ]; do
85 ARCHIVE=${1##sync:archive:}
86 # We do not like / or . in the remotely supplied archive name.
87 ARCHIVE=${ARCHIVE//\/}
97 echo "Unknown option
${1} ignored
"
100 shift # Check next set of parameters.
104 # All the stuff we want to do when we exit, no matter where
106 trap - ERR TERM HUP INT QUIT EXIT
107 # all done. Mail the log, exit.
108 log "Mirrorsync
done";
110 # Lets get a statistical value
112 if [ -f "${LOGDIR}/rsync-
${NAME}.log
" ]; then
114 SPEEDLINE=$(egrep '[0-9.]+ bytes/sec' "${LOGDIR}/rsync-
${NAME}.log
")
115 set "nothing
" ${SPEEDLINE}
118 if [ -n "${SPEED}" ]; then
120 SPEED=$(( $SPEED / 1024 ))
123 log "Rsync transfer speed
: ${SPEED} KB
/s
"
125 if [ -n "${MAILTO}" ]; then
126 # In case rsync had something on stderr
127 if [ -s "${LOGDIR}/rsync-
${NAME}.error
" ]; then
128 mail -e -s "[${PROGRAM}@$
(hostname
-s
)] ($$
) rsync ERROR on $
(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-
${NAME}.error
"
130 if [ "x
${ERRORSONLY}x
" = "xfalsex
" ]; then
133 if [ "x
${FULLLOGS}x
" = "xtruex
" ]; then
134 # Someone wants full logs including rsync
135 MAILFILES="${MAILFILES} ${LOGDIR}/rsync-
${NAME}.log
"
137 cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$
(hostname
-s
)] archive sync finished on $
(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
141 savelog "${LOGDIR}/rsync-
${NAME}.log
"
142 savelog "${LOGDIR}/rsync-
${NAME}.error
"
143 savelog "$LOG" > /dev/null
148 # Check rsyncs return value
153 # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
154 # and us re-running. If it's not, uplink is broken anyways.
161 error "ERROR
: ${msg}"
167 ########################################################################
168 ########################################################################
171 # As what are we called?
172 NAME="$
(basename $0)"
173 # The original command line arguments need to be saved!
174 if [ $# -gt 0 ]; then
180 SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""}
181 # Now, check if we got told about stuff via ssh
182 if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then
183 # We deliberately add "nothing
" and ignore it right again, to avoid
184 # people from outside putting some set options in the first place,
185 # making us parse them...
186 set "nothing
" "${SSH_ORIGINAL_COMMAND}"
188 # Yes, unqouted $* here. Or the function will only see it as one
189 # parameter, which doesnt help the case in it.
193 # Now, we can locally override all the above variables by just putting
194 # them into the .ssh/authorized_keys file forced command.
195 if [ -n "${ORIGINAL_COMMAND}" ]; then
196 set ${ORIGINAL_COMMAND}
200 # If we have been told to do stuff for a different archive than default,
201 # set the name accordingly.
202 ARCHIVE=${ARCHIVE:-""}
203 if [ -n "${ARCHIVE}" ]; then
204 NAME="${NAME}-${ARCHIVE}"
207 # Now source the config for the archive we run on.
208 # (Yes, people can also overwrite the options above in the config file
210 if [ -f "${BASEDIR}/etc
/${NAME}.conf
" ]; then
211 . "${BASEDIR}/etc
/${NAME}.conf
"
213 echo "Nono
, you can
't tell us about random archives. Bad boy!"
217 ########################################################################
218 # Config options go here. Feel free to overwrite them in the config #
219 # file if you need to. #
220 # On debian.org machines the defaults should be ok. #
222 # The following extra variables can be defined in the config file: #
225 # can be used to exclude a complete architecture from #
226 # mirrorring. Use as space seperated list. #
227 # Possible values are: #
228 # alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, #
229 # mipsel, mips, powerpc, s390, sparc, kfreebsd-i386, kfreebsd-amd64 #
231 # eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc" #
233 # An unset value will mirror all architectures #
234 ########################################################################
236 ########################################################################
237 # There should be nothing to edit here, use the config file #
238 ########################################################################
239 MIRRORNAME=${MIRRORNAME:-$(hostname -f)}
240 # Where to put logfiles in
241 LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
243 LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
245 # Where should we put all the mirrored files?
246 TO=${TO:-"/org/ftp.debian.org/ftp/"}
248 # used by log() and error()
249 PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
251 # Where to send mails about mirroring to?
252 if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
253 # We are not on a debian.org host
254 MAILTO=${MAILTO:-"root"}
256 # Yay, on a .debian.org host
257 MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
259 # Want errors only or every log?
260 ERRORSONLY=${ERRORSONLY:-"true"}
261 # Want full logs, ie. including the rsync one?
262 FULLLOGS=${FULLLOGS:-"false"}
264 # How many logfiles to keep
265 LOGROTATE=${LOGROTATE:-14}
268 LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"}
269 # timeout for the lockfile, in case we have bash older than v4 (and no /proc)
270 LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
271 # Do we need another rsync run?
272 UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
273 # Trace file for mirror stats and checks (make sure we get full hostname)
274 TRACE=${TRACE:-"project/trace/${MIRRORNAME}"}
277 RSYNC=${RSYNC:-rsync}
278 # Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
280 RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"}
281 # limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited
282 RSYNC_BW=${RSYNC_BW:-0}
283 # Default rsync options for *every* rsync call
284 RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
285 # Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files
286 RSYNC_OPTIONS1
=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"}
287 # Options for the second pass, where we do want everything, including deletion of old and now unused files
288 RSYNC_OPTIONS2
=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
289 # Which rsync share to use on our upstream mirror?
290 RSYNC_PATH
=${RSYNC_PATH:-"ftp"}
292 # Now add the bwlimit option. As default is 0 we always add it, rsync interprets
293 # 0 as unlimited, so this is safe.
294 RSYNC_OPTIONS
="--bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}"
296 # We have no default host to sync from, but will error out if its unset
297 RSYNC_HOST
=${RSYNC_HOST:-""}
298 # Error out if we have no host to sync from
299 if [ -z
"${RSYNC_HOST}" ]; then
300 error
"Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf"
303 # our username for the rsync share
304 RSYNC_USER
=${RSYNC_USER:-""}
306 RSYNC_PASSWORD
=${RSYNC_PASSWORD:-""}
309 RSYNC_PROXY
=${RSYNC_PROXY:-""}
312 SYNCSTAGE1
=${SYNCSTAGE1:-"false"}
314 SYNCSTAGE2
=${SYNCSTAGE2:-"false"}
316 SYNCALL
=${SYNCALL:-"true"}
317 # Do we have a mhop sync?
318 SYNCMHOP
=${SYNCMHOP:-"false"}
320 SYNCCALLBACK
=${SYNCCALLBACK:-"false"}
321 # If we call back we need some more options defined in the config file.
322 CALLBACKUSER
=${CALLBACKUSER:-"archvsync"}
323 CALLBACKHOST
=${CALLBACKHOST:-"none"}
324 CALLBACKKEY
=${CALLBACKKEY:-"none"}
326 # General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that!
327 EXCLUDE
=${EXCLUDE:-""}
329 # The temp directory used by rsync --delay-updates is not
330 # world-readable remotely. Always exclude it to avoid errors.
331 EXCLUDE
="${EXCLUDE} --exclude .~tmp~/"
333 SOURCE_EXCLUDE
=${SOURCE_EXCLUDE:-""}
334 ARCH_EXCLUDE
=${ARCH_EXCLUDE:-""}
335 # Exclude architectures defined in $ARCH_EXCLUDE
336 for ARCH
in ${ARCH_EXCLUDE}; do
337 EXCLUDE
="${EXCLUDE} --exclude binary-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes"
338 if [ "${ARCH}" = "source" ]; then
339 if [ -z
${SOURCE_EXCLUDE} ]; then
340 SOURCE_EXCLUDE
=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc "
355 ########################################################################
356 # Really nothing to see below here. Only code follows. #
357 ########################################################################
358 ########################################################################
364 # If we are here for the first time, create the
365 # destination and the trace directory
366 mkdir
-p
"${TO}/project/trace"
368 # Used to make sure we will have the archive fully and completly synced before
369 # we stop, even if we get multiple pushes while this script is running.
370 # Otherwise we can end up with a half-synced archive:
372 # - sync, while locked
373 # - get another push. Of course no extra sync run then happens, we are locked.
374 # - done. Archive not correctly synced, we don't have all the changes from the second push.
375 touch "${UPDATEREQUIRED}"
377 # Check to see if another sync is in progress
378 if ! ( set -o noclobber
; echo "$$" > "${LOCK}") 2> /dev
/null
; then
379 if [ ${BASH_VERSINFO[0]} -gt
3 ] ||
[ -L
/proc
/self
]; then
380 # We have a recent enough bash version, lets do it the easy way,
381 # the lock will contain the right pid, thanks to $BASHPID
382 if ! $
(kill -0 $
(cat ${LOCK}) 2>/dev
/null
); then
383 # Process does either not exist or is not owned by us.
384 echo "$$" > "${LOCK}"
386 echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
390 # Old bash, means we dont have the right pid in our lockfile
391 # So take a different way - guess if it is still there by comparing its age.
392 # Not optimal, but hey.
393 stamptime
=$
(date --reference
="${LOCK}" +%s
)
395 difference
=$
(( $unixtime - $stamptime ))
396 if [ ${difference} -ge
${LOCKTIMEOUT} ]; then
397 # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock
398 echo "$$" > "${LOCK}"
400 echo "Unable to start rsync, lock file younger than one hour"
406 # When we exit normally we call cleanup on our own. Otherwise we want it called by
407 # this trap. (We can not trap on EXIT, because that is called when the main script
408 # exits. Which also happens when we background the mainroutine, ie. while we still
410 trap cleanup ERR TERM HUP INT QUIT
412 # Start log by redirecting stdout and stderr there and closing stdin
413 exec >"$LOG" 2>&1 <&-
414 log
"Mirrorsync start"
416 # Look who pushed us and note that in the log.
417 PUSHFROM
="${SSH_CONNECTION%%\ *}"
418 if [ -n
"${PUSHFROM}" ]; then
419 log
"We got pushed from ${PUSHFROM}"
422 if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
423 if [ "xnonex" = "x${CALLBACKHOST}x" ] ||
[ "xnonex" = "x${CALLBACKKEY}x" ]; then
425 error
"We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
435 # Now, we might want to sync from anonymous too.
436 # This is that deep in this script so hook1 could, if wanted, change things!
437 if [ -z
${RSYNC_USER} ]; then
438 RSYNCPTH
="${RSYNC_HOST}"
440 RSYNCPTH
="${RSYNC_USER}@${RSYNC_HOST}"
443 # Now do the actual mirroring, and run as long as we have an updaterequired file.
444 export RSYNC_PASSWORD
447 while [ -e
"${UPDATEREQUIRED}" ]; do
448 log
"Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
450 # if we want stage1 *or* all
451 if [ "xtruex" = "x${SYNCSTAGE1}x" ] ||
[ "xtruex" = "x${SYNCALL}x" ]; then
452 while [ -e
"${UPDATEREQUIRED}" ]; do
453 rm -f
"${UPDATEREQUIRED}"
454 log
"Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
457 # Step one, sync everything except Packages/Releases
458 ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
459 ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
463 log
"Back from rsync with returncode ${result}"
466 # Fake a good resultcode
469 rm -f
"${UPDATEREQUIRED}"
472 check_rsync
$result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
475 if [ ${GO} -eq
2 ] && [ -e
"${UPDATEREQUIRED}" ]; then
476 log
"We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
477 elif [ ${GO} -ne
0 ]; then
487 # if we want stage2 *or* all
488 if [ "xtruex" = "x${SYNCSTAGE2}x" ] ||
[ "xtruex" = "x${SYNCALL}x" ]; then
489 log
"Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
492 # We are lucky, it worked. Now do step 2 and sync again, this time including
493 # the packages/releases files
494 ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
495 ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
499 log
"Back from rsync with returncode ${result}"
501 # Fake a good resultcode
506 check_rsync
$result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
509 if [ ${GO} -eq
2 ] && [ -e
"${UPDATEREQUIRED}" ]; then
510 log
"We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
511 elif [ ${GO} -ne
0 ]; then
522 # We only update our tracefile when we had a stage2 or an all sync.
523 # Otherwise we would update it after stage1 already, which is wrong.
524 if [ "xtruex" = "x${SYNCSTAGE2}x" ] ||
[ "xtruex" = "x${SYNCALL}x" ]; then
525 if [ -d
"$(dirname "${TO}/${TRACE}")" ]; then
526 LC_ALL
=POSIX LANG
=POSIX
date -u
> "${TO}/${TRACE}"
527 echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
528 echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
538 if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
540 callback
${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
544 # Remove the Archive-Update-in-Progress file before we push our downstreams.
547 if [ x
${HUB} = "xtrue" ]; then
548 # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
549 if [ "xtruex" = "x${SYNCSTAGE2}x" ] ||
[ "xtruex" = "x${SYNCALL}x" ] ||
[ "xtruex" = "x${SYNCMHOP}x" ]; then
551 if [ -n
"${ARCHIVE}" ]; then
552 # We tell runmirrors about the archive we are running on.
553 RUNMIRRORARGS
="-a ${ARCHIVE}"
555 # We also tell runmirrors that we are running it from within ftpsync, so it can change
556 # the way it works with mhop based on that.
557 RUNMIRRORARGS
="${RUNMIRRORARGS} -f"
559 if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
560 # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
561 # not get to this point.
562 # So if that happens, tell runmirrors we are doing mhop
563 RUNMIRRORARGS
="${RUNMIRRORARGS} -k mhop"
564 elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
565 RUNMIRRORARGS
="${RUNMIRRORARGS} -k stage2"
566 elif [ "xtruex" = "x${SYNCALL}x" ]; then
567 RUNMIRRORARGS
="${RUNMIRRORARGS} -k all"
569 log
"Trigger slave mirrors using ${RUNMIRRORARGS}"
570 ${BASEDIR}/bin
/runmirrors
${RUNMIRRORARGS}
571 log
"Trigger slave done"
581 # All done, lets call cleanup