5 ### (c) 2012 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This file is part of the `rsync-backup' program.
12 ### rsync-backup is free software; you can redistribute it and/or modify
13 ### it under the terms of the GNU General Public License as published by
14 ### the Free Software Foundation; either version 2 of the License, or
15 ### (at your option) any later version.
17 ### rsync-backup is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ### GNU General Public License for more details.
22 ### You should have received a copy of the GNU General Public License
23 ### along with rsync-backup; if not, write to the Free Software Foundation,
24 ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
28 thishost=$(hostname -s)
35 conf=@sysconfdir@/rsync-backup.conf
40 ###--------------------------------------------------------------------------
41 ### Utility functions.
46 ## Run rsync(1) in an appropriate manner. Configuration should ovrride
47 ## this or set $RSYNCOPTS if it wants to do something weirder. Arguments
48 ## to this function are passed on to rsync.
51 --archive --hard-links --numeric-ids --del \
56 --filter="dir-merge .rsync-backup" \
66 now=$(date +"%Y-%m-%d %H:%M:%S %z")
73 ## Run CMD, if this isn't a dry run.
76 t) echo >&2 " +++ $*" ;;
82 tag=$1 cmd=$2; shift 2
83 ## Run CMD, logging its output in a pleasing manner.
87 echo >&2 " *** RUN $tag"
88 echo >&2 " +++ $cmd $*"
95 "$cmd" "$@" 3>&- 4>&- 5>&- 9>&-
97 while IFS= read line; do echo "| $line"; done >&4; } 2>&1 |
98 while IFS= read line; do echo "* $line"; done >&4; } 4>&1 |
99 cat >&9; } 5>&1 </dev/null
103 *) log "FAIL $tag (rc = $rc)" ;;
112 ## Answer whether H is a local host.
115 "$thishost") return 0 ;;
122 ## Run CMD on the current host. If the host seems local then run the
123 ## command through a local shell; otherwise run it through ssh(1). Either
124 ## way it will be processed by a shell.
126 if localp $host; then run "@$host: $tag" sh -c "$cmd"
127 else run "@$host: $tag" ssh $host "$cmd"
133 ## Like hostrun, but without the complicated logging, and targetted at a
136 if localp $h; then sh -c "$cmd"
143 ## Output (to stdout) either PATH or HOST:PATH, choosing the former if the
144 ## current host is local.
146 if localp $host; then echo $path
147 else echo $host:$path
151 ###--------------------------------------------------------------------------
152 ### Database operations.
154 INDEXDB=@pkglocalstatedir@/index.db
157 host=$1 fs=$2 date=$3 vol=$4
159 if [ -f "$INDEXDB" ]; then
160 sqlite3 "$INDEXDB" <<EOF
161 INSERT INTO idx (host, fs, date, vol)
162 VALUES ('$host', '$fs', '$date', '$vol');
168 host=$1 fs=$2 date=$3
170 if [ -f "$INDEXDB" ]; then
171 sqlite3 "$INDEXDB" <<EOF
172 DELETE FROM idx WHERE
173 host = '$host' AND fs = '$fs' AND date = '$date';
178 ###--------------------------------------------------------------------------
179 ### Snapshot handling.
181 ## Snapshot protocol. Each snapshot type has a pair of functions snap_TYPE
182 ## and unsnap_TYPE. Each is given the current snapshot arguments and the
183 ## filesystem name to back up. The snap_TYPE function should create and
184 ## mount the snapshot and output an rsync(1) path to where the filesystem can
185 ## be copied; the unsnap_TYPE function should unmount and tear down the
188 ## Fake snapshot by not doing anything. Use only if you have no choice.
189 snap_live () { hostpath "$2"; }
190 unsnap_live () { :; }
192 ## Fake snapshot by remounting a live filesystem read-only. Useful if the
193 ## underlying storage isn't in LVM.
198 ## Place a marker in the filesystem so we know why it was made readonly.
199 ## (Also this serves to ensure that the filesystem was writable before.)
200 hostrun "snap-ro $mnt" "
201 echo rsync-backup >$mnt/.lock
202 mount -oremount,ro $mnt" || return $?
211 ## Check that the filesystem still has our lock marker.
212 hostrun "unsnap-ro $mnt" "
213 case \$(cat $mnt/.lock) in
215 *) echo unlocked by someone else; exit 31 ;;
217 mount -oremount,rw $mnt
218 rm $mnt/.lock" || return $?
221 ## Snapshot using LVM.
223 SNAPSIZE="-l10%ORIGIN"
224 SNAPDIR=@mntbkpdir@/snap
229 ## Make the snapshot.
230 hostrun "snap-lvm $vg/$lv" "
231 lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv
232 mkdir -p $SNAPDIR/$lv
233 mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv" || return $?
236 hostpath $SNAPDIR/$lv
242 ## Remove the snapshot. Sometimes LVM doesn't notice that the snapshot is
243 ## no longer in open immdiately, so try several times.
244 hostrun "unsnap-lvm $vg/$lv" "
248 if lvremove -f $vg/$lv.bkp; then rc=0; break; fi
251 exit $rc" || return $?
254 ## Complicated snapshot using LVM, where the volume group and filesystem are
255 ## owned by different machines, so they need to be synchronized during the
259 lvhost=$1 vg=$2 lv=$3 fshost=$4 fsdir=$5
261 ## Engage in the rfreezefs protocol with the filesystem host. This
262 ## involves some hairy plumbing. We want to get exit statuses out of both
265 ssh $fshost rfreezefs $fsdir | {
268 ## Read the codebook from the remote end.
274 TOKEN) eval tok_$2=$3 ;;
275 READY) ready=t; break ;;
277 echo >&2 "$quis: unexpected keyword $1 (rfreezefs to $rhost)"
284 echo >&2 "$quis: unexpected eof (rfreezefs to $rhost)"
289 ## Connect to the filesystem host's TCP port and get it to freeze its
291 exec 3<>/dev/tcp/$fshost/$port
297 echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
302 ## Get the volume host to create the snapshot.
304 _hostrun >&2 3>&- $lvhost \
305 "lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv"
309 ## The filesystem can thaw now.
315 _hostrun >&2 3>&- $lvhost "lvremove -f $vg/$lv.bkp" || :
316 echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
325 ## Sift through the wreckage to find out what happened.
326 rc_rfreezefs=${PIPESTATUS[0]} rc_snapshot=${PIPESTATUS[1]}
328 case $rc_rfreezefs:$rc_snapshot in
332 echo >&2 "$quis: EMERGENCY failed to thaw $fsdir on $fshost!"
336 echo >&2 "$quis: failed to snapshot $vg/$lv ($fsdir on $fshost)"
341 ## Mount the snapshot on the volume host.
342 _hostrun >&2 $lvhost "
343 mkdir -p $SNAPDIR/$lv
344 mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv"
348 rhost=$1 vg=$2 lv=$3 rfs=$4
351 run "snap-rfreezefs $host:$vg/$lv $rhost:$rfs" \
352 do_rfreezefs $host $vg $lv $rhost $rfs || return $?
353 hostpath $SNAPDIR/$lv
356 unsnap_rfreezefs () {
358 ## Unshapping is the same as for plain LVM.
359 rhost=$1 vg=$2 lv=$3 rfs=$4
363 ###--------------------------------------------------------------------------
364 ### Expiry computations.
368 ## Parse an ISO8601 DATE, and set YEAR, MONTH, DAY appropriately (and
369 ## without leading zeros).
371 ## Extract the components of the date and trim leading zeros (which will
372 ## cause things to be interpreted as octal and fail).
373 year=${date%%-*} rest=${date#*-}; month=${rest%%-*} day=${rest#*-}
374 year=${year#0} month=${month#0} day=${day#0}
379 ## Convert an ISO8601 DATE to a Julian Day Number.
383 ## The actual calculation: convert a (proleptic) Gregorian calendar date
384 ## into a Julian day number. This is taken from Wikipedia's page
385 ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary
386 ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24
387 ## proleptic Gregorian.
389 ## If the MONTH is January or February then set a = 1, otherwise set a = 0.
390 a=$(( (14 - $month)/12 ))
392 ## Compute a year offset relative to 4799BC-03-01. This puts the leap day
393 ## as the very last day in a year, which is very convenient. The offset
394 ## here is sufficient to make all y values positive (within the range of
395 ## the JDN calendar), and is a multiple of 400, which is the Gregorian
397 y=$(( $year + 4800 - $a ))
399 ## Compute the offset month number in that year. These months count from
401 m=$(( $month + 12*$a - 3 ))
403 ## Now for the main event. The (153 m + 2)/5 term is a surprising but
404 ## correct trick for obtaining the number of days in the first m months of
405 ## the (shifted) year). The magic offset 32045 is what you get when you
406 ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the
408 jdn=$(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 ))
414 ## Read dates on stdin; write to stdout `EXPIRE date' for dates which
415 ## should be expired and `RETAIN date' for dates which should be retained.
417 ## Get the current date and convert it into useful forms.
418 now=$(date +%Y-%m-%d)
420 now_jdn=$(julian $now) now_year=$year now_month=$month now_day=$day
423 ## Work through each date in the input.
427 ## Convert the date into a useful form.
431 ## Work through the policy list.
432 if [ $jdn -le $now_jdn ]; then
433 while read ival age; do
435 ## Decide whether the policy entry applies to this date.
442 if [ $year -eq $now_year ] ||
443 ([ $year -eq $(( $now_year - 1 )) ] &&
444 [ $month -ge $now_month ])
448 if ([ $month -eq $now_month ] && [ $year -eq $now_year ]) ||
449 ((([ $month -eq $(( $now_month - 1 )) ] &&
450 [ $year -eq $now_year ]) ||
451 ([ $month -eq 12 ] && [ $now_month -eq 1 ] &&
452 [ $year -eq $(( $now_year - 1 )) ])) &&
453 [ $day -ge $now_day ])
457 if [ $jdn -ge $(( $now_jdn - 7 )) ]; then apply=t; fi
460 echo >&2 "$quis: unknown age symbol \`$age'"
464 case $apply in nil) continue ;; esac
466 ## Find the interval marker for this date.
472 ydn=$(julian $year-01-01)
473 wk=$(( ($jdn - $ydn)/7 + 1 ))
483 echo >&2 "$quis: unknown interval symbol \`$ival'"
488 ## See if we've alredy retained something in this interval.
491 *) keep=t kept=$kept$marker: ;;
500 t) echo RETAIN $date ;;
501 *) echo EXPIRE $date ;;
507 ###--------------------------------------------------------------------------
508 ### Actually taking backups of filesystems.
510 STOREDIR=@mntbkpdir@/store
511 METADIR=@mntbkpdir@/meta
523 echo \"*** $host $fs $date\"; echo
524 rsync -rx --filter='dir-merge .rsync-backup' ./ |
525 fshash -c$fshashdir/$fs.bkp -a -H$HASH -frsync
530 { echo "*** $host $fs $date"; echo
531 fshash -c$STOREDIR/fshash.cache -H$HASH new/
539 case $i in *[!-0-9]*) continue ;; esac
540 case $seen in *:"$i":*) continue ;; esac
545 while read op date; do
548 echo >&2 " --- keep $date"
551 echo >&2 " --- delete $date"
558 $verbose -n " expire $date..."
560 delete_index $host $fs $date
567 backup_precommit_hook () {
568 host=$1 fs=$2 date=$3
569 ## Override this hook in the configuration file for special effects.
574 backup_commit_hook () {
575 host=$1 fs=$2 date=$3
576 ## Override this hook in the configuration file for special effects.
582 date=$1 fs=$2 fsarg=$3
583 ## Back up FS on the current host.
588 ## Report the start of this attempt.
589 log "START BACKUP of $host:$fs"
591 ## Maybe we need to retry the backup.
594 ## Create and mount the remote snapshot.
597 maybe snap_$snap $fs $fsarg
601 snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
604 $verbose " create snapshot"
606 ## Build the list of hardlink sources.
608 for i in $host $like; do
609 d=$STOREDIR/$i/$fs/last/
610 if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
613 ## Copy files from the remote snapshot.
616 t) $verbose " running rsync" ;;
617 nil) $verbose -n " running rsync..." ;;
620 run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
626 case $dryrun in nil) $verbose " done" ;; esac
628 ## Collect a map of the snapshot for verification purposes.
631 t) $verbose " remote fshash" ;;
632 nil) $verbose -n " remote fshash..." ;;
634 run "@$host: fshash $fs" remote_fshash
637 case $dryrun in nil) $verbose " done" ;; esac
639 ## Remove the snapshot.
640 maybe unsnap_$snap $snapargs $fs $fsarg
641 $verbose " remove snapshot"
643 ## If we failed to copy, then give up.
644 case $rc_rsync:$rc_fshash in
646 0:*) return $rc_fshash ;;
647 *) return $rc_rsync ;;
650 ## Get a matching map of the files received.
651 maybe mkdir -m750 -p $STOREDIR/tmp/
652 localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
654 t) $verbose " local fshash" ;;
655 nil) $verbose -n " local fshash..." ;;
657 run "local fshash $host:$fs" local_fshash || return $?
658 case $dryrun in nil) $verbose " done" ;; esac
660 ## Compare the two maps.
662 run "compare fshash maps for $host:$fs" diff -u new.fshash $localmap
670 if [ $attempt -ge $retry ]; then return $rc; fi
671 $verbose " fshash mismatch; retrying"
672 attempt=$(( $attempt + 1 ))
681 maybe rm -f $localmap
682 $verbose " fshash match"
684 ## Commit this backup.
687 backup_precommit_hook $host $fs $date
689 mv new.fshash $date.fshash
690 insert_index $host $fs $date $VOLUME
691 backup_commit_hook $host $fs $date
693 ln -s $date hack/last
700 ## Expire old backups.
701 case "${expire_policy+t},${default_policy+t}" in
702 ,t) expire_policy=$default_policy ;;
704 case "${expire_policy+t},$dryrun" in
705 t,nil) run "expiry for $host:$fs" expire_backups ;;
706 t,t) expire_backups ;;
712 t) log "END BACKUP of $host:$fs" ;;
713 nil) log "SUCCESSFUL BACKUP of $host:$fs" ;;
718 ## backup FS[:ARG] ...
720 ## Back up the filesystems on the currently selected host using the
721 ## currently selected snapshot type.
723 ## Make sure that there's a store volume. We must do this here rather than
724 ## in the main body of the script, since the configuration file needs a
725 ## chance to override STOREDIR.
726 if ! [ -r $STOREDIR/.rsync-backup-store ]; then
727 echo >&2 "$quis: no backup volume mounted"
731 ## Read the volume name if we don't have one already. Again, this allows
732 ## the configuration file to provide a volume name.
733 case "${VOLUME+t}${VOLUME-nil}" in
734 nil) VOLUME=$(cat $METADIR/volume) ;;
737 ## Back up each requested file system in turn.
740 ## Parse the argument.
742 *:*) fsarg=${fs#*:} fs=${fs%%:*} ;;
745 $verbose " filesystem $fs"
747 ## Move to the store directory and set up somewhere to put this backup.
751 if [ ! -d $host ]; then
753 chown root:root $host
755 if [ ! -d $host/$fs ]; then
756 mkdir -m750 $host/$fs
757 chown root:backup $host/$fs
763 ## Find out if we've already copied this filesystem today.
764 date=$(date +%Y-%m-%d)
765 if [ $dryrun = nil ] && [ -d $date ]; then
766 $verbose " already dumped"
770 ## Find a name for the log file. In unusual circumstances, we may have
771 ## deleted old logs from today, so just checking for an unused sequence
772 ## number is insufficient. Instead, check all of the logfiles for today,
773 ## and use a sequence number that's larger than any of them.
780 for i in "$logdir/$host/$fs.$date#"*; do
782 case "$tail" in [!1-9]* | *[!0-9]*) continue ;; esac
783 if [ -f "$i" -a $tail -ge $seq ]; then seq=$(( tail + 1 )); fi
785 log="$logdir/$host/$fs.$date#$seq"
789 ## Do the backup of this filesystem.
790 case $dryrun in nil) mkdir -p $logdir/$host ;; esac
791 if ! do_backup $date $fs $fsarg 9>$log 1>&9; then
793 echo >&2 "$quis: backup of $host:$fs FAILED!"
797 ## Count up the logfiles.
799 for i in "$logdir/$host/$fs".*; do
800 if [ ! -f "$i" ]; then continue; fi
804 ## If there are too many, go through and delete some early ones.
805 if [ $dryrun = nil ] && [ $nlog -gt $MAXLOG ]; then
806 n=$(( nlog - MAXLOG ))
807 for i in "$logdir/$host/$fs".*; do
808 if [ ! -f "$i" ]; then continue; fi
811 if [ $n -eq 0 ]; then break; fi
817 ###--------------------------------------------------------------------------
818 ### Configuration functions.
823 case "${expire_policy+t},${default_policy+t}" in
824 t,) default_policy=$expire_policy ;;
827 $verbose "host $host"
830 snaptype () { snap=$1; shift; snapargs="$*"; retry=0; }
831 rsyncargs () { rsyncargs="$*"; }
832 like () { like="$*"; }
833 retry () { retry="$*"; }
836 case $clear_policy in t) unset expire_policy; clear_policy=nil ;; esac
837 expire_policy="${expire_policy+$expire_policy
841 ###--------------------------------------------------------------------------
842 ### Read the configuration and we're done.
845 echo "usage: $quis [-v] [-c CONF]"
849 echo "$quis version $VERSION"
856 mntbkpdir = $mntbkpdir
857 fshashdir = $fshashdir
862 whine () { echo >&8 "$@"; }
864 while getopts "hVvc:n" opt; do
867 V) version; config; exit 0 ;;
874 shift $((OPTIND - 1))
875 case $# in 0) ;; *) usage >&2; exit 1 ;; esac
880 0) $verbose "All backups successful" ;;
881 *) $verbose "Backups FAILED" ;;
884 ###----- That's all, folks --------------------------------------------------