5 ### (c) 2012 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This file is part of the `rsync-backup' program.
12 ### rsync-backup is free software; you can redistribute it and/or modify
13 ### it under the terms of the GNU General Public License as published by
14 ### the Free Software Foundation; either version 2 of the License, or
15 ### (at your option) any later version.
17 ### rsync-backup is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ### GNU General Public License for more details.
22 ### You should have received a copy of the GNU General Public License
23 ### along with rsync-backup; if not, write to the Free Software Foundation,
24 ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
28 thishost=$(hostname -s)
35 ###--------------------------------------------------------------------------
36 ### Utility functions.
41 ## Run rsync(1) in an appropriate manner. Configuration should ovrride
42 ## this or set $RSYNCOPTS if it wants to do something weirder. Arguments
43 ## to this function are passed on to rsync.
46 --archive --hard-links --numeric-ids --del \
51 --filter="dir-merge .rsync-backup" \
61 now=$(date +"%Y-%m-%d %H:%M:%S %z")
68 ## Run CMD, if this isn't a dry run.
71 t) echo >&2 " +++ $*" ;;
78 ## Copy lines from stdin to stdout, adding PREFIX.
80 while IFS= read -r line; do
81 printf "%s %s\n" "$prefix" "$line"
89 -stdin) stdinp=t; shift ;;
94 tag=$1 cmd=$2; shift 2
95 ## Run CMD, logging its output in a pleasing manner.
99 echo >&2 " *** RUN $tag"
100 echo >&2 " +++ $cmd $*"
106 case $stdinp in nil) exec </dev/null ;; esac
108 "$cmd" "$@" 3>&- 4>&- 5>&- 9>&-
110 copy "|" >&4; } 2>&1 |
111 copy "*" >&4; } 4>&1 |
116 *) log "FAIL $tag (rc = $rc)" ;;
125 ## Write a unified diff from OLD to NEW, to OUT.
127 set +e; diff -u "$old" "$new" >"$out"; rc=$?; set -e
128 case $rc in 1) cat "$out" ;; esac
134 ## Answer whether H is a local host.
137 "$thishost") return 0 ;;
144 ## Run CMD on the current host. If the host seems local then run the
145 ## command through a local shell; otherwise run it through ssh(1). Either
146 ## way it will be processed by a shell.
148 if localp $host; then run "@$host: $tag" sh -c "$cmd"
149 else run "@$host: $tag" ssh $userat$host "$cmd"
155 ## Like hostrun, but without the complicated logging, and targetted at a
158 if localp $h; then sh -c "$cmd"
165 ## Output (to stdout) either PATH or HOST:PATH, choosing the former if the
166 ## current host is local.
168 if localp $host; then echo $path
169 else echo $userat$host:$path
175 ## Define a hook called HOOK.
182 ## Add command CMD to the hook HOOK.
184 eval old=\$hk_$hook; new="$old $cmd"
190 ## Invoke HOOK, passing it the remaining arguments.
194 if ! $cmd "$@"; then return $?; fi
198 remove_old_logfiles () {
200 ## Remove old logfiles with names of the form BASE.DATE#N, so that there
201 ## are at most $MAXLOG of them.
203 ## Count up the logfiles.
205 for i in "$base".*; do
206 if [ ! -f "$i" ]; then continue; fi
210 ## If there are too many, go through and delete some early ones.
211 if [ $dryrun = nil ] && [ $nlog -gt $MAXLOG ]; then
212 n=$(( nlog - MAXLOG ))
213 for i in "$base".*; do
214 if [ ! -f "$i" ]; then continue; fi
217 if [ $n -eq 0 ]; then break; fi
222 ###--------------------------------------------------------------------------
223 ### Database operations.
226 host=$1 fs=$2 date=$3 vol=$4
228 if [ -f "$INDEXDB" ]; then
229 sqlite3 "$INDEXDB" <<EOF
230 INSERT INTO idx (host, fs, date, vol)
231 VALUES ('$host', '$fs', '$date', '$vol');
237 host=$1 fs=$2 date=$3
239 if [ -f "$INDEXDB" ]; then
240 sqlite3 "$INDEXDB" <<EOF
241 DELETE FROM idx WHERE
242 host = '$host' AND fs = '$fs' AND date = '$date';
247 ###--------------------------------------------------------------------------
248 ### Snapshot handling.
250 ## Snapshot protocol. Each snapshot type has a pair of functions snap_TYPE
251 ## and unsnap_TYPE. Each is given the current snapshot arguments and the
252 ## filesystem name to back up. The snap_TYPE function should create and
253 ## mount the snapshot and output an rsync(1) path to where the filesystem can
254 ## be copied; the unsnap_TYPE function should unmount and tear down the
257 ## Fake snapshot by not doing anything. Use only if you have no choice.
258 snap_live () { hostpath "$2"; }
259 unsnap_live () { :; }
261 ## Fake snapshot by remounting a live filesystem read-only. Useful if the
262 ## underlying storage isn't in LVM.
267 ## Place a marker in the filesystem so we know why it was made readonly.
268 ## (Also this serves to ensure that the filesystem was writable before.)
269 hostrun "snap-ro $mnt" "
270 echo rsync-backup >$mnt/.lock
271 mount -oremount,ro $mnt" || return $?
280 ## Check that the filesystem still has our lock marker.
281 hostrun "unsnap-ro $mnt" "
282 case \$(cat $mnt/.lock) in
284 *) echo unlocked by someone else; exit 31 ;;
286 mount -oremount,rw $mnt
287 rm $mnt/.lock" || return $?
290 ## Snapshot using LVM.
292 SNAPSIZE="-l10%ORIGIN"
297 ## Make the snapshot.
298 hostrun "snap-lvm $vg/$lv" "
299 lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv
300 mkdir -p $SNAPDIR/$lv
301 mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv" || return $?
304 hostpath $SNAPDIR/$lv
310 ## Remove the snapshot. Sometimes LVM doesn't notice that the snapshot is
311 ## no longer in open immdiately, so try several times.
312 hostrun "unsnap-lvm $vg/$lv" "
316 if lvremove -f $vg/$lv.bkp; then rc=0; break; fi
319 exit $rc" || return $?
322 ## Complicated snapshot using LVM, where the volume group and filesystem are
323 ## owned by different machines, so they need to be synchronized during the
327 lvhost=$1 vg=$2 lv=$3 fshost=$4 fsdir=$5
329 ## Engage in the rfreezefs protocol with the filesystem host. This
330 ## involves some hairy plumbing. We want to get exit statuses out of both
333 ssh $fshost rfreezefs $fsdir | {
336 ## Read the codebook from the remote end.
342 TOKEN) eval tok_$2=$3 ;;
343 READY) ready=t; break ;;
345 echo >&2 "$quis: unexpected keyword $1 (rfreezefs to $rhost)"
352 echo >&2 "$quis: unexpected eof (rfreezefs to $rhost)"
357 ## Connect to the filesystem host's TCP port and get it to freeze its
359 exec 3<>/dev/tcp/$fshost/$port
365 echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
370 ## Get the volume host to create the snapshot.
372 _hostrun >&2 3>&- $userat$lvhost \
373 "lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv"
377 ## The filesystem can thaw now.
383 _hostrun >&2 3>&- $userat$lvhost "lvremove -f $vg/$lv.bkp" || :
384 echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
393 ## Sift through the wreckage to find out what happened.
394 rc_rfreezefs=${PIPESTATUS[0]} rc_snapshot=${PIPESTATUS[1]}
396 case $rc_rfreezefs:$rc_snapshot in
400 echo >&2 "$quis: EMERGENCY failed to thaw $fsdir on $fshost!"
404 echo >&2 "$quis: failed to snapshot $vg/$lv ($fsdir on $fshost)"
409 ## Mount the snapshot on the volume host.
410 _hostrun >&2 $userat$lvhost "
411 mkdir -p $SNAPDIR/$lv
412 mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv"
416 rhost=$1 vg=$2 lv=$3 rfs=$4
419 run "snap-rfreezefs $host:$vg/$lv $rhost:$rfs" \
420 do_rfreezefs $host $vg $lv $rhost $rfs || return $?
421 hostpath $SNAPDIR/$lv
424 unsnap_rfreezefs () {
426 ## Unshapping is the same as for plain LVM.
427 rhost=$1 vg=$2 lv=$3 rfs=$4
431 ###--------------------------------------------------------------------------
432 ### Expiry computations.
435 ## Read dates on stdin; write to stdout `EXPIRE date' for dates which
436 ## should be expired and `RETAIN date' for dates which should be retained.
438 ## Get the current date and convert it into useful forms.
439 now=$(date +%Y-%m-%d)
441 now_jdn=$(julian $now) now_year=$year now_month=$month now_day=$day
444 ## Work through each date in the input.
448 ## Convert the date into a useful form.
452 ## Work through the policy list.
453 if [ $jdn -le $now_jdn ]; then
454 while read ival age; do
456 ## Decide whether the policy entry applies to this date.
463 if [ $year -eq $now_year ] ||
464 ([ $year -eq $(( $now_year - 1 )) ] &&
465 [ $month -ge $now_month ])
469 if ([ $month -eq $now_month ] && [ $year -eq $now_year ]) ||
470 ((([ $month -eq $(( $now_month - 1 )) ] &&
471 [ $year -eq $now_year ]) ||
472 ([ $month -eq 12 ] && [ $now_month -eq 1 ] &&
473 [ $year -eq $(( $now_year - 1 )) ])) &&
474 [ $day -ge $now_day ])
478 if [ $jdn -ge $(( $now_jdn - 7 )) ]; then apply=t; fi
481 echo >&2 "$quis: unknown age symbol \`$age'"
485 case $apply in nil) continue ;; esac
487 ## Find the interval marker for this date.
493 ydn=$(julian $year-01-01)
494 wk=$(( ($jdn - $ydn)/7 + 1 ))
504 echo >&2 "$quis: unknown interval symbol \`$ival'"
509 ## See if we've alredy retained something in this interval.
512 *) keep=t kept=$kept$marker: ;;
521 t) echo RETAIN $date ;;
522 *) echo EXPIRE $date ;;
528 ###--------------------------------------------------------------------------
529 ### Actually taking backups of filesystems.
541 md5 | sha1 | sha224 | sha256 | sha384 | sha512)
542 set -- $(${HASH}sum <"$file")
546 set -- $(openssl dgst -$HASH <"$file")
553 _hostrun $userat$host "
557 echo \"*** $host $fs $date\"; echo
558 rsync -rx --filter='dir-merge .rsync-backup' ./ |
559 fshash -c$fshashdir/$fs.bkp -a -H$HASH -frsync
564 { echo "*** $host $fs $date"; echo
565 fshash -c$STOREDIR/fshash.cache -H$HASH new/
573 case $i in *[!-0-9]*) continue ;; esac
574 case $seen in *:"$i":*) continue ;; esac
579 while read op date; do
582 echo >&2 " --- keep $date"
585 echo >&2 " --- delete $date"
592 $verbose -n " expire $date..."
594 delete_index $host $fs $date
606 backup_precommit_hook () {
607 host=$1 fs=$2 date=$3
608 ## Compatibility: You can override this hook in the configuration file for
609 ## special effects; but it's better to use `addhook precommit'.
613 addhook precommit backup_precommit_hook
615 backup_commit_hook () {
616 host=$1 fs=$2 date=$3
617 ## Compatibility: You can override this hook in the configuration file for
618 ## special effects; but it's better to use `addhook commit'.
622 addhook commit backup_commit_hook
625 date=$1 fs=$2 fsarg=$3
626 ## Back up FS on the current host.
632 ## Run a hook beforehand.
633 set +e; runhook setup $host $fs $date; rc=$?; set -e
636 99) log "BACKUP of $host:$fs SKIPPED by hook"; return 0 ;;
637 *) log "BACKUP of $host:$fs FAILED (hook returns $?)"; return $? ;;
640 ## Report the start of this attempt.
641 log "START BACKUP of $host:$fs"
643 ## Maybe we need to retry the backup.
646 ## Rig checksum variables to mismatch unless they're set later.
647 hrfs=REMOTE hlfs=LOCAL
649 ## Create and mount the remote snapshot.
652 maybe snap_$snap $fs $fsarg
656 snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
659 $verbose " create snapshot"
661 ## If we had a fshash-mismatch, then clear out the potentially stale
662 ## entries, both locally and remotely.
666 $verbose " prune cache"
667 run -stdin "local prune fshash" \
668 fshash -u -c$STOREDIR/fshash.cache -H$HASH new/ <$fshash_diff
669 run -stdin "@$host: prune fshash" \
670 _hostrun $userat$host <$fshash_diff \
671 "fshash -u -c$fshashdir/$fs.bkp -H$HASH ${snapmnt#*:}"
675 ## Build the list of hardlink sources.
677 for i in $host $like; do
678 d=$STOREDIR/$i/$fs/last/
679 if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
682 ## Copy files from the remote snapshot.
685 t) $verbose " running rsync" ;;
686 nil) $verbose -n " running rsync..." ;;
689 run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
695 case $dryrun in nil) $verbose " done" ;; esac
697 ## Collect a map of the snapshot for verification purposes.
700 t) $verbose " remote fshash" ;;
701 nil) $verbose -n " remote fshash..." ;;
703 run "@$host: fshash $fs" remote_fshash
708 hrfs=$(hash_file "new.fshash")
709 log "remote fshash $HASH checksum: $hrfs"
717 ## Remove the snapshot.
718 maybe unsnap_$snap $snapargs $fs $fsarg
719 $verbose " remove snapshot"
721 ## If we failed to copy, then give up.
722 case $rc_rsync:$rc_fshash in
724 0:*) return $rc_fshash ;;
725 *) return $rc_rsync ;;
728 ## Get a matching map of the files received.
729 maybe mkdir -m750 -p $STOREDIR/tmp/
730 localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
732 t) $verbose " local fshash" ;;
733 nil) $verbose -n " local fshash..." ;;
735 run "local fshash $host:$fs" local_fshash || return $?
738 hlfs=$(hash_file "$localmap")
739 log "local fshash $HASH checksum: $hlfs"
747 ## Compare the two maps.
749 fshash_diff=$STOREDIR/tmp/fshash-diff.$host.$fs.$date
750 run "compare fshash maps for $host:$fs" \
751 run_diff $fshash_diff new.fshash $localmap
759 if [ $attempt -ge $retry ]; then return $rc; fi
760 $verbose " fshash mismatch; retrying"
761 attempt=$(( $attempt + 1 ))
769 ## Double-check the checksums.
770 if [ $hrfs != $hlfs ]; then
772 $0: INTERNAL ERROR: fshash $HASH checksum mismatch -- aborting
773 remote fshash checksum = $hrfs
774 local fshash checksum = $hlfs
780 maybe rm -f $localmap
781 case $fshash_diff in nil) ;; *) maybe rm -f $fshash_diff ;; esac
782 $verbose " fshash match"
784 ## Commit this backup.
787 runhook precommit $host $fs $date
789 mv new.fshash $date.fshash
790 insert_index $host $fs $date $VOLUME
791 runhook commit $host $fs $date
793 ln -s $date hack/last
800 ## Expire old backups.
801 case "${expire_policy+t},${default_policy+t}" in
802 ,t) expire_policy=$default_policy ;;
804 case "${expire_policy+t},$dryrun" in
805 t,nil) run "expiry for $host:$fs" expire_backups ;;
806 t,t) expire_backups ;;
812 t) log "END BACKUP of $host:$fs" ;;
813 nil) log "SUCCESSFUL BACKUP of $host:$fs" ;;
818 fs=$1 date=$2 cmd=$3; shift 3
819 ## try_backup FS DATE COMMAND ARGS ...
821 ## Run COMMAND ARGS to back up filesystem FS on the current host,
822 ## maintaining a log, and checking whether it worked. The caller has
823 ## usually worked out the DATE in order to set up the filesystem, and we
824 ## need it to name the log file properly.
826 ## Find a name for the log file. In unusual circumstances, we may have
827 ## deleted old logs from today, so just checking for an unused sequence
828 ## number is insufficient. Instead, check all of the logfiles for today,
829 ## and use a sequence number that's larger than any of them.
836 for i in "$logdir/$host/$fs.$date#"*; do
838 case "$tail" in [!1-9]* | *[!0-9]*) continue ;; esac
839 if [ -f "$i" -a $tail -ge $seq ]; then seq=$(( tail + 1 )); fi
841 log="$logdir/$host/$fs.$date#$seq"
845 ## Run the backup command.
846 case $dryrun in nil) mkdir -p $logdir/$host ;; esac
847 if ! "$cmd" "$@" 9>$log 1>&9; then
849 echo >&2 "$quis: backup of $host:$fs FAILED!"
853 ## Clear away any old logfiles.
854 remove_old_logfiles "$logdir/$host/$fs"
858 ## backup FS[:ARG] ...
860 ## Back up the filesystems on the currently selected host using the
861 ## currently selected snapshot type.
863 ## Make sure that there's a store volume. We must do this here rather than
864 ## in the main body of the script, since the configuration file needs a
865 ## chance to override STOREDIR.
866 if ! [ -r $STOREDIR/.rsync-backup-store ]; then
867 echo >&2 "$quis: no backup volume mounted"
871 ## Read the volume name if we don't have one already. Again, this allows
872 ## the configuration file to provide a volume name.
873 case "${VOLUME+t}${VOLUME-nil}" in
874 nil) VOLUME=$(cat $METADIR/volume) ;;
877 ## Back up each requested file system in turn.
880 ## Parse the argument.
882 *:*) fsarg=${fs#*:} fs=${fs%%:*} ;;
885 $verbose " filesystem $fs"
887 ## Move to the store directory and set up somewhere to put this backup.
891 if [ ! -d $host ]; then
893 chown root:root $host
895 if [ ! -d $host/$fs ]; then
896 mkdir -m750 $host/$fs
897 chown root:backup $host/$fs
903 ## Find out if we've already copied this filesystem today.
904 date=$(date +%Y-%m-%d)
905 if [ $dryrun = nil ] && [ -d $date ]; then
906 $verbose " already dumped"
910 ## Do the backup of this filesystem.
911 run_backup_cmd $fs $date do_backup $date $fs $fsarg
915 ###--------------------------------------------------------------------------
916 ### Configuration functions.
921 done_first_host_p=nil
926 case $done_first_host_p in
927 nil) runhook start; done_first_host_p=t ;;
929 case "${expire_policy+t},${default_policy+t}" in
930 t,) default_policy=$expire_policy ;;
933 $verbose "host $host"
936 snaptype () { snap=$1; shift; snapargs="$*"; retry=1; }
937 rsyncargs () { rsyncargs="$*"; }
938 like () { like="$*"; }
939 retry () { retry="$*"; }
940 user () { userat="$*@"; }
943 case $clear_policy in t) unset expire_policy; clear_policy=nil ;; esac
944 expire_policy="${expire_policy+$expire_policy
948 ###--------------------------------------------------------------------------
949 ### Read the configuration and we're done.
952 echo "usage: $quis [-nv] [-c CONF]"
956 echo "$quis version $VERSION"
959 whine () { echo >&8 "$@"; }
961 while getopts "hVvc:n" opt; do
964 V) version; config; exit 0 ;;
971 shift $((OPTIND - 1))
972 case $# in 0) ;; *) usage >&2; exit 1 ;; esac
979 0) $verbose "All backups successful" ;;
980 *) $verbose "Backups FAILED" ;;
983 ###----- That's all, folks --------------------------------------------------