5 ### (c) 2012 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This file is part of the `rsync-backup' program.
12 ### rsync-backup is free software; you can redistribute it and/or modify
13 ### it under the terms of the GNU General Public License as published by
14 ### the Free Software Foundation; either version 2 of the License, or
15 ### (at your option) any later version.
17 ### rsync-backup is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ### GNU General Public License for more details.
22 ### You should have received a copy of the GNU General Public License
23 ### along with rsync-backup; if not, write to the Free Software Foundation,
24 ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
28 thishost=$(hostname -s)
35 conf=@sysconfdir@/rsync-backup.conf
40 ###--------------------------------------------------------------------------
41 ### Utility functions.
46 ## Run rsync(1) in an appropriate manner. Configuration should ovrride
47 ## this or set $RSYNCOPTS if it wants to do something weirder. Arguments
48 ## to this function are passed on to rsync.
51 --archive --hard-links --numeric-ids --del \
56 --filter="dir-merge .rsync-backup" \
66 now=$(date +"%Y-%m-%d %H:%M:%S %z")
73 ## Run CMD, if this isn't a dry run.
76 t) echo >&2 " +++ $*" ;;
83 ## Copy lines from stdin to stdout, adding PREFIX.
85 while IFS= read -r line; do
86 printf "%s %s\n" "$prefix" "$line"
91 tag=$1 cmd=$2; shift 2
92 ## Run CMD, logging its output in a pleasing manner.
96 echo >&2 " *** RUN $tag"
97 echo >&2 " +++ $cmd $*"
104 "$cmd" "$@" 3>&- 4>&- 5>&- 9>&-
106 copy "|" >&4; } 2>&1 |
107 copy "*" >&4; } 4>&1 |
108 cat >&9; } 5>&1 </dev/null
112 *) log "FAIL $tag (rc = $rc)" ;;
121 ## Answer whether H is a local host.
124 "$thishost") return 0 ;;
131 ## Run CMD on the current host. If the host seems local then run the
132 ## command through a local shell; otherwise run it through ssh(1). Either
133 ## way it will be processed by a shell.
135 if localp $host; then run "@$host: $tag" sh -c "$cmd"
136 else run "@$host: $tag" ssh $host "$cmd"
142 ## Like hostrun, but without the complicated logging, and targetted at a
145 if localp $h; then sh -c "$cmd"
152 ## Output (to stdout) either PATH or HOST:PATH, choosing the former if the
153 ## current host is local.
155 if localp $host; then echo $path
156 else echo $host:$path
160 ###--------------------------------------------------------------------------
161 ### Database operations.
163 INDEXDB=@pkglocalstatedir@/index.db
166 host=$1 fs=$2 date=$3 vol=$4
168 if [ -f "$INDEXDB" ]; then
169 sqlite3 "$INDEXDB" <<EOF
170 INSERT INTO idx (host, fs, date, vol)
171 VALUES ('$host', '$fs', '$date', '$vol');
177 host=$1 fs=$2 date=$3
179 if [ -f "$INDEXDB" ]; then
180 sqlite3 "$INDEXDB" <<EOF
181 DELETE FROM idx WHERE
182 host = '$host' AND fs = '$fs' AND date = '$date';
187 ###--------------------------------------------------------------------------
188 ### Snapshot handling.
190 ## Snapshot protocol. Each snapshot type has a pair of functions snap_TYPE
191 ## and unsnap_TYPE. Each is given the current snapshot arguments and the
192 ## filesystem name to back up. The snap_TYPE function should create and
193 ## mount the snapshot and output an rsync(1) path to where the filesystem can
194 ## be copied; the unsnap_TYPE function should unmount and tear down the
197 ## Fake snapshot by not doing anything. Use only if you have no choice.
198 snap_live () { hostpath "$2"; }
199 unsnap_live () { :; }
201 ## Fake snapshot by remounting a live filesystem read-only. Useful if the
202 ## underlying storage isn't in LVM.
207 ## Place a marker in the filesystem so we know why it was made readonly.
208 ## (Also this serves to ensure that the filesystem was writable before.)
209 hostrun "snap-ro $mnt" "
210 echo rsync-backup >$mnt/.lock
211 mount -oremount,ro $mnt" || return $?
220 ## Check that the filesystem still has our lock marker.
221 hostrun "unsnap-ro $mnt" "
222 case \$(cat $mnt/.lock) in
224 *) echo unlocked by someone else; exit 31 ;;
226 mount -oremount,rw $mnt
227 rm $mnt/.lock" || return $?
230 ## Snapshot using LVM.
232 SNAPSIZE="-l10%ORIGIN"
233 SNAPDIR=@mntbkpdir@/snap
238 ## Make the snapshot.
239 hostrun "snap-lvm $vg/$lv" "
240 lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv
241 mkdir -p $SNAPDIR/$lv
242 mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv" || return $?
245 hostpath $SNAPDIR/$lv
251 ## Remove the snapshot. Sometimes LVM doesn't notice that the snapshot is
252 ## no longer in open immdiately, so try several times.
253 hostrun "unsnap-lvm $vg/$lv" "
257 if lvremove -f $vg/$lv.bkp; then rc=0; break; fi
260 exit $rc" || return $?
263 ## Complicated snapshot using LVM, where the volume group and filesystem are
264 ## owned by different machines, so they need to be synchronized during the
268 lvhost=$1 vg=$2 lv=$3 fshost=$4 fsdir=$5
270 ## Engage in the rfreezefs protocol with the filesystem host. This
271 ## involves some hairy plumbing. We want to get exit statuses out of both
274 ssh $fshost rfreezefs $fsdir | {
277 ## Read the codebook from the remote end.
283 TOKEN) eval tok_$2=$3 ;;
284 READY) ready=t; break ;;
286 echo >&2 "$quis: unexpected keyword $1 (rfreezefs to $rhost)"
293 echo >&2 "$quis: unexpected eof (rfreezefs to $rhost)"
298 ## Connect to the filesystem host's TCP port and get it to freeze its
300 exec 3<>/dev/tcp/$fshost/$port
306 echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
311 ## Get the volume host to create the snapshot.
313 _hostrun >&2 3>&- $lvhost \
314 "lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv"
318 ## The filesystem can thaw now.
324 _hostrun >&2 3>&- $lvhost "lvremove -f $vg/$lv.bkp" || :
325 echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
334 ## Sift through the wreckage to find out what happened.
335 rc_rfreezefs=${PIPESTATUS[0]} rc_snapshot=${PIPESTATUS[1]}
337 case $rc_rfreezefs:$rc_snapshot in
341 echo >&2 "$quis: EMERGENCY failed to thaw $fsdir on $fshost!"
345 echo >&2 "$quis: failed to snapshot $vg/$lv ($fsdir on $fshost)"
350 ## Mount the snapshot on the volume host.
351 _hostrun >&2 $lvhost "
352 mkdir -p $SNAPDIR/$lv
353 mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv"
357 rhost=$1 vg=$2 lv=$3 rfs=$4
360 run "snap-rfreezefs $host:$vg/$lv $rhost:$rfs" \
361 do_rfreezefs $host $vg $lv $rhost $rfs || return $?
362 hostpath $SNAPDIR/$lv
365 unsnap_rfreezefs () {
367 ## Unshapping is the same as for plain LVM.
368 rhost=$1 vg=$2 lv=$3 rfs=$4
372 ###--------------------------------------------------------------------------
373 ### Expiry computations.
377 ## Parse an ISO8601 DATE, and set YEAR, MONTH, DAY appropriately (and
378 ## without leading zeros).
380 ## Extract the components of the date and trim leading zeros (which will
381 ## cause things to be interpreted as octal and fail).
382 year=${date%%-*} rest=${date#*-}; month=${rest%%-*} day=${rest#*-}
383 year=${year#0} month=${month#0} day=${day#0}
388 ## Convert an ISO8601 DATE to a Julian Day Number.
392 ## The actual calculation: convert a (proleptic) Gregorian calendar date
393 ## into a Julian day number. This is taken from Wikipedia's page
394 ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary
395 ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24
396 ## proleptic Gregorian.
398 ## If the MONTH is January or February then set a = 1, otherwise set a = 0.
399 a=$(( (14 - $month)/12 ))
401 ## Compute a year offset relative to 4799BC-03-01. This puts the leap day
402 ## as the very last day in a year, which is very convenient. The offset
403 ## here is sufficient to make all y values positive (within the range of
404 ## the JDN calendar), and is a multiple of 400, which is the Gregorian
406 y=$(( $year + 4800 - $a ))
408 ## Compute the offset month number in that year. These months count from
410 m=$(( $month + 12*$a - 3 ))
412 ## Now for the main event. The (153 m + 2)/5 term is a surprising but
413 ## correct trick for obtaining the number of days in the first m months of
414 ## the (shifted) year). The magic offset 32045 is what you get when you
415 ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the
417 jdn=$(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 ))
423 ## Read dates on stdin; write to stdout `EXPIRE date' for dates which
424 ## should be expired and `RETAIN date' for dates which should be retained.
426 ## Get the current date and convert it into useful forms.
427 now=$(date +%Y-%m-%d)
429 now_jdn=$(julian $now) now_year=$year now_month=$month now_day=$day
432 ## Work through each date in the input.
436 ## Convert the date into a useful form.
440 ## Work through the policy list.
441 if [ $jdn -le $now_jdn ]; then
442 while read ival age; do
444 ## Decide whether the policy entry applies to this date.
451 if [ $year -eq $now_year ] ||
452 ([ $year -eq $(( $now_year - 1 )) ] &&
453 [ $month -ge $now_month ])
457 if ([ $month -eq $now_month ] && [ $year -eq $now_year ]) ||
458 ((([ $month -eq $(( $now_month - 1 )) ] &&
459 [ $year -eq $now_year ]) ||
460 ([ $month -eq 12 ] && [ $now_month -eq 1 ] &&
461 [ $year -eq $(( $now_year - 1 )) ])) &&
462 [ $day -ge $now_day ])
466 if [ $jdn -ge $(( $now_jdn - 7 )) ]; then apply=t; fi
469 echo >&2 "$quis: unknown age symbol \`$age'"
473 case $apply in nil) continue ;; esac
475 ## Find the interval marker for this date.
481 ydn=$(julian $year-01-01)
482 wk=$(( ($jdn - $ydn)/7 + 1 ))
492 echo >&2 "$quis: unknown interval symbol \`$ival'"
497 ## See if we've alredy retained something in this interval.
500 *) keep=t kept=$kept$marker: ;;
509 t) echo RETAIN $date ;;
510 *) echo EXPIRE $date ;;
516 ###--------------------------------------------------------------------------
517 ### Actually taking backups of filesystems.
519 STOREDIR=@mntbkpdir@/store
520 METADIR=@mntbkpdir@/meta
532 echo \"*** $host $fs $date\"; echo
533 rsync -rx --filter='dir-merge .rsync-backup' ./ |
534 fshash -c$fshashdir/$fs.bkp -a -H$HASH -frsync
539 { echo "*** $host $fs $date"; echo
540 fshash -c$STOREDIR/fshash.cache -H$HASH new/
548 case $i in *[!-0-9]*) continue ;; esac
549 case $seen in *:"$i":*) continue ;; esac
554 while read op date; do
557 echo >&2 " --- keep $date"
560 echo >&2 " --- delete $date"
567 $verbose -n " expire $date..."
569 delete_index $host $fs $date
576 backup_precommit_hook () {
577 host=$1 fs=$2 date=$3
578 ## Override this hook in the configuration file for special effects.
583 backup_commit_hook () {
584 host=$1 fs=$2 date=$3
585 ## Override this hook in the configuration file for special effects.
591 date=$1 fs=$2 fsarg=$3
592 ## Back up FS on the current host.
597 ## Report the start of this attempt.
598 log "START BACKUP of $host:$fs"
600 ## Maybe we need to retry the backup.
603 ## Create and mount the remote snapshot.
606 maybe snap_$snap $fs $fsarg
610 snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
613 $verbose " create snapshot"
615 ## Build the list of hardlink sources.
617 for i in $host $like; do
618 d=$STOREDIR/$i/$fs/last/
619 if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
622 ## Copy files from the remote snapshot.
625 t) $verbose " running rsync" ;;
626 nil) $verbose -n " running rsync..." ;;
629 run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
635 case $dryrun in nil) $verbose " done" ;; esac
637 ## Collect a map of the snapshot for verification purposes.
640 t) $verbose " remote fshash" ;;
641 nil) $verbose -n " remote fshash..." ;;
643 run "@$host: fshash $fs" remote_fshash
646 case $dryrun in nil) $verbose " done" ;; esac
648 ## Remove the snapshot.
649 maybe unsnap_$snap $snapargs $fs $fsarg
650 $verbose " remove snapshot"
652 ## If we failed to copy, then give up.
653 case $rc_rsync:$rc_fshash in
655 0:*) return $rc_fshash ;;
656 *) return $rc_rsync ;;
659 ## Get a matching map of the files received.
660 maybe mkdir -m750 -p $STOREDIR/tmp/
661 localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
663 t) $verbose " local fshash" ;;
664 nil) $verbose -n " local fshash..." ;;
666 run "local fshash $host:$fs" local_fshash || return $?
667 case $dryrun in nil) $verbose " done" ;; esac
669 ## Compare the two maps.
671 run "compare fshash maps for $host:$fs" diff -u new.fshash $localmap
679 if [ $attempt -ge $retry ]; then return $rc; fi
680 $verbose " fshash mismatch; retrying"
681 attempt=$(( $attempt + 1 ))
690 maybe rm -f $localmap
691 $verbose " fshash match"
693 ## Commit this backup.
696 backup_precommit_hook $host $fs $date
698 mv new.fshash $date.fshash
699 insert_index $host $fs $date $VOLUME
700 backup_commit_hook $host $fs $date
702 ln -s $date hack/last
709 ## Expire old backups.
710 case "${expire_policy+t},${default_policy+t}" in
711 ,t) expire_policy=$default_policy ;;
713 case "${expire_policy+t},$dryrun" in
714 t,nil) run "expiry for $host:$fs" expire_backups ;;
715 t,t) expire_backups ;;
721 t) log "END BACKUP of $host:$fs" ;;
722 nil) log "SUCCESSFUL BACKUP of $host:$fs" ;;
727 fs=$1 date=$2 cmd=$3; shift 3
728 ## try_backup FS DATE COMMAND ARGS ...
730 ## Run COMMAND ARGS to back up filesystem FS on the current host,
731 ## maintaining a log, and checking whether it worked. The caller has
732 ## usually worked out the DATE in order to set up the filesystem, and we
733 ## need it to name the log file properly.
735 ## Find a name for the log file. In unusual circumstances, we may have
736 ## deleted old logs from today, so just checking for an unused sequence
737 ## number is insufficient. Instead, check all of the logfiles for today,
738 ## and use a sequence number that's larger than any of them.
745 for i in "$logdir/$host/$fs.$date#"*; do
747 case "$tail" in [!1-9]* | *[!0-9]*) continue ;; esac
748 if [ -f "$i" -a $tail -ge $seq ]; then seq=$(( tail + 1 )); fi
750 log="$logdir/$host/$fs.$date#$seq"
754 ## Run the backup command.
755 case $dryrun in nil) mkdir -p $logdir/$host ;; esac
756 if ! "$cmd" "$@" 9>$log 1>&9; then
758 echo >&2 "$quis: backup of $host:$fs FAILED!"
762 ## Count up the logfiles.
764 for i in "$logdir/$host/$fs".*; do
765 if [ ! -f "$i" ]; then continue; fi
769 ## If there are too many, go through and delete some early ones.
770 if [ $dryrun = nil ] && [ $nlog -gt $MAXLOG ]; then
771 n=$(( nlog - MAXLOG ))
772 for i in "$logdir/$host/$fs".*; do
773 if [ ! -f "$i" ]; then continue; fi
776 if [ $n -eq 0 ]; then break; fi
782 ## backup FS[:ARG] ...
784 ## Back up the filesystems on the currently selected host using the
785 ## currently selected snapshot type.
787 ## Make sure that there's a store volume. We must do this here rather than
788 ## in the main body of the script, since the configuration file needs a
789 ## chance to override STOREDIR.
790 if ! [ -r $STOREDIR/.rsync-backup-store ]; then
791 echo >&2 "$quis: no backup volume mounted"
795 ## Read the volume name if we don't have one already. Again, this allows
796 ## the configuration file to provide a volume name.
797 case "${VOLUME+t}${VOLUME-nil}" in
798 nil) VOLUME=$(cat $METADIR/volume) ;;
801 ## Back up each requested file system in turn.
804 ## Parse the argument.
806 *:*) fsarg=${fs#*:} fs=${fs%%:*} ;;
809 $verbose " filesystem $fs"
811 ## Move to the store directory and set up somewhere to put this backup.
815 if [ ! -d $host ]; then
817 chown root:root $host
819 if [ ! -d $host/$fs ]; then
820 mkdir -m750 $host/$fs
821 chown root:backup $host/$fs
827 ## Find out if we've already copied this filesystem today.
828 date=$(date +%Y-%m-%d)
829 if [ $dryrun = nil ] && [ -d $date ]; then
830 $verbose " already dumped"
834 ## Do the backup of this filesystem.
835 run_backup_cmd $fs $date do_backup $date $fs $fsarg
839 ###--------------------------------------------------------------------------
840 ### Configuration functions.
845 case "${expire_policy+t},${default_policy+t}" in
846 t,) default_policy=$expire_policy ;;
849 $verbose "host $host"
852 snaptype () { snap=$1; shift; snapargs="$*"; retry=0; }
853 rsyncargs () { rsyncargs="$*"; }
854 like () { like="$*"; }
855 retry () { retry="$*"; }
858 case $clear_policy in t) unset expire_policy; clear_policy=nil ;; esac
859 expire_policy="${expire_policy+$expire_policy
863 ###--------------------------------------------------------------------------
864 ### Read the configuration and we're done.
867 echo "usage: $quis [-v] [-c CONF]"
871 echo "$quis version $VERSION"
878 mntbkpdir = $mntbkpdir
879 fshashdir = $fshashdir
884 whine () { echo >&8 "$@"; }
886 while getopts "hVvc:n" opt; do
889 V) version; config; exit 0 ;;
896 shift $((OPTIND - 1))
897 case $# in 0) ;; *) usage >&2; exit 1 ;; esac
902 0) $verbose "All backups successful" ;;
903 *) $verbose "Backups FAILED" ;;
906 ###----- That's all, folks --------------------------------------------------