conf=@sysconfdir@/rsync-backup.conf
verbose=:
+dryrun=nil
###--------------------------------------------------------------------------
### Utility functions.
}
log () {
- now=$(date +"%Y-%m-%d %H:%M:%S %z")
- echo >&9 "$now $*"
+ case $dryrun in
+ t)
+ echo >&2 " *** $*"
+ ;;
+ nil)
+ now=$(date +"%Y-%m-%d %H:%M:%S %z")
+ echo >&9 "$now $*"
+ ;;
+ esac
+}
+
+maybe () {
+ ## Run CMD, if this isn't a dry run.
+
+ case $dryrun in
+ t) echo >&2 " +++ $*" ;;
+ nil) "$@" ;;
+ esac
+}
+
+copy () {
+ prefix=$1
+ ## Copy lines from stdin to stdout, adding PREFIX.
+
+ while IFS= read -r line; do
+ printf "%s %s\n" "$prefix" "$line"
+ done
}
run () {
tag=$1 cmd=$2; shift 2
## Run CMD, logging its output in a pleasing manner.
- log "BEGIN $tag"
- rc=$(
- { { { ( set +e
- "$cmd" "$@" 3>&- 4>&- 5>&- 9>&-
- echo $? >&5; ) |
- while IFS= read line; do echo "| $line"; done >&4; } 2>&1 |
- while IFS= read line; do echo "* $line"; done >&4; } 4>&1 |
- cat >&9; } 5>&1 </dev/null
- )
- case $rc in
- 0) log "END $tag" ;;
- *) log "FAIL $tag (rc = $rc)" ;;
+ case $dryrun in
+ t)
+ echo >&2 " *** RUN $tag"
+ echo >&2 " +++ $cmd $*"
+ rc=0
+ ;;
+ nil)
+ log "BEGIN $tag"
+ rc=$(
+ { { { ( set +e
+ "$cmd" "$@" 3>&- 4>&- 5>&- 9>&-
+ echo $? >&5; ) |
+ copy "|" >&4; } 2>&1 |
+ copy "*" >&4; } 4>&1 |
+ cat >&9; } 5>&1 </dev/null
+ )
+ case $rc in
+ 0) log "END $tag" ;;
+ *) log "FAIL $tag (rc = $rc)" ;;
+ esac
+ ;;
esac
return $rc
}
## way it will be processed by a shell.
if localp $host; then run "@$host: $tag" sh -c "$cmd"
- else run "@$host: $tag" ssh $host "$cmd"
+ else run "@$host: $tag" ssh $userat$host "$cmd"
fi
}
_hostrun () {
h=$1 cmd=$2
- ## Like hostrun, but without the complicated logging, but targetted at a
+ ## Like hostrun, but without the complicated logging, and targetted at a
## specific host.
if localp $h; then sh -c "$cmd"
## current host is local.
if localp $host; then echo $path
- else echo $host:$path
+ else echo $userat$host:$path
+ fi
+}
+
+###--------------------------------------------------------------------------
+### Database operations.
+
+INDEXDB=@pkglocalstatedir@/index.db
+
+insert_index () {
+ host=$1 fs=$2 date=$3 vol=$4
+
+ if [ -f "$INDEXDB" ]; then
+ sqlite3 "$INDEXDB" <<EOF
+INSERT INTO idx (host, fs, date, vol)
+ VALUES ('$host', '$fs', '$date', '$vol');
+EOF
+ fi
+}
+
+delete_index () {
+ host=$1 fs=$2 date=$3
+
+ if [ -f "$INDEXDB" ]; then
+ sqlite3 "$INDEXDB" <<EOF
+DELETE FROM idx WHERE
+ host = '$host' AND fs = '$fs' AND date = '$date';
+EOF
fi
}
## Get the volume host to create the snapshot.
set +e
- _hostrun >&2 3>&- $lvhost \
+ _hostrun >&2 3>&- $userat$lvhost \
"lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv"
snaprc=$?
set -e
case $tok in
"$tok_THAWED") ;;
*)
- _hostrun >&2 3>&- $lvhost "lvremove -f $vg/$lv.bkp" || :
+ _hostrun >&2 3>&- $userat$lvhost "lvremove -f $vg/$lv.bkp" || :
echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
exit 1
;;
esac
## Mount the snapshot on the volume host.
- _hostrun >&2 $lvhost "
+ _hostrun >&2 $userat$lvhost "
mkdir -p $SNAPDIR/$lv
mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv"
}
### Actually taking backups of filesystems.
STOREDIR=@mntbkpdir@/store
+METADIR=@mntbkpdir@/meta
MAXLOG=14
HASH=sha256
+unset VOLUME
bkprc=0
remote_fshash () {
- _hostrun $host "
+ _hostrun $userat$host "
umask 077
mkdir -p $fshashdir
cd ${snapmnt#*:}
done; } |
expire |
while read op date; do
- case $op in
- RETAIN)
+ case $op,$dryrun in
+ RETAIN,t)
+ echo >&2 " --- keep $date"
+ ;;
+ EXPIRE,t)
+ echo >&2 " --- delete $date"
+ ;;
+ RETAIN,nil)
echo "keep $date"
;;
- EXPIRE)
+ EXPIRE,nil)
echo "delete $date"
$verbose -n " expire $date..."
rm -rf $date $date.*
+ delete_index $host $fs $date
$verbose " done"
;;
esac
## Back up FS on the current host.
set -e
+ attempt=0
## Report the start of this attempt.
log "START BACKUP of $host:$fs"
- ## Create and mount the remote snapshot.
- snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
- $verbose " create snapshot"
+ ## Maybe we need to retry the backup.
+ while :; do
- ## Build the list of hardlink sources.
- linkdests=""
- for i in $host $like; do
- d=$STOREDIR/$i/$fs/last/
- if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
- done
+ ## Create and mount the remote snapshot.
+ case $dryrun in
+ t)
+ maybe snap_$snap $fs $fsarg
+ snapmnt="<snapshot>"
+ ;;
+ nil)
+ snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
+ ;;
+ esac
+ $verbose " create snapshot"
- ## Copy files from the remote snapshot.
- mkdir -p new/
- $verbose -n " running rsync..."
- set +e
- run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
- $linkdests \
- $rsyncargs \
- $snapmnt/ new/
- rc_rsync=$?
- set -e
- $verbose " done"
+ ## Build the list of hardlink sources.
+ linkdests=""
+ for i in $host $like; do
+ d=$STOREDIR/$i/$fs/last/
+ if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
+ done
- ## Collect a map of the snapshot for verification purposes.
- set +e
- $verbose -n " remote fshash..."
- run "@$host: fshash $fs" remote_fshash
- rc_fshash=$?
- set -e
- $verbose " done"
+ ## Copy files from the remote snapshot.
+ maybe mkdir -p new/
+ case $dryrun in
+ t) $verbose " running rsync" ;;
+ nil) $verbose -n " running rsync..." ;;
+ esac
+ set +e
+ run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
+ $linkdests \
+ $rsyncargs \
+ $snapmnt/ new/
+ rc_rsync=$?
+ set -e
+ case $dryrun in nil) $verbose " done" ;; esac
+
+ ## Collect a map of the snapshot for verification purposes.
+ set +e
+ case $dryrun in
+ t) $verbose " remote fshash" ;;
+ nil) $verbose -n " remote fshash..." ;;
+ esac
+ run "@$host: fshash $fs" remote_fshash
+ rc_fshash=$?
+ set -e
+ case $dryrun in nil) $verbose " done" ;; esac
- ## Remove the snapshot.
- unsnap_$snap $snapargs $fs $fsarg
- $verbose " remove snapshot"
+ ## Remove the snapshot.
+ maybe unsnap_$snap $snapargs $fs $fsarg
+ $verbose " remove snapshot"
- ## If we failed to copy, then give up.
- case $rc_rsync:$rc_fshash in
- 0:0) ;;
- 0:*) return $rc_fshash ;;
- *) return $rc_rsync ;;
- esac
+ ## If we failed to copy, then give up.
+ case $rc_rsync:$rc_fshash in
+ 0:0) ;;
+ 0:*) return $rc_fshash ;;
+ *) return $rc_rsync ;;
+ esac
+
+ ## Get a matching map of the files received.
+ maybe mkdir -m750 -p $STOREDIR/tmp/
+ localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
+ case $dryrun in
+ t) $verbose " local fshash" ;;
+ nil) $verbose -n " local fshash..." ;;
+ esac
+ run "local fshash $host:$fs" local_fshash || return $?
+ case $dryrun in nil) $verbose " done" ;; esac
- ## Get a matching map of the files received.
- mkdir -m750 -p $STOREDIR/tmp
- localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
- $verbose -n " local fshash..."
- run "local fshash $host:$fs" local_fshash || return $?
- $verbose " done"
-
- ## Compare the two maps.
- run "compare fshash maps for $host:$fs" \
- diff -u new.fshash $localmap || return $?
- rm -f $localmap
+ ## Compare the two maps.
+ set +e
+ run "compare fshash maps for $host:$fs" diff -u new.fshash $localmap
+ rc_diff=$?
+ set -e
+ case $rc_diff in
+ 0)
+ break
+ ;;
+ 1)
+ if [ $attempt -ge $retry ]; then return $rc; fi
+ $verbose " fshash mismatch; retrying"
+ attempt=$(( $attempt + 1 ))
+ ;;
+ *)
+ return $rc_diff
+ ;;
+ esac
+ done
+
+ ## Glorious success.
+ maybe rm -f $localmap
$verbose " fshash match"
## Commit this backup.
- backup_precommit_hook $host $fs $date
- mv new $date
- mv new.fshash $date.fshash
- backup_commit_hook $host $fs $date
- mkdir hack
- ln -s $date hack/last
- mv hack/last .
- rmdir hack
+ case $dryrun in
+ nil)
+ backup_precommit_hook $host $fs $date
+ mv new $date
+ mv new.fshash $date.fshash
+ insert_index $host $fs $date $VOLUME
+ backup_commit_hook $host $fs $date
+ mkdir hack
+ ln -s $date hack/last
+ mv hack/last .
+ rmdir hack
+ ;;
+ esac
$verbose " commit"
## Expire old backups.
- case "${expire_policy+t}" in
- t) run "expiry for $host:$fs" expire_backups ;;
+ case "${expire_policy+t},${default_policy+t}" in
+ ,t) expire_policy=$default_policy ;;
+ esac
+ case "${expire_policy+t},$dryrun" in
+ t,nil) run "expiry for $host:$fs" expire_backups ;;
+ t,t) expire_backups ;;
esac
+ clear_policy=t
## Report success.
- log "SUCCESSFUL BACKUP of $host:$fs"
+ case $dryrun in
+ t) log "END BACKUP of $host:$fs" ;;
+ nil) log "SUCCESSFUL BACKUP of $host:$fs" ;;
+ esac
+}
+
+run_backup_cmd () {
+ fs=$1 date=$2 cmd=$3; shift 3
+ ## try_backup FS DATE COMMAND ARGS ...
+ ##
+ ## Run COMMAND ARGS to back up filesystem FS on the current host,
+ ## maintaining a log, and checking whether it worked. The caller has
+ ## usually worked out the DATE in order to set up the filesystem, and we
+ ## need it to name the log file properly.
+
+ ## Find a name for the log file. In unusual circumstances, we may have
+ ## deleted old logs from today, so just checking for an unused sequence
+ ## number is insufficient. Instead, check all of the logfiles for today,
+ ## and use a sequence number that's larger than any of them.
+ case $dryrun in
+ t)
+ log=/dev/null
+ ;;
+ nil)
+ seq=1
+ for i in "$logdir/$host/$fs.$date#"*; do
+ tail=${i##*#}
+ case "$tail" in [!1-9]* | *[!0-9]*) continue ;; esac
+ if [ -f "$i" -a $tail -ge $seq ]; then seq=$(( tail + 1 )); fi
+ done
+ log="$logdir/$host/$fs.$date#$seq"
+ ;;
+ esac
+
+ ## Run the backup command.
+ case $dryrun in nil) mkdir -p $logdir/$host ;; esac
+ if ! "$cmd" "$@" 9>$log 1>&9; then
+ echo >&2
+ echo >&2 "$quis: backup of $host:$fs FAILED!"
+ bkprc=1
+ fi
+
+ ## Count up the logfiles.
+ nlog=0
+ for i in "$logdir/$host/$fs".*; do
+ if [ ! -f "$i" ]; then continue; fi
+ nlog=$(( nlog + 1 ))
+ done
+
+ ## If there are too many, go through and delete some early ones.
+ if [ $dryrun = nil ] && [ $nlog -gt $MAXLOG ]; then
+ n=$(( nlog - MAXLOG ))
+ for i in "$logdir/$host/$fs".*; do
+ if [ ! -f "$i" ]; then continue; fi
+ rm -f "$i"
+ n=$(( n - 1 ))
+ if [ $n -eq 0 ]; then break; fi
+ done
+ fi
}
backup () {
## Back up the filesystems on the currently selected host using the
## currently selected snapshot type.
+ ## Make sure that there's a store volume. We must do this here rather than
+ ## in the main body of the script, since the configuration file needs a
+ ## chance to override STOREDIR.
+ if ! [ -r $STOREDIR/.rsync-backup-store ]; then
+ echo >&2 "$quis: no backup volume mounted"
+ exit 15
+ fi
+
+ ## Read the volume name if we don't have one already. Again, this allows
+ ## the configuration file to provide a volume name.
+ case "${VOLUME+t}${VOLUME-nil}" in
+ nil) VOLUME=$(cat $METADIR/volume) ;;
+ esac
+
+ ## Back up each requested file system in turn.
for fs in "$@"; do
## Parse the argument.
## Move to the store directory and set up somewhere to put this backup.
cd $STOREDIR
- if [ ! -d $host ]; then
- mkdir -m755 $host
- chown root:root $host
- fi
- if [ ! -d $host/$fs ]; then
- mkdir -m750 $host/$fs
- chown root:backup $host/$fs
- fi
+ case $dryrun in
+ nil)
+ if [ ! -d $host ]; then
+ mkdir -m755 $host
+ chown root:root $host
+ fi
+ if [ ! -d $host/$fs ]; then
+ mkdir -m750 $host/$fs
+ chown root:backup $host/$fs
+ fi
+ ;;
+ esac
cd $host/$fs
## Find out if we've already copied this filesystem today.
date=$(date +%Y-%m-%d)
- if [ -d $date ]; then
+ if [ $dryrun = nil ] && [ -d $date ]; then
$verbose " already dumped"
continue
fi
- ## Find a name for the log file. In unusual circumstances, we may have
- ## deleted old logs from today, so just checking for an unused sequence
- ## number is insufficient. Instead, check all of the logfiles for today,
- ## and use a sequence number that's larger than any of them.
- seq=1
- for i in "$logdir/$host/$fs.$date#"*; do
- tail=${i##*#}
- case "$tail" in [!1-9]* | *[!0-9]*) continue ;; esac
- if [ -f "$i" -a $tail -ge $seq ]; then seq=$(( tail + 1 )); fi
- done
- log="$logdir/$host/$fs.$date#$seq"
-
## Do the backup of this filesystem.
- mkdir -p $logdir/$host
- if ! do_backup $date $fs $fsarg 9>$log 1>&9; then
- echo >&2
- echo >&2 "$quis: backup of $host:$fs FAILED!"
- bkprc=1
- fi
-
- ## Count up the logfiles.
- nlog=0
- for i in "$logdir/$host/$fs".*; do
- if [ ! -f "$i" ]; then continue; fi
- nlog=$(( nlog + 1 ))
- done
-
- ## If there are too many, go through and delete some early ones.
- if [ $nlog -gt $MAXLOG ]; then
- n=$(( nlog - MAXLOG ))
- for i in "$logdir/$host/$fs".*; do
- if [ ! -f "$i" ]; then continue; fi
- rm -f "$i"
- n=$(( n - 1 ))
- if [ $n -eq 0 ]; then break; fi
- done
- fi
+ run_backup_cmd $fs $date do_backup $date $fs $fsarg
done
}
###--------------------------------------------------------------------------
### Configuration functions.
-host () { host=$1; like=; $verbose "host $host"; }
-snaptype () { snap=$1; shift; snapargs="$*"; }
+host () {
+ host=$1
+ like= userat=
+ case "${expire_policy+t},${default_policy+t}" in
+ t,) default_policy=$expire_policy ;;
+ esac
+ unset expire_policy
+ $verbose "host $host"
+}
+
+snaptype () { snap=$1; shift; snapargs="$*"; retry=0; }
rsyncargs () { rsyncargs="$*"; }
like () { like="$*"; }
+retry () { retry="$*"; }
+user () { userat="$*@"; }
retain () {
+ case $clear_policy in t) unset expire_policy; clear_policy=nil ;; esac
expire_policy="${expire_policy+$expire_policy
}$*"
}
whine () { echo >&8 "$@"; }
-while getopts "hVvc:" opt; do
+while getopts "hVvc:n" opt; do
case "$opt" in
h) usage; exit 0 ;;
V) version; config; exit 0 ;;
v) verbose=whine ;;
c) conf=$OPTARG ;;
+ n) dryrun=t ;;
*) exit 1 ;;
esac
done
exec 8>&1
. "$conf"
+case "$bkprc" in
+ 0) $verbose "All backups successful" ;;
+ *) $verbose "Backups FAILED" ;;
+esac
###----- That's all, folks --------------------------------------------------
~mdw / rsync-backup / blobdiff