rsync-backup.in, rsync-backup.8: Retry backups which fail fshash check.

[rsync-backup] / rsync-backup.in

diff --git a/rsync-backup.in b/rsync-backup.in
index 5c9a54c..ade7231 100644 (file)
--- a/rsync-backup.in
+++ b/rsync-backup.in
@@ -521,62 +521,84 @@ do_backup () {
   ## Back up FS on the current host.
 
   set -e
+  attempt=0
 
   ## Report the start of this attempt.
   log "START BACKUP of $host:$fs"
 
-  ## Create and mount the remote snapshot.
-  snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
-  $verbose "   create snapshot"
+  ## Maybe we need to retry the backup.
+  while :; do
 
-  ## Build the list of hardlink sources.
-  linkdests=""
-  for i in $host $like; do
-    d=$STOREDIR/$i/$fs/last/
-    if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
-  done
+    ## Create and mount the remote snapshot.
+    snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
+    $verbose " create snapshot"
 
-  ## Copy files from the remote snapshot.
-  mkdir -p new/
-  $verbose -n "        running rsync..."
-  set +e
-  run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
-    $linkdests \
-    $rsyncargs \
-    $snapmnt/ new/
-  rc_rsync=$?
-  set -e
-  $verbose " done"
+    ## Build the list of hardlink sources.
+    linkdests=""
+    for i in $host $like; do
+      d=$STOREDIR/$i/$fs/last/
+      if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
+    done
 
-  ## Collect a map of the snapshot for verification purposes.
-  set +e
-  $verbose -n "        remote fshash..."
-  run "@$host: fshash $fs" remote_fshash
-  rc_fshash=$?
-  set -e
-  $verbose " done"
+    ## Copy files from the remote snapshot.
+    mkdir -p new/
+    $verbose -n "      running rsync..."
+    set +e
+    run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
+      $linkdests \
+      $rsyncargs \
+      $snapmnt/ new/
+    rc_rsync=$?
+    set -e
+    $verbose " done"
+
+    ## Collect a map of the snapshot for verification purposes.
+    set +e
+    $verbose -n "      remote fshash..."
+    run "@$host: fshash $fs" remote_fshash
+    rc_fshash=$?
+    set -e
+    $verbose " done"
 
-  ## Remove the snapshot.
-  unsnap_$snap $snapargs $fs $fsarg
-  $verbose "   remove snapshot"
+    ## Remove the snapshot.
+    unsnap_$snap $snapargs $fs $fsarg
+    $verbose " remove snapshot"
 
-  ## If we failed to copy, then give up.
-  case $rc_rsync:$rc_fshash in
-    0:0) ;;
-    0:*) return $rc_fshash ;;
-    *) return $rc_rsync ;;
-  esac
+    ## If we failed to copy, then give up.
+    case $rc_rsync:$rc_fshash in
+      0:0) ;;
+      0:*) return $rc_fshash ;;
+      *) return $rc_rsync ;;
+    esac
+
+    ## Get a matching map of the files received.
+    mkdir -m750 -p $STOREDIR/tmp
+    localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
+    $verbose -n "      local fshash..."
+    run "local fshash $host:$fs" local_fshash || return $?
+    $verbose " done"
 
-  ## Get a matching map of the files received.
-  mkdir -m750 -p $STOREDIR/tmp
-  localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
-  $verbose -n "        local fshash..."
-  run "local fshash $host:$fs" local_fshash || return $?
-  $verbose " done"
+    ## Compare the two maps.
+    set +e
+    run "compare fshash maps for $host:$fs" diff -u new.fshash $localmap
+    rc_diff=$?
+    set -e
+    case $rc_diff in
+      0)
+       break
+       ;;
+      1)
+       if [ $attempt -ge $retry ]; then return $rc; fi
+       $verbose "      fshash mismatch; retrying"
+       attempt=$(( $attempt + 1 ))
+       ;;
+      *)
+       return $rc_diff
+       ;;
+    esac
+  done
 
-  ## Compare the two maps.
-  run "compare fshash maps for $host:$fs" \
-    diff -u new.fshash $localmap || return $?
+  ## Glorious success.
   rm -f $localmap
   $verbose "   fshash match"
 
@@ -606,6 +628,15 @@ backup () {
   ## Back up the filesystems on the currently selected host using the
   ## currently selected snapshot type.
 
+  ## Make sure that there's a store volume.  We must do this here rather than
+  ## in the main body of the script, since the configuration file needs a
+  ## chance to override STOREDIR.
+  if ! [ -r $STOREDIR/.rsync-backup-store ]; then
+    echo >&2 "$quis: no backup volume mounted"
+    exit 15
+  fi
+
+  ## Back up each requested file system in turn.
   for fs in "$@"; do
 
     ## Parse the argument.
@@ -678,9 +709,10 @@ backup () {
 ### Configuration functions.
 
 host () { host=$1; like=; $verbose "host $host"; }
-snaptype () { snap=$1; shift; snapargs="$*"; }
+snaptype () { snap=$1; shift; snapargs="$*"; retry=0; }
 rsyncargs () { rsyncargs="$*"; }
 like () { like="$*"; }
+retry () { retry="$*"; }
 
 retain () {
   expire_policy="${expire_policy+$expire_policy