X-Git-Url: https://git.distorted.org.uk/~mdw/rsync-backup/blobdiff_plain/9b1d71c6944756a0a732b8553d0516c3f0e720e2..9653f45d4e6e3009f86f912c7bee4441e9ae2b06:/rsync-backup.in?ds=sidebyside diff --git a/rsync-backup.in b/rsync-backup.in index e232bc8..2c1e3b6 100644 --- a/rsync-backup.in +++ b/rsync-backup.in @@ -27,12 +27,7 @@ set -e thishost=$(hostname -s) quis=${0##*/} - -VERSION=@VERSION@ -mntbkpdir=@mntbkpdir@ -logdir=@logdir@ -fshashdir=@fshashdir@ -conf=@sysconfdir@/rsync-backup.conf +. @pkgdatadir@/lib.sh verbose=: dryrun=nil @@ -88,6 +83,14 @@ copy () { } run () { + stdinp=nil + while :; do + case $1 in + -stdin) stdinp=t; shift ;; + --) shift; break ;; + *) break ;; + esac + done tag=$1 cmd=$2; shift 2 ## Run CMD, logging its output in a pleasing manner. @@ -100,12 +103,13 @@ run () { nil) log "BEGIN $tag" rc=$( + case $stdinp in nil) exec &- 4>&- 5>&- 9>&- echo $? >&5; ) | copy "|" >&4; } 2>&1 | copy "*" >&4; } 4>&1 | - cat >&9; } 5>&1 &9; } 5>&1 ) case $rc in 0) log "END $tag" ;; @@ -116,6 +120,15 @@ run () { return $rc } +run_diff () { + out=$1 old=$2 new=$3 + ## Write a unified diff from OLD to NEW, to OUT. + + set +e; diff -u "$old" "$new" >"$out"; rc=$?; set -e + case $rc in 1) cat "$out" ;; esac + return $rc +} + localp () { h=$1 ## Answer whether H is a local host. @@ -182,11 +195,33 @@ runhook () { done } +remove_old_logfiles () { + base=$1 + ## Remove old logfiles with names of the form BASE.DATE#N, so that there + ## are at most $MAXLOG of them. + + ## Count up the logfiles. + nlog=0 + for i in "$base".*; do + if [ ! -f "$i" ]; then continue; fi + nlog=$(( nlog + 1 )) + done + + ## If there are too many, go through and delete some early ones. + if [ $dryrun = nil ] && [ $nlog -gt $MAXLOG ]; then + n=$(( nlog - MAXLOG )) + for i in "$base".*; do + if [ ! -f "$i" ]; then continue; fi + rm -f "$i" + n=$(( n - 1 )) + if [ $n -eq 0 ]; then break; fi + done + fi +} + ###-------------------------------------------------------------------------- ### Database operations. -INDEXDB=@pkglocalstatedir@/index.db - insert_index () { host=$1 fs=$2 date=$3 vol=$4 @@ -255,7 +290,6 @@ unsnap_ro () { ## Snapshot using LVM. SNAPSIZE="-l10%ORIGIN" -SNAPDIR=@mntbkpdir@/snap snap_lvm () { vg=$1 lv=$2 @@ -397,53 +431,6 @@ unsnap_rfreezefs () { ###-------------------------------------------------------------------------- ### Expiry computations. -parsedate () { - date=$1 - ## Parse an ISO8601 DATE, and set YEAR, MONTH, DAY appropriately (and - ## without leading zeros). - - ## Extract the components of the date and trim leading zeros (which will - ## cause things to be interpreted as octal and fail). - year=${date%%-*} rest=${date#*-}; month=${rest%%-*} day=${rest#*-} - year=${year#0} month=${month#0} day=${day#0} -} - -julian () { - date=$1 - ## Convert an ISO8601 DATE to a Julian Day Number. - - parsedate $date - - ## The actual calculation: convert a (proleptic) Gregorian calendar date - ## into a Julian day number. This is taken from Wikipedia's page - ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary - ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24 - ## proleptic Gregorian. - - ## If the MONTH is January or February then set a = 1, otherwise set a = 0. - a=$(( (14 - $month)/12 )) - - ## Compute a year offset relative to 4799BC-03-01. This puts the leap day - ## as the very last day in a year, which is very convenient. The offset - ## here is sufficient to make all y values positive (within the range of - ## the JDN calendar), and is a multiple of 400, which is the Gregorian - ## cycle length. - y=$(( $year + 4800 - $a )) - - ## Compute the offset month number in that year. These months count from - ## zero, not one. - m=$(( $month + 12*$a - 3 )) - - ## Now for the main event. The (153 m + 2)/5 term is a surprising but - ## correct trick for obtaining the number of days in the first m months of - ## the (shifted) year). The magic offset 32045 is what you get when you - ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the - ## above machinery. - jdn=$(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 )) - - echo $jdn -} - expire () { ## Read dates on stdin; write to stdout `EXPIRE date' for dates which ## should be expired and `RETAIN date' for dates which should be retained. @@ -541,8 +528,6 @@ EOF ###-------------------------------------------------------------------------- ### Actually taking backups of filesystems. -STOREDIR=@mntbkpdir@/store -METADIR=@mntbkpdir@/meta MAXLOG=14 HASH=sha256 unset VOLUME @@ -627,6 +612,7 @@ do_backup () { set -e attempt=0 + fshash_diff=nil ## Run a hook beforehand. set +e; runhook setup $host $fs $date; rc=$?; set -e @@ -654,6 +640,20 @@ do_backup () { esac $verbose " create snapshot" + ## If we had a fshash-mismatch, then clear out the potentially stale + ## entries, both locally and remotely. + case $fshash_diff in + nil) ;; + *) + $verbose " prune cache" + run -stdin "local prune fshash" \ + fshash -u -c$STOREDIR/fshash.cache -H$HASH new/ <$fshash_diff + run -stdin "@$host: prune fshash" \ + _hostrun $userat$host <$fshash_diff \ + "fshash -u -c$fshashdir/$fs.bkp -H$HASH ${snapmnt#*:}" + ;; + esac + ## Build the list of hardlink sources. linkdests="" for i in $host $like; do @@ -710,7 +710,9 @@ do_backup () { ## Compare the two maps. set +e - run "compare fshash maps for $host:$fs" diff -u new.fshash $localmap + fshash_diff=$STOREDIR/tmp/fshash-diff.$host.$fs.$date + run "compare fshash maps for $host:$fs" \ + run_diff $fshash_diff new.fshash $localmap rc_diff=$? set -e case $rc_diff in @@ -730,6 +732,7 @@ do_backup () { ## Glorious success. maybe rm -f $localmap + case $fshash_diff in nil) ;; *) maybe rm -f $fshash_diff ;; esac $verbose " fshash match" ## Commit this backup. @@ -801,23 +804,8 @@ run_backup_cmd () { bkprc=1 fi - ## Count up the logfiles. - nlog=0 - for i in "$logdir/$host/$fs".*; do - if [ ! -f "$i" ]; then continue; fi - nlog=$(( nlog + 1 )) - done - - ## If there are too many, go through and delete some early ones. - if [ $dryrun = nil ] && [ $nlog -gt $MAXLOG ]; then - n=$(( nlog - MAXLOG )) - for i in "$logdir/$host/$fs".*; do - if [ ! -f "$i" ]; then continue; fi - rm -f "$i" - n=$(( n - 1 )) - if [ $n -eq 0 ]; then break; fi - done - fi + ## Clear away any old logfiles. + remove_old_logfiles "$logdir/$host/$fs" } backup () { @@ -899,7 +887,7 @@ host () { $verbose "host $host" } -snaptype () { snap=$1; shift; snapargs="$*"; retry=0; } +snaptype () { snap=$1; shift; snapargs="$*"; retry=1; } rsyncargs () { rsyncargs="$*"; } like () { like="$*"; } retry () { retry="$*"; } @@ -922,16 +910,6 @@ version () { echo "$quis version $VERSION" } -config () { - echo - cat <&8 "$@"; } while getopts "hVvc:n" opt; do