Maintain an index of backup artifacts.

There's a new program `update-bkp-index' to create and refresh the
database from a backup volume, and some new pieces of `rsync-backup' to
update the index incrementally as artifacts are committed and expired.

diff --git a/Makefile.am b/Makefile.am
index f116932..4fa8227 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -47,6 +47,7 @@ SUBSTVARS = \
        sysconfdir="$(sysconfdir)" \
        mntbkpdir="$(mntbkpdir)" \
        fshashdir="$(fshashdir)" \
        sysconfdir="$(sysconfdir)" \
        mntbkpdir="$(mntbkpdir)" \
        fshashdir="$(fshashdir)" \

+       pkglocalstatedir="$(localstatedir)/lib/bkp" \

        logdir="$(logdir)"
 
 V_SUBST = $(V_SUBST_$V)
        logdir="$(logdir)"
 
 V_SUBST = $(V_SUBST_$V)


@@ -73,6 +74,16 @@ rsync-backup: rsync-backup.in Makefile
                chmod +x rsync-backup.new && \
                mv rsync-backup.new rsync-backup
 
                chmod +x rsync-backup.new && \
                mv rsync-backup.new rsync-backup
 

+sbin_SCRIPTS           += update-bkp-index
+dist_man_MANS          += update-bkp-index.8
+CLEANFILES             += update-bkp-index
+EXTRA_DIST             += update-bkp-index.in
+update-bkp-index: update-bkp-index.in Makefile
+       $(SUBST) >update-bkp-index.new \
+                       $(srcdir)/update-bkp-index.in $(SUBSTVARS) && \
+               chmod +x update-bkp-index.new && \
+               mv update-bkp-index.new update-bkp-index
+

 bin_SCRIPTS            += fshash
 dist_man_MANS          += fshash.1
 CLEANFILES             += fshash
 bin_SCRIPTS            += fshash
 dist_man_MANS          += fshash.1
 CLEANFILES             += fshash

diff --git a/rsync-backup.8 b/rsync-backup.8
index 271a9d6..aa7821f 100644 (file)
--- a/rsync-backup.8
+++ b/rsync-backup.8
@@ -265,11 +265,30 @@ module.
 The default is
 .BR sha256 .
 .TP
 The default is
 .BR sha256 .
 .TP

+.B INDEXDB
+The name of a SQLite database initialized by
+.BR update-bkp-index (8)
+in which an index is maintained of which dumps are on which backup
+volumes.  If the file doesn't exist, then no index is maintained.  The
+default is
+.IB localstatedir /lib/bkp/index.db
+where
+.I localstatedir
+is the state directory configured at build time.
+.TP

 .B MAXLOG
 The number of log files to be kept for each filesystem.  Old logfiles
 are deleted to keep the total number below this bound.  The default
 value is 14.
 .TP
 .B MAXLOG
 The number of log files to be kept for each filesystem.  Old logfiles
 are deleted to keep the total number below this bound.  The default
 value is 14.
 .TP

+.B METADIR
+The metadata directory for the currently mounted backup volume.
+The default is
+.IB mntbkpdir /meta
+where
+.I mntbkpdir
+is the backup mount directory configured at build time.
+.TP

 .B RSYNCOPTS
 Command-line options to pass to
 .BR rsync (1)
 .B RSYNCOPTS
 Command-line options to pass to
 .BR rsync (1)


@@ -315,6 +334,11 @@ where
 .I mntbkpdir
 is the backup mount directory configured at build time.
 .TP
 .I mntbkpdir
 is the backup mount directory configured at build time.
 .TP

+.B VOLUME
+The name of the current volume.  If this is left unset, the volume name
+is read from the file
+.IB METADIR /volume
+once at the start of the backup run.

 .SS Hook functions
 The configuration file may define shell functions to perform custom
 actions at various points in the backup process.
 .SS Hook functions
 The configuration file may define shell functions to perform custom
 actions at various points in the backup process.


@@ -451,6 +475,7 @@ format), together with associated files named
 .BR lvm (8),
 .BR rfreezefs (8),
 .BR rsync (1),
 .BR lvm (8),
 .BR rfreezefs (8),
 .BR rsync (1),

-.BR ssh (1).
+.BR ssh (1),
+.BR update-bkp-index (8).

 .SH AUTHOR
 Mark Wooding, <mdw@distorted.org.uk>
 .SH AUTHOR
 Mark Wooding, <mdw@distorted.org.uk>

diff --git a/rsync-backup.in b/rsync-backup.in
index 7519b12..ac9cfdc 100644 (file)
--- a/rsync-backup.in
+++ b/rsync-backup.in
@@ -149,6 +149,33 @@ hostpath () {
 }
 
 ###--------------------------------------------------------------------------
 }
 
 ###--------------------------------------------------------------------------

+### Database operations.
+
+INDEXDB=@pkglocalstatedir@/index.db
+
+insert_index () {
+  host=$1 fs=$2 date=$3 vol=$4
+
+  if [ -f "$INDEXDB" ]; then
+    sqlite3 "$INDEXDB" <<EOF
+INSERT INTO idx (host, fs, date, vol)
+       VALUES ('$host', '$fs', '$date', '$vol');
+EOF
+  fi
+}
+
+delete_index () {
+  host=$1 fs=$2 date=$3
+
+  if [ -f "$INDEXDB" ]; then
+    sqlite3 "$INDEXDB" <<EOF
+DELETE FROM idx WHERE
+       host = '$host' AND fs = '$fs' AND $date = '$date';
+EOF
+  fi
+}
+
+###--------------------------------------------------------------------------

 ### Snapshot handling.
 
 ## Snapshot protocol.  Each snapshot type has a pair of functions snap_TYPE
 ### Snapshot handling.
 
 ## Snapshot protocol.  Each snapshot type has a pair of functions snap_TYPE


@@ -481,8 +508,10 @@ EOF
 ### Actually taking backups of filesystems.
 
 STOREDIR=@mntbkpdir@/store
 ### Actually taking backups of filesystems.
 
 STOREDIR=@mntbkpdir@/store

+METADIR=@mntbkpdir@/meta

 MAXLOG=14
 HASH=sha256
 MAXLOG=14
 HASH=sha256

+unset VOLUME

 
 bkprc=0
 
 
 bkprc=0
 


@@ -528,6 +557,7 @@ expire_backups () {
        echo "delete $date"
        $verbose -n "   expire $date..."
        rm -rf $date $date.*
        echo "delete $date"
        $verbose -n "   expire $date..."
        rm -rf $date $date.*

+       delete_index $host $fs $date

        $verbose " done"
        ;;
     esac
        $verbose " done"
        ;;
     esac


@@ -657,6 +687,7 @@ do_backup () {
       backup_precommit_hook $host $fs $date
       mv new $date
       mv new.fshash $date.fshash
       backup_precommit_hook $host $fs $date
       mv new $date
       mv new.fshash $date.fshash

+      insert_index $host $fs $date $VOLUME

       backup_commit_hook $host $fs $date
       mkdir hack
       ln -s $date hack/last
       backup_commit_hook $host $fs $date
       mkdir hack
       ln -s $date hack/last


@@ -697,6 +728,12 @@ backup () {
     exit 15
   fi
 
     exit 15
   fi
 

+  ## Read the volume name if we don't have one already.  Again, this allows
+  ## the configuration file to provide a volume name.
+  case "${VOLUME+t}${VOLUME-nil}" in
+    nil) VOLUME=$(cat $METADIR/volume) ;;
+  esac
+

   ## Back up each requested file system in turn.
   for fs in "$@"; do
 
   ## Back up each requested file system in turn.
   for fs in "$@"; do
 

diff --git a/update-bkp-index.8 b/update-bkp-index.8
new file mode 100644 (file)
index 0000000..91f82c8
--- /dev/null
+++ b/update-bkp-index.8
@@ -0,0 +1,61 @@
+.ie t .ds o \(bu
+.el .ds o o
+.de hP
+.IP
+\h'-\w'\fB\\$1\ \fP'u'\fB\\$1\ \fP\c
+..
+.TH update-bkp-index 8 "25 January 2013" rsync-backup
+.SH NAME
+update-bkp-index \- create or update the rsync-backup index database
+.SH SYNOPSIS
+The
+.BR rsync-backup (8)
+program can maintain an index database which keeps track of which dumps
+are on which backup volumes.
+.PP
+The
+.B update-bkp-index
+program updates this database from the currently mounted backup volume,
+creating or upgrading it if necessary.
+.PP
+The program ignores any command-line arguments it's given, but it makes
+use of a number of environment variables.
+.TP
+.B INDEXDB
+The name of a SQLite database initialized by
+.BR update-bkp-index (8)
+in which an index is maintained of which dumps are on which backup
+volumes.  If the file doesn't exist, then no index is maintained.  The
+default is
+.IB localstatedir /lib/bkp/index.db
+where
+.I localstatedir
+is the state directory configured at build time.
+.TP
+.B METADIR
+The metadata directory for the currently mounted backup volume.
+The default is
+.IB mntbkpdir /meta
+where
+.I mntbkpdir
+is the backup mount directory configured at build time.
+.TP
+.B STOREDIR
+Where the actual backup trees should be stored.  See the section on
+.B Archive structure
+below.
+The default is
+.IB mntbkpdir /store
+where
+.I mntbkpdir
+is the backup mount directory configured at build time.
+.TP
+.B VOLUME
+The name of the current volume.  The default is to read this from the
+file
+.IB METADIR /volume
+once at the start of the backup run.
+.SH SEE ALSO
+.BR rsync-backup (8).
+.SH AUTHOR
+Mark Wooding, <mdw@distorted.org.uk>

diff --git a/update-bkp-index.in b/update-bkp-index.in
new file mode 100644 (file)
index 0000000..23b64bd
--- /dev/null
+++ b/update-bkp-index.in
@@ -0,0 +1,100 @@
+#! @BASH@
+###
+### Backup script
+###
+### (c) 2012 Mark Wooding
+###
+
+###----- Licensing notice ---------------------------------------------------
+###
+### This file is part of the `rsync-backup' program.
+###
+### rsync-backup is free software; you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation; either version 2 of the License, or
+### (at your option) any later version.
+###
+### rsync-backup is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with rsync-backup; if not, write to the Free Software Foundation,
+### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+set -e
+
+mkdir -p @pkglocalstatedir@
+INDEXDB=@pkglocalstatedir@/index.db
+: ${STOREDIR=@mntbkpdir@/store}
+: ${METADIR=@mntbkpdir@/meta}
+
+if [ ! -f $STOREDIR/.rsync-backup-store ]; then
+  echo >&2 "$quis: no backup volume mounted"
+  exit 15
+fi
+: ${VOLUME=$(cat $METADIR/volume)}
+
+## If the database exists then we're OK.  (This will turn into a version
+## check and upgrade if the schema changes.)
+if [ ! -f "$INDEXDB" ]; then
+
+  ## Create the database.
+  rm -f "$INDEXDB.new"
+  sqlite3 "$INDEXDB.new" <<EOF
+CREATE TABLE meta (
+       version INTEGER NOT NULL);
+INSERT INTO meta (version) VALUES (0);
+
+CREATE TABLE idx (
+       host TEXT NOT NULL,
+       fs TEXT NOT NULL,
+       date TEXT NOT NULL,
+       vol TEXT NOT NULL,
+       PRIMARY KEY (host, fs, date));
+CREATE INDEX idx_byvol ON idx (vol);
+EOF
+
+  ## Done.
+  mv "$INDEXDB.new" "$INDEXDB"
+fi
+
+{
+  ## Do everything in a single transaction.  SQLite is pretty good at this,
+  ## and also it'll avoid updating the database until it sees a `COMMIT'
+  ## command, so if we fail halfway through we're still OK.  So it's safe to
+  ## start by removing all of the current records referring to this volume.
+  cat <<EOF
+BEGIN;
+DELETE FROM idx WHERE vol = '$VOLUME';
+EOF
+
+  ## Now work through the various filesystems.  This is a slightly cheesy way
+  ## of finding them.
+  for i in $STOREDIR/*/*/last; do
+
+    ## Parse out the host and filesystem names.
+    i=${i%/*}
+    fs=${i##*/} i=${i%/*}
+    host=${i##*/} i=${i%/*}
+
+    ## And work through the date list.
+    for j in $STOREDIR/$host/$fs/*; do
+      if [ -L "$j" ] || [ ! -d "$j" ]; then continue; fi
+      j=${j%/}
+      date=${j##*/}
+      cat <<EOF
+INSERT INTO idx (host, fs, date, vol)
+       VALUES ('$host', '$fs', '$date', '$VOLUME');
+EOF
+    done
+  done
+
+  ## Done.
+  cat <<EOF
+COMMIT;
+EOF
+} | sqlite3 "$INDEXDB"
+
+###----- That's all, folks --------------------------------------------------