From: Mark Wooding <mwooding@good.com>
Date: Fri, 25 Jan 2013 18:33:03 +0000 (+0000)
Subject: Maintain an index of backup artifacts.
X-Git-Tag: 1.0.0~11^2
X-Git-Url: https://git.distorted.org.uk/~mdw/rsync-backup/commitdiff_plain/a8447303fe9883920c783acae85c76caeaa78c32

Maintain an index of backup artifacts.

There's a new program `update-bkp-index' to create and refresh the
database from a backup volume, and some new pieces of `rsync-backup' to
update the index incrementally as artifacts are committed and expired.
---

diff --git a/Makefile.am b/Makefile.am
index f116932..4fa8227 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -47,6 +47,7 @@ SUBSTVARS = \
 	sysconfdir="$(sysconfdir)" \
 	mntbkpdir="$(mntbkpdir)" \
 	fshashdir="$(fshashdir)" \
+	pkglocalstatedir="$(localstatedir)/lib/bkp" \
 	logdir="$(logdir)"
 
 V_SUBST = $(V_SUBST_$V)
@@ -73,6 +74,16 @@ rsync-backup: rsync-backup.in Makefile
 		chmod +x rsync-backup.new && \
 		mv rsync-backup.new rsync-backup
 
+sbin_SCRIPTS		+= update-bkp-index
+dist_man_MANS		+= update-bkp-index.8
+CLEANFILES		+= update-bkp-index
+EXTRA_DIST		+= update-bkp-index.in
+update-bkp-index: update-bkp-index.in Makefile
+	$(SUBST) >update-bkp-index.new \
+			$(srcdir)/update-bkp-index.in $(SUBSTVARS) && \
+		chmod +x update-bkp-index.new && \
+		mv update-bkp-index.new update-bkp-index
+
 bin_SCRIPTS		+= fshash
 dist_man_MANS		+= fshash.1
 CLEANFILES		+= fshash
diff --git a/rsync-backup.8 b/rsync-backup.8
index 271a9d6..aa7821f 100644
--- a/rsync-backup.8
+++ b/rsync-backup.8
@@ -265,11 +265,30 @@ module.
 The default is
 .BR sha256 .
 .TP
+.B INDEXDB
+The name of a SQLite database initialized by
+.BR update-bkp-index (8)
+in which an index is maintained of which dumps are on which backup
+volumes.  If the file doesn't exist, then no index is maintained.  The
+default is
+.IB localstatedir /lib/bkp/index.db
+where
+.I localstatedir
+is the state directory configured at build time.
+.TP
 .B MAXLOG
 The number of log files to be kept for each filesystem.  Old logfiles
 are deleted to keep the total number below this bound.  The default
 value is 14.
 .TP
+.B METADIR
+The metadata directory for the currently mounted backup volume.
+The default is
+.IB mntbkpdir /meta
+where
+.I mntbkpdir
+is the backup mount directory configured at build time.
+.TP
 .B RSYNCOPTS
 Command-line options to pass to
 .BR rsync (1)
@@ -315,6 +334,11 @@ where
 .I mntbkpdir
 is the backup mount directory configured at build time.
 .TP
+.B VOLUME
+The name of the current volume.  If this is left unset, the volume name
+is read from the file
+.IB METADIR /volume
+once at the start of the backup run.
 .SS Hook functions
 The configuration file may define shell functions to perform custom
 actions at various points in the backup process.
@@ -451,6 +475,7 @@ format), together with associated files named
 .BR lvm (8),
 .BR rfreezefs (8),
 .BR rsync (1),
-.BR ssh (1).
+.BR ssh (1),
+.BR update-bkp-index (8).
 .SH AUTHOR
 Mark Wooding, <mdw@distorted.org.uk>
diff --git a/rsync-backup.in b/rsync-backup.in
index 7519b12..ac9cfdc 100644
--- a/rsync-backup.in
+++ b/rsync-backup.in
@@ -149,6 +149,33 @@ hostpath () {
 }
 
 ###--------------------------------------------------------------------------
+### Database operations.
+
+INDEXDB=@pkglocalstatedir@/index.db
+
+insert_index () {
+  host=$1 fs=$2 date=$3 vol=$4
+
+  if [ -f "$INDEXDB" ]; then
+    sqlite3 "$INDEXDB" <<EOF
+INSERT INTO idx (host, fs, date, vol)
+	VALUES ('$host', '$fs', '$date', '$vol');
+EOF
+  fi
+}
+
+delete_index () {
+  host=$1 fs=$2 date=$3
+
+  if [ -f "$INDEXDB" ]; then
+    sqlite3 "$INDEXDB" <<EOF
+DELETE FROM idx WHERE
+	host = '$host' AND fs = '$fs' AND $date = '$date';
+EOF
+  fi
+}
+
+###--------------------------------------------------------------------------
 ### Snapshot handling.
 
 ## Snapshot protocol.  Each snapshot type has a pair of functions snap_TYPE
@@ -481,8 +508,10 @@ EOF
 ### Actually taking backups of filesystems.
 
 STOREDIR=@mntbkpdir@/store
+METADIR=@mntbkpdir@/meta
 MAXLOG=14
 HASH=sha256
+unset VOLUME
 
 bkprc=0
 
@@ -528,6 +557,7 @@ expire_backups () {
 	echo "delete $date"
 	$verbose -n "	expire $date..."
 	rm -rf $date $date.*
+	delete_index $host $fs $date
 	$verbose " done"
 	;;
     esac
@@ -657,6 +687,7 @@ do_backup () {
       backup_precommit_hook $host $fs $date
       mv new $date
       mv new.fshash $date.fshash
+      insert_index $host $fs $date $VOLUME
       backup_commit_hook $host $fs $date
       mkdir hack
       ln -s $date hack/last
@@ -697,6 +728,12 @@ backup () {
     exit 15
   fi
 
+  ## Read the volume name if we don't have one already.  Again, this allows
+  ## the configuration file to provide a volume name.
+  case "${VOLUME+t}${VOLUME-nil}" in
+    nil) VOLUME=$(cat $METADIR/volume) ;;
+  esac
+
   ## Back up each requested file system in turn.
   for fs in "$@"; do
 
diff --git a/update-bkp-index.8 b/update-bkp-index.8
new file mode 100644
index 0000000..91f82c8
--- /dev/null
+++ b/update-bkp-index.8
@@ -0,0 +1,61 @@
+.ie t .ds o \(bu
+.el .ds o o
+.de hP
+.IP
+\h'-\w'\fB\\$1\ \fP'u'\fB\\$1\ \fP\c
+..
+.TH update-bkp-index 8 "25 January 2013" rsync-backup
+.SH NAME
+update-bkp-index \- create or update the rsync-backup index database
+.SH SYNOPSIS
+The
+.BR rsync-backup (8)
+program can maintain an index database which keeps track of which dumps
+are on which backup volumes.
+.PP
+The
+.B update-bkp-index
+program updates this database from the currently mounted backup volume,
+creating or upgrading it if necessary.
+.PP
+The program ignores any command-line arguments it's given, but it makes
+use of a number of environment variables.
+.TP
+.B INDEXDB
+The name of a SQLite database initialized by
+.BR update-bkp-index (8)
+in which an index is maintained of which dumps are on which backup
+volumes.  If the file doesn't exist, then no index is maintained.  The
+default is
+.IB localstatedir /lib/bkp/index.db
+where
+.I localstatedir
+is the state directory configured at build time.
+.TP
+.B METADIR
+The metadata directory for the currently mounted backup volume.
+The default is
+.IB mntbkpdir /meta
+where
+.I mntbkpdir
+is the backup mount directory configured at build time.
+.TP
+.B STOREDIR
+Where the actual backup trees should be stored.  See the section on
+.B Archive structure
+below.
+The default is
+.IB mntbkpdir /store
+where
+.I mntbkpdir
+is the backup mount directory configured at build time.
+.TP
+.B VOLUME
+The name of the current volume.  The default is to read this from the
+file
+.IB METADIR /volume
+once at the start of the backup run.
+.SH SEE ALSO
+.BR rsync-backup (8).
+.SH AUTHOR
+Mark Wooding, <mdw@distorted.org.uk>
diff --git a/update-bkp-index.in b/update-bkp-index.in
new file mode 100644
index 0000000..23b64bd
--- /dev/null
+++ b/update-bkp-index.in
@@ -0,0 +1,100 @@
+#! @BASH@
+###
+### Backup script
+###
+### (c) 2012 Mark Wooding
+###
+
+###----- Licensing notice ---------------------------------------------------
+###
+### This file is part of the `rsync-backup' program.
+###
+### rsync-backup is free software; you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation; either version 2 of the License, or
+### (at your option) any later version.
+###
+### rsync-backup is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with rsync-backup; if not, write to the Free Software Foundation,
+### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+set -e
+
+mkdir -p @pkglocalstatedir@
+INDEXDB=@pkglocalstatedir@/index.db
+: ${STOREDIR=@mntbkpdir@/store}
+: ${METADIR=@mntbkpdir@/meta}
+
+if [ ! -f $STOREDIR/.rsync-backup-store ]; then
+  echo >&2 "$quis: no backup volume mounted"
+  exit 15
+fi
+: ${VOLUME=$(cat $METADIR/volume)}
+
+## If the database exists then we're OK.  (This will turn into a version
+## check and upgrade if the schema changes.)
+if [ ! -f "$INDEXDB" ]; then
+
+  ## Create the database.
+  rm -f "$INDEXDB.new"
+  sqlite3 "$INDEXDB.new" <<EOF
+CREATE TABLE meta (
+	version INTEGER NOT NULL);
+INSERT INTO meta (version) VALUES (0);
+
+CREATE TABLE idx (
+	host TEXT NOT NULL,
+	fs TEXT NOT NULL,
+	date TEXT NOT NULL,
+	vol TEXT NOT NULL,
+	PRIMARY KEY (host, fs, date));
+CREATE INDEX idx_byvol ON idx (vol);
+EOF
+
+  ## Done.
+  mv "$INDEXDB.new" "$INDEXDB"
+fi
+
+{
+  ## Do everything in a single transaction.  SQLite is pretty good at this,
+  ## and also it'll avoid updating the database until it sees a `COMMIT'
+  ## command, so if we fail halfway through we're still OK.  So it's safe to
+  ## start by removing all of the current records referring to this volume.
+  cat <<EOF
+BEGIN;
+DELETE FROM idx WHERE vol = '$VOLUME';
+EOF
+
+  ## Now work through the various filesystems.  This is a slightly cheesy way
+  ## of finding them.
+  for i in $STOREDIR/*/*/last; do
+
+    ## Parse out the host and filesystem names.
+    i=${i%/*}
+    fs=${i##*/} i=${i%/*}
+    host=${i##*/} i=${i%/*}
+
+    ## And work through the date list.
+    for j in $STOREDIR/$host/$fs/*; do
+      if [ -L "$j" ] || [ ! -d "$j" ]; then continue; fi
+      j=${j%/}
+      date=${j##*/}
+      cat <<EOF
+INSERT INTO idx (host, fs, date, vol)
+	VALUES ('$host', '$fs', '$date', '$VOLUME');
+EOF
+    done
+  done
+
+  ## Done.
+  cat <<EOF
+COMMIT;
+EOF
+} | sqlite3 "$INDEXDB"
+
+###----- That's all, folks --------------------------------------------------