3 ### Manage the backup archive structure
5 ### (c) 2011 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This program is free software; you can redistribute it and/or modify
11 ### it under the terms of the GNU General Public License as published by
12 ### the Free Software Foundation; either version 2 of the License, or
13 ### (at your option) any later version.
15 ### This program is distributed in the hope that it will be useful,
16 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ### GNU General Public License for more details.
20 ### You should have received a copy of the GNU General Public License
21 ### along with this program; if not, write to the Free Software Foundation,
22 ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 ## Configuration and testing.
27 : ${BKP=/mnt/bkp} ${META=/mnt/bkpmeta}
30 case $
(id
-u
) in 0) ;; *) exec userv root bkpadmin
"$@" ;; esac
32 ###--------------------------------------------------------------------------
39 ## Print a complaint to standard error.
45 ## Print a complaint and exit.
54 ## Add a cleanup command CMD to the list.
60 trap 'rc=$?; for c in $cleanups; do $c; done; exit $rc' \
64 cleanups
=${cleanups+$cleanups }$cmd
67 rmtmp
() { case ${tmpdir+t} in t
) rm -rf
"$tmpdir" ;; esac }
70 ## Make a temporary directory and output its name.
78 r
=$
(openssl rand
-base64
12)
79 tmpdir
=${TMPDIR-/tmp}/$quis.$$.
$r
80 if mkdir
-m700
"$tmpdir" >/dev
/null
2>&1; then break; fi
81 case $i in ???
) die
"failed to create temporary directory" ;; esac
89 ###--------------------------------------------------------------------------
92 case "${USERV_USER+t}" in t
) uservp
=t
;; *) uservp
=nil
;; esac
94 USAGE
="COMMAND [ARGUMENT ...]"
99 name
=$1; shift; args
=$
*
100 ## Define a command unconditionally.
107 ## Define a command for privileged users only.
109 case $uservp in nil
) _defcmd
"$@" ;; esac
113 ## Define a command usable via userv.
119 ## Write a usage message for the current command.
121 echo "usage: $quis${cmdname:+ $cmdname}${cmdargs:+ $cmdargs}"
125 ## Fail with a usage error.
133 ## Try to loop up the command CMD.
135 while read cmdname cmdargs
; do
136 case $cmdname in "$cmd") return ;; esac
140 die
"unknown command \`$cmd'"
145 case $# in 0) ;; *) usage_err
;; esac
148 $quis, version $version
154 while read cmd args
; do
155 echo " $cmd${args:+ $args}"
161 ###--------------------------------------------------------------------------
162 ### Utility functions.
166 ## Sign the named FILE, producing a signature FILE.sig.
168 seccure-sign
-F
$KEYS/priv
/backup-auth
-cp256
-s
"$file.sig" <"$file"
172 ## Check that a host is defined.
175 t
) ;; *) die
"no host defined (use \`-H')" ;;
180 thing
=$1 good
=$2 what
=$3 string
=$4
181 ## Check that STRING is a valid THING -- i.e., it only consists of GOOD
186 die
"bad $thing \`$string' given for $what"
193 ## Check that STRING is at least plausibly numeric.
195 checkthing number
"0-9" "$what" "$string"
200 ## Check that STRING is a plausible pathname.
203 .
* |
*/.
* |
*[!-a-zA-Z0-9.
,_
#!%^+=@/:]*)
204 die
"bad pathname \`$string' given for $what"
211 ## Check that THING doesn't need shell quoting, and doesn't interfere with
212 ## other common delimiter characters. (Colons aren't allowed because they
213 ## mess up /etc/passwd; slashes aren't allowed because they're directory
214 ## separators. Leading dots aren't allowed either. Hashes seem OK.)
216 checkthing word
"-a-zA-Z0-9.,_#!%^+=@" "$what" "$string"
220 dir
=$1 owner
=$2 mode
=$3
221 ## Make a directory and set permissions on it.
228 ###--------------------------------------------------------------------------
229 ### Volume and volume group maintenance.
232 ## Output the tag of the mounted backup volume group.
236 /dev
/mapper
/cbkp-
*) echo "${dev#*-}"; return ;;
237 *) die
"failed to parse tag from device name \`$dev'" ;;
242 ## Guess and print the tag of the available backup volume group. If there
243 ## is not exactly one volume group available, print an error and fail.
245 LVM_SUPPRESS_FD_WARNINGS
=t vgs @backup
--noheadings
-o name
,attr |
{
247 while read name attr
; do
248 case "$name" in bkp-
*) ;; *) continue ;; esac
249 case "$attr" in ??x
*) continue ;; esac
250 match
="$match${match:+ }${name#bkp-}"
253 x
) die
"no backup volume groups available" ;;
254 x
*\
*) die
"multiple backup volume groups available: $match" ;;
262 ## Output a device name for the filesystem mounted on DIR.
264 dev
=$
(mountpoint
-d
"$dir")
265 devname
=$
(udevadm info
--query
=name
--path
="/dev/block/$dev")
268 devname
=mapper
/$
(dmsetup info
-c
--noheadings
-oname
"/dev/$devname")
276 ## Mount the metadata volume of the backup volume group named TAG.
278 if ! mountpoint
-q
$META; then
279 mount
"/dev/bkp-$tag/meta" $META
284 ## Decrypt and output the key for the encrypted volume. This assumes that
285 ## the metadata volume is already mounted on /mnt/bkpmeta.
287 seccure-decrypt
-q
-m128
-cp256
-F
$KEYS/priv
/backup-disk
<$META/cur
/blob
292 ## Decrypt but don't mount the encrypted volume of the backup volume group
296 if [ ! -b
"/dev/mapper/cbkp-$tag" ]; then
297 cryptkey | cryptsetup luksOpen
--key-file
=- \
298 "/dev/bkp-$tag/crypt" "cbkp-$tag"
304 ## Mount the encrypted subvolume of the backup volume group named TAG. The
305 ## metadata volume will be mounted if necessary.
308 if ! mountpoint
-q
$BKP; then
309 mount
"/dev/mapper/cbkp-$tag" $BKP
314 ## Unmounts a backup volume group: both the encrypted and metadata volumes
317 if mountpoint
-q
$BKP; then
318 tag
=$
(currenttag
) cryptclosep
=t
322 for i
in bkp bkpmeta
; do
323 if mountpoint
-q
/mnt
/$i; then umount
/mnt
/$i; fi
327 if [ -b
"/dev/mapper/cbkp-$tag" ]; then
328 cryptsetup luksClose
"cbkp-$tag"
333 defcmd initvol TAG DEVICE
335 case $# in 2) ;; *) usage_err
;; esac
338 vgcreate
--addtag @backup
"bkp-$tag" "$dev"
340 lvcreate
-L4M
-nmeta
"bkp-$tag"
341 mkfs
-text2
-Lmeta
"/dev/bkp-$tag/meta"
344 mkdir
-m755
$META/new
345 dd if=/dev
/random bs
=1 count
=512 |
346 seccure-encrypt
-m128 $
(cat $KEYS/pub
/backup-disk.pub
) >$META/new
/blob
347 mv $META/new
$META/cur
349 lvcreate
-l100
%FREE
-ncrypt
"bkp-$tag"
350 cryptkey | cryptsetup luksFormat \
351 --cipher
=twofish-xts-benbi
:sha256
--hash=sha256 \
352 "/dev/bkp-$tag/crypt" -
354 mkfs
-text2
-Lbackup
-i1048576
"/dev/mapper/cbkp-$tag"
358 defucmd mount
"[TAG]"
361 0) tag
=$
(guesstag
) check
=nil
;;
366 if mountpoint
-q
$BKP; then
368 case "$check,$curtag" in "t,$tag") ;; t
*) exit 1 ;; esac
376 case $# in 0) ;; *) usage_err
;; esac
378 for i
in bkp bkpmeta
; do
379 if mountpoint
-q
/mnt
/$i; then mntp
=t
; fi
382 nil
) die
"backup volume not mounted" ;;
387 ###--------------------------------------------------------------------------
388 ### Archive maintenance.
392 ## Check a directory which has `hashes' and `hashes.sig' files.
394 if ! seccure-verify
-q
-i
"$dir/hashes" -- \
395 $
(cat "$KEYS/$key") $
(cat "$dir/hashes.sig")
397 die
"failed to verify signature for \`$dir'"
401 sha256sum
--quiet
-c hashes
404 find .
-type f
-print |
sed 's:^\./::' |
sort >"$tmpdir/present"
407 sed 's/^[a-f0-9]*[* ] //' hashes
408 } |
sort >"$tmpdir/checked"
410 diff -u checked present
414 dir
=$1 owner
=$2 fmode
=$3 dmode
=$4
415 ## Fix the directory tree DIR so that everything is owned by OWNER (a
416 ## USER:GROUP pair) and has modes FMODE for files and DMODE for
419 ## Change all of the ownerships. This will prevent anyone else from
420 ## changing the permissions on the files. This assumes that chown(1) is
421 ## secure in recursive mode; I've checked that GNU chown seems correct.
422 chown
-R
$owner "$dir"
424 ## Paranoia: check that we correctly changed all of the files.
425 u
=${owner%:*} g
=${owner#*:}
426 (cd "$dir"; find .
! \
( -user
$u -group
$g \
) -ls) |
428 moan
"failed to fix permssions on \`$dir'"
429 { echo $line; cat; } |
sed 's/^/ /'
433 ## Now get to work on the file and directory permissions.
434 find "$dir" -type d
-print0 |
xargs -0r
chmod $dmode
435 find "$dir" ! -type d
-print0 |
xargs -0r
chmod $fmode
440 ## Commit an `prepare' directory DIR, moving its `incoming' files to
441 ## TARGET. This will choose the correct name for the directory, but
442 ## assumes that it's already correctly laid out. We assume that the
443 ## permissions on this directory are safe (e.g., they've already been fixed
444 ## using `fixperms'). On successful exit, DIR won't exist any more. The
445 ## shell variable `label' is set to the resulting archive name.
447 ## If there's no `incoming' directory, then there's nothing to do. Just
448 ## zap the directory and move on.
449 if [ ! -d
"$dir/incoming" ]; then
454 ## Find the datestamp and level numbers to use for this directory. These
455 ## are created before the `incoming' directory, so they ought to exist.
456 read level
date time tz
<"$dir/meta"
458 ## Find a suitable sequence number for the target. This is rather ugly;
463 for i
in "$target"/"$date#$seq".
*; do
464 if [ -e
"$i" ]; then anyp
=t
; break; fi
466 case $anyp in nil
) break ;; esac
470 ## Move the directory.
471 label
="$date#$seq.$level"
472 mv "$dir/incoming" "$target/$label"
475 ## Update the catalogue. Replace an existing dump at the same level.
476 ## Assume that dates are monotonically increasing: add the new entry at the
479 while read lab l d t
; do
480 if [ $l -ne
$level ]; then echo $label $l $d $t; fi
481 done <"$target"/CATALOGUE
482 echo $level $date $time $tz
483 } >"$target"/CATALOGUE.new
484 mv "$target"/CATALOGUE.new
"$target"/CATALOGUE
489 case $# in 0) ;; *) usage_err
;; esac
491 ## Make a `new' directory and start recording our files.
497 ## Copy the blob from the existing metadata.
501 ## Archive the key recovery information.
503 tar cfz
$META/new
/keys.tgz pub
/ recov
/
506 ## Copy user and group information.
508 for i
in passwd group
; do
509 grep -E
'^(root|backup|bkp-[[:alnum:]]+):' /etc
/$i >$i
513 ## Build the hashes file, and sign it.
519 ## Replace the old metadata.
528 case $# in 0) ;; *) usage_err
;; esac
530 checkdir pub
/backup-auth.pub
$META/cur
534 ## Report the current date, as ISO8601. Allow an override.
536 case "${forceday+t}" in t
) echo "$forceday" ;; *) date +%Y-
%m-
%d
;; esac
539 defucmd prep ASSET LEVEL \
[DATE TIME TZ
]
542 2) set -- "$@" $
(today
) $
(date +%H
:%M
:%S
) $
(date +%z
) ;;
546 asset
=$1 level
=$2 date=$3 time=$4 tz
=$5
548 checkword asset
"$asset"
549 checknum level
"$level"
550 checkthing
date -0-9 date "$date"
551 checkthing
time :0-9 time "$time"
552 checkthing timezone
-+0-9 tz
"$tz"
554 ## Make the host and asset directories if necessary.
556 for i
in $host $asset; do
557 if [ ! -d
$i ]; then domkdir
$i root
:root
755; fi
560 if [ ! -d failed
]; then domkdir failed root
:root
755; fi
561 for i
in . failed
; do
562 if [ ! -f
$i/CATALOGUE
]; then
564 chown root
:root
$i/CATALOGUE
565 chmod 644 $i/CATALOGUE
569 ## If an existing dump is in progress then archive it as a failure.
570 if [ -d prepare
]; then
571 if [ -d prepare
/incoming
]; then
572 fixperms prepare
/incoming root
:root
640 755
574 commitdir prepare failed
/
577 ## Make a new preparation directory.
578 domkdir prepare root
:bkp-
$host 755
579 echo $level $date $time $tz >prepare
/meta
580 domkdir prepare
/incoming bkp-
$host:bkp-
$host 2775
582 ## Print the directory name.
583 echo $BKP/$host/$asset/prepare
/incoming
588 case $# in 1) ;; *) usage_err
;; esac
591 checkword asset
"$asset"
593 ## Check that there's something to abort.
595 if [ ! -d
$host/$asset/prepare
]; then
596 die
"no dump in progress for $host/$asset"
599 ## Just throw it away.
600 rm -rf
$host/$asset/prepare
605 case $# in 1) ;; *) usage_err
;; esac
608 checkword asset
"$asset"
610 ## Check that there's something to fail.
612 if [ ! -d
$host/$asset/prepare
]; then
613 die
"no dump in progress for $host/$asset"
616 ## Archive the failure. This shouldn't be used to determine dump levels or
617 ## we'll have gaps when things get sorted out.
619 if [ -d prepare
/incoming
]; then
620 fixperms prepare
/incoming root
:root
640 755
622 commitdir prepare failed
/
627 ## Convert an ISO8601 DATE to a Julian Day Number.
629 ## Extract the components of the date and trim leading zeros (which will
630 ## cause things to be interpreted as octal and fail).
631 year
=${date%%-*} rest
=${date#*-}; month
=${rest%%-*} day
=${rest#*-}
632 year
=${year#0} month
=${month#0} day
=${day#0}
634 ## The actual calculation: convert a (proleptic) Gregorian calendar date
635 ## into a Julian day number. This is taken from Wikipedia's page
636 ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary
637 ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24
638 ## proleptic Gregorian.
640 ## If the MONTH is January or February then set a = 1, otherwise set a = 0.
641 a
=$
(( (14 - $month)/12 ))
643 ## Compute a year offset relative to 4799BC-03-01. This puts the leap day
644 ## as the very last day in a year, which is very convenient. The offset
645 ## here is sufficient to make all y values positive (within the range of
646 ## the JDN calendar), and is a multiple of 400, which is the Gregorian
648 y
=$
(( $year + 4800 - $a ))
650 ## Compute the offset month number in that year. These months count from
652 m
=$
(( $month + 12*$a - 3 ))
654 ## Now for the main event. The (153 m + 2)/5 term is a surprising but
655 ## correct trick for obtaining the number of days in the first m months of
656 ## the (shifted) year). The magic offset 32045 is what you get when you
657 ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the
659 jdn
=$
(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 ))
665 fulldate
=$1 lastdate
=$2
666 ## Return the dump level, given that the most recent full dump occurred on
667 ## FULLDATE and the most revent dump of any kind occurred on LASTDATE.
669 ## Actually, we're much more interested in the day difference between these
671 fulljdn
=$
(julian
$fulldate)
672 lastjdn
=$
(julian
$lastdate)
673 now
=$
(today
); nowjdn
=$
(julian
$now)
674 lastday
=$
(( $lastjdn - $fulljdn ))
675 today
=$
(( $nowjdn - $fulljdn ))
677 ## If the difference is greater than 512 then we know we should do a full
678 ## dump. (This provides an upper bound for the search below. It should
679 ## never happen in practice, of course.)
680 if [ $
(( $today - $lastday )) -ge
512 ]; then echo 0; return; fi
682 ## Now we work out the correct dump level. This will assume that the
683 ## previous dump had a sensible level. If dumps are omitted, then we will
684 ## choose a lower (more comprehensive) dump level than the schedule calls
685 ## for; such an overestimation will mean that we will probably end up
686 ## dumping too much again. This is the right error to make.
688 ## We use a Towers of Hanoi schedule. If we're doing dumps every day, then
689 ## on day n since the last full dump, we work out the dump level as
690 ## follows: write n = 2^s t where t is odd (i.e., s is the number of
691 ## trailing zero bits in the binary representation of n); then the dump
692 ## level on day n is 9 - s. This is enough for 512 days without a full
693 ## dump, and it fails gracefully anyway.
695 ## Now we have to deal with the problem of skipping dumps. Suppose the
696 ## last dump was on day m = 2^u v, and it's now day n = 2^s t. We ought to
697 ## take the lowest dump level of any intervening day, i.e., the dump level
698 ## is 9 - a for the largest a such that there exists b with m < l = 2^a b
699 ## <= n. We claim that such an l is unique. Suppose, to the contrary,
700 ## that m < 2^a b < 2^a b' <= n, with both b and b' odd. Then m < 2^{a+1}
701 ## (b + 1)/2 <= n, contradicting maximality of a.
703 ## How does this help? Observe that n = 2^s t = 2^a b + o, for some o <
704 ## 2^a: if o >= 2^a then 2^a (b + 1) <= n contradicting uniqueness of l.
705 ## Similarly, m = 2^u v = 2^a b - r, for some r <= 2^a (otherwise m <
706 ## 2^a (b - 1), again contradicting uniqueness). Therefore, m and n are
707 ## identical from bit a + 1 onwards, and differ at bit a. In other words,
708 ## a is the position of the most significant set bit in m XOR n.
709 diff=$
(( lastday ^ today
))
711 ## We know that the bit position must be less than 16.
713 while [ $diff -gt
1 ]; do
714 xx
=$
(( $diff >> $t ))
715 if [ $xx -gt
0 ]; then
716 diff=$xx n
=$
(( $n + $t ))
726 case $# in 1) ;; *) usage_err
;; esac
729 checkword asset
"$asset"
731 ## Set the correct directory. If it doesn't exist then we obviously need a
734 full
="0 1970-01-01 00:00:00 +0000"
735 if [ ! -d
$host/$asset ]; then echo $full; return; fi
738 ## We need the time of the most recent dump of any kind, and the most
739 ## recent level-zero dump.
740 fulldate
=none lastdate
=none
741 while read label level
date time tz
; do
742 if [ $level -eq
0 ]; then fulldate
=$date; fi
745 case $fulldate in none
) echo $full; return ;; esac
746 level
=$
(dumplevel
$fulldate $lastdate)
748 ## Determine the time of the most recent dump of the same or more inclusive
751 while read lab l d t
; do
752 if [ $l -le
$level ]; then date=$d time=$t; fi
754 echo $level $date $time $tz
757 defucmd
hash ASSET FILE HASH
759 case $# in 3) ;; *) usage_err
;; esac
760 asset
=$1 file=$2 hash=$3
761 checkword asset
"$asset"
762 checkpath
file "$file"
763 checkword
hash "$hash"
765 cd $BKP/$host/$asset/prepare
767 if [ -f hashes
]; then
769 case "$f" in "$file") die
"file \`$file' already hashed" ;; esac
773 echo "$hash $file" >>hashes.new
779 case $# in 1) ;; *) usage_err
;; esac
782 checkword asset
"$asset"
784 cd $BKP/$host/$asset/prepare
785 fixperms incoming root
:bkp-
$host 640 755
788 if [ -f hashes
]; then
789 while read hash name
; do
790 if [ ! -f
"incoming/$name" ]; then
791 die
"precomputed hash for nonexistent or non-file \`$name'"
793 findargs
="$findargs ! -path incoming/$name"
795 cp hashes hashes.calc
798 find incoming
-type f
$findargs -print0 | \
799 xargs -0r sha256sum | \
800 sed 's: incoming/: :' \
802 sort -k2 hashes.calc
>incoming
/hashes
804 chmod 640 incoming
/hashes incoming
/hashes.sig
805 chown root
:bkp-
$host incoming
/hashes incoming
/hashes.sig
812 defucmd check ASSET LABEL
814 case $# in 2) ;; *) usage_err
;; esac
817 checkword asset
"$asset"
818 checkword label
"$label"
820 checkdir pub
/backup-auth.pub
$BKP/$host/$asset/$label
823 defucmd catalogue ASSET
825 case $# in 1) ;; *) usage_err
;; esac
828 checkword asset
"$asset"
830 cat $BKP/$host/$asset/CATALOGUE
833 defucmd outdated ASSET
835 case $# in 1) ;; *) usage_err
;; esac
838 checkword asset
"$asset"
841 for i
in [0-9]*#*.*; do
842 if [ -d
"$i" ]; then echo "$i"; fi
847 date=${tag%%#*} level
=${tag##*.}
848 if [ $level -le
$best ]
856 ###--------------------------------------------------------------------------
859 defcmd
test CMD
'[ARGS ...]'
860 cmd_test
() { "$@"; }
864 host=${USERV_USER#bkp-}
873 while getopts "$opts" opt
; do
877 D
) forceday
=$OPTARG ;;
881 shift $
(( $OPTIND - 1 ))
883 case $# in 0) usage_err
;; esac
884 lookupcmd
"$1"; shift
887 ###----- That's all, folks --------------------------------------------------