Commit | Line | Data |
---|---|---|
f6b4ffdc MW |
1 | #! @BASH@ |
2 | ### | |
3 | ### Backup script | |
4 | ### | |
5 | ### (c) 2012 Mark Wooding | |
6 | ### | |
7 | ||
8 | ###----- Licensing notice --------------------------------------------------- | |
9 | ### | |
10 | ### This file is part of the `rsync-backup' program. | |
11 | ### | |
12 | ### rsync-backup is free software; you can redistribute it and/or modify | |
13 | ### it under the terms of the GNU General Public License as published by | |
14 | ### the Free Software Foundation; either version 2 of the License, or | |
15 | ### (at your option) any later version. | |
16 | ### | |
17 | ### rsync-backup is distributed in the hope that it will be useful, | |
18 | ### but WITHOUT ANY WARRANTY; without even the implied warranty of | |
19 | ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
20 | ### GNU General Public License for more details. | |
21 | ### | |
22 | ### You should have received a copy of the GNU General Public License | |
23 | ### along with rsync-backup; if not, write to the Free Software Foundation, | |
24 | ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
25 | ||
26 | set -e | |
27 | ||
28 | thishost=$(hostname -s) | |
29 | quis=${0##*/} | |
e0ff797b | 30 | . @pkgdatadir@/lib.sh |
f6b4ffdc MW |
31 | |
32 | verbose=: | |
3f496b2b | 33 | dryrun=nil |
f6b4ffdc MW |
34 | |
35 | ###-------------------------------------------------------------------------- | |
36 | ### Utility functions. | |
37 | ||
38 | RSYNCOPTS="--verbose" | |
39 | ||
40 | do_rsync () { | |
41 | ## Run rsync(1) in an appropriate manner. Configuration should ovrride | |
42 | ## this or set $RSYNCOPTS if it wants to do something weirder. Arguments | |
43 | ## to this function are passed on to rsync. | |
44 | ||
45 | rsync \ | |
46 | --archive --hard-links --numeric-ids --del \ | |
47 | --sparse --compress \ | |
48 | --one-file-system \ | |
49 | --partial \ | |
50 | $RSYNCOPTS \ | |
51 | --filter="dir-merge .rsync-backup" \ | |
52 | "$@" | |
53 | } | |
54 | ||
55 | log () { | |
3f496b2b MW |
56 | case $dryrun in |
57 | t) | |
58 | echo >&2 " *** $*" | |
59 | ;; | |
60 | nil) | |
61 | now=$(date +"%Y-%m-%d %H:%M:%S %z") | |
62 | echo >&9 "$now $*" | |
63 | ;; | |
64 | esac | |
65 | } | |
66 | ||
67 | maybe () { | |
68 | ## Run CMD, if this isn't a dry run. | |
69 | ||
70 | case $dryrun in | |
71 | t) echo >&2 " +++ $*" ;; | |
72 | nil) "$@" ;; | |
73 | esac | |
f6b4ffdc MW |
74 | } |
75 | ||
4f618c54 MW |
76 | copy () { |
77 | prefix=$1 | |
78 | ## Copy lines from stdin to stdout, adding PREFIX. | |
79 | ||
80 | while IFS= read -r line; do | |
81 | printf "%s %s\n" "$prefix" "$line" | |
82 | done | |
83 | } | |
84 | ||
f6b4ffdc | 85 | run () { |
d482de4c MW |
86 | stdinp=nil |
87 | while :; do | |
88 | case $1 in | |
89 | -stdin) stdinp=t; shift ;; | |
90 | --) shift; break ;; | |
91 | *) break ;; | |
92 | esac | |
93 | done | |
f6b4ffdc MW |
94 | tag=$1 cmd=$2; shift 2 |
95 | ## Run CMD, logging its output in a pleasing manner. | |
96 | ||
3f496b2b MW |
97 | case $dryrun in |
98 | t) | |
99 | echo >&2 " *** RUN $tag" | |
100 | echo >&2 " +++ $cmd $*" | |
101 | rc=0 | |
102 | ;; | |
103 | nil) | |
104 | log "BEGIN $tag" | |
105 | rc=$( | |
d482de4c | 106 | case $stdinp in nil) exec </dev/null ;; esac |
3f496b2b MW |
107 | { { { ( set +e |
108 | "$cmd" "$@" 3>&- 4>&- 5>&- 9>&- | |
109 | echo $? >&5; ) | | |
4f618c54 MW |
110 | copy "|" >&4; } 2>&1 | |
111 | copy "*" >&4; } 4>&1 | | |
d482de4c | 112 | cat >&9; } 5>&1 |
3f496b2b MW |
113 | ) |
114 | case $rc in | |
115 | 0) log "END $tag" ;; | |
116 | *) log "FAIL $tag (rc = $rc)" ;; | |
117 | esac | |
118 | ;; | |
f6b4ffdc MW |
119 | esac |
120 | return $rc | |
121 | } | |
122 | ||
1c0d8611 MW |
123 | run_diff () { |
124 | out=$1 old=$2 new=$3 | |
125 | ## Write a unified diff from OLD to NEW, to OUT. | |
126 | ||
127 | set +e; diff -u "$old" "$new" >"$out"; rc=$?; set -e | |
128 | case $rc in 1) cat "$out" ;; esac | |
129 | return $rc | |
130 | } | |
131 | ||
f6b4ffdc MW |
132 | localp () { |
133 | h=$1 | |
134 | ## Answer whether H is a local host. | |
135 | ||
136 | case $h in | |
137 | "$thishost") return 0 ;; | |
138 | *) return 1 ;; | |
139 | esac | |
140 | } | |
141 | ||
142 | hostrun () { | |
143 | tag=$1 cmd=$2 | |
144 | ## Run CMD on the current host. If the host seems local then run the | |
145 | ## command through a local shell; otherwise run it through ssh(1). Either | |
146 | ## way it will be processed by a shell. | |
147 | ||
148 | if localp $host; then run "@$host: $tag" sh -c "$cmd" | |
fdd73e22 | 149 | else run "@$host: $tag" ssh $userat$host "$cmd" |
f6b4ffdc MW |
150 | fi |
151 | } | |
152 | ||
153 | _hostrun () { | |
154 | h=$1 cmd=$2 | |
f8d0b27d | 155 | ## Like hostrun, but without the complicated logging, and targetted at a |
f6b4ffdc MW |
156 | ## specific host. |
157 | ||
158 | if localp $h; then sh -c "$cmd" | |
159 | else ssh $h "$cmd" | |
160 | fi | |
161 | } | |
162 | ||
163 | hostpath () { | |
164 | path=$1 | |
165 | ## Output (to stdout) either PATH or HOST:PATH, choosing the former if the | |
166 | ## current host is local. | |
167 | ||
168 | if localp $host; then echo $path | |
fdd73e22 | 169 | else echo $userat$host:$path |
f6b4ffdc MW |
170 | fi |
171 | } | |
172 | ||
9b1d71c6 MW |
173 | defhook () { |
174 | hook=$1 | |
175 | ## Define a hook called HOOK. | |
176 | ||
177 | eval hk_$hook= | |
178 | } | |
179 | ||
180 | addhook () { | |
181 | hook=$1 cmd=$2 | |
182 | ## Add command CMD to the hook HOOK. | |
183 | ||
184 | eval old=\$hk_$hook; new="$old $cmd" | |
185 | eval hk_$hook=\$new | |
186 | } | |
187 | ||
188 | runhook () { | |
189 | hook=$1; shift 1 | |
190 | ## Invoke HOOK, passing it the remaining arguments. | |
191 | ||
192 | eval cmds=\$hk_$hook | |
193 | for cmd in $cmds; do | |
194 | if ! $cmd "$@"; then return $?; fi | |
195 | done | |
196 | } | |
197 | ||
c4098e64 MW |
198 | remove_old_logfiles () { |
199 | base=$1 | |
200 | ## Remove old logfiles with names of the form BASE.DATE#N, so that there | |
201 | ## are at most $MAXLOG of them. | |
202 | ||
203 | ## Count up the logfiles. | |
204 | nlog=0 | |
205 | for i in "$base".*; do | |
206 | if [ ! -f "$i" ]; then continue; fi | |
207 | nlog=$(( nlog + 1 )) | |
208 | done | |
209 | ||
210 | ## If there are too many, go through and delete some early ones. | |
211 | if [ $dryrun = nil ] && [ $nlog -gt $MAXLOG ]; then | |
212 | n=$(( nlog - MAXLOG )) | |
213 | for i in "$base".*; do | |
214 | if [ ! -f "$i" ]; then continue; fi | |
215 | rm -f "$i" | |
216 | n=$(( n - 1 )) | |
217 | if [ $n -eq 0 ]; then break; fi | |
218 | done | |
219 | fi | |
220 | } | |
221 | ||
f6b4ffdc | 222 | ###-------------------------------------------------------------------------- |
a8447303 MW |
223 | ### Database operations. |
224 | ||
a8447303 MW |
225 | insert_index () { |
226 | host=$1 fs=$2 date=$3 vol=$4 | |
227 | ||
228 | if [ -f "$INDEXDB" ]; then | |
229 | sqlite3 "$INDEXDB" <<EOF | |
230 | INSERT INTO idx (host, fs, date, vol) | |
231 | VALUES ('$host', '$fs', '$date', '$vol'); | |
232 | EOF | |
233 | fi | |
234 | } | |
235 | ||
236 | delete_index () { | |
237 | host=$1 fs=$2 date=$3 | |
238 | ||
239 | if [ -f "$INDEXDB" ]; then | |
240 | sqlite3 "$INDEXDB" <<EOF | |
241 | DELETE FROM idx WHERE | |
1da0cd47 | 242 | host = '$host' AND fs = '$fs' AND date = '$date'; |
a8447303 MW |
243 | EOF |
244 | fi | |
245 | } | |
246 | ||
247 | ###-------------------------------------------------------------------------- | |
f6b4ffdc MW |
248 | ### Snapshot handling. |
249 | ||
250 | ## Snapshot protocol. Each snapshot type has a pair of functions snap_TYPE | |
251 | ## and unsnap_TYPE. Each is given the current snapshot arguments and the | |
252 | ## filesystem name to back up. The snap_TYPE function should create and | |
253 | ## mount the snapshot and output an rsync(1) path to where the filesystem can | |
254 | ## be copied; the unsnap_TYPE function should unmount and tear down the | |
255 | ## snapshot. | |
256 | ||
257 | ## Fake snapshot by not doing anything. Use only if you have no choice. | |
258 | snap_live () { hostpath "$2"; } | |
259 | unsnap_live () { :; } | |
260 | ||
261 | ## Fake snapshot by remounting a live filesystem read-only. Useful if the | |
262 | ## underlying storage isn't in LVM. | |
263 | ||
264 | snap_ro () { | |
265 | fs=$1 mnt=$2 | |
266 | ||
267 | ## Place a marker in the filesystem so we know why it was made readonly. | |
268 | ## (Also this serves to ensure that the filesystem was writable before.) | |
269 | hostrun "snap-ro $mnt" " | |
270 | echo rsync-backup >$mnt/.lock | |
271 | mount -oremount,ro $mnt" || return $? | |
272 | ||
273 | ## Done. | |
274 | hostpath $mnt | |
275 | } | |
276 | ||
277 | unsnap_ro () { | |
278 | fs=$1 mnt=$2 | |
279 | ||
280 | ## Check that the filesystem still has our lock marker. | |
281 | hostrun "unsnap-ro $mnt" " | |
282 | case \$(cat $mnt/.lock) in | |
283 | rsync-backup) ;; | |
284 | *) echo unlocked by someone else; exit 31 ;; | |
285 | esac | |
286 | mount -oremount,rw $mnt | |
287 | rm $mnt/.lock" || return $? | |
288 | } | |
289 | ||
290 | ## Snapshot using LVM. | |
291 | ||
292 | SNAPSIZE="-l10%ORIGIN" | |
f6b4ffdc MW |
293 | |
294 | snap_lvm () { | |
295 | vg=$1 lv=$2 | |
296 | ||
297 | ## Make the snapshot. | |
298 | hostrun "snap-lvm $vg/$lv" " | |
299 | lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv | |
300 | mkdir -p $SNAPDIR/$lv | |
301 | mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv" || return $? | |
302 | ||
303 | ## Done. | |
304 | hostpath $SNAPDIR/$lv | |
305 | } | |
306 | ||
307 | unsnap_lvm () { | |
308 | vg=$1 lv=$2 | |
309 | ||
310 | ## Remove the snapshot. Sometimes LVM doesn't notice that the snapshot is | |
0c220a40 MW |
311 | ## no longer in open immdiately, so try several times. Sometimes, more |
312 | ## mysteriously, something is keeping the filesystem from being unmounted, | |
313 | ## so try that several times and report on things keeping the filesystem | |
314 | ## open. | |
f6b4ffdc | 315 | hostrun "unsnap-lvm $vg/$lv" " |
0c220a40 MW |
316 | for i in 1 2 3 4; do |
317 | echo \";;; BEGIN fuser -mv $SNAPDIR/$lv\" | |
318 | fuser -mv $SNAPDIR/$lv | sed 's/^/;;; /' | |
319 | echo \";;; END fuser -mv $SNAPDIR/$lv\" | |
320 | echo \";;; BEGIN lsof $SNAPDIR/$lv\" | |
321 | lsof $SNAPDIR/$lv | sed 's/^/;;; /' | |
322 | echo \";;; END lsof $SNAPDIR/$lv\" | |
323 | if umount $SNAPDIR/$lv; then break; fi | |
324 | sleep 2 | |
325 | done | |
f6b4ffdc MW |
326 | rc=1 |
327 | for i in 1 2 3 4; do | |
328 | if lvremove -f $vg/$lv.bkp; then rc=0; break; fi | |
329 | sleep 2 | |
330 | done | |
331 | exit $rc" || return $? | |
332 | } | |
333 | ||
334 | ## Complicated snapshot using LVM, where the volume group and filesystem are | |
335 | ## owned by different machines, so they need to be synchronized during the | |
336 | ## snapshot. | |
337 | ||
338 | do_rfreezefs () { | |
339 | lvhost=$1 vg=$2 lv=$3 fshost=$4 fsdir=$5 | |
340 | ||
341 | ## Engage in the rfreezefs protocol with the filesystem host. This | |
342 | ## involves some hairy plumbing. We want to get exit statuses out of both | |
343 | ## halves. | |
344 | set +e | |
345 | ssh $fshost rfreezefs $fsdir | { | |
346 | set -e | |
347 | ||
348 | ## Read the codebook from the remote end. | |
349 | ready=nil | |
350 | while read line; do | |
351 | set -- $line | |
352 | case "$1" in | |
353 | PORT) port=$2 ;; | |
354 | TOKEN) eval tok_$2=$3 ;; | |
355 | READY) ready=t; break ;; | |
356 | *) | |
357 | echo >&2 "$quis: unexpected keyword $1 (rfreezefs to $rhost)" | |
358 | exit 1 | |
359 | ;; | |
360 | esac | |
361 | done | |
362 | case $ready in | |
363 | nil) | |
364 | echo >&2 "$quis: unexpected eof (rfreezefs to $rhost)" | |
365 | exit 1 | |
366 | ;; | |
367 | esac | |
368 | ||
369 | ## Connect to the filesystem host's TCP port and get it to freeze its | |
370 | ## filesystem. | |
371 | exec 3<>/dev/tcp/$fshost/$port | |
372 | echo $tok_FREEZE >&3 | |
373 | read tok <&3 | |
374 | case $tok in | |
375 | "$tok_FROZEN") ;; | |
376 | *) | |
377 | echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)" | |
378 | exit 1 | |
379 | ;; | |
380 | esac | |
381 | ||
382 | ## Get the volume host to create the snapshot. | |
383 | set +e | |
fdd73e22 | 384 | _hostrun >&2 3>&- $userat$lvhost \ |
f6b4ffdc MW |
385 | "lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv" |
386 | snaprc=$? | |
387 | set -e | |
388 | ||
389 | ## The filesystem can thaw now. | |
390 | echo $tok_THAW >&3 | |
391 | read tok <&3 | |
392 | case $tok in | |
393 | "$tok_THAWED") ;; | |
394 | *) | |
fdd73e22 | 395 | _hostrun >&2 3>&- $userat$lvhost "lvremove -f $vg/$lv.bkp" || : |
f6b4ffdc MW |
396 | echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)" |
397 | exit 1 | |
398 | ;; | |
399 | esac | |
400 | ||
401 | ## Done. | |
402 | exit $snaprc | |
403 | } | |
404 | ||
405 | ## Sift through the wreckage to find out what happened. | |
406 | rc_rfreezefs=${PIPESTATUS[0]} rc_snapshot=${PIPESTATUS[1]} | |
407 | set -e | |
408 | case $rc_rfreezefs:$rc_snapshot in | |
409 | 0:0) | |
410 | ;; | |
411 | 112:*) | |
412 | echo >&2 "$quis: EMERGENCY failed to thaw $fsdir on $fshost!" | |
413 | exit 112 | |
414 | ;; | |
415 | *) | |
416 | echo >&2 "$quis: failed to snapshot $vg/$lv ($fsdir on $fshost)" | |
417 | exit 1 | |
418 | ;; | |
419 | esac | |
420 | ||
421 | ## Mount the snapshot on the volume host. | |
fdd73e22 | 422 | _hostrun >&2 $userat$lvhost " |
f6b4ffdc MW |
423 | mkdir -p $SNAPDIR/$lv |
424 | mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv" | |
425 | } | |
426 | ||
427 | snap_rfreezefs () { | |
428 | rhost=$1 vg=$2 lv=$3 rfs=$4 | |
429 | ||
430 | set -e | |
431 | run "snap-rfreezefs $host:$vg/$lv $rhost:$rfs" \ | |
432 | do_rfreezefs $host $vg $lv $rhost $rfs || return $? | |
433 | hostpath $SNAPDIR/$lv | |
434 | } | |
435 | ||
436 | unsnap_rfreezefs () { | |
437 | ||
438 | ## Unshapping is the same as for plain LVM. | |
439 | rhost=$1 vg=$2 lv=$3 rfs=$4 | |
440 | unsnap_lvm $vg $lv | |
441 | } | |
442 | ||
443 | ###-------------------------------------------------------------------------- | |
444 | ### Expiry computations. | |
445 | ||
f6b4ffdc MW |
446 | expire () { |
447 | ## Read dates on stdin; write to stdout `EXPIRE date' for dates which | |
448 | ## should be expired and `RETAIN date' for dates which should be retained. | |
449 | ||
450 | ## Get the current date and convert it into useful forms. | |
451 | now=$(date +%Y-%m-%d) | |
452 | parsedate $now | |
453 | now_jdn=$(julian $now) now_year=$year now_month=$month now_day=$day | |
454 | kept=: | |
455 | ||
456 | ## Work through each date in the input. | |
457 | while read date; do | |
458 | keep=nil | |
459 | ||
460 | ## Convert the date into a useful form. | |
461 | jdn=$(julian $date) | |
462 | parsedate $date | |
463 | ||
464 | ## Work through the policy list. | |
465 | if [ $jdn -le $now_jdn ]; then | |
466 | while read ival age; do | |
467 | ||
468 | ## Decide whether the policy entry applies to this date. | |
469 | apply=nil | |
470 | case $age in | |
471 | forever) | |
472 | apply=t | |
473 | ;; | |
474 | year) | |
475 | if [ $year -eq $now_year ] || | |
476 | ([ $year -eq $(( $now_year - 1 )) ] && | |
477 | [ $month -ge $now_month ]) | |
478 | then apply=t; fi | |
479 | ;; | |
480 | month) | |
481 | if ([ $month -eq $now_month ] && [ $year -eq $now_year ]) || | |
482 | ((([ $month -eq $(( $now_month - 1 )) ] && | |
483 | [ $year -eq $now_year ]) || | |
484 | ([ $month -eq 12 ] && [ $now_month -eq 1 ] && | |
485 | [ $year -eq $(( $now_year - 1 )) ])) && | |
486 | [ $day -ge $now_day ]) | |
487 | then apply=t; fi | |
488 | ;; | |
489 | week) | |
490 | if [ $jdn -ge $(( $now_jdn - 7 )) ]; then apply=t; fi | |
491 | ;; | |
492 | *) | |
493 | echo >&2 "$quis: unknown age symbol \`$age'" | |
494 | exit 1 | |
495 | ;; | |
496 | esac | |
497 | case $apply in nil) continue ;; esac | |
498 | ||
499 | ## Find the interval marker for this date. | |
500 | case $ival in | |
501 | daily) | |
502 | marker=$date | |
503 | ;; | |
504 | weekly) | |
505 | ydn=$(julian $year-01-01) | |
506 | wk=$(( ($jdn - $ydn)/7 + 1 )) | |
507 | marker=$year-w$wk | |
508 | ;; | |
509 | monthly) | |
510 | marker=$year-$month | |
511 | ;; | |
512 | annually | yearly) | |
513 | marker=$year | |
514 | ;; | |
515 | *) | |
516 | echo >&2 "$quis: unknown interval symbol \`$ival'" | |
517 | exit 1 | |
518 | ;; | |
519 | esac | |
520 | ||
521 | ## See if we've alredy retained something in this interval. | |
522 | case $kept in | |
523 | *:"$marker":*) ;; | |
524 | *) keep=t kept=$kept$marker: ;; | |
525 | esac | |
526 | ||
527 | done <<EOF | |
528 | $expire_policy | |
529 | EOF | |
530 | fi | |
531 | ||
532 | case $keep in | |
533 | t) echo RETAIN $date ;; | |
534 | *) echo EXPIRE $date ;; | |
535 | esac | |
536 | ||
537 | done | |
538 | } | |
539 | ||
540 | ###-------------------------------------------------------------------------- | |
541 | ### Actually taking backups of filesystems. | |
542 | ||
f6b4ffdc MW |
543 | MAXLOG=14 |
544 | HASH=sha256 | |
a8447303 | 545 | unset VOLUME |
f6b4ffdc MW |
546 | |
547 | bkprc=0 | |
548 | ||
ec1d1af2 MW |
549 | hash_file () { |
550 | file=$1 | |
551 | ||
552 | case $HASH in | |
553 | md5 | sha1 | sha224 | sha256 | sha384 | sha512) | |
554 | set -- $(${HASH}sum <"$file") | |
555 | echo "$1" | |
556 | ;; | |
557 | *) | |
558 | set -- $(openssl dgst -$HASH <"$file") | |
559 | echo "$2" | |
560 | ;; | |
561 | esac | |
562 | } | |
563 | ||
f6b4ffdc | 564 | remote_fshash () { |
fdd73e22 | 565 | _hostrun $userat$host " |
f6b4ffdc MW |
566 | umask 077 |
567 | mkdir -p $fshashdir | |
568 | cd ${snapmnt#*:} | |
569 | echo \"*** $host $fs $date\"; echo | |
570 | rsync -rx --filter='dir-merge .rsync-backup' ./ | | |
571 | fshash -c$fshashdir/$fs.bkp -a -H$HASH -frsync | |
572 | " >new.fshash | |
573 | } | |
574 | ||
575 | local_fshash () { | |
576 | { echo "*** $host $fs $date"; echo | |
577 | fshash -c$STOREDIR/fshash.cache -H$HASH new/ | |
578 | } >$localmap | |
579 | } | |
580 | ||
581 | expire_backups () { | |
582 | { seen=: | |
583 | for i in *-*-*; do | |
584 | i=${i%%.*} | |
585 | case $i in *[!-0-9]*) continue ;; esac | |
586 | case $seen in *:"$i":*) continue ;; esac | |
587 | seen=$seen$i: | |
588 | echo $i | |
589 | done; } | | |
590 | expire | | |
591 | while read op date; do | |
3f496b2b MW |
592 | case $op,$dryrun in |
593 | RETAIN,t) | |
594 | echo >&2 " --- keep $date" | |
595 | ;; | |
596 | EXPIRE,t) | |
597 | echo >&2 " --- delete $date" | |
598 | ;; | |
599 | RETAIN,nil) | |
f6b4ffdc MW |
600 | echo "keep $date" |
601 | ;; | |
3f496b2b | 602 | EXPIRE,nil) |
f6b4ffdc MW |
603 | echo "delete $date" |
604 | $verbose -n " expire $date..." | |
605 | rm -rf $date $date.* | |
a8447303 | 606 | delete_index $host $fs $date |
f6b4ffdc MW |
607 | $verbose " done" |
608 | ;; | |
609 | esac | |
610 | done | |
611 | } | |
612 | ||
9b1d71c6 MW |
613 | ## Backup hooks. |
614 | defhook setup | |
615 | defhook precommit | |
616 | defhook postcommit | |
617 | ||
f6b4ffdc MW |
618 | backup_precommit_hook () { |
619 | host=$1 fs=$2 date=$3 | |
9b1d71c6 MW |
620 | ## Compatibility: You can override this hook in the configuration file for |
621 | ## special effects; but it's better to use `addhook precommit'. | |
f6b4ffdc MW |
622 | |
623 | : | |
624 | } | |
9b1d71c6 | 625 | addhook precommit backup_precommit_hook |
f6b4ffdc MW |
626 | |
627 | backup_commit_hook () { | |
628 | host=$1 fs=$2 date=$3 | |
9b1d71c6 MW |
629 | ## Compatibility: You can override this hook in the configuration file for |
630 | ## special effects; but it's better to use `addhook commit'. | |
f6b4ffdc MW |
631 | |
632 | : | |
633 | } | |
9b1d71c6 | 634 | addhook commit backup_commit_hook |
f6b4ffdc MW |
635 | |
636 | do_backup () { | |
637 | date=$1 fs=$2 fsarg=$3 | |
638 | ## Back up FS on the current host. | |
639 | ||
640 | set -e | |
5675acda | 641 | attempt=0 |
1c0d8611 | 642 | fshash_diff=nil |
f6b4ffdc | 643 | |
9b1d71c6 MW |
644 | ## Run a hook beforehand. |
645 | set +e; runhook setup $host $fs $date; rc=$?; set -e | |
646 | case $? in | |
647 | 0) ;; | |
648 | 99) log "BACKUP of $host:$fs SKIPPED by hook"; return 0 ;; | |
649 | *) log "BACKUP of $host:$fs FAILED (hook returns $?)"; return $? ;; | |
650 | esac | |
651 | ||
f6b4ffdc MW |
652 | ## Report the start of this attempt. |
653 | log "START BACKUP of $host:$fs" | |
654 | ||
5675acda MW |
655 | ## Maybe we need to retry the backup. |
656 | while :; do | |
f6b4ffdc | 657 | |
ec1d1af2 MW |
658 | ## Rig checksum variables to mismatch unless they're set later. |
659 | hrfs=REMOTE hlfs=LOCAL | |
660 | ||
5675acda | 661 | ## Create and mount the remote snapshot. |
3f496b2b MW |
662 | case $dryrun in |
663 | t) | |
664 | maybe snap_$snap $fs $fsarg | |
665 | snapmnt="<snapshot>" | |
666 | ;; | |
667 | nil) | |
668 | snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $? | |
669 | ;; | |
670 | esac | |
5675acda | 671 | $verbose " create snapshot" |
f6b4ffdc | 672 | |
1c0d8611 MW |
673 | ## If we had a fshash-mismatch, then clear out the potentially stale |
674 | ## entries, both locally and remotely. | |
675 | case $fshash_diff in | |
676 | nil) ;; | |
677 | *) | |
678 | $verbose " prune cache" | |
679 | run -stdin "local prune fshash" \ | |
680 | fshash -u -c$STOREDIR/fshash.cache -H$HASH new/ <$fshash_diff | |
681 | run -stdin "@$host: prune fshash" \ | |
682 | _hostrun $userat$host <$fshash_diff \ | |
683 | "fshash -u -c$fshashdir/$fs.bkp -H$HASH ${snapmnt#*:}" | |
684 | ;; | |
685 | esac | |
686 | ||
5675acda MW |
687 | ## Build the list of hardlink sources. |
688 | linkdests="" | |
689 | for i in $host $like; do | |
690 | d=$STOREDIR/$i/$fs/last/ | |
691 | if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi | |
692 | done | |
f6b4ffdc | 693 | |
5675acda | 694 | ## Copy files from the remote snapshot. |
3f496b2b MW |
695 | maybe mkdir -p new/ |
696 | case $dryrun in | |
697 | t) $verbose " running rsync" ;; | |
698 | nil) $verbose -n " running rsync..." ;; | |
699 | esac | |
5675acda MW |
700 | set +e |
701 | run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \ | |
702 | $linkdests \ | |
703 | $rsyncargs \ | |
704 | $snapmnt/ new/ | |
705 | rc_rsync=$? | |
706 | set -e | |
3f496b2b | 707 | case $dryrun in nil) $verbose " done" ;; esac |
f6b4ffdc | 708 | |
5675acda MW |
709 | ## Collect a map of the snapshot for verification purposes. |
710 | set +e | |
3f496b2b MW |
711 | case $dryrun in |
712 | t) $verbose " remote fshash" ;; | |
713 | nil) $verbose -n " remote fshash..." ;; | |
714 | esac | |
5675acda MW |
715 | run "@$host: fshash $fs" remote_fshash |
716 | rc_fshash=$? | |
717 | set -e | |
ec1d1af2 MW |
718 | case $dryrun in |
719 | nil) | |
7a16a237 MW |
720 | hrfs=$(hash_file "new.fshash") |
721 | log "remote fshash $HASH checksum: $hrfs" | |
ec1d1af2 MW |
722 | $verbose " done" |
723 | ;; | |
724 | t) | |
725 | hrfs=UNSET | |
726 | ;; | |
727 | esac | |
f6b4ffdc | 728 | |
5675acda | 729 | ## Remove the snapshot. |
3f496b2b | 730 | maybe unsnap_$snap $snapargs $fs $fsarg |
5675acda | 731 | $verbose " remove snapshot" |
f6b4ffdc | 732 | |
5675acda MW |
733 | ## If we failed to copy, then give up. |
734 | case $rc_rsync:$rc_fshash in | |
735 | 0:0) ;; | |
736 | 0:*) return $rc_fshash ;; | |
737 | *) return $rc_rsync ;; | |
738 | esac | |
739 | ||
740 | ## Get a matching map of the files received. | |
3f496b2b | 741 | maybe mkdir -m750 -p $STOREDIR/tmp/ |
5675acda | 742 | localmap=$STOREDIR/tmp/fshash.$host.$fs.$date |
3f496b2b MW |
743 | case $dryrun in |
744 | t) $verbose " local fshash" ;; | |
745 | nil) $verbose -n " local fshash..." ;; | |
746 | esac | |
5675acda | 747 | run "local fshash $host:$fs" local_fshash || return $? |
ec1d1af2 MW |
748 | case $dryrun in |
749 | nil) | |
750 | hlfs=$(hash_file "$localmap") | |
7a16a237 | 751 | log "local fshash $HASH checksum: $hlfs" |
ec1d1af2 MW |
752 | $verbose " done" |
753 | ;; | |
754 | t) | |
755 | hlfs=UNSET | |
756 | ;; | |
757 | esac | |
5675acda MW |
758 | |
759 | ## Compare the two maps. | |
760 | set +e | |
1c0d8611 MW |
761 | fshash_diff=$STOREDIR/tmp/fshash-diff.$host.$fs.$date |
762 | run "compare fshash maps for $host:$fs" \ | |
763 | run_diff $fshash_diff new.fshash $localmap | |
5675acda MW |
764 | rc_diff=$? |
765 | set -e | |
766 | case $rc_diff in | |
767 | 0) | |
768 | break | |
769 | ;; | |
770 | 1) | |
771 | if [ $attempt -ge $retry ]; then return $rc; fi | |
772 | $verbose " fshash mismatch; retrying" | |
773 | attempt=$(( $attempt + 1 )) | |
774 | ;; | |
775 | *) | |
776 | return $rc_diff | |
777 | ;; | |
778 | esac | |
779 | done | |
f6b4ffdc | 780 | |
ec1d1af2 MW |
781 | ## Double-check the checksums. |
782 | if [ $hrfs != $hlfs ]; then | |
783 | cat >&2 <<EOF | |
784 | $0: INTERNAL ERROR: fshash $HASH checksum mismatch -- aborting | |
785 | remote fshash checksum = $hrfs | |
786 | local fshash checksum = $hlfs | |
787 | EOF | |
788 | exit 127 | |
789 | fi | |
790 | ||
5675acda | 791 | ## Glorious success. |
3f496b2b | 792 | maybe rm -f $localmap |
1c0d8611 | 793 | case $fshash_diff in nil) ;; *) maybe rm -f $fshash_diff ;; esac |
f6b4ffdc MW |
794 | $verbose " fshash match" |
795 | ||
796 | ## Commit this backup. | |
3f496b2b MW |
797 | case $dryrun in |
798 | nil) | |
9b1d71c6 | 799 | runhook precommit $host $fs $date |
3f496b2b MW |
800 | mv new $date |
801 | mv new.fshash $date.fshash | |
a8447303 | 802 | insert_index $host $fs $date $VOLUME |
9b1d71c6 | 803 | runhook commit $host $fs $date |
3f496b2b MW |
804 | mkdir hack |
805 | ln -s $date hack/last | |
806 | mv hack/last . | |
807 | rmdir hack | |
808 | ;; | |
809 | esac | |
f6b4ffdc MW |
810 | $verbose " commit" |
811 | ||
812 | ## Expire old backups. | |
f8d0b27d MW |
813 | case "${expire_policy+t},${default_policy+t}" in |
814 | ,t) expire_policy=$default_policy ;; | |
815 | esac | |
3f496b2b MW |
816 | case "${expire_policy+t},$dryrun" in |
817 | t,nil) run "expiry for $host:$fs" expire_backups ;; | |
818 | t,t) expire_backups ;; | |
f6b4ffdc | 819 | esac |
f8d0b27d | 820 | clear_policy=t |
f6b4ffdc MW |
821 | |
822 | ## Report success. | |
3f496b2b MW |
823 | case $dryrun in |
824 | t) log "END BACKUP of $host:$fs" ;; | |
825 | nil) log "SUCCESSFUL BACKUP of $host:$fs" ;; | |
826 | esac | |
f6b4ffdc MW |
827 | } |
828 | ||
3aa4fd30 MW |
829 | run_backup_cmd () { |
830 | fs=$1 date=$2 cmd=$3; shift 3 | |
831 | ## try_backup FS DATE COMMAND ARGS ... | |
832 | ## | |
833 | ## Run COMMAND ARGS to back up filesystem FS on the current host, | |
834 | ## maintaining a log, and checking whether it worked. The caller has | |
835 | ## usually worked out the DATE in order to set up the filesystem, and we | |
836 | ## need it to name the log file properly. | |
837 | ||
838 | ## Find a name for the log file. In unusual circumstances, we may have | |
839 | ## deleted old logs from today, so just checking for an unused sequence | |
840 | ## number is insufficient. Instead, check all of the logfiles for today, | |
841 | ## and use a sequence number that's larger than any of them. | |
842 | case $dryrun in | |
843 | t) | |
844 | log=/dev/null | |
845 | ;; | |
846 | nil) | |
847 | seq=1 | |
848 | for i in "$logdir/$host/$fs.$date#"*; do | |
849 | tail=${i##*#} | |
850 | case "$tail" in [!1-9]* | *[!0-9]*) continue ;; esac | |
851 | if [ -f "$i" -a $tail -ge $seq ]; then seq=$(( tail + 1 )); fi | |
852 | done | |
853 | log="$logdir/$host/$fs.$date#$seq" | |
854 | ;; | |
855 | esac | |
856 | ||
857 | ## Run the backup command. | |
858 | case $dryrun in nil) mkdir -p $logdir/$host ;; esac | |
859 | if ! "$cmd" "$@" 9>$log 1>&9; then | |
860 | echo >&2 | |
861 | echo >&2 "$quis: backup of $host:$fs FAILED!" | |
862 | bkprc=1 | |
863 | fi | |
864 | ||
c4098e64 MW |
865 | ## Clear away any old logfiles. |
866 | remove_old_logfiles "$logdir/$host/$fs" | |
3aa4fd30 MW |
867 | } |
868 | ||
f6b4ffdc MW |
869 | backup () { |
870 | ## backup FS[:ARG] ... | |
871 | ## | |
872 | ## Back up the filesystems on the currently selected host using the | |
873 | ## currently selected snapshot type. | |
874 | ||
6037bdb3 MW |
875 | ## Make sure that there's a store volume. We must do this here rather than |
876 | ## in the main body of the script, since the configuration file needs a | |
877 | ## chance to override STOREDIR. | |
878 | if ! [ -r $STOREDIR/.rsync-backup-store ]; then | |
879 | echo >&2 "$quis: no backup volume mounted" | |
880 | exit 15 | |
881 | fi | |
882 | ||
a8447303 MW |
883 | ## Read the volume name if we don't have one already. Again, this allows |
884 | ## the configuration file to provide a volume name. | |
885 | case "${VOLUME+t}${VOLUME-nil}" in | |
886 | nil) VOLUME=$(cat $METADIR/volume) ;; | |
887 | esac | |
888 | ||
6037bdb3 | 889 | ## Back up each requested file system in turn. |
f6b4ffdc MW |
890 | for fs in "$@"; do |
891 | ||
892 | ## Parse the argument. | |
893 | case $fs in | |
894 | *:*) fsarg=${fs#*:} fs=${fs%%:*} ;; | |
895 | *) fsarg="" ;; | |
896 | esac | |
897 | $verbose " filesystem $fs" | |
898 | ||
899 | ## Move to the store directory and set up somewhere to put this backup. | |
900 | cd $STOREDIR | |
3f496b2b MW |
901 | case $dryrun in |
902 | nil) | |
903 | if [ ! -d $host ]; then | |
904 | mkdir -m755 $host | |
905 | chown root:root $host | |
906 | fi | |
907 | if [ ! -d $host/$fs ]; then | |
908 | mkdir -m750 $host/$fs | |
909 | chown root:backup $host/$fs | |
910 | fi | |
911 | ;; | |
912 | esac | |
f6b4ffdc MW |
913 | cd $host/$fs |
914 | ||
915 | ## Find out if we've already copied this filesystem today. | |
916 | date=$(date +%Y-%m-%d) | |
3f496b2b | 917 | if [ $dryrun = nil ] && [ -d $date ]; then |
f6b4ffdc MW |
918 | $verbose " already dumped" |
919 | continue | |
920 | fi | |
921 | ||
f6b4ffdc | 922 | ## Do the backup of this filesystem. |
3aa4fd30 | 923 | run_backup_cmd $fs $date do_backup $date $fs $fsarg |
f6b4ffdc MW |
924 | done |
925 | } | |
926 | ||
927 | ###-------------------------------------------------------------------------- | |
928 | ### Configuration functions. | |
929 | ||
9b1d71c6 MW |
930 | defhook start |
931 | defhook end | |
932 | ||
933 | done_first_host_p=nil | |
934 | ||
f8d0b27d MW |
935 | host () { |
936 | host=$1 | |
fdd73e22 | 937 | like= userat= |
9b1d71c6 MW |
938 | case $done_first_host_p in |
939 | nil) runhook start; done_first_host_p=t ;; | |
940 | esac | |
f8d0b27d MW |
941 | case "${expire_policy+t},${default_policy+t}" in |
942 | t,) default_policy=$expire_policy ;; | |
943 | esac | |
944 | unset expire_policy | |
945 | $verbose "host $host" | |
946 | } | |
947 | ||
1c0d8611 | 948 | snaptype () { snap=$1; shift; snapargs="$*"; retry=1; } |
f6b4ffdc MW |
949 | rsyncargs () { rsyncargs="$*"; } |
950 | like () { like="$*"; } | |
5675acda | 951 | retry () { retry="$*"; } |
fdd73e22 | 952 | user () { userat="$*@"; } |
f6b4ffdc MW |
953 | |
954 | retain () { | |
f8d0b27d | 955 | case $clear_policy in t) unset expire_policy; clear_policy=nil ;; esac |
f6b4ffdc MW |
956 | expire_policy="${expire_policy+$expire_policy |
957 | }$*" | |
958 | } | |
959 | ||
960 | ###-------------------------------------------------------------------------- | |
961 | ### Read the configuration and we're done. | |
962 | ||
963 | usage () { | |
5b4c55ab | 964 | echo "usage: $quis [-nv] [-c CONF]" |
f6b4ffdc MW |
965 | } |
966 | ||
967 | version () { | |
968 | echo "$quis version $VERSION" | |
969 | } | |
970 | ||
f6b4ffdc MW |
971 | whine () { echo >&8 "$@"; } |
972 | ||
3f496b2b | 973 | while getopts "hVvc:n" opt; do |
f6b4ffdc MW |
974 | case "$opt" in |
975 | h) usage; exit 0 ;; | |
976 | V) version; config; exit 0 ;; | |
977 | v) verbose=whine ;; | |
978 | c) conf=$OPTARG ;; | |
3f496b2b | 979 | n) dryrun=t ;; |
f6b4ffdc MW |
980 | *) exit 1 ;; |
981 | esac | |
982 | done | |
983 | shift $((OPTIND - 1)) | |
984 | case $# in 0) ;; *) usage >&2; exit 1 ;; esac | |
985 | exec 8>&1 | |
986 | ||
987 | . "$conf" | |
9b1d71c6 MW |
988 | |
989 | runhook end $bkprc | |
1da0cd47 MW |
990 | case "$bkprc" in |
991 | 0) $verbose "All backups successful" ;; | |
992 | *) $verbose "Backups FAILED" ;; | |
993 | esac | |
f6b4ffdc MW |
994 | |
995 | ###----- That's all, folks -------------------------------------------------- | |
996 | ||
997 | exit $bkprc |