| 1 | #! /bin/bash |
| 2 | |
| 3 | set -e |
| 4 | set -u |
| 5 | |
| 6 | # runmirrors script for Debian |
| 7 | # Based losely on existing scripts, written by an unknown number of |
| 8 | # different people over the years. |
| 9 | # |
| 10 | # Copyright (C) 2008, 2009 Joerg Jaspert <joerg@debian.org> |
| 11 | # |
| 12 | # This program is free software; you can redistribute it and/or |
| 13 | # modify it under the terms of the GNU General Public License as |
| 14 | # published by the Free Software Foundation; version 2. |
| 15 | # |
| 16 | # This program is distributed in the hope that it will be useful, but |
| 17 | # WITHOUT ANY WARRANTY; without even the implied warranty of |
| 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 19 | # General Public License for more details. |
| 20 | # |
| 21 | # You should have received a copy of the GNU General Public License |
| 22 | # along with this program; if not, write to the Free Software |
| 23 | # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
| 24 | |
| 25 | # In case the admin somehow wants to have this script located someplace else, |
| 26 | # he can set BASEDIR, and we will take that. If it is unset we take ${HOME} |
| 27 | BASEDIR=${BASEDIR:-"${HOME}"} |
| 28 | |
| 29 | NAME="$(basename $0)" |
| 30 | |
| 31 | HELP="$0, (C) 2008, 2009 by Joerg Jaspert <joerg@debian.org>\n |
| 32 | Usage:\n\n |
| 33 | |
| 34 | 1.) a single parameter with NO leading -.\n |
| 35 | \t This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n |
| 36 | \t have us look for ${NAME}-security.{conf,mirror} files.\n\n |
| 37 | |
| 38 | 2.) using getopt style parameters:\n |
| 39 | \t -a [NAME] - Same as 1.) above, used for the config files. Default empty.\n |
| 40 | \t -k [TYPE] - Type of push. all, stage2, mhop. Default mhop.\n |
| 41 | \t -f - Run from within the mirrorscript ftpsync. Don't use from commandline!\n |
| 42 | \t -h - Print this help and exit |
| 43 | " |
| 44 | # If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle |
| 45 | # it will not start with a -. If we find oldstyle we assume its only one, the config |
| 46 | # name we run on. |
| 47 | if [ $# -gt 0 ]; then |
| 48 | if [ "x${1:0:1}x" != "x-x" ]; then |
| 49 | # Yes, does not start with a -, so use it for the config name. |
| 50 | CONF=${1:-""} |
| 51 | if [ -n "${CONF}" ]; then |
| 52 | NAME="${NAME}-${CONF}" |
| 53 | fi |
| 54 | else |
| 55 | # Yeah well, new style, starting with - for getopts |
| 56 | while getopts ':a:k:fh' OPTION ; do |
| 57 | case $OPTION in |
| 58 | a) CONF="${OPTARG}" |
| 59 | if [ -n "${CONF}" ]; then |
| 60 | NAME="${NAME}-${CONF}" |
| 61 | fi |
| 62 | ;; |
| 63 | k) PUSHKIND="${OPTARG}" |
| 64 | ;; |
| 65 | f) FROMFTPSYNC="true" |
| 66 | ;; |
| 67 | h) echo -e $HELP |
| 68 | exit 0 |
| 69 | ;; |
| 70 | |
| 71 | *) echo "Invalid usage" |
| 72 | echo -e $HELP |
| 73 | exit 1 |
| 74 | ;; |
| 75 | esac |
| 76 | done |
| 77 | fi |
| 78 | fi |
| 79 | # Make sure the values are always defined, even if there was no commandline option |
| 80 | # for them |
| 81 | # Default config is empty |
| 82 | CONF=${CONF:-""} |
| 83 | |
| 84 | # Set the default to all, if we didnt get told about it. Currently |
| 85 | # valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1, |
| 86 | # stage2 - staged push, second phase. Default is mhop. |
| 87 | PUSHKIND=${PUSHKIND:-"mhop"} |
| 88 | |
| 89 | # If we are pushed from within ftpsync. Default false. |
| 90 | FROMFTPSYNC=${FROMFTPSYNC:-"false"} |
| 91 | |
| 92 | ######################################################################## |
| 93 | # Read our config file |
| 94 | . "${BASEDIR}/etc/${NAME}.conf" |
| 95 | |
| 96 | # Source our common functions |
| 97 | . "${BASEDIR}/etc/common" |
| 98 | |
| 99 | # Set sane defaults if the configfile didn't do that for us. |
| 100 | # The directory for our logfiles |
| 101 | LOGDIR=${LOGDIR:-"${BASEDIR}/log"} |
| 102 | # Our own logfile |
| 103 | LOG=${LOG:-"${LOGDIR}/${NAME}.log"} |
| 104 | # Our lockfile directory |
| 105 | LOCKDIR=${LOCKDIR:-"${BASEDIR}/locks"} |
| 106 | # How many logfiles to keep |
| 107 | LOGROTATE=${LOGROTATE:-14} |
| 108 | # Our mirrorfile |
| 109 | MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"} |
| 110 | # used by log() |
| 111 | PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} |
| 112 | # extra ssh options we might want hostwide |
| 113 | SSH_OPTS=${SSH_OPTS:-"-o StrictHostKeyChecking=no"} |
| 114 | # Whats our archive name? We will also tell our leafs about it |
| 115 | PUSHARCHIVE=${PUSHARCHIVE:-"${CONF}"} |
| 116 | # How long to wait for mirrors to do stage1 if we have multi-stage syncing |
| 117 | PUSHDELAY=${PUSHDELAY:-600} |
| 118 | # Which ssh key to use? |
| 119 | KEYFILE=${KEYFILE:-".ssh/pushmirror"} |
| 120 | # where to send mails to |
| 121 | if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then |
| 122 | # We are not on a debian.org host |
| 123 | MAILTO=${MAILTO:-"root"} |
| 124 | else |
| 125 | # Yay, on a .debian.org host |
| 126 | MAILTO=${MAILTO:-"mirrorlogs@debian.org"} |
| 127 | fi |
| 128 | |
| 129 | if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then |
| 130 | error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> "${LOG}" |
| 131 | exit 5 |
| 132 | fi |
| 133 | |
| 134 | # Hooks |
| 135 | HOOK1=${HOOK1:-""} |
| 136 | HOOK2=${HOOK2:-""} |
| 137 | HOOK3=${HOOK3:-""} |
| 138 | |
| 139 | ######################################################################## |
| 140 | |
| 141 | # Some sane defaults |
| 142 | cd "${BASEDIR}" |
| 143 | umask 022 |
| 144 | |
| 145 | # Make sure we have our log and lock directories |
| 146 | mkdir -p "${LOGDIR}" |
| 147 | mkdir -p "${LOCKDIR}" |
| 148 | |
| 149 | trap 'log "Mirrorpush done" >> "${LOG}"; savelog "${LOG}" > /dev/null' EXIT |
| 150 | |
| 151 | log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> "${LOG}" |
| 152 | |
| 153 | HOOK=( |
| 154 | HOOKNR=1 |
| 155 | HOOKSCR=${HOOK1} |
| 156 | ) |
| 157 | hook $HOOK |
| 158 | |
| 159 | # From here on we do *NOT* want to exit on errors. We don't want to |
| 160 | # stop pushing mirrors just because we can't reach one of them. |
| 161 | set +e |
| 162 | |
| 163 | # Built up our list of 2-stage mirrors. |
| 164 | PUSHLOCKS="" |
| 165 | PUSHLOCKS=$(get2stage) |
| 166 | |
| 167 | # In case we have it - remove. It is used to synchronize multi-stage mirroring |
| 168 | rm -f "${LOCKDIR}/all_stage1" |
| 169 | |
| 170 | # Now read our mirrorfile and push the mirrors defined in there. |
| 171 | # We use grep to easily sort out all lines having a # in front of them or are empty. |
| 172 | egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" | |
| 173 | while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do |
| 174 | if [ "x${MTYPE}x" = "xDELAYx" ]; then |
| 175 | # We should wait a bit. |
| 176 | if [ -z ${MLNAME} ]; then |
| 177 | MLNAME=600 |
| 178 | fi |
| 179 | log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}" |
| 180 | sleep ${MLNAME} |
| 181 | continue |
| 182 | fi |
| 183 | |
| 184 | # If we are told we have a mhop sync to do and are called from within ftpsync, |
| 185 | # we will only look at staged/mhop entries and ignore the rest. |
| 186 | if [ "x${PUSHKIND}x" = "xmhopx" ] && [ "x${FROMFTPSYNC}x" = "xtruex" ]; then |
| 187 | if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then |
| 188 | continue |
| 189 | fi |
| 190 | fi |
| 191 | |
| 192 | # Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken |
| 193 | # as a set of options to give to ssh, we pass it without doing anything with it. |
| 194 | # If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use, |
| 195 | # and also means we look if there is one value more after a space. That value would then |
| 196 | # be the ssh keyfile we use with -i. That gives us full flexibility for all |
| 197 | # ssh options but doesn't destroy backwards compatibility. |
| 198 | # If it is empty we assume proto 2 and the default keyfile. |
| 199 | # |
| 200 | # There is one bug in here. We will give out the master keyfile, even if there is a |
| 201 | # "-i /bla/bla" in the options. ssh stuffs them together and presents two keys to the |
| 202 | # target server. In the case both keys do some actions- the first one presented wins. |
| 203 | # And this might not be what one wants. |
| 204 | # |
| 205 | # The only sane way to go around this, i think, is by dropping backward compability. |
| 206 | # Which I don't really like. |
| 207 | if [ -n "${MSSHOPT}" ]; then |
| 208 | # So its not empty, lets check if it starts with a - and as such is a "new-style" |
| 209 | # ssh options set. |
| 210 | if [ "x${MSSHOPT:0:1}x" = "x-x" ]; then |
| 211 | # Yes we start with a - |
| 212 | SSHOPT="${MSSHOPT}" |
| 213 | MPROTO="99" |
| 214 | MKEYFILE="${BASEDIR}/${KEYFILE}" |
| 215 | elif [ ${MSSHOPT:0:1} -eq 1 ] || [ ${MSSHOPT:0:1} -eq 2 ]; then |
| 216 | # We do seem to have oldstyle options here. |
| 217 | MPROTO=${MSSHOPT:0:1} |
| 218 | MKEYFILE=${MSSHOPT:1} |
| 219 | SSHOPT="" |
| 220 | else |
| 221 | error "I don't know what is configured for mirror ${MLNAME}" |
| 222 | continue |
| 223 | fi |
| 224 | else |
| 225 | MPROTO=2 |
| 226 | MKEYFILE="${BASEDIR}/${KEYFILE}" |
| 227 | SSHOPT="" |
| 228 | fi |
| 229 | |
| 230 | # Built our array |
| 231 | SIGNAL_OPTS=( |
| 232 | MIRROR="${MLNAME}" |
| 233 | HOSTNAME="${MHOSTNAME}" |
| 234 | USERNAME="${MUSER}" |
| 235 | SSHPROTO="${MPROTO}" |
| 236 | SSHKEY="${MKEYFILE}" |
| 237 | SSHOPTS="${SSHOPT/ /#}" |
| 238 | PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" |
| 239 | PUSHTYPE="${MTYPE}" |
| 240 | PUSHARCHIVE=${PUSHARCHIVE} |
| 241 | PUSHKIND=${PUSHKIND} |
| 242 | FROMFTPSYNC=${FROMFTPSYNC} |
| 243 | ) |
| 244 | |
| 245 | # And finally, push the mirror |
| 246 | log "Trigger ${MLNAME}" >> "${LOG}" |
| 247 | signal "${SIGNAL_OPTS}" & |
| 248 | log "Trigger for ${MLNAME} done" >> "${LOG}" |
| 249 | |
| 250 | HOOK=( |
| 251 | HOOKNR=2 |
| 252 | HOOKSCR=${HOOK2} |
| 253 | ) |
| 254 | hook $HOOK |
| 255 | set +e |
| 256 | done |
| 257 | |
| 258 | # If we are run from within ftpsync *and* have an mhop push to send on, we have |
| 259 | # to wait until the push is gone through and they all returned, or we will exit |
| 260 | # much too early. |
| 261 | # As the signal routine touches $LOCKDIR/all_stage1 when all are done, its |
| 262 | # easy enough just to wait for that to appear. Of course we do the same game |
| 263 | # with PUSHDELAY to not wait forever. |
| 264 | if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then |
| 265 | tries=0 |
| 266 | # We do not wait forever |
| 267 | while [ ${tries} -lt ${PUSHDELAY} ]; do |
| 268 | if [ -f "${LOCKDIR}/all_stage1" ]; then |
| 269 | break |
| 270 | fi |
| 271 | tries=$((tries + 5)) |
| 272 | sleep 5 |
| 273 | done |
| 274 | |
| 275 | if [ ${tries} -ge ${PUSHDELAY} ]; then |
| 276 | error "Failed to wait for our mirrors when sending mhop push down." >> "${LOG}" |
| 277 | fi |
| 278 | fi |
| 279 | |
| 280 | HOOK=( |
| 281 | HOOKNR=3 |
| 282 | HOOKSCR=${HOOK3} |
| 283 | ) |
| 284 | hook $HOOK |
| 285 | |
| 286 | exit 0 |