mdw@git.distorted.org.uk Git - tripe/blob - peerdb/tripe-newpeers.in

   1 #! @PYTHON@
   2 ### -*-python-*-
   3 ###
   4 ### Build a CDB file from configuration file
   5 ###
   6 ### (c) 2007 Straylight/Edgeware
   7 ###
   8
   9 ###----- Licensing notice ---------------------------------------------------
  10 ###
  11 ### This file is part of Trivial IP Encryption (TrIPE).
  12 ###
  13 ### TrIPE is free software: you can redistribute it and/or modify it under
  14 ### the terms of the GNU General Public License as published by the Free
  15 ### Software Foundation; either version 3 of the License, or (at your
  16 ### option) any later version.
  17 ###
  18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
  19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  20 ### FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  21 ### for more details.
  22 ###
  23 ### You should have received a copy of the GNU General Public License
  24 ### along with TrIPE.  If not, see <https://www.gnu.org/licenses/>.
  25
  26 VERSION = '@VERSION@'
  27
  28 ###--------------------------------------------------------------------------
  29 ### External dependencies.
  30
  31 import ConfigParser as CP
  32 import mLib as M
  33 from optparse import OptionParser
  34 import cdb as CDB
  35 from sys import stdin, stdout, exit, argv
  36 import re as RX
  37 import os as OS
  38
  39 ###--------------------------------------------------------------------------
  40 ### Utilities.
  41
  42 class CDBFake (object):
  43   """Like cdbmake, but just outputs data suitable for cdb-map."""
  44   def __init__(me, file = stdout):
  45     me.file = file
  46   def add(me, key, value):
  47     me.file.write('%s:%s\n' % (key, value))
  48   def finish(me):
  49     pass
  50
  51 ###--------------------------------------------------------------------------
  52 ### A bulk DNS resolver.
  53
  54 class BulkResolver (object):
  55   """
  56   Resolve a number of DNS names in parallel.
  57
  58   The BulkResovler resolves a number of hostnames in parallel.  Using it
  59   works in three phases:
  60
  61     1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
  62        you're interested in.
  63
  64     2. You call run() to actually drive the resolver.
  65
  66     3. You call lookup(HOSTNAME) to get the address you wanted.  This will
  67        fail with KeyError if the resolver couldn't resolve the HOSTNAME.
  68   """
  69
  70   def __init__(me):
  71     """Initialize the resolver."""
  72     me._resolvers = {}
  73     me._namemap = {}
  74
  75   def prepare(me, host):
  76     """Prime the resolver to resolve the name HOST."""
  77     if host not in me._resolvers:
  78       me._resolvers[host] = M.SelResolveByName \
  79                             (host,
  80                              lambda name, alias, addr:
  81                                me._resolved(host, addr[0]),
  82                              lambda: me._resolved(host, None))
  83
  84   def run(me):
  85     """Run the background DNS resolver until it's finished."""
  86     while me._resolvers:
  87       M.select()
  88
  89   def lookup(me, host):
  90     """
  91     Fetch the address corresponding to HOST.
  92     """
  93     addr = me._namemap[host]
  94     if addr is None:
  95       raise KeyError(host)
  96     return addr
  97
  98   def _resolved(me, host, addr):
  99     """Callback function: remember that ADDR is the address for HOST."""
 100     me._namemap[host] = addr
 101     del me._resolvers[host]
 102
 103 ###--------------------------------------------------------------------------
 104 ### The configuration parser.
 105
 106 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
 107 r_ref = RX.compile(r'\$\(([^)]+)\)')
 108
 109 ## Match a $[HOST] name resolution reference; group 1 is the HOST.
 110 r_resolve = RX.compile(r'\$\[([^]]+)\]')
 111
 112 def _fmt_path(path):
 113   return ' -> '.join(["`%s'" % hop for hop in path])
 114
 115 class AmbiguousOptionError (Exception):
 116   def __init__(me, key, patha, vala, pathb, valb):
 117     me.key = key
 118     me.patha, me.vala = patha, vala
 119     me.pathb, me.valb = pathb, valb
 120   def __str__(me):
 121     return "Ambiguous answer resolving key `%s': " \
 122         "path %s yields `%s' but %s yields `%s'" % \
 123         (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
 124
 125 class InheritanceCycleError (Exception):
 126   def __init__(me, key, path):
 127     me.key = key
 128     me.path = path
 129   def __str__(me):
 130     return "Found a cycle %s looking up key `%s'" % \
 131         (_fmt_path(me.path), me.key)
 132
 133 class MissingKeyException (Exception):
 134   def __init__(me, sec, key):
 135     me.sec = sec
 136     me.key = key
 137   def __str__(me):
 138     return "Key `%s' not found in section `%s'" % (me.key, me.sec)
 139
 140 class MyConfigParser (CP.RawConfigParser):
 141   """
 142   A more advanced configuration parser.
 143
 144   This has three major enhancements over the standard ConfigParser which are
 145   relevant to us.
 146
 147     * It recognizes `@inherits' keys and follows them when expanding a
 148       value.
 149
 150     * It recognizes `$(VAR)' references to configuration variables during
 151       expansion and processes them correctly.
 152
 153     * It recognizes `$[HOST]' name-resolver requests and handles them
 154       correctly.
 155
 156   Use:
 157
 158     1. Call read(FILENAME) and/or read(FP, [FILENAME]) to slurp in the
 159        configuration data.
 160
 161     2. Call resolve() to collect the hostnames which need to be resolved and
 162        actually do the name resolution.
 163
 164     3. Call get(SECTION, ITEM) to collect the results, or items(SECTION) to
 165        iterate over them.
 166   """
 167
 168   def __init__(me):
 169     """
 170     Initialize a new, empty configuration parser.
 171     """
 172     CP.RawConfigParser.__init__(me)
 173     me._resolver = BulkResolver()
 174
 175   def resolve(me):
 176     """
 177     Works out all of the hostnames which need resolving and resolves them.
 178
 179     Until you call this, attempts to fetch configuration items which need to
 180     resolve hostnames will fail!
 181     """
 182     for sec in me.sections():
 183       for key, value in me.items(sec, resolvep = False):
 184         for match in r_resolve.finditer(value):
 185           me._resolver.prepare(match.group(1))
 186     me._resolver.run()
 187
 188   def _expand(me, sec, string, resolvep):
 189     """
 190     Expands $(...) and (optionally) $[...] placeholders in STRING.
 191
 192     The SEC is the configuration section from which to satisfy $(...)
 193     requests.  RESOLVEP is a boolean switch: do we bother to tax the resolver
 194     or not?  This is turned off by the resolve() method while it's collecting
 195     hostnames to be resolved.
 196     """
 197     string = r_ref.sub \
 198              (lambda m: me.get(sec, m.group(1), resolvep), string)
 199     if resolvep:
 200       string = r_resolve.sub(lambda m: me._resolver.lookup(m.group(1)),
 201                              string)
 202     return string
 203
 204   def has_option(me, sec, key):
 205     """
 206     Decide whether section SEC has a configuration key KEY.
 207
 208     This version of the method properly handles the @inherit key.
 209     """
 210     return key == 'name' or me._get(sec, key)[0] is not None
 211
 212   def _get(me, sec, key, map = None, path = None):
 213     """
 214     Low-level option-fetching method.
 215
 216     Fetch the value for the named KEY from section SEC, or maybe
 217     (recursively) a section which SEC inherits from.
 218
 219     Returns a pair VALUE, PATH.  The value is not expanded; nor do we check
 220     for the special `name' key.  The caller is expected to do these things.
 221     Returns None if no value could be found.
 222     """
 223
 224     ## If we weren't given a memoization map or path, then we'd better make
 225     ## one.
 226     if map is None: map = {}
 227     if path is None: path = []
 228
 229     ## Extend the path to cover the lookup section, but remember to remove us
 230     ## again when we've finished.  If we need to pass the current path back
 231     ## upwards, then remember to take a copy.
 232     path.append(sec)
 233     try:
 234
 235       ## If we've been this way before on another pass through then return
 236       ## the value we found then.  If we're still thinking about it then
 237       ## we've found a cycle.
 238       try:
 239         threadp, value = map[sec]
 240       except KeyError:
 241         pass
 242       else:
 243         if threadp:
 244           raise InheritanceCycleError(key, path[:])
 245
 246       ## See whether the answer is ready waiting for us.
 247       try:
 248         v = CP.RawConfigParser.get(me, sec, key)
 249       except CP.NoOptionError:
 250         pass
 251       else:
 252         return v, path[:]
 253
 254       ## No, apparently, not.  Find out our list of parents.
 255       try:
 256         parents = CP.RawConfigParser.get(me, sec, '@inherit').\
 257             replace(',', ' ').split()
 258       except CP.NoOptionError:
 259         parents = []
 260
 261       ## Initially we have no idea.
 262       value = None
 263       winner = None
 264
 265       ## Go through our parents and ask them what they think.
 266       map[sec] = True, None
 267       for p in parents:
 268
 269         ## See whether we get an answer.  If not, keep on going.
 270         v, pp = me._get(p, key, map, path)
 271         if v is None: continue
 272
 273         ## If we got an answer, check that it matches any previous ones.
 274         if value is None:
 275           value = v
 276           winner = pp
 277         elif value != v:
 278           raise AmbiguousOptionError(key, winner, value, pp, v)
 279
 280       ## That's the best we could manage.
 281       map[sec] = False, value
 282       return value, winner
 283
 284     finally:
 285       ## Remove us from the path again.
 286       path.pop()
 287
 288   def get(me, sec, key, resolvep = True):
 289     """
 290     Retrieve the value of KEY from section SEC.
 291     """
 292
 293     ## Special handling for the `name' key.
 294     if key == 'name':
 295       try: value = CP.RawConfigParser.get(me, sec, key)
 296       except CP.NoOptionError: value = sec
 297     else:
 298       value, _ = me._get(sec, key)
 299       if value is None:
 300         raise MissingKeyException(sec, key)
 301
 302     ## Expand the value and return it.
 303     return me._expand(sec, value, resolvep)
 304
 305   def items(me, sec, resolvep = True):
 306     """
 307     Return a list of (NAME, VALUE) items in section SEC.
 308
 309     This extends the default method by handling the inheritance chain.
 310     """
 311
 312     ## Initialize for a depth-first walk of the inheritance graph.
 313     d = {}
 314     visited = {}
 315     basesec = sec
 316     stack = [sec]
 317
 318     ## Visit nodes, collecting their keys.  Don't believe the values:
 319     ## resolving inheritance is too hard to do like this.
 320     while stack:
 321       sec = stack.pop()
 322       if sec in visited: continue
 323       visited[sec] = True
 324
 325       for key, value in CP.RawConfigParser.items(me, sec):
 326         if key == '@inherit': stack += value.replace(',', ' ').split()
 327         else: d[key] = None
 328
 329     ## Now collect the values for the known keys, one by one.
 330     items = []
 331     for key in d: items.append((key, me.get(basesec, key, resolvep)))
 332
 333     ## And we're done.
 334     return items
 335
 336 ###--------------------------------------------------------------------------
 337 ### Command-line handling.
 338
 339 def inputiter(things):
 340   """
 341   Iterate over command-line arguments, returning corresponding open files.
 342
 343   If none were given, or one is `-', assume standard input; if one is a
 344   directory, scan it for files other than backups; otherwise return the
 345   opened files.
 346   """
 347
 348   if not things:
 349     if OS.isatty(stdin.fileno()):
 350       M.die('no input given, and stdin is a terminal')
 351     yield stdin
 352   else:
 353     for thing in things:
 354       if thing == '-':
 355         yield stdin
 356       elif OS.path.isdir(thing):
 357         for item in OS.listdir(thing):
 358           if item.endswith('~') or item.endswith('#'):
 359             continue
 360           name = OS.path.join(thing, item)
 361           if not OS.path.isfile(name):
 362             continue
 363           yield file(name)
 364       else:
 365         yield file(thing)
 366
 367 def parse_options(argv = argv):
 368   """
 369   Parse command-line options, returning a pair (OPTS, ARGS).
 370   """
 371   M.ego(argv[0])
 372   op = OptionParser(usage = '%prog [-c CDB] INPUT...',
 373                     version = '%%prog (tripe, version %s)' % VERSION)
 374   op.add_option('-c', '--cdb', metavar = 'CDB',
 375                 dest = 'cdbfile', default = None,
 376                 help = 'Compile output into a CDB file.')
 377   opts, args = op.parse_args(argv)
 378   return opts, args
 379
 380 ###--------------------------------------------------------------------------
 381 ### Main code.
 382
 383 def getconf(args):
 384   """
 385   Read the configuration files and return the accumulated result.
 386
 387   We make sure that all hostnames have been properly resolved.
 388   """
 389   conf = MyConfigParser()
 390   for f in inputiter(args):
 391     conf.readfp(f)
 392   conf.resolve()
 393   return conf
 394
 395 def output(conf, cdb):
 396   """
 397   Output the configuration information CONF to the database CDB.
 398
 399   This is where the special `user' and `auto' database entries get set.
 400   """
 401   auto = []
 402   for sec in sorted(conf.sections()):
 403     if sec.startswith('@'):
 404       continue
 405     elif sec.startswith('$'):
 406       label = sec
 407     else:
 408       label = 'P%s' % sec
 409       if conf.has_option(sec, 'auto') and \
 410          conf.get(sec, 'auto') in ('y', 'yes', 't', 'true', '1', 'on'):
 411         auto.append(sec)
 412       if conf.has_option(sec, 'user'):
 413         cdb.add('U%s' % conf.get(sec, 'user'), sec)
 414     url = M.URLEncode(laxp = True, semip = True)
 415     for key, value in sorted(conf.items(sec), key = lambda (k, v): k):
 416       if not key.startswith('@'):
 417         url.encode(key, ' '.join(M.split(value)[0]))
 418     cdb.add(label, url.result)
 419   cdb.add('%AUTO', ' '.join(auto))
 420   cdb.finish()
 421
 422 def main():
 423   """Main program."""
 424   opts, args = parse_options()
 425   if opts.cdbfile:
 426     cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
 427   else:
 428     cdb = CDBFake()
 429   conf = getconf(args[1:])
 430   output(conf, cdb)
 431
 432 if __name__ == '__main__':
 433   main()
 434
 435 ###----- That's all, folks --------------------------------------------------