mdw@git.distorted.org.uk Git - tripe/blob - peerdb/tripe-newpeers.in

   1 #! @PYTHON@
   2 ### -*-python-*-
   3 ###
   4 ### Build a CDB file from configuration file
   5 ###
   6 ### (c) 2007 Straylight/Edgeware
   7 ###
   8
   9 ###----- Licensing notice ---------------------------------------------------
  10 ###
  11 ### This file is part of Trivial IP Encryption (TrIPE).
  12 ###
  13 ### TrIPE is free software: you can redistribute it and/or modify it under
  14 ### the terms of the GNU General Public License as published by the Free
  15 ### Software Foundation; either version 3 of the License, or (at your
  16 ### option) any later version.
  17 ###
  18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
  19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  20 ### FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  21 ### for more details.
  22 ###
  23 ### You should have received a copy of the GNU General Public License
  24 ### along with TrIPE.  If not, see <https://www.gnu.org/licenses/>.
  25
  26 VERSION = '@VERSION@'
  27
  28 ###--------------------------------------------------------------------------
  29 ### External dependencies.
  30
  31 import mLib as M
  32 from optparse import OptionParser
  33 import cdb as CDB
  34 from sys import stdin, stdout, exit, argv
  35 import re as RX
  36 import os as OS
  37 from cStringIO import StringIO
  38
  39 ###--------------------------------------------------------------------------
  40 ### Utilities.
  41
  42 class CDBFake (object):
  43   """Like cdbmake, but just outputs data suitable for cdb-map."""
  44   def __init__(me, file = stdout):
  45     me.file = file
  46   def add(me, key, value):
  47     me.file.write('%s:%s\n' % (key, value))
  48   def finish(me):
  49     pass
  50
  51 ###--------------------------------------------------------------------------
  52 ### A bulk DNS resolver.
  53
  54 class BulkResolver (object):
  55   """
  56   Resolve a number of DNS names in parallel.
  57
  58   The BulkResovler resolves a number of hostnames in parallel.  Using it
  59   works in three phases:
  60
  61     1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
  62        you're interested in.
  63
  64     2. You call run() to actually drive the resolver.
  65
  66     3. You call lookup(HOSTNAME) to get the address you wanted.  This will
  67        fail with KeyError if the resolver couldn't resolve the HOSTNAME.
  68   """
  69
  70   def __init__(me):
  71     """Initialize the resolver."""
  72     me._resolvers = {}
  73     me._namemap = {}
  74
  75   def prepare(me, host):
  76     """Prime the resolver to resolve the name HOST."""
  77     if host not in me._resolvers:
  78       me._resolvers[host] = M.SelResolveByName \
  79                             (host,
  80                              lambda name, alias, addr:
  81                                me._resolved(host, addr[0]),
  82                              lambda: me._resolved(host, None))
  83
  84   def run(me):
  85     """Run the background DNS resolver until it's finished."""
  86     while me._resolvers:
  87       M.select()
  88
  89   def lookup(me, host):
  90     """
  91     Fetch the address corresponding to HOST.
  92     """
  93     addr = me._namemap[host]
  94     if addr is None:
  95       raise KeyError(host)
  96     return addr
  97
  98   def _resolved(me, host, addr):
  99     """Callback function: remember that ADDR is the address for HOST."""
 100     me._namemap[host] = addr
 101     del me._resolvers[host]
 102
 103 ###--------------------------------------------------------------------------
 104 ### The configuration parser.
 105
 106 ## Match a comment or empty line.
 107 RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])')
 108
 109 ## Match a section group header.
 110 RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $')
 111
 112 ## Match an assignment line.
 113 RX_ASSGN = RX.compile(r'''(?x) ^
 114         ([^\s:=] (?: [^:=]* [^\s:=])?)
 115         \s* [:=] \s*
 116         (| \S | \S.*\S)
 117         \s* $''')
 118
 119 ## Match a continuation line.
 120 RX_CONT = RX.compile(r'''(?x) ^ \s+
 121         (| \S | \S.*\S)
 122         \s* $''')
 123
 124 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
 125 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
 126
 127 ## Match a $[HOST] name resolution reference; group 1 is the HOST.
 128 RX_RESOLVE = RX.compile(r'(?x) \$ \[ ([^]]+) \]')
 129
 130 class ConfigSyntaxError (Exception):
 131   def __init__(me, fname, lno, msg):
 132     me.fname = fname
 133     me.lno = lno
 134     me.msg = msg
 135   def __str__(me):
 136     return '%s:%d: %s' % (me.fname, me.lno, me.msg)
 137
 138 def _fmt_path(path):
 139   return ' -> '.join(["`%s'" % hop for hop in path])
 140
 141 class AmbiguousOptionError (Exception):
 142   def __init__(me, key, patha, vala, pathb, valb):
 143     me.key = key
 144     me.patha, me.vala = patha, vala
 145     me.pathb, me.valb = pathb, valb
 146   def __str__(me):
 147     return "Ambiguous answer resolving key `%s': " \
 148         "path %s yields `%s' but %s yields `%s'" % \
 149         (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
 150
 151 class InheritanceCycleError (Exception):
 152   def __init__(me, key, path):
 153     me.key = key
 154     me.path = path
 155   def __str__(me):
 156     return "Found a cycle %s looking up key `%s'" % \
 157         (_fmt_path(me.path), me.key)
 158
 159 class MissingSectionException (Exception):
 160   def __init__(me, sec):
 161     me.key = key
 162   def __str__(me):
 163     return "Section `%s' not found" % (me.sec)
 164
 165 class MissingKeyException (Exception):
 166   def __init__(me, sec, key):
 167     me.sec = sec
 168     me.key = key
 169   def __str__(me):
 170     return "Key `%s' not found in section `%s'" % (me.key, me.sec)
 171
 172 class ConfigSection (object):
 173   """
 174   A section in a configuration parser.
 175
 176   This is where a lot of the nitty-gritty stuff actually happens.  The
 177   `MyConfigParser' knows a lot about the internals of this class, which saves
 178   on building a complicated interface.
 179   """
 180
 181   def __init__(me, name, cp):
 182     """Initialize a new, empty section with a given NAME and parent CP."""
 183     me.name = name
 184     me._itemmap = dict()
 185     me._cp = cp
 186
 187   def _expand(me, string, resolvep):
 188     """
 189     Expands $(...) and (optionally) $[...] placeholders in STRING.
 190
 191     RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
 192     This is turned off by MyConfigParser's resolve() method while it's
 193     collecting hostnames to be resolved.
 194     """
 195     string = RX_REF.sub \
 196              (lambda m: me.get(m.group(1), resolvep), string)
 197     if resolvep:
 198       string = RX_RESOLVE.sub(lambda m: me._cp._resolver.lookup(m.group(1)),
 199                               string)
 200     return string
 201
 202   def has_option(me, key):
 203     """
 204     Decide whether this section has a configuration key KEY.
 205
 206     This version of the method properly handles the @inherit key.
 207     """
 208     return key == 'name' or me._get(key)[0] is not None
 209
 210   def _get(me, key, map = None, path = None):
 211     """
 212     Low-level option-fetching method.
 213
 214     Fetch the value for the named KEY in this section, or maybe (recursively)
 215     a section which it inherits from.
 216
 217     Returns a pair VALUE, PATH.  The value is not expanded; nor do we check
 218     for the special `name' key.  The caller is expected to do these things.
 219     Returns None if no value could be found.
 220     """
 221
 222     ## If we weren't given a memoization map or path, then we'd better make
 223     ## one.
 224     if map is None: map = {}
 225     if path is None: path = []
 226
 227     ## Extend the path to cover us, but remember to remove us again when
 228     ## we've finished.  If we need to pass the current path back upwards,
 229     ## then remember to take a copy.
 230     path.append(me.name)
 231     try:
 232
 233       ## If we've been this way before on another pass through then return
 234       ## the value we found then.  If we're still thinking about it then
 235       ## we've found a cycle.
 236       try: threadp, value = map[me.name]
 237       except KeyError: pass
 238       else:
 239         if threadp: raise InheritanceCycleError(key, path[:])
 240
 241       ## See whether the answer is ready waiting for us.
 242       try: v = me._itemmap[key]
 243       except KeyError: pass
 244       else: return v, path[:]
 245
 246       ## No, apparently, not.  Find out our list of parents.
 247       try:
 248         parents = [me._cp.section(p) for p in
 249                    me._itemmap['@inherit'].replace(',', ' ').split()]
 250       except KeyError:
 251         parents = []
 252
 253       ## Initially we have no idea.
 254       value = None
 255       winner = None
 256
 257       ## Go through our parents and ask them what they think.
 258       map[me.name] = True, None
 259       for p in parents:
 260
 261         ## See whether we get an answer.  If not, keep on going.
 262         v, pp = p._get(key, map, path)
 263         if v is None: continue
 264
 265         ## If we got an answer, check that it matches any previous ones.
 266         if value is None:
 267           value = v
 268           winner = pp
 269         elif value != v:
 270           raise AmbiguousOptionError(key, winner, value, pp, v)
 271
 272       ## That's the best we could manage.
 273       map[me.name] = False, value
 274       return value, winner
 275
 276     finally:
 277       ## Remove us from the path again.
 278       path.pop()
 279
 280   def get(me, key, resolvep = True):
 281     """
 282     Retrieve the value of KEY from this section.
 283     """
 284
 285     ## Special handling for the `name' key.
 286     if key == 'name':
 287       value = me._itemmap.get('name', me.name)
 288     else:
 289       value, _ = me._get(key)
 290       if value is None:
 291         raise MissingKeyException(me.name, key)
 292
 293     ## Expand the value and return it.
 294     return me._expand(value, resolvep)
 295
 296   def items(me, resolvep = True):
 297     """
 298     Return a list of (NAME, VALUE) items in this section.
 299
 300     This extends the default method by handling the inheritance chain.
 301     """
 302
 303     ## Initialize for a depth-first walk of the inheritance graph.
 304     d = {}
 305     visited = {}
 306     stack = [me.name]
 307
 308     ## Visit nodes, collecting their keys.  Don't believe the values:
 309     ## resolving inheritance is too hard to do like this.
 310     while stack:
 311       sec = me._cp.section(stack.pop())
 312       if sec.name in visited: continue
 313       visited[sec.name] = True
 314
 315       for key, value in sec._itemmap.iteritems():
 316         if key == '@inherit': stack += value.replace(',', ' ').split()
 317         else: d[key] = None
 318
 319     ## Now collect the values for the known keys, one by one.
 320     items = []
 321     for key in d: items.append((key, me.get(key, resolvep)))
 322
 323     ## And we're done.
 324     return items
 325
 326 class MyConfigParser (object):
 327   """
 328   A more advanced configuration parser.
 329
 330   This has four major enhancements over the standard ConfigParser which are
 331   relevant to us.
 332
 333     * It recognizes `@inherits' keys and follows them when expanding a
 334       value.
 335
 336     * It recognizes `$(VAR)' references to configuration variables during
 337       expansion and processes them correctly.
 338
 339     * It recognizes `$[HOST]' name-resolver requests and handles them
 340       correctly.
 341
 342     * Its parsing behaviour is well-defined.
 343
 344   Use:
 345
 346     1. Call parse(FILENAME) to slurp in the configuration data.
 347
 348     2. Call resolve() to collect the hostnames which need to be resolved and
 349        actually do the name resolution.
 350
 351     3. Call sections() to get a list of the configuration sections, or
 352        section(NAME) to find a named section.
 353
 354     4. Call get(ITEM) on a section to collect the results, or items() to
 355        iterate over them.
 356   """
 357
 358   def __init__(me):
 359     """
 360     Initialize a new, empty configuration parser.
 361     """
 362     me._sectmap = dict()
 363     me._resolver = BulkResolver()
 364
 365   def parse(me, f):
 366     """
 367     Parse configuration from a file F.
 368     """
 369
 370     ## Initial parser state.
 371     sect = None
 372     key = None
 373     val = None
 374     lno = 0
 375
 376     ## An unpleasant hack.  Python makes it hard to capture a value in a
 377     ## variable and examine it in a single action, and this is the best that
 378     ## I came up with.
 379     m = [None]
 380     def match(rx): m[0] = rx.match(line); return m[0]
 381
 382     ## Commit a key's value when we've determined that there are no further
 383     ## continuation lines.
 384     def flush():
 385       if key is not None: sect._itemmap[key] = val.getvalue()
 386
 387     ## Work through all of the input lines.
 388     for line in f:
 389       lno += 1
 390
 391       if match(RX_COMMENT):
 392         ## A comment or a blank line.  Nothing doing.  (This means that we
 393         ## leave out blank lines which look like they might be continuation
 394         ## lines.)
 395
 396         pass
 397
 398       elif match(RX_GRPHDR):
 399         ## A section header.  Flush out any previous value and set up the new
 400         ## group.
 401
 402         flush()
 403         name = m[0].group(1)
 404         try: sect = me._sectmap[name]
 405         except KeyError: sect = me._sectmap[name] = ConfigSection(name, me)
 406         key = None
 407
 408       elif match(RX_ASSGN):
 409         ## A new assignment.  Flush out the old one, and set up to store this
 410         ## one.
 411
 412         if sect is None:
 413           raise ConfigSyntaxError(f.name, lno, 'no active section to update')
 414         flush()
 415         key = m[0].group(1)
 416         val = StringIO(); val.write(m[0].group(2))
 417
 418       elif match(RX_CONT):
 419         ## A continuation line.  Accumulate the value.
 420
 421         if key is None:
 422           raise ConfigSyntaxError(f.name, lno, 'no config value to continue')
 423         val.write('\n'); val.write(m[0].group(1))
 424
 425       else:
 426         ## Something else.
 427
 428         raise ConfigSyntaxError(f.name, lno, 'incomprehensible line')
 429
 430     ## Don't forget to commit any final value material.
 431     flush()
 432
 433   def section(me, name):
 434     """Return a ConfigSection with the given NAME."""
 435     try: return me._sectmap[name]
 436     except KeyError: raise MissingSectionException(name)
 437
 438   def sections(me):
 439     """Yield the known sections."""
 440     return me._sectmap.itervalues()
 441
 442   def resolve(me):
 443     """
 444     Works out all of the hostnames which need resolving and resolves them.
 445
 446     Until you call this, attempts to fetch configuration items which need to
 447     resolve hostnames will fail!
 448     """
 449     for sec in me.sections():
 450       for key, value in sec.items(resolvep = False):
 451         for match in RX_RESOLVE.finditer(value):
 452           me._resolver.prepare(match.group(1))
 453     me._resolver.run()
 454
 455 ###--------------------------------------------------------------------------
 456 ### Command-line handling.
 457
 458 def inputiter(things):
 459   """
 460   Iterate over command-line arguments, returning corresponding open files.
 461
 462   If none were given, or one is `-', assume standard input; if one is a
 463   directory, scan it for files other than backups; otherwise return the
 464   opened files.
 465   """
 466
 467   if not things:
 468     if OS.isatty(stdin.fileno()):
 469       M.die('no input given, and stdin is a terminal')
 470     yield stdin
 471   else:
 472     for thing in things:
 473       if thing == '-':
 474         yield stdin
 475       elif OS.path.isdir(thing):
 476         for item in OS.listdir(thing):
 477           if item.endswith('~') or item.endswith('#'):
 478             continue
 479           name = OS.path.join(thing, item)
 480           if not OS.path.isfile(name):
 481             continue
 482           yield file(name)
 483       else:
 484         yield file(thing)
 485
 486 def parse_options(argv = argv):
 487   """
 488   Parse command-line options, returning a pair (OPTS, ARGS).
 489   """
 490   M.ego(argv[0])
 491   op = OptionParser(usage = '%prog [-c CDB] INPUT...',
 492                     version = '%%prog (tripe, version %s)' % VERSION)
 493   op.add_option('-c', '--cdb', metavar = 'CDB',
 494                 dest = 'cdbfile', default = None,
 495                 help = 'Compile output into a CDB file.')
 496   opts, args = op.parse_args(argv)
 497   return opts, args
 498
 499 ###--------------------------------------------------------------------------
 500 ### Main code.
 501
 502 def getconf(args):
 503   """
 504   Read the configuration files and return the accumulated result.
 505
 506   We make sure that all hostnames have been properly resolved.
 507   """
 508   conf = MyConfigParser()
 509   for f in inputiter(args):
 510     conf.parse(f)
 511   conf.resolve()
 512   return conf
 513
 514 def output(conf, cdb):
 515   """
 516   Output the configuration information CONF to the database CDB.
 517
 518   This is where the special `user' and `auto' database entries get set.
 519   """
 520   auto = []
 521   for sec in sorted(conf.sections(), key = lambda sec: sec.name):
 522     if sec.name.startswith('@'):
 523       continue
 524     elif sec.name.startswith('$'):
 525       label = sec.name
 526     else:
 527       label = 'P%s' % sec.name
 528       if sec.has_option('auto') and \
 529          sec.get('auto') in ('y', 'yes', 't', 'true', '1', 'on'):
 530         auto.append(sec.name)
 531       if sec.has_option('user'):
 532         cdb.add('U%s' % sec.get('user'))
 533     url = M.URLEncode(laxp = True, semip = True)
 534     for key, value in sorted(sec.items(), key = lambda (k, v): k):
 535       if not key.startswith('@'):
 536         url.encode(key, ' '.join(M.split(value)[0]))
 537     cdb.add(label, url.result)
 538   cdb.add('%AUTO', ' '.join(auto))
 539   cdb.finish()
 540
 541 def main():
 542   """Main program."""
 543   opts, args = parse_options()
 544   if opts.cdbfile:
 545     cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
 546   else:
 547     cdb = CDBFake()
 548   conf = getconf(args[1:])
 549   output(conf, cdb)
 550
 551 if __name__ == '__main__':
 552   main()
 553
 554 ###----- That's all, folks --------------------------------------------------