mdw@git.distorted.org.uk Git - tripe/blob - peerdb/tripe-newpeers.in

   1 #! @PYTHON@
   2 ### -*-python-*-
   3 ###
   4 ### Build a CDB file from configuration file
   5 ###
   6 ### (c) 2007 Straylight/Edgeware
   7 ###
   8
   9 ###----- Licensing notice ---------------------------------------------------
  10 ###
  11 ### This file is part of Trivial IP Encryption (TrIPE).
  12 ###
  13 ### TrIPE is free software: you can redistribute it and/or modify it under
  14 ### the terms of the GNU General Public License as published by the Free
  15 ### Software Foundation; either version 3 of the License, or (at your
  16 ### option) any later version.
  17 ###
  18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
  19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  20 ### FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  21 ### for more details.
  22 ###
  23 ### You should have received a copy of the GNU General Public License
  24 ### along with TrIPE.  If not, see <https://www.gnu.org/licenses/>.
  25
  26 VERSION = '@VERSION@'
  27
  28 ###--------------------------------------------------------------------------
  29 ### External dependencies.
  30
  31 import ConfigParser as CP
  32 import mLib as M
  33 from optparse import OptionParser
  34 import cdb as CDB
  35 from sys import stdin, stdout, exit, argv
  36 import re as RX
  37 import os as OS
  38
  39 ###--------------------------------------------------------------------------
  40 ### Utilities.
  41
  42 class CDBFake (object):
  43   """Like cdbmake, but just outputs data suitable for cdb-map."""
  44   def __init__(me, file = stdout):
  45     me.file = file
  46   def add(me, key, value):
  47     me.file.write('%s:%s\n' % (key, value))
  48   def finish(me):
  49     pass
  50
  51 ###--------------------------------------------------------------------------
  52 ### A bulk DNS resolver.
  53
  54 class BulkResolver (object):
  55   """
  56   Resolve a number of DNS names in parallel.
  57
  58   The BulkResovler resolves a number of hostnames in parallel.  Using it
  59   works in three phases:
  60
  61     1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
  62        you're interested in.
  63
  64     2. You call run() to actually drive the resolver.
  65
  66     3. You call lookup(HOSTNAME) to get the address you wanted.  This will
  67        fail with KeyError if the resolver couldn't resolve the HOSTNAME.
  68   """
  69
  70   def __init__(me):
  71     """Initialize the resolver."""
  72     me._resolvers = {}
  73     me._namemap = {}
  74
  75   def prepare(me, host):
  76     """Prime the resolver to resolve the name HOST."""
  77     if host not in me._resolvers:
  78       me._resolvers[host] = M.SelResolveByName \
  79                             (host,
  80                              lambda name, alias, addr:
  81                                me._resolved(host, addr[0]),
  82                              lambda: me._resolved(host, None))
  83
  84   def run(me):
  85     """Run the background DNS resolver until it's finished."""
  86     while me._resolvers:
  87       M.select()
  88
  89   def lookup(me, host):
  90     """
  91     Fetch the address corresponding to HOST.
  92     """
  93     addr = me._namemap[host]
  94     if addr is None:
  95       raise KeyError(host)
  96     return addr
  97
  98   def _resolved(me, host, addr):
  99     """Callback function: remember that ADDR is the address for HOST."""
 100     me._namemap[host] = addr
 101     del me._resolvers[host]
 102
 103 ###--------------------------------------------------------------------------
 104 ### The configuration parser.
 105
 106 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
 107 r_ref = RX.compile(r'\$\(([^)]+)\)')
 108
 109 ## Match a $[HOST] name resolution reference; group 1 is the HOST.
 110 r_resolve = RX.compile(r'\$\[([^]]+)\]')
 111
 112 def _fmt_path(path):
 113   return ' -> '.join(["`%s'" % hop for hop in path])
 114
 115 class AmbiguousOptionError (Exception):
 116   def __init__(me, key, patha, vala, pathb, valb):
 117     me.key = key
 118     me.patha, me.vala = patha, vala
 119     me.pathb, me.valb = pathb, valb
 120   def __str__(me):
 121     return "Ambiguous answer resolving key `%s': " \
 122         "path %s yields `%s' but %s yields `%s'" % \
 123         (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
 124
 125 class InheritanceCycleError (Exception):
 126   def __init__(me, key, path):
 127     me.key = key
 128     me.path = path
 129   def __str__(me):
 130     return "Found a cycle %s looking up key `%s'" % \
 131         (_fmt_path(me.path), me.key)
 132
 133 class MissingKeyException (Exception):
 134   def __init__(me, sec, key):
 135     me.sec = sec
 136     me.key = key
 137   def __str__(me):
 138     return "Key `%s' not found in section `%s'" % (me.key, me.sec)
 139
 140 class MyConfigParser (CP.RawConfigParser):
 141   """
 142   A more advanced configuration parser.
 143
 144   This has three major enhancements over the standard ConfigParser which are
 145   relevant to us.
 146
 147     * It recognizes `@inherits' keys and follows them when expanding a
 148       value.
 149
 150     * It recognizes `$(VAR)' references to configuration variables during
 151       expansion and processes them correctly.
 152
 153     * It recognizes `$[HOST]' name-resolver requests and handles them
 154       correctly.
 155
 156   Use:
 157
 158     1. Call read(FILENAME) and/or read(FP, [FILENAME]) to slurp in the
 159        configuration data.
 160
 161     2. Call resolve() to collect the hostnames which need to be resolved and
 162        actually do the name resolution.
 163
 164     3. Call get(SECTION, ITEM) to collect the results, or items(SECTION) to
 165        iterate over them.
 166   """
 167
 168   def __init__(me):
 169     """
 170     Initialize a new, empty configuration parser.
 171     """
 172     CP.RawConfigParser.__init__(me)
 173     me._resolver = BulkResolver()
 174
 175   def resolve(me):
 176     """
 177     Works out all of the hostnames which need resolving and resolves them.
 178
 179     Until you call this, attempts to fetch configuration items which need to
 180     resolve hostnames will fail!
 181     """
 182     for sec in me.sections():
 183       for key, value in me.items(sec, resolvep = False):
 184         for match in r_resolve.finditer(value):
 185           me._resolver.prepare(match.group(1))
 186     me._resolver.run()
 187
 188   def _expand(me, sec, string, resolvep):
 189     """
 190     Expands $(...) and (optionally) $[...] placeholders in STRING.
 191
 192     The SEC is the configuration section from which to satisfy $(...)
 193     requests.  RESOLVEP is a boolean switch: do we bother to tax the resolver
 194     or not?  This is turned off by the resolve() method while it's collecting
 195     hostnames to be resolved.
 196     """
 197     string = r_ref.sub \
 198              (lambda m: me.get(sec, m.group(1), resolvep), string)
 199     if resolvep:
 200       string = r_resolve.sub(lambda m: me._resolver.lookup(m.group(1)),
 201                              string)
 202     return string
 203
 204   def has_option(me, sec, key):
 205     """
 206     Decide whether section SEC has a configuration key KEY.
 207
 208     This version of the method properly handles the @inherit key.
 209     """
 210     return key == 'name' or me._get(sec, key)[0] is not None
 211
 212   def _get(me, sec, key, map = None, path = None):
 213     """
 214     Low-level option-fetching method.
 215
 216     Fetch the value for the named KEY from section SEC, or maybe
 217     (recursively) a section which SEC inherits from.
 218
 219     Returns a pair VALUE, PATH.  The value is not expanded; nor do we check
 220     for the special `name' key.  The caller is expected to do these things.
 221     Returns None if no value could be found.
 222     """
 223
 224     ## If we weren't given a memoization map or path, then we'd better make
 225     ## one.
 226     if map is None: map = {}
 227     if path is None: path = []
 228
 229     ## If we've been this way before on another pass through then return the
 230     ## value we found then.  If we're still thinking about it then we've
 231     ## found a cycle.
 232     path.append(sec)
 233     try:
 234       threadp, value = map[sec]
 235     except KeyError:
 236       pass
 237     else:
 238       if threadp:
 239         raise InheritanceCycleError(key, path)
 240
 241     ## See whether the answer is ready waiting for us.
 242     try:
 243       v = CP.RawConfigParser.get(me, sec, key)
 244     except CP.NoOptionError:
 245       pass
 246     else:
 247       p = path[:]
 248       path.pop()
 249       return v, p
 250
 251     ## No, apparently, not.  Find out our list of parents.
 252     try:
 253       parents = CP.RawConfigParser.get(me, sec, '@inherit').\
 254           replace(',', ' ').split()
 255     except CP.NoOptionError:
 256       parents = []
 257
 258     ## Initially we have no idea.
 259     value = None
 260     winner = None
 261
 262     ## Go through our parents and ask them what they think.
 263     map[sec] = True, None
 264     for p in parents:
 265
 266       ## See whether we get an answer.  If not, keep on going.
 267       v, pp = me._get(p, key, map, path)
 268       if v is None: continue
 269
 270       ## If we got an answer, check that it matches any previous ones.
 271       if value is None:
 272         value = v
 273         winner = pp
 274       elif value != v:
 275         raise AmbiguousOptionError(key, winner, value, pp, v)
 276
 277     ## That's the best we could manage.
 278     path.pop()
 279     map[sec] = False, value
 280     return value, winner
 281
 282   def get(me, sec, key, resolvep = True):
 283     """
 284     Retrieve the value of KEY from section SEC.
 285     """
 286
 287     ## Special handling for the `name' key.
 288     if key == 'name':
 289       try: value = CP.RawConfigParser.get(me, sec, key)
 290       except CP.NoOptionError: value = sec
 291     else:
 292       value, _ = me._get(sec, key)
 293       if value is None:
 294         raise MissingKeyException(sec, key)
 295
 296     ## Expand the value and return it.
 297     return me._expand(sec, value, resolvep)
 298
 299   def items(me, sec, resolvep = True):
 300     """
 301     Return a list of (NAME, VALUE) items in section SEC.
 302
 303     This extends the default method by handling the inheritance chain.
 304     """
 305
 306     ## Initialize for a depth-first walk of the inheritance graph.
 307     d = {}
 308     visited = {}
 309     basesec = sec
 310     stack = [sec]
 311
 312     ## Visit nodes, collecting their keys.  Don't believe the values:
 313     ## resolving inheritance is too hard to do like this.
 314     while stack:
 315       sec = stack.pop()
 316       if sec in visited: continue
 317       visited[sec] = True
 318
 319       for key, value in CP.RawConfigParser.items(me, sec):
 320         if key == '@inherit': stack += value.replace(',', ' ').split()
 321         else: d[key] = None
 322
 323     ## Now collect the values for the known keys, one by one.
 324     items = []
 325     for key in d: items.append((key, me.get(basesec, key, resolvep)))
 326
 327     ## And we're done.
 328     return items
 329
 330 ###--------------------------------------------------------------------------
 331 ### Command-line handling.
 332
 333 def inputiter(things):
 334   """
 335   Iterate over command-line arguments, returning corresponding open files.
 336
 337   If none were given, or one is `-', assume standard input; if one is a
 338   directory, scan it for files other than backups; otherwise return the
 339   opened files.
 340   """
 341
 342   if not things:
 343     if OS.isatty(stdin.fileno()):
 344       M.die('no input given, and stdin is a terminal')
 345     yield stdin
 346   else:
 347     for thing in things:
 348       if thing == '-':
 349         yield stdin
 350       elif OS.path.isdir(thing):
 351         for item in OS.listdir(thing):
 352           if item.endswith('~') or item.endswith('#'):
 353             continue
 354           name = OS.path.join(thing, item)
 355           if not OS.path.isfile(name):
 356             continue
 357           yield file(name)
 358       else:
 359         yield file(thing)
 360
 361 def parse_options(argv = argv):
 362   """
 363   Parse command-line options, returning a pair (OPTS, ARGS).
 364   """
 365   M.ego(argv[0])
 366   op = OptionParser(usage = '%prog [-c CDB] INPUT...',
 367                     version = '%%prog (tripe, version %s)' % VERSION)
 368   op.add_option('-c', '--cdb', metavar = 'CDB',
 369                 dest = 'cdbfile', default = None,
 370                 help = 'Compile output into a CDB file.')
 371   opts, args = op.parse_args(argv)
 372   return opts, args
 373
 374 ###--------------------------------------------------------------------------
 375 ### Main code.
 376
 377 def getconf(args):
 378   """
 379   Read the configuration files and return the accumulated result.
 380
 381   We make sure that all hostnames have been properly resolved.
 382   """
 383   conf = MyConfigParser()
 384   for f in inputiter(args):
 385     conf.readfp(f)
 386   conf.resolve()
 387   return conf
 388
 389 def output(conf, cdb):
 390   """
 391   Output the configuration information CONF to the database CDB.
 392
 393   This is where the special `user' and `auto' database entries get set.
 394   """
 395   auto = []
 396   for sec in sorted(conf.sections()):
 397     if sec.startswith('@'):
 398       continue
 399     elif sec.startswith('$'):
 400       label = sec
 401     else:
 402       label = 'P%s' % sec
 403       if conf.has_option(sec, 'auto') and \
 404          conf.get(sec, 'auto') in ('y', 'yes', 't', 'true', '1', 'on'):
 405         auto.append(sec)
 406       if conf.has_option(sec, 'user'):
 407         cdb.add('U%s' % conf.get(sec, 'user'), sec)
 408     url = M.URLEncode(laxp = True, semip = True)
 409     for key, value in sorted(conf.items(sec), key = lambda (k, v): k):
 410       if not key.startswith('@'):
 411         url.encode(key, ' '.join(M.split(value)[0]))
 412     cdb.add(label, url.result)
 413   cdb.add('%AUTO', ' '.join(auto))
 414   cdb.finish()
 415
 416 def main():
 417   """Main program."""
 418   opts, args = parse_options()
 419   if opts.cdbfile:
 420     cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
 421   else:
 422     cdb = CDBFake()
 423   conf = getconf(args[1:])
 424   output(conf, cdb)
 425
 426 if __name__ == '__main__':
 427   main()
 428
 429 ###----- That's all, folks --------------------------------------------------