4 ### Build a CDB file from configuration file
6 ### (c) 2007 Straylight/Edgeware
9 ###----- Licensing notice ---------------------------------------------------
11 ### This file is part of Trivial IP Encryption (TrIPE).
13 ### TrIPE is free software: you can redistribute it and/or modify it under
14 ### the terms of the GNU General Public License as published by the Free
15 ### Software Foundation; either version 3 of the License, or (at your
16 ### option) any later version.
18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
20 ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
23 ### You should have received a copy of the GNU General Public License
24 ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
28 ###--------------------------------------------------------------------------
29 ### External dependencies.
32 from optparse import OptionParser
34 from sys import stdin, stdout, exit, argv
37 from cStringIO import StringIO
39 ###--------------------------------------------------------------------------
42 class CDBFake (object):
43 """Like cdbmake, but just outputs data suitable for cdb-map."""
44 def __init__(me, file = stdout):
46 def add(me, key, value):
47 me.file.write('%s:%s\n' % (key, value))
51 class ExpectedError (Exception): pass
53 ###--------------------------------------------------------------------------
54 ### A bulk DNS resolver.
56 class ResolverFailure (ExpectedError):
57 def __init__(me, host, msg):
61 return "failed to resolve `%s': %s" % (me.host, me.msg)
63 class ResolvingHost (object):
65 A host name which is being looked up by a bulk-resolver instance.
67 Most notably, this is where the flag-handling logic lives for the
68 $FLAGS[HOSTNAME] syntax.
71 def __init__(me, name):
72 """Make a new resolving-host object for the host NAME."""
77 def addaddr(me, addr):
78 """Add the address ADDR."""
83 Report that resolution of this host failed, with a human-readable MSG.
88 """Return a list of addresses according to the FLAGS string."""
89 if me.failure is not None: raise ResolverFailure(me.name, me.failure)
93 if ch == '*': all = True
94 else: raise ValueError("unknown address-resolution flag `%s'" % ch)
95 if not aa: raise ResolverFailure(me.name, 'no matching addresses found')
96 if not all: aa = [aa[0]]
99 class BulkResolver (object):
101 Resolve a number of DNS names in parallel.
103 The BulkResovler resolves a number of hostnames in parallel. Using it
104 works in three phases:
106 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
107 you're interested in.
109 2. You call run() to actually drive the resolver.
111 3. You call lookup(HOSTNAME) to get the address you wanted. This will
112 fail with KeyError if the resolver couldn't resolve the HOSTNAME.
116 """Initialize the resolver."""
120 def prepare(me, name):
121 """Prime the resolver to resolve the given host NAME."""
122 if name not in me._namemap:
123 me._namemap[name] = host = ResolvingHost(name)
124 host._resolv = M.SelResolveByName(
126 lambda cname, alias, addr: me._resolved(host, cname, addr),
127 lambda: me._resolved(host, None, []))
131 """Run the background DNS resolver until it's finished."""
132 while me._noutstand: M.select()
134 def lookup(me, name, flags):
135 """Fetch the address corresponding to the host NAME."""
136 return me._namemap[name].get(flags)
138 def _resolved(me, host, cname, addr):
139 """Callback function: remember that ADDRs are the addresses for HOST."""
141 host.failed('(unknown failure)')
143 if cname is not None: host.name = cname
144 for a in addr: host.addaddr(a)
148 ###--------------------------------------------------------------------------
149 ### The configuration parser.
151 ## Match a comment or empty line.
152 RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])')
154 ## Match a section group header.
155 RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $')
157 ## Match an assignment line.
158 RX_ASSGN = RX.compile(r'''(?x) ^
159 ([^\s:=] (?: [^:=]* [^\s:=])?)
164 ## Match a continuation line.
165 RX_CONT = RX.compile(r'''(?x) ^ \s+
169 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
170 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
172 ## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags;
173 ## group 2 is the HOST.
174 RX_RESOLVE = RX.compile(r'(?x) \$ ([*]*) \[ ([^]]+) \]')
176 class ConfigSyntaxError (ExpectedError):
177 def __init__(me, fname, lno, msg):
182 return '%s:%d: %s' % (me.fname, me.lno, me.msg)
185 return ' -> '.join(["`%s'" % hop for hop in path])
187 class AmbiguousOptionError (ExpectedError):
188 def __init__(me, key, patha, vala, pathb, valb):
190 me.patha, me.vala = patha, vala
191 me.pathb, me.valb = pathb, valb
193 return "Ambiguous answer resolving key `%s': " \
194 "path %s yields `%s' but %s yields `%s'" % \
195 (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
197 class InheritanceCycleError (ExpectedError):
198 def __init__(me, key, path):
202 return "Found a cycle %s looking up key `%s'" % \
203 (_fmt_path(me.path), me.key)
205 class MissingSectionException (ExpectedError):
206 def __init__(me, sec):
209 return "Section `%s' not found" % (me.sec)
211 class MissingKeyException (ExpectedError):
212 def __init__(me, sec, key):
216 return "Key `%s' not found in section `%s'" % (me.key, me.sec)
218 class ConfigSection (object):
220 A section in a configuration parser.
222 This is where a lot of the nitty-gritty stuff actually happens. The
223 `MyConfigParser' knows a lot about the internals of this class, which saves
224 on building a complicated interface.
227 def __init__(me, name, cp):
228 """Initialize a new, empty section with a given NAME and parent CP."""
230 ## The cache maps item keys to entries, which consist of a pair of
231 ## objects. There are four possible states for a cache entry:
233 ## * missing -- there is no entry at all with this key, so we must
236 ## * None, None -- we are actively trying to resolve this key, so if we
237 ## encounter this state, we have found a cycle in the inheritance
240 ## * None, [] -- we know that this key isn't reachable through any of
243 ## * VALUE, PATH -- we know that the key resolves to VALUE, along the
244 ## PATH from us (exclusive) to the defining parent (inclusive).
250 def _expand(me, string, resolvep):
252 Expands $(...) and (optionally) $FLAGS[...] placeholders in STRING.
254 RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
255 This is turned off by MyConfigParser's resolve() method while it's
256 collecting hostnames to be resolved.
258 string = RX_REF.sub(lambda m: me.get(m.group(1), resolvep), string)
260 string = RX_RESOLVE.sub(
261 lambda m: ' '.join(me._cp._resolver.lookup(m.group(2), m.group(1))),
266 """Yield this section's parents."""
267 try: names = me._itemmap['@inherit']
268 except KeyError: return
269 for name in names.replace(',', ' ').split():
270 yield me._cp.section(name)
272 def _get(me, key, path = None):
274 Low-level option-fetching method.
276 Fetch the value for the named KEY in this section, or maybe (recursively)
277 a section which it inherits from.
279 Returns a pair VALUE, PATH. The value is not expanded; nor do we check
280 for the special `name' key. The caller is expected to do these things.
281 Returns None if no value could be found.
284 ## If we weren't given a path, then we'd better make one.
285 if path is None: path = []
287 ## Extend the path to cover us, but remember to remove us again when
288 ## we've finished. If we need to pass the current path back upwards,
289 ## then remember to take a copy.
293 ## If we've been this way before on another pass through then return the
294 ## value we found then. If we're still thinking about it then we've
296 try: v, p = me._cache[key]
297 except KeyError: pass
299 if p is None: raise InheritanceCycleError(key, path[:])
300 else: return v, path + p
302 ## See whether the answer is ready waiting for us.
303 try: v = me._itemmap[key]
304 except KeyError: pass
307 me._cache[key] = v, []
310 ## Initially we have no idea.
314 ## Go through our parents and ask them what they think.
315 me._cache[key] = None, None
316 for p in me._parents():
318 ## See whether we get an answer. If not, keep on going.
319 v, pp = p._get(key, path)
320 if v is None: continue
322 ## If we got an answer, check that it matches any previous ones.
327 raise AmbiguousOptionError(key, winner, value, pp, v)
329 ## That's the best we could manage.
330 me._cache[key] = value, winner[len(path):]
334 ## Remove us from the path again.
337 def get(me, key, resolvep = True):
339 Retrieve the value of KEY from this section.
342 ## Special handling for the `name' key.
344 value = me._itemmap.get('name', me.name)
345 elif key == '@inherits':
346 try: return me._itemmap['@inherits']
347 except KeyError: raise MissingKeyException(me.name, key)
349 value, _ = me._get(key)
351 raise MissingKeyException(me.name, key)
353 ## Expand the value and return it.
354 return me._expand(value, resolvep)
356 def items(me, resolvep = True):
358 Yield a list of item names in the section.
361 ## Initialize for a depth-first walk of the inheritance graph.
362 seen = { 'name': True }
363 visiting = { me.name: True }
366 ## Visit nodes, collecting their keys. Don't believe the values:
367 ## resolving inheritance is too hard to do like this.
370 for p in sec._parents():
371 if p.name not in visiting:
372 stack.append(p); visiting[p.name] = True
374 for key in sec._itemmap.iterkeys(): seen[key] = None
377 return seen.iterkeys()
379 class MyConfigParser (object):
381 A more advanced configuration parser.
383 This has four major enhancements over the standard ConfigParser which are
386 * It recognizes `@inherits' keys and follows them when expanding a
389 * It recognizes `$(VAR)' references to configuration variables during
390 expansion and processes them correctly.
392 * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them
393 correctly. FLAGS may be empty, or `*' (all addresses, space-separated,
394 rather than just the first).
396 * Its parsing behaviour is well-defined.
400 1. Call parse(FILENAME) to slurp in the configuration data.
402 2. Call resolve() to collect the hostnames which need to be resolved and
403 actually do the name resolution.
405 3. Call sections() to get a list of the configuration sections, or
406 section(NAME) to find a named section.
408 4. Call get(ITEM) on a section to collect the results, or items() to
414 Initialize a new, empty configuration parser.
417 me._resolver = BulkResolver()
421 Parse configuration from a file F.
424 ## Initial parser state.
430 ## An unpleasant hack. Python makes it hard to capture a value in a
431 ## variable and examine it in a single action, and this is the best that
434 def match(rx): m[0] = rx.match(line); return m[0]
436 ## Commit a key's value when we've determined that there are no further
437 ## continuation lines.
439 if key is not None: sect._itemmap[key] = val.getvalue()
441 ## Work through all of the input lines.
445 if match(RX_COMMENT):
446 ## A comment or a blank line. Nothing doing. (This means that we
447 ## leave out blank lines which look like they might be continuation
452 elif match(RX_GRPHDR):
453 ## A section header. Flush out any previous value and set up the new
458 try: sect = me._sectmap[name]
459 except KeyError: sect = me._sectmap[name] = ConfigSection(name, me)
462 elif match(RX_ASSGN):
463 ## A new assignment. Flush out the old one, and set up to store this
467 raise ConfigSyntaxError(f.name, lno, 'no active section to update')
470 val = StringIO(); val.write(m[0].group(2))
473 ## A continuation line. Accumulate the value.
476 raise ConfigSyntaxError(f.name, lno, 'no config value to continue')
477 val.write('\n'); val.write(m[0].group(1))
482 raise ConfigSyntaxError(f.name, lno, 'incomprehensible line')
484 ## Don't forget to commit any final value material.
487 def section(me, name):
488 """Return a ConfigSection with the given NAME."""
489 try: return me._sectmap[name]
490 except KeyError: raise MissingSectionException(name)
493 """Yield the known sections."""
494 return me._sectmap.itervalues()
498 Works out all of the hostnames which need resolving and resolves them.
500 Until you call this, attempts to fetch configuration items which need to
501 resolve hostnames will fail!
503 for sec in me.sections():
504 for key in sec.items():
505 value = sec.get(key, resolvep = False)
506 for match in RX_RESOLVE.finditer(value):
507 me._resolver.prepare(match.group(2))
510 ###--------------------------------------------------------------------------
511 ### Command-line handling.
513 def inputiter(things):
515 Iterate over command-line arguments, returning corresponding open files.
517 If none were given, or one is `-', assume standard input; if one is a
518 directory, scan it for files other than backups; otherwise return the
523 if OS.isatty(stdin.fileno()):
524 M.die('no input given, and stdin is a terminal')
530 elif OS.path.isdir(thing):
531 for item in OS.listdir(thing):
532 if item.endswith('~') or item.endswith('#'):
534 name = OS.path.join(thing, item)
535 if not OS.path.isfile(name):
541 def parse_options(argv = argv):
543 Parse command-line options, returning a pair (OPTS, ARGS).
546 op = OptionParser(usage = '%prog [-c CDB] INPUT...',
547 version = '%%prog (tripe, version %s)' % VERSION)
548 op.add_option('-c', '--cdb', metavar = 'CDB',
549 dest = 'cdbfile', default = None,
550 help = 'Compile output into a CDB file.')
551 opts, args = op.parse_args(argv)
554 ###--------------------------------------------------------------------------
559 Read the configuration files and return the accumulated result.
561 We make sure that all hostnames have been properly resolved.
563 conf = MyConfigParser()
564 for f in inputiter(args):
569 def output(conf, cdb):
571 Output the configuration information CONF to the database CDB.
573 This is where the special `user' and `auto' database entries get set.
576 for sec in sorted(conf.sections(), key = lambda sec: sec.name):
577 if sec.name.startswith('@'):
579 elif sec.name.startswith('$'):
582 label = 'P%s' % sec.name
583 try: a = sec.get('auto')
584 except MissingKeyException: pass
586 if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name)
587 try: u = sec.get('user')
588 except MissingKeyException: pass
589 else: cdb.add('U%s' % u)
590 url = M.URLEncode(semip = True)
591 for key in sorted(sec.items()):
592 if not key.startswith('@'):
593 url.encode(key, sec.get(key))
594 cdb.add(label, url.result)
595 cdb.add('%AUTO', ' '.join(auto))
600 opts, args = parse_options()
602 cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
606 conf = getconf(args[1:])
608 except ExpectedError, e:
612 if __name__ == '__main__':
615 ###----- That's all, folks --------------------------------------------------