4 ### Build a CDB file from configuration file
6 ### (c) 2007 Straylight/Edgeware
9 ###----- Licensing notice ---------------------------------------------------
11 ### This file is part of Trivial IP Encryption (TrIPE).
13 ### TrIPE is free software: you can redistribute it and/or modify it under
14 ### the terms of the GNU General Public License as published by the Free
15 ### Software Foundation; either version 3 of the License, or (at your
16 ### option) any later version.
18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
20 ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
23 ### You should have received a copy of the GNU General Public License
24 ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
28 ###--------------------------------------------------------------------------
29 ### External dependencies.
32 from optparse import OptionParser
34 from sys import stdin, stdout, exit, argv
37 from cStringIO import StringIO
39 ###--------------------------------------------------------------------------
42 class CDBFake (object):
43 """Like cdbmake, but just outputs data suitable for cdb-map."""
44 def __init__(me, file = stdout):
46 def add(me, key, value):
47 me.file.write('%s:%s\n' % (key, value))
51 class ExpectedError (Exception): pass
53 ###--------------------------------------------------------------------------
54 ### A bulk DNS resolver.
56 class ResolverFailure (ExpectedError):
57 def __init__(me, host, msg):
61 return "failed to resolve `%s': %s" % (me.host, me.msg)
63 class ResolvingHost (object):
65 A host name which is being looked up by a bulk-resolver instance.
67 Most notably, this is where the flag-handling logic lives for the
68 $FLAGS[HOSTNAME] syntax.
71 def __init__(me, name):
72 """Make a new resolving-host object for the host NAME."""
74 me.addr = { 'INET': [] }
77 def addaddr(me, af, addr):
79 Add the address ADDR with address family AF.
81 The address family must currently be `INET'.
83 me.addr[af].append(addr)
87 Report that resolution of this host failed, with a human-readable MSG.
92 """Return a list of addresses according to the FLAGS string."""
93 if me.failure is not None: raise ResolverFailure(me.name, me.failure)
96 all, any = False, False
98 if ch == '*': all = True
99 elif ch == '4': aa += a4; any = True
100 else: raise ValueError("unknown address-resolution flag `%s'" % ch)
102 if not aa: raise ResolverFailure(me.name, 'no matching addresses found')
103 if not all: aa = [aa[0]]
106 class BulkResolver (object):
108 Resolve a number of DNS names in parallel.
110 The BulkResovler resolves a number of hostnames in parallel. Using it
111 works in three phases:
113 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
114 you're interested in.
116 2. You call run() to actually drive the resolver.
118 3. You call lookup(HOSTNAME) to get the address you wanted. This will
119 fail with KeyError if the resolver couldn't resolve the HOSTNAME.
123 """Initialize the resolver."""
127 def _prepare(me, host, name):
128 """Arrange to resolve a NAME, reporting the results to HOST."""
129 host._resolv = M.SelResolveByName(
131 lambda cname, alias, addr: me._resolved(host, cname, addr),
132 lambda: me._resolved(host, None, []))
135 def prepare(me, name):
136 """Prime the resolver to resolve the given host NAME."""
137 if name not in me._namemap:
138 me._namemap[name] = host = ResolvingHost(name)
139 me._prepare(host, name)
142 """Run the background DNS resolver until it's finished."""
143 while me._noutstand: M.select()
145 def lookup(me, name, flags):
146 """Fetch the address corresponding to the host NAME."""
147 return me._namemap[name].get(flags)
149 def _resolved(me, host, cname, addr):
150 """Callback function: remember that ADDRs are the addresses for HOST."""
152 host.failed('(unknown failure)')
154 if cname is not None: host.name = cname
155 for a in addr: host.addaddr('INET', a)
159 ###--------------------------------------------------------------------------
160 ### The configuration parser.
162 ## Match a comment or empty line.
163 RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])')
165 ## Match a section group header.
166 RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $')
168 ## Match an assignment line.
169 RX_ASSGN = RX.compile(r'''(?x) ^
170 ([^\s:=] (?: [^:=]* [^\s:=])?)
175 ## Match a continuation line.
176 RX_CONT = RX.compile(r'''(?x) ^ \s+
180 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
181 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
183 ## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags;
184 ## group 2 is the HOST.
185 RX_RESOLVE = RX.compile(r'(?x) \$ ([4*]*) \[ ([^]]+) \]')
187 class ConfigSyntaxError (ExpectedError):
188 def __init__(me, fname, lno, msg):
193 return '%s:%d: %s' % (me.fname, me.lno, me.msg)
196 return ' -> '.join(["`%s'" % hop for hop in path])
198 class AmbiguousOptionError (ExpectedError):
199 def __init__(me, key, patha, vala, pathb, valb):
201 me.patha, me.vala = patha, vala
202 me.pathb, me.valb = pathb, valb
204 return "Ambiguous answer resolving key `%s': " \
205 "path %s yields `%s' but %s yields `%s'" % \
206 (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
208 class InheritanceCycleError (ExpectedError):
209 def __init__(me, key, path):
213 return "Found a cycle %s looking up key `%s'" % \
214 (_fmt_path(me.path), me.key)
216 class MissingSectionException (ExpectedError):
217 def __init__(me, sec):
220 return "Section `%s' not found" % (me.sec)
222 class MissingKeyException (ExpectedError):
223 def __init__(me, sec, key):
227 return "Key `%s' not found in section `%s'" % (me.key, me.sec)
229 class ConfigSection (object):
231 A section in a configuration parser.
233 This is where a lot of the nitty-gritty stuff actually happens. The
234 `MyConfigParser' knows a lot about the internals of this class, which saves
235 on building a complicated interface.
238 def __init__(me, name, cp):
239 """Initialize a new, empty section with a given NAME and parent CP."""
241 ## The cache maps item keys to entries, which consist of a pair of
242 ## objects. There are four possible states for a cache entry:
244 ## * missing -- there is no entry at all with this key, so we must
247 ## * None, None -- we are actively trying to resolve this key, so if we
248 ## encounter this state, we have found a cycle in the inheritance
251 ## * None, [] -- we know that this key isn't reachable through any of
254 ## * VALUE, PATH -- we know that the key resolves to VALUE, along the
255 ## PATH from us (exclusive) to the defining parent (inclusive).
261 def _expand(me, string, resolvep):
263 Expands $(...) and (optionally) $FLAGS[...] placeholders in STRING.
265 RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
266 This is turned off by MyConfigParser's resolve() method while it's
267 collecting hostnames to be resolved.
269 string = RX_REF.sub(lambda m: me.get(m.group(1), resolvep), string)
271 string = RX_RESOLVE.sub(
272 lambda m: ' '.join(me._cp._resolver.lookup(m.group(2), m.group(1))),
277 """Yield this section's parents."""
278 try: names = me._itemmap['@inherit']
279 except KeyError: return
280 for name in names.replace(',', ' ').split():
281 yield me._cp.section(name)
283 def _get(me, key, path = None):
285 Low-level option-fetching method.
287 Fetch the value for the named KEY in this section, or maybe (recursively)
288 a section which it inherits from.
290 Returns a pair VALUE, PATH. The value is not expanded; nor do we check
291 for the special `name' key. The caller is expected to do these things.
292 Returns None if no value could be found.
295 ## If we weren't given a path, then we'd better make one.
296 if path is None: path = []
298 ## Extend the path to cover us, but remember to remove us again when
299 ## we've finished. If we need to pass the current path back upwards,
300 ## then remember to take a copy.
304 ## If we've been this way before on another pass through then return the
305 ## value we found then. If we're still thinking about it then we've
307 try: v, p = me._cache[key]
308 except KeyError: pass
310 if p is None: raise InheritanceCycleError(key, path[:])
311 else: return v, path + p
313 ## See whether the answer is ready waiting for us.
314 try: v = me._itemmap[key]
315 except KeyError: pass
318 me._cache[key] = v, []
321 ## Initially we have no idea.
325 ## Go through our parents and ask them what they think.
326 me._cache[key] = None, None
327 for p in me._parents():
329 ## See whether we get an answer. If not, keep on going.
330 v, pp = p._get(key, path)
331 if v is None: continue
333 ## If we got an answer, check that it matches any previous ones.
338 raise AmbiguousOptionError(key, winner, value, pp, v)
340 ## That's the best we could manage.
341 me._cache[key] = value, winner[len(path):]
345 ## Remove us from the path again.
348 def get(me, key, resolvep = True):
350 Retrieve the value of KEY from this section.
353 ## Special handling for the `name' key.
355 value = me._itemmap.get('name', me.name)
356 elif key == '@inherits':
357 try: return me._itemmap['@inherits']
358 except KeyError: raise MissingKeyException(me.name, key)
360 value, _ = me._get(key)
362 raise MissingKeyException(me.name, key)
364 ## Expand the value and return it.
365 return me._expand(value, resolvep)
367 def items(me, resolvep = True):
369 Yield a list of item names in the section.
372 ## Initialize for a depth-first walk of the inheritance graph.
373 seen = { 'name': True }
374 visiting = { me.name: True }
377 ## Visit nodes, collecting their keys. Don't believe the values:
378 ## resolving inheritance is too hard to do like this.
381 for p in sec._parents():
382 if p.name not in visiting:
383 stack.append(p); visiting[p.name] = True
385 for key in sec._itemmap.iterkeys(): seen[key] = None
388 return seen.iterkeys()
390 class MyConfigParser (object):
392 A more advanced configuration parser.
394 This has four major enhancements over the standard ConfigParser which are
397 * It recognizes `@inherits' keys and follows them when expanding a
400 * It recognizes `$(VAR)' references to configuration variables during
401 expansion and processes them correctly.
403 * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them
404 correctly. FLAGS consists of characters `4' (IPv4 addresses), and `*'
405 (all addresses, space-separated, rather than just the first).
407 * Its parsing behaviour is well-defined.
411 1. Call parse(FILENAME) to slurp in the configuration data.
413 2. Call resolve() to collect the hostnames which need to be resolved and
414 actually do the name resolution.
416 3. Call sections() to get a list of the configuration sections, or
417 section(NAME) to find a named section.
419 4. Call get(ITEM) on a section to collect the results, or items() to
425 Initialize a new, empty configuration parser.
428 me._resolver = BulkResolver()
432 Parse configuration from a file F.
435 ## Initial parser state.
441 ## An unpleasant hack. Python makes it hard to capture a value in a
442 ## variable and examine it in a single action, and this is the best that
445 def match(rx): m[0] = rx.match(line); return m[0]
447 ## Commit a key's value when we've determined that there are no further
448 ## continuation lines.
450 if key is not None: sect._itemmap[key] = val.getvalue()
452 ## Work through all of the input lines.
456 if match(RX_COMMENT):
457 ## A comment or a blank line. Nothing doing. (This means that we
458 ## leave out blank lines which look like they might be continuation
463 elif match(RX_GRPHDR):
464 ## A section header. Flush out any previous value and set up the new
469 try: sect = me._sectmap[name]
470 except KeyError: sect = me._sectmap[name] = ConfigSection(name, me)
473 elif match(RX_ASSGN):
474 ## A new assignment. Flush out the old one, and set up to store this
478 raise ConfigSyntaxError(f.name, lno, 'no active section to update')
481 val = StringIO(); val.write(m[0].group(2))
484 ## A continuation line. Accumulate the value.
487 raise ConfigSyntaxError(f.name, lno, 'no config value to continue')
488 val.write('\n'); val.write(m[0].group(1))
493 raise ConfigSyntaxError(f.name, lno, 'incomprehensible line')
495 ## Don't forget to commit any final value material.
498 def section(me, name):
499 """Return a ConfigSection with the given NAME."""
500 try: return me._sectmap[name]
501 except KeyError: raise MissingSectionException(name)
504 """Yield the known sections."""
505 return me._sectmap.itervalues()
509 Works out all of the hostnames which need resolving and resolves them.
511 Until you call this, attempts to fetch configuration items which need to
512 resolve hostnames will fail!
514 for sec in me.sections():
515 for key in sec.items():
516 value = sec.get(key, resolvep = False)
517 for match in RX_RESOLVE.finditer(value):
518 me._resolver.prepare(match.group(2))
521 ###--------------------------------------------------------------------------
522 ### Command-line handling.
524 def inputiter(things):
526 Iterate over command-line arguments, returning corresponding open files.
528 If none were given, or one is `-', assume standard input; if one is a
529 directory, scan it for files other than backups; otherwise return the
534 if OS.isatty(stdin.fileno()):
535 M.die('no input given, and stdin is a terminal')
541 elif OS.path.isdir(thing):
542 for item in OS.listdir(thing):
543 if item.endswith('~') or item.endswith('#'):
545 name = OS.path.join(thing, item)
546 if not OS.path.isfile(name):
552 def parse_options(argv = argv):
554 Parse command-line options, returning a pair (OPTS, ARGS).
557 op = OptionParser(usage = '%prog [-c CDB] INPUT...',
558 version = '%%prog (tripe, version %s)' % VERSION)
559 op.add_option('-c', '--cdb', metavar = 'CDB',
560 dest = 'cdbfile', default = None,
561 help = 'Compile output into a CDB file.')
562 opts, args = op.parse_args(argv)
565 ###--------------------------------------------------------------------------
570 Read the configuration files and return the accumulated result.
572 We make sure that all hostnames have been properly resolved.
574 conf = MyConfigParser()
575 for f in inputiter(args):
580 def output(conf, cdb):
582 Output the configuration information CONF to the database CDB.
584 This is where the special `user' and `auto' database entries get set.
587 for sec in sorted(conf.sections(), key = lambda sec: sec.name):
588 if sec.name.startswith('@'):
590 elif sec.name.startswith('$'):
593 label = 'P%s' % sec.name
594 try: a = sec.get('auto')
595 except MissingKeyException: pass
597 if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name)
598 try: u = sec.get('user')
599 except MissingKeyException: pass
600 else: cdb.add('U%s' % u)
601 url = M.URLEncode(semip = True)
602 for key in sorted(sec.items()):
603 if not key.startswith('@'):
604 url.encode(key, sec.get(key))
605 cdb.add(label, url.result)
606 cdb.add('%AUTO', ' '.join(auto))
611 opts, args = parse_options()
613 cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
617 conf = getconf(args[1:])
619 except ExpectedError, e:
623 if __name__ == '__main__':
626 ###----- That's all, folks --------------------------------------------------