X-Git-Url: https://git.distorted.org.uk/~mdw/tripe/blobdiff_plain/4251f8ad09fe48e87afd36a172649250c48acd94..3c8803fa37cd6a0fcad90c467227c9fff9bca85e:/peerdb/tripe-newpeers.in diff --git a/peerdb/tripe-newpeers.in b/peerdb/tripe-newpeers.in index 8b4d800c..8297d569 100644 --- a/peerdb/tripe-newpeers.in +++ b/peerdb/tripe-newpeers.in @@ -48,9 +48,54 @@ class CDBFake (object): def finish(me): pass +class ExpectedError (Exception): pass + ###-------------------------------------------------------------------------- ### A bulk DNS resolver. +class ResolverFailure (ExpectedError): + def __init__(me, host, msg): + me.host = host + me.msg = msg + def __str__(me): + return "failed to resolve `%s': %s" % (me.host, me.msg) + +class ResolvingHost (object): + """ + A host name which is being looked up by a bulk-resolver instance. + + Most notably, this is where the flag-handling logic lives for the + $FLAGS[HOSTNAME] syntax. + """ + + def __init__(me, name): + """Make a new resolving-host object for the host NAME.""" + me.name = name + me.addr = [] + me.failure = None + + def addaddr(me, addr): + """Add the address ADDR.""" + me.addr.append(addr) + + def failed(me, msg): + """ + Report that resolution of this host failed, with a human-readable MSG. + """ + me.failure = msg + + def get(me, flags): + """Return a list of addresses according to the FLAGS string.""" + if me.failure is not None: raise ResolverFailure(me.name, me.failure) + aa = me.addr + all = False + for ch in flags: + if ch == '*': all = True + else: raise ValueError("unknown address-resolution flag `%s'" % ch) + if not aa: raise ResolverFailure(me.name, 'no matching addresses found') + if not all: aa = [aa[0]] + return aa + class BulkResolver (object): """ Resolve a number of DNS names in parallel. @@ -69,36 +114,36 @@ class BulkResolver (object): def __init__(me): """Initialize the resolver.""" - me._resolvers = {} me._namemap = {} - - def prepare(me, host): - """Prime the resolver to resolve the name HOST.""" - if host not in me._resolvers: - me._resolvers[host] = M.SelResolveByName \ - (host, - lambda name, alias, addr: - me._resolved(host, addr[0]), - lambda: me._resolved(host, None)) + me._noutstand = 0 + + def prepare(me, name): + """Prime the resolver to resolve the given host NAME.""" + if name not in me._namemap: + me._namemap[name] = host = ResolvingHost(name) + host._resolv = M.SelResolveByName( + name, + lambda cname, alias, addr: me._resolved(host, cname, addr), + lambda: me._resolved(host, None, [])) + me._noutstand += 1 def run(me): """Run the background DNS resolver until it's finished.""" - while me._resolvers: - M.select() + while me._noutstand: M.select() - def lookup(me, host): - """ - Fetch the address corresponding to HOST. - """ - addr = me._namemap[host] - if addr is None: - raise KeyError(host) - return addr + def lookup(me, name, flags): + """Fetch the address corresponding to the host NAME.""" + return me._namemap[name].get(flags) - def _resolved(me, host, addr): - """Callback function: remember that ADDR is the address for HOST.""" - me._namemap[host] = addr - del me._resolvers[host] + def _resolved(me, host, cname, addr): + """Callback function: remember that ADDRs are the addresses for HOST.""" + if not addr: + host.failed('(unknown failure)') + else: + if cname is not None: host.name = cname + for a in addr: host.addaddr(a) + host._resolv = None + me._noutstand -= 1 ###-------------------------------------------------------------------------- ### The configuration parser. @@ -124,10 +169,11 @@ RX_CONT = RX.compile(r'''(?x) ^ \s+ ## Match a $(VAR) configuration variable reference; group 1 is the VAR. RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)') -## Match a $[HOST] name resolution reference; group 1 is the HOST. -RX_RESOLVE = RX.compile(r'(?x) \$ \[ ([^]]+) \]') +## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags; +## group 2 is the HOST. +RX_RESOLVE = RX.compile(r'(?x) \$ ([*]*) \[ ([^]]+) \]') -class ConfigSyntaxError (Exception): +class ConfigSyntaxError (ExpectedError): def __init__(me, fname, lno, msg): me.fname = fname me.lno = lno @@ -138,7 +184,7 @@ class ConfigSyntaxError (Exception): def _fmt_path(path): return ' -> '.join(["`%s'" % hop for hop in path]) -class AmbiguousOptionError (Exception): +class AmbiguousOptionError (ExpectedError): def __init__(me, key, patha, vala, pathb, valb): me.key = key me.patha, me.vala = patha, vala @@ -148,7 +194,7 @@ class AmbiguousOptionError (Exception): "path %s yields `%s' but %s yields `%s'" % \ (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb) -class InheritanceCycleError (Exception): +class InheritanceCycleError (ExpectedError): def __init__(me, key, path): me.key = key me.path = path @@ -156,13 +202,13 @@ class InheritanceCycleError (Exception): return "Found a cycle %s looking up key `%s'" % \ (_fmt_path(me.path), me.key) -class MissingSectionException (Exception): +class MissingSectionException (ExpectedError): def __init__(me, sec): - me.key = key + me.sec = sec def __str__(me): return "Section `%s' not found" % (me.sec) -class MissingKeyException (Exception): +class MissingKeyException (ExpectedError): def __init__(me, sec, key): me.sec = sec me.key = key @@ -180,23 +226,40 @@ class ConfigSection (object): def __init__(me, name, cp): """Initialize a new, empty section with a given NAME and parent CP.""" + + ## The cache maps item keys to entries, which consist of a pair of + ## objects. There are four possible states for a cache entry: + ## + ## * missing -- there is no entry at all with this key, so we must + ## search for it; + ## + ## * None, None -- we are actively trying to resolve this key, so if we + ## encounter this state, we have found a cycle in the inheritance + ## graph; + ## + ## * None, [] -- we know that this key isn't reachable through any of + ## our parents; + ## + ## * VALUE, PATH -- we know that the key resolves to VALUE, along the + ## PATH from us (exclusive) to the defining parent (inclusive). me.name = name me._itemmap = dict() + me._cache = dict() me._cp = cp def _expand(me, string, resolvep): """ - Expands $(...) and (optionally) $[...] placeholders in STRING. + Expands $(...) and (optionally) $FLAGS[...] placeholders in STRING. RESOLVEP is a boolean switch: do we bother to tax the resolver or not? This is turned off by MyConfigParser's resolve() method while it's collecting hostnames to be resolved. """ - string = RX_REF.sub \ - (lambda m: me.get(m.group(1), resolvep), string) + string = RX_REF.sub(lambda m: me.get(m.group(1), resolvep), string) if resolvep: - string = RX_RESOLVE.sub(lambda m: me._cp._resolver.lookup(m.group(1)), - string) + string = RX_RESOLVE.sub( + lambda m: ' '.join(me._cp._resolver.lookup(m.group(2), m.group(1))), + string) return string def _parents(me): @@ -206,7 +269,7 @@ class ConfigSection (object): for name in names.replace(',', ' ').split(): yield me._cp.section(name) - def _get(me, key, map = None, path = None): + def _get(me, key, path = None): """ Low-level option-fetching method. @@ -218,9 +281,7 @@ class ConfigSection (object): Returns None if no value could be found. """ - ## If we weren't given a memoization map or path, then we'd better make - ## one. - if map is None: map = {} + ## If we weren't given a path, then we'd better make one. if path is None: path = [] ## Extend the path to cover us, but remember to remove us again when @@ -229,29 +290,33 @@ class ConfigSection (object): path.append(me.name) try: - ## If we've been this way before on another pass through then return - ## the value we found then. If we're still thinking about it then - ## we've found a cycle. - try: threadp, value = map[me.name] + ## If we've been this way before on another pass through then return the + ## value we found then. If we're still thinking about it then we've + ## found a cycle. + try: v, p = me._cache[key] except KeyError: pass else: - if threadp: raise InheritanceCycleError(key, path[:]) + if p is None: raise InheritanceCycleError(key, path[:]) + else: return v, path + p ## See whether the answer is ready waiting for us. try: v = me._itemmap[key] except KeyError: pass - else: return v, path[:] + else: + p = path[:] + me._cache[key] = v, [] + return v, p ## Initially we have no idea. value = None - winner = None + winner = [] ## Go through our parents and ask them what they think. - map[me.name] = True, None + me._cache[key] = None, None for p in me._parents(): ## See whether we get an answer. If not, keep on going. - v, pp = p._get(key, map, path) + v, pp = p._get(key, path) if v is None: continue ## If we got an answer, check that it matches any previous ones. @@ -262,7 +327,7 @@ class ConfigSection (object): raise AmbiguousOptionError(key, winner, value, pp, v) ## That's the best we could manage. - map[me.name] = False, value + me._cache[key] = value, winner[len(path):] return value, winner finally: @@ -277,6 +342,9 @@ class ConfigSection (object): ## Special handling for the `name' key. if key == 'name': value = me._itemmap.get('name', me.name) + elif key == '@inherits': + try: return me._itemmap['@inherits'] + except KeyError: raise MissingKeyException(me.name, key) else: value, _ = me._get(key) if value is None: @@ -291,23 +359,22 @@ class ConfigSection (object): """ ## Initialize for a depth-first walk of the inheritance graph. - d = {} - visited = {} + seen = { 'name': True } + visiting = { me.name: True } stack = [me] ## Visit nodes, collecting their keys. Don't believe the values: ## resolving inheritance is too hard to do like this. while stack: sec = stack.pop() - if sec.name in visited: continue - visited[sec.name] = True - stack += sec._parents() + for p in sec._parents(): + if p.name not in visiting: + stack.append(p); visiting[p.name] = True - for key in sec._itemmap.iterkeys(): - if key != '@inherit': d[key] = None + for key in sec._itemmap.iterkeys(): seen[key] = None ## And we're done. - return d.iterkeys() + return seen.iterkeys() class MyConfigParser (object): """ @@ -322,8 +389,9 @@ class MyConfigParser (object): * It recognizes `$(VAR)' references to configuration variables during expansion and processes them correctly. - * It recognizes `$[HOST]' name-resolver requests and handles them - correctly. + * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them + correctly. FLAGS may be empty, or `*' (all addresses, space-separated, + rather than just the first). * Its parsing behaviour is well-defined. @@ -436,7 +504,7 @@ class MyConfigParser (object): for key in sec.items(): value = sec.get(key, resolvep = False) for match in RX_RESOLVE.finditer(value): - me._resolver.prepare(match.group(1)) + me._resolver.prepare(match.group(2)) me._resolver.run() ###-------------------------------------------------------------------------- @@ -534,8 +602,12 @@ def main(): cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new') else: cdb = CDBFake() - conf = getconf(args[1:]) - output(conf, cdb) + try: + conf = getconf(args[1:]) + output(conf, cdb) + except ExpectedError, e: + M.moan(str(e)) + exit(2) if __name__ == '__main__': main()