X-Git-Url: https://git.distorted.org.uk/~mdw/tripe/blobdiff_plain/ef7d7afbc62c58537caabba75b36d9938eb49b94..5891679d126ad426ed4250aeec75901bc854bf41:/peerdb/tripe-newpeers.in diff --git a/peerdb/tripe-newpeers.in b/peerdb/tripe-newpeers.in index 1f17811e..39529531 100644 --- a/peerdb/tripe-newpeers.in +++ b/peerdb/tripe-newpeers.in @@ -32,8 +32,12 @@ import mLib as M from optparse import OptionParser import cdb as CDB from sys import stdin, stdout, exit, argv +import subprocess as SUB import re as RX import os as OS +import errno as E +import fcntl as F +import socket as S from cStringIO import StringIO ###-------------------------------------------------------------------------- @@ -71,12 +75,16 @@ class ResolvingHost (object): def __init__(me, name): """Make a new resolving-host object for the host NAME.""" me.name = name - me.addr = [] + me.addr = { 'INET': [], 'INET6': [] } me.failure = None - def addaddr(me, addr): - """Add the address ADDR.""" - me.addr.append(addr) + def addaddr(me, af, addr): + """ + Add the address ADDR with address family AF. + + The address family may be `INET' or `INET6'. + """ + me.addr[af].append(addr) def failed(me, msg): """ @@ -87,16 +95,21 @@ class ResolvingHost (object): def get(me, flags): """Return a list of addresses according to the FLAGS string.""" if me.failure is not None: raise ResolverFailure(me.name, me.failure) - aa = me.addr - all = False + aa = [] + a4 = me.addr['INET'] + a6 = me.addr['INET6'] + all, any = False, False for ch in flags: if ch == '*': all = True + elif ch == '4': aa += a4; any = True + elif ch == '6': aa += a6; any = True else: raise ValueError("unknown address-resolution flag `%s'" % ch) + if not any: aa = a4 + a6 if not aa: raise ResolverFailure(me.name, 'no matching addresses found') if not all: aa = [aa[0]] return aa -class BulkResolver (object): +class BaseBulkResolver (object): """ Resolve a number of DNS names in parallel. @@ -115,35 +128,193 @@ class BulkResolver (object): def __init__(me): """Initialize the resolver.""" me._namemap = {} - me._noutstand = 0 def prepare(me, name): """Prime the resolver to resolve the given host NAME.""" if name not in me._namemap: me._namemap[name] = host = ResolvingHost(name) - host._resolv = M.SelResolveByName( - name, - lambda cname, alias, addr: me._resolved(host, addr), - lambda: me._resolved(host, [])) - me._noutstand += 1 - - def run(me): - """Run the background DNS resolver until it's finished.""" - while me._noutstand: M.select() + try: + ailist = S.getaddrinfo(name, None, S.AF_UNSPEC, S.SOCK_DGRAM, 0, + S.AI_NUMERICHOST | S.AI_NUMERICSERV) + except S.gaierror: + me._prepare(host, name) + else: + for af, skty, proto, cname, sa in ailist: + if af == S.AF_INET: host.addaddr('INET', sa[0]) + elif af == S.AF_INET6: host.addaddr('INET6', sa[0]) def lookup(me, name, flags): """Fetch the address corresponding to the host NAME.""" return me._namemap[name].get(flags) - def _resolved(me, host, addr): +class BresBulkResolver (BaseBulkResolver): + """ + A BulkResolver using mLib's `bres' background resolver. + + This is always available (and might use ADNS), but only does IPv4. + """ + + def __init__(me): + super(BresBulkResolver, me).__init__() + """Initialize the resolver.""" + me._noutstand = 0 + + def _prepare(me, host, name): + """Arrange to resolve a NAME, reporting the results to HOST.""" + host._resolv = M.SelResolveByName( + name, + lambda cname, alias, addr: me._resolved(host, cname, addr), + lambda: me._resolved(host, None, [])) + me._noutstand += 1 + + def run(me): + """Run the background DNS resolver until it's finished.""" + while me._noutstand: M.select() + + def _resolved(me, host, cname, addr): """Callback function: remember that ADDRs are the addresses for HOST.""" if not addr: host.failed('(unknown failure)') else: - for a in addr: host.addaddr(a) + if cname is not None: host.name = cname + for a in addr: host.addaddr('INET', a) host._resolv = None me._noutstand -= 1 +class AdnsBulkResolver (BaseBulkResolver): + """ + A BulkResolver using ADNS, via the `adnshost' command-line tool. + + This can do simultaneous IPv4 and IPv6 lookups and is quite shiny. + """ + + def __init__(me): + """Initialize the resolver.""" + + super(AdnsBulkResolver, me).__init__() + + ## Start the external resolver process. + me._kid = SUB.Popen(['adnshost', '-afs'], + stdin = SUB.PIPE, stdout = SUB.PIPE) + + ## Set up the machinery for feeding input to the resolver. + me._in = me._kid.stdin + M.fdflags(me._in, fbic = OS.O_NONBLOCK, fxor = OS.O_NONBLOCK) + me._insel = M.SelFile(me._in.fileno(), M.SEL_WRITE, me._write) + me._inbuf, me._inoff, me._inlen = '', 0, 0 + me._idmap = {} + me._nextid = 0 + + ## Set up the machinery for collecting the resolver's output. + me._out = me._kid.stdout + M.fdflags(me._out, fbic = OS.O_NONBLOCK, fxor = OS.O_NONBLOCK) + me._outline = M.SelLineBuffer(me._out, + lineproc = me._hostline, eofproc = me._eof) + me._outline.enable() + + ## It's not finished yet. + me._done = False + + def _prepare(me, host, name): + """Arrange for the resolver to resolve the name NAME.""" + + ## Work out the next job id, and associate that with the host record. + host.id = me._nextid; me._nextid += 1 + me._namemap[name] = me._idmap[host.id] = host + + ## Feed the name to the resolver process. + me._inbuf += name + '\n' + me._inlen += len(name) + 1 + if not me._insel.activep: me._insel.enable() + while me._inoff < me._inlen: M.select() + + def _write(me): + """Write material from `_inbuf' to the resolver when it's ready.""" + + ## Try to feed some more material to the resolver. + try: n = OS.write(me._in.fileno(), me._inbuf[me._inoff:]) + except OSError, e: + if e.errno == E.EAGAIN or e.errno == E.EWOULDBLOCK: return + else: raise + + ## If we're done, then clear the buffer. + me._inoff += n + if me._inoff >= me._inlen: + me._insel.disable() + me._inbuf, me._inoff, me._inlen = '', 0, 0 + + def _eof(me): + """Notice that the resolver has finished.""" + me._outline.disable() + me._done = True + me._kid.wait() + + def run(me): + """ + Tell the resolver it has all of our input now, and wait for it to finish. + """ + me._in.close() + while not me._done: M.select() + if me._idmap: + raise Exception('adnshost failed to process all the requests') + + def _hostline(me, line): + """Handle a host line from the resolver.""" + + ## Parse the line into fields. + (id, nrrs, stty, stocde, stmsg, owner, cname, ststr), _ = \ + M.split(line, quotep = True) + id, nrrs = int(id), int(nrrs) + + ## Find the right record. + host = me._idmap[id] + if stty != 'ok': host.failed(ststr) + + ## Stash away the canonical name of the host. + host.name = cname == '$' and owner or cname + + ## If there are no record lines to come, then remove this record from the + ## list of outstanding jobs. Otherwise, switch to the handler for record + ## lines. + if not nrrs: + del me._idmap[id] + else: + me._outline.lineproc = me._rrline + me._nrrs = nrrs + me._outhost = host + + def _rrline(me, line): + """Handle a record line from the resolver.""" + + ## Parse the line into fields. + ww, _ = M.split(line, quotep = True) + owner, type, af = ww[:3] + + ## If this is an address record, and it looks like an interesting address + ## type, then stash the address. + if type == 'A' and (af == 'INET' or af == 'INET6'): + me._outhost.addaddr(af, ww[3]) + + ## Update the parser state. If there are no more records for this job + ## then mark the job as done and switch back to expecting a host line. + me._nrrs -= 1 + if not me._nrrs: + me._outline.lineproc = me._hostline + del me._idmap[me._outhost.id] + me._outhost = None + +## Select a bulk resolver. If `adnshost' exists then we might as well use +## it. +BulkResolver = BresBulkResolver +try: + p = SUB.Popen(['adnshost', '--version'], + stdin = SUB.PIPE, stdout = SUB.PIPE, stderr = SUB.PIPE) + _out, _err = p.communicate() + st = p.wait() + if st == 0: BulkResolver = AdnsBulkResolver +except OSError: + pass + ###-------------------------------------------------------------------------- ### The configuration parser. @@ -170,7 +341,7 @@ RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)') ## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags; ## group 2 is the HOST. -RX_RESOLVE = RX.compile(r'(?x) \$ ([*]*) \[ ([^]]+) \]') +RX_RESOLVE = RX.compile(r'(?x) \$ ([46*]*) \[ ([^]]+) \]') class ConfigSyntaxError (ExpectedError): def __init__(me, fname, lno, msg): @@ -289,9 +460,9 @@ class ConfigSection (object): path.append(me.name) try: - ## If we've been this way before on another pass through then return the - ## value we found then. If we're still thinking about it then we've - ## found a cycle. + ## If we've been this way before on another pass through then return + ## the value we found then. If we're still thinking about it then + ## we've found a cycle. try: v, p = me._cache[key] except KeyError: pass else: @@ -389,8 +560,9 @@ class MyConfigParser (object): expansion and processes them correctly. * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them - correctly. FLAGS may be empty, or `*' (all addresses, space-separated, - rather than just the first). + correctly. FLAGS consists of characters `4' (IPv4 addresses), `6' + (IPv6 addresses), and `*' (all, space-separated, rather than just the + first). * Its parsing behaviour is well-defined. @@ -585,7 +757,7 @@ def output(conf, cdb): if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name) try: u = sec.get('user') except MissingKeyException: pass - else: cdb.add('U%s' % u) + else: cdb.add('U%s' % u, sec.name) url = M.URLEncode(semip = True) for key in sorted(sec.items()): if not key.startswith('@'):