peerdb/tripe-newpeers.in: Split out a resolver base class.

[tripe] / peerdb / tripe-newpeers.in

diff --git a/peerdb/tripe-newpeers.in b/peerdb/tripe-newpeers.in
index 0ba9cb6..202cea8 100644 (file)
--- a/peerdb/tripe-newpeers.in
+++ b/peerdb/tripe-newpeers.in
@@ -34,6 +34,7 @@ import cdb as CDB
 from sys import stdin, stdout, exit, argv
 import re as RX
 import os as OS
 from sys import stdin, stdout, exit, argv
 import re as RX
 import os as OS

+import socket as S

 from cStringIO import StringIO
 
 ###--------------------------------------------------------------------------
 from cStringIO import StringIO
 
 ###--------------------------------------------------------------------------


@@ -48,17 +49,64 @@ class CDBFake (object):
   def finish(me):
     pass
 
   def finish(me):
     pass
 

+class ExpectedError (Exception): pass
+

 ###--------------------------------------------------------------------------
 ### A bulk DNS resolver.
 
 ###--------------------------------------------------------------------------
 ### A bulk DNS resolver.
 

-class ResolverFailure (Exception):
+class ResolverFailure (ExpectedError):

   def __init__(me, host, msg):
     me.host = host
     me.msg = msg
   def __str__(me):
     return "failed to resolve `%s': %s" % (me.host, me.msg)
 
   def __init__(me, host, msg):
     me.host = host
     me.msg = msg
   def __str__(me):
     return "failed to resolve `%s': %s" % (me.host, me.msg)
 

-class BulkResolver (object):
+class ResolvingHost (object):
+  """
+  A host name which is being looked up by a bulk-resolver instance.
+
+  Most notably, this is where the flag-handling logic lives for the
+  $FLAGS[HOSTNAME] syntax.
+  """
+
+  def __init__(me, name):
+    """Make a new resolving-host object for the host NAME."""
+    me.name = name
+    me.addr = { 'INET': [], 'INET6': [] }
+    me.failure = None
+
+  def addaddr(me, af, addr):
+    """
+    Add the address ADDR with address family AF.
+
+    The address family may be `INET' or `INET6'.
+    """
+    me.addr[af].append(addr)
+
+  def failed(me, msg):
+    """
+    Report that resolution of this host failed, with a human-readable MSG.
+    """
+    me.failure = msg
+
+  def get(me, flags):
+    """Return a list of addresses according to the FLAGS string."""
+    if me.failure is not None: raise ResolverFailure(me.name, me.failure)
+    aa = []
+    a4 = me.addr['INET']
+    a6 = me.addr['INET6']
+    all, any = False, False
+    for ch in flags:
+      if ch == '*': all = True
+      elif ch == '4': aa += a4; any = True
+      elif ch == '6': aa += a6; any = True
+      else: raise ValueError("unknown address-resolution flag `%s'" % ch)
+    if not any: aa = a4 + a6
+    if not aa: raise ResolverFailure(me.name, 'no matching addresses found')
+    if not all: aa = [aa[0]]
+    return aa
+
+class BaseBulkResolver (object):

   """
   Resolve a number of DNS names in parallel.
 
   """
   Resolve a number of DNS names in parallel.
 


@@ -76,36 +124,62 @@ class BulkResolver (object):
 
   def __init__(me):
     """Initialize the resolver."""
 
   def __init__(me):
     """Initialize the resolver."""

-    me._resolvers = {}

     me._namemap = {}
 
     me._namemap = {}
 

-  def prepare(me, host):
-    """Prime the resolver to resolve the name HOST."""
-    if host not in me._resolvers:
-      me._resolvers[host] = M.SelResolveByName \
-                            (host,
-                             lambda name, alias, addr:
-                               me._resolved(host, addr[0]),
-                             lambda: me._resolved(host, None))
+  def prepare(me, name):
+    """Prime the resolver to resolve the given host NAME."""
+    if name not in me._namemap:
+      me._namemap[name] = host = ResolvingHost(name)
+      try:
+        ailist = S.getaddrinfo(name, None, S.AF_UNSPEC, S.SOCK_DGRAM, 0,
+                               S.AI_NUMERICHOST | S.AI_NUMERICSERV)
+      except S.gaierror:
+        me._prepare(host, name)
+      else:
+        for af, skty, proto, cname, sa in ailist:
+          if af == S.AF_INET: host.addaddr('INET', sa[0])
+          elif af == S.AF_INET6: host.addaddr('INET6', sa[0])
+
+  def lookup(me, name, flags):
+    """Fetch the address corresponding to the host NAME."""
+    return me._namemap[name].get(flags)
+
+class BresBulkResolver (BaseBulkResolver):
+  """
+  A BulkResolver using mLib's `bres' background resolver.
+
+  This is always available (and might use ADNS), but only does IPv4.
+  """
+
+  def __init__(me):
+    super(BresBulkResolver, me).__init__()
+    """Initialize the resolver."""
+    me._noutstand = 0
+
+  def _prepare(me, host, name):
+    """Arrange to resolve a NAME, reporting the results to HOST."""
+    host._resolv = M.SelResolveByName(
+      name,
+      lambda cname, alias, addr: me._resolved(host, cname, addr),
+      lambda: me._resolved(host, None, []))
+    me._noutstand += 1

 
   def run(me):
     """Run the background DNS resolver until it's finished."""
 
   def run(me):
     """Run the background DNS resolver until it's finished."""

-    while me._resolvers:
-      M.select()
+    while me._noutstand: M.select()

 
 

-  def lookup(me, host):
-    """
-    Fetch the address corresponding to HOST.
-    """
-    addr = me._namemap[host]
-    if addr is None:
-      raise ResolverFailure(host, '(unknown failure)')
-    return addr
+  def _resolved(me, host, cname, addr):
+    """Callback function: remember that ADDRs are the addresses for HOST."""
+    if not addr:
+      host.failed('(unknown failure)')
+    else:
+      if cname is not None: host.name = cname
+      for a in addr: host.addaddr('INET', a)
+    host._resolv = None
+    me._noutstand -= 1

 
 

-  def _resolved(me, host, addr):
-    """Callback function: remember that ADDR is the address for HOST."""
-    me._namemap[host] = addr
-    del me._resolvers[host]
+## Select a bulk resolver.  Currently, there's only one choice.
+BulkResolver = BresBulkResolver

 
 ###--------------------------------------------------------------------------
 ### The configuration parser.
 
 ###--------------------------------------------------------------------------
 ### The configuration parser.


@@ -131,10 +205,11 @@ RX_CONT = RX.compile(r'''(?x) ^ \s+
 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
 
 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
 

-## Match a $[HOST] name resolution reference; group 1 is the HOST.
-RX_RESOLVE = RX.compile(r'(?x) \$ \[ ([^]]+) \]')
+## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags;
+## group 2 is the HOST.
+RX_RESOLVE = RX.compile(r'(?x) \$ ([46*]*) \[ ([^]]+) \]')

 
 

-class ConfigSyntaxError (Exception):
+class ConfigSyntaxError (ExpectedError):

   def __init__(me, fname, lno, msg):
     me.fname = fname
     me.lno = lno
   def __init__(me, fname, lno, msg):
     me.fname = fname
     me.lno = lno


@@ -145,7 +220,7 @@ class ConfigSyntaxError (Exception):
 def _fmt_path(path):
   return ' -> '.join(["`%s'" % hop for hop in path])
 
 def _fmt_path(path):
   return ' -> '.join(["`%s'" % hop for hop in path])
 

-class AmbiguousOptionError (Exception):
+class AmbiguousOptionError (ExpectedError):

   def __init__(me, key, patha, vala, pathb, valb):
     me.key = key
     me.patha, me.vala = patha, vala
   def __init__(me, key, patha, vala, pathb, valb):
     me.key = key
     me.patha, me.vala = patha, vala


@@ -155,7 +230,7 @@ class AmbiguousOptionError (Exception):
         "path %s yields `%s' but %s yields `%s'" % \
         (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
 
         "path %s yields `%s' but %s yields `%s'" % \
         (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
 

-class InheritanceCycleError (Exception):
+class InheritanceCycleError (ExpectedError):

   def __init__(me, key, path):
     me.key = key
     me.path = path
   def __init__(me, key, path):
     me.key = key
     me.path = path


@@ -163,13 +238,13 @@ class InheritanceCycleError (Exception):
     return "Found a cycle %s looking up key `%s'" % \
         (_fmt_path(me.path), me.key)
 
     return "Found a cycle %s looking up key `%s'" % \
         (_fmt_path(me.path), me.key)
 

-class MissingSectionException (Exception):
+class MissingSectionException (ExpectedError):

   def __init__(me, sec):
   def __init__(me, sec):

-    me.key = key
+    me.sec = sec

   def __str__(me):
     return "Section `%s' not found" % (me.sec)
 
   def __str__(me):
     return "Section `%s' not found" % (me.sec)
 

-class MissingKeyException (Exception):
+class MissingKeyException (ExpectedError):

   def __init__(me, sec, key):
     me.sec = sec
     me.key = key
   def __init__(me, sec, key):
     me.sec = sec
     me.key = key


@@ -210,17 +285,17 @@ class ConfigSection (object):
 
   def _expand(me, string, resolvep):
     """
 
   def _expand(me, string, resolvep):
     """

-    Expands $(...) and (optionally) $[...] placeholders in STRING.
+    Expands $(...) and (optionally) $FLAGS[...] placeholders in STRING.

 
     RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
     This is turned off by MyConfigParser's resolve() method while it's
     collecting hostnames to be resolved.
     """
 
     RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
     This is turned off by MyConfigParser's resolve() method while it's
     collecting hostnames to be resolved.
     """

-    string = RX_REF.sub \
-             (lambda m: me.get(m.group(1), resolvep), string)
+    string = RX_REF.sub(lambda m: me.get(m.group(1), resolvep), string)

     if resolvep:
     if resolvep:

-      string = RX_RESOLVE.sub(lambda m: me._cp._resolver.lookup(m.group(1)),
-                              string)
+      string = RX_RESOLVE.sub(
+        lambda m: ' '.join(me._cp._resolver.lookup(m.group(2), m.group(1))),
+        string)

     return string
 
   def _parents(me):
     return string
 
   def _parents(me):


@@ -350,8 +425,10 @@ class MyConfigParser (object):
     * It recognizes `$(VAR)' references to configuration variables during
       expansion and processes them correctly.
 
     * It recognizes `$(VAR)' references to configuration variables during
       expansion and processes them correctly.
 

-    * It recognizes `$[HOST]' name-resolver requests and handles them
-      correctly.
+    * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them
+      correctly.  FLAGS consists of characters `4' (IPv4 addresses), `6'
+      (IPv6 addresses), and `*' (all, space-separated, rather than just the
+      first).

 
     * Its parsing behaviour is well-defined.
 
 
     * Its parsing behaviour is well-defined.
 


@@ -464,7 +541,7 @@ class MyConfigParser (object):
       for key in sec.items():
         value = sec.get(key, resolvep = False)
         for match in RX_RESOLVE.finditer(value):
       for key in sec.items():
         value = sec.get(key, resolvep = False)
         for match in RX_RESOLVE.finditer(value):

-          me._resolver.prepare(match.group(1))
+          me._resolver.prepare(match.group(2))

     me._resolver.run()
 
 ###--------------------------------------------------------------------------
     me._resolver.run()
 
 ###--------------------------------------------------------------------------


@@ -562,8 +639,12 @@ def main():
     cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
   else:
     cdb = CDBFake()
     cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
   else:
     cdb = CDBFake()

-  conf = getconf(args[1:])
-  output(conf, cdb)
+  try:
+    conf = getconf(args[1:])
+    output(conf, cdb)
+  except ExpectedError, e:
+    M.moan(str(e))
+    exit(2)

 
 if __name__ == '__main__':
   main()
 
 if __name__ == '__main__':
   main()