peerdb/tripe-newpeers.in: Split out a resolver base class.

[tripe] / peerdb / tripe-newpeers.in

diff --git a/peerdb/tripe-newpeers.in b/peerdb/tripe-newpeers.in
index b8cf681..202cea8 100644 (file)
--- a/peerdb/tripe-newpeers.in
+++ b/peerdb/tripe-newpeers.in
@@ -34,6 +34,7 @@ import cdb as CDB
 from sys import stdin, stdout, exit, argv
 import re as RX
 import os as OS
 from sys import stdin, stdout, exit, argv
 import re as RX
 import os as OS

+import socket as S

 from cStringIO import StringIO
 
 ###--------------------------------------------------------------------------
 from cStringIO import StringIO
 
 ###--------------------------------------------------------------------------


@@ -48,10 +49,64 @@ class CDBFake (object):
   def finish(me):
     pass
 
   def finish(me):
     pass
 

+class ExpectedError (Exception): pass
+

 ###--------------------------------------------------------------------------
 ### A bulk DNS resolver.
 
 ###--------------------------------------------------------------------------
 ### A bulk DNS resolver.
 

-class BulkResolver (object):
+class ResolverFailure (ExpectedError):
+  def __init__(me, host, msg):
+    me.host = host
+    me.msg = msg
+  def __str__(me):
+    return "failed to resolve `%s': %s" % (me.host, me.msg)
+
+class ResolvingHost (object):
+  """
+  A host name which is being looked up by a bulk-resolver instance.
+
+  Most notably, this is where the flag-handling logic lives for the
+  $FLAGS[HOSTNAME] syntax.
+  """
+
+  def __init__(me, name):
+    """Make a new resolving-host object for the host NAME."""
+    me.name = name
+    me.addr = { 'INET': [], 'INET6': [] }
+    me.failure = None
+
+  def addaddr(me, af, addr):
+    """
+    Add the address ADDR with address family AF.
+
+    The address family may be `INET' or `INET6'.
+    """
+    me.addr[af].append(addr)
+
+  def failed(me, msg):
+    """
+    Report that resolution of this host failed, with a human-readable MSG.
+    """
+    me.failure = msg
+
+  def get(me, flags):
+    """Return a list of addresses according to the FLAGS string."""
+    if me.failure is not None: raise ResolverFailure(me.name, me.failure)
+    aa = []
+    a4 = me.addr['INET']
+    a6 = me.addr['INET6']
+    all, any = False, False
+    for ch in flags:
+      if ch == '*': all = True
+      elif ch == '4': aa += a4; any = True
+      elif ch == '6': aa += a6; any = True
+      else: raise ValueError("unknown address-resolution flag `%s'" % ch)
+    if not any: aa = a4 + a6
+    if not aa: raise ResolverFailure(me.name, 'no matching addresses found')
+    if not all: aa = [aa[0]]
+    return aa
+
+class BaseBulkResolver (object):

   """
   Resolve a number of DNS names in parallel.
 
   """
   Resolve a number of DNS names in parallel.
 


@@ -69,36 +124,62 @@ class BulkResolver (object):
 
   def __init__(me):
     """Initialize the resolver."""
 
   def __init__(me):
     """Initialize the resolver."""

-    me._resolvers = {}

     me._namemap = {}
 
     me._namemap = {}
 

-  def prepare(me, host):
-    """Prime the resolver to resolve the name HOST."""
-    if host not in me._resolvers:
-      me._resolvers[host] = M.SelResolveByName \
-                            (host,
-                             lambda name, alias, addr:
-                               me._resolved(host, addr[0]),
-                             lambda: me._resolved(host, None))
+  def prepare(me, name):
+    """Prime the resolver to resolve the given host NAME."""
+    if name not in me._namemap:
+      me._namemap[name] = host = ResolvingHost(name)
+      try:
+        ailist = S.getaddrinfo(name, None, S.AF_UNSPEC, S.SOCK_DGRAM, 0,
+                               S.AI_NUMERICHOST | S.AI_NUMERICSERV)
+      except S.gaierror:
+        me._prepare(host, name)
+      else:
+        for af, skty, proto, cname, sa in ailist:
+          if af == S.AF_INET: host.addaddr('INET', sa[0])
+          elif af == S.AF_INET6: host.addaddr('INET6', sa[0])
+
+  def lookup(me, name, flags):
+    """Fetch the address corresponding to the host NAME."""
+    return me._namemap[name].get(flags)
+
+class BresBulkResolver (BaseBulkResolver):
+  """
+  A BulkResolver using mLib's `bres' background resolver.
+
+  This is always available (and might use ADNS), but only does IPv4.
+  """
+
+  def __init__(me):
+    super(BresBulkResolver, me).__init__()
+    """Initialize the resolver."""
+    me._noutstand = 0
+
+  def _prepare(me, host, name):
+    """Arrange to resolve a NAME, reporting the results to HOST."""
+    host._resolv = M.SelResolveByName(
+      name,
+      lambda cname, alias, addr: me._resolved(host, cname, addr),
+      lambda: me._resolved(host, None, []))
+    me._noutstand += 1

 
   def run(me):
     """Run the background DNS resolver until it's finished."""
 
   def run(me):
     """Run the background DNS resolver until it's finished."""

-    while me._resolvers:
-      M.select()
+    while me._noutstand: M.select()

 
 

-  def lookup(me, host):
-    """
-    Fetch the address corresponding to HOST.
-    """
-    addr = me._namemap[host]
-    if addr is None:
-      raise KeyError(host)
-    return addr
+  def _resolved(me, host, cname, addr):
+    """Callback function: remember that ADDRs are the addresses for HOST."""
+    if not addr:
+      host.failed('(unknown failure)')
+    else:
+      if cname is not None: host.name = cname
+      for a in addr: host.addaddr('INET', a)
+    host._resolv = None
+    me._noutstand -= 1

 
 

-  def _resolved(me, host, addr):
-    """Callback function: remember that ADDR is the address for HOST."""
-    me._namemap[host] = addr
-    del me._resolvers[host]
+## Select a bulk resolver.  Currently, there's only one choice.
+BulkResolver = BresBulkResolver

 
 ###--------------------------------------------------------------------------
 ### The configuration parser.
 
 ###--------------------------------------------------------------------------
 ### The configuration parser.


@@ -124,10 +205,11 @@ RX_CONT = RX.compile(r'''(?x) ^ \s+
 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
 
 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
 

-## Match a $[HOST] name resolution reference; group 1 is the HOST.
-RX_RESOLVE = RX.compile(r'(?x) \$ \[ ([^]]+) \]')
+## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags;
+## group 2 is the HOST.
+RX_RESOLVE = RX.compile(r'(?x) \$ ([46*]*) \[ ([^]]+) \]')

 
 

-class ConfigSyntaxError (Exception):
+class ConfigSyntaxError (ExpectedError):

   def __init__(me, fname, lno, msg):
     me.fname = fname
     me.lno = lno
   def __init__(me, fname, lno, msg):
     me.fname = fname
     me.lno = lno


@@ -138,7 +220,7 @@ class ConfigSyntaxError (Exception):
 def _fmt_path(path):
   return ' -> '.join(["`%s'" % hop for hop in path])
 
 def _fmt_path(path):
   return ' -> '.join(["`%s'" % hop for hop in path])
 

-class AmbiguousOptionError (Exception):
+class AmbiguousOptionError (ExpectedError):

   def __init__(me, key, patha, vala, pathb, valb):
     me.key = key
     me.patha, me.vala = patha, vala
   def __init__(me, key, patha, vala, pathb, valb):
     me.key = key
     me.patha, me.vala = patha, vala


@@ -148,7 +230,7 @@ class AmbiguousOptionError (Exception):
         "path %s yields `%s' but %s yields `%s'" % \
         (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
 
         "path %s yields `%s' but %s yields `%s'" % \
         (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
 

-class InheritanceCycleError (Exception):
+class InheritanceCycleError (ExpectedError):

   def __init__(me, key, path):
     me.key = key
     me.path = path
   def __init__(me, key, path):
     me.key = key
     me.path = path


@@ -156,13 +238,13 @@ class InheritanceCycleError (Exception):
     return "Found a cycle %s looking up key `%s'" % \
         (_fmt_path(me.path), me.key)
 
     return "Found a cycle %s looking up key `%s'" % \
         (_fmt_path(me.path), me.key)
 

-class MissingSectionException (Exception):
+class MissingSectionException (ExpectedError):

   def __init__(me, sec):
   def __init__(me, sec):

-    me.key = key
+    me.sec = sec

   def __str__(me):
     return "Section `%s' not found" % (me.sec)
 
   def __str__(me):
     return "Section `%s' not found" % (me.sec)
 

-class MissingKeyException (Exception):
+class MissingKeyException (ExpectedError):

   def __init__(me, sec, key):
     me.sec = sec
     me.key = key
   def __init__(me, sec, key):
     me.sec = sec
     me.key = key


@@ -180,26 +262,50 @@ class ConfigSection (object):
 
   def __init__(me, name, cp):
     """Initialize a new, empty section with a given NAME and parent CP."""
 
   def __init__(me, name, cp):
     """Initialize a new, empty section with a given NAME and parent CP."""

+
+    ## The cache maps item keys to entries, which consist of a pair of
+    ## objects.  There are four possible states for a cache entry:
+    ##
+    ##   * missing -- there is no entry at all with this key, so we must
+    ##     search for it;
+    ##
+    ##   * None, None -- we are actively trying to resolve this key, so if we
+    ##     encounter this state, we have found a cycle in the inheritance
+    ##     graph;
+    ##
+    ##   * None, [] -- we know that this key isn't reachable through any of
+    ##     our parents;
+    ##
+    ##   * VALUE, PATH -- we know that the key resolves to VALUE, along the
+    ##     PATH from us (exclusive) to the defining parent (inclusive).

     me.name = name
     me._itemmap = dict()
     me.name = name
     me._itemmap = dict()

+    me._cache = dict()

     me._cp = cp
 
   def _expand(me, string, resolvep):
     """
     me._cp = cp
 
   def _expand(me, string, resolvep):
     """

-    Expands $(...) and (optionally) $[...] placeholders in STRING.
+    Expands $(...) and (optionally) $FLAGS[...] placeholders in STRING.

 
     RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
     This is turned off by MyConfigParser's resolve() method while it's
     collecting hostnames to be resolved.
     """
 
     RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
     This is turned off by MyConfigParser's resolve() method while it's
     collecting hostnames to be resolved.
     """

-    string = RX_REF.sub \
-             (lambda m: me.get(m.group(1), resolvep), string)
+    string = RX_REF.sub(lambda m: me.get(m.group(1), resolvep), string)

     if resolvep:
     if resolvep:

-      string = RX_RESOLVE.sub(lambda m: me._cp._resolver.lookup(m.group(1)),
-                              string)
+      string = RX_RESOLVE.sub(
+        lambda m: ' '.join(me._cp._resolver.lookup(m.group(2), m.group(1))),
+        string)

     return string
 
     return string
 

-  def _get(me, key, map = None, path = None):
+  def _parents(me):
+    """Yield this section's parents."""
+    try: names = me._itemmap['@inherit']
+    except KeyError: return
+    for name in names.replace(',', ' ').split():
+      yield me._cp.section(name)
+
+  def _get(me, key, path = None):

     """
     Low-level option-fetching method.
 
     """
     Low-level option-fetching method.
 


@@ -211,9 +317,7 @@ class ConfigSection (object):
     Returns None if no value could be found.
     """
 
     Returns None if no value could be found.
     """
 

-    ## If we weren't given a memoization map or path, then we'd better make
-    ## one.
-    if map is None: map = {}
+    ## If we weren't given a path, then we'd better make one.

     if path is None: path = []
 
     ## Extend the path to cover us, but remember to remove us again when
     if path is None: path = []
 
     ## Extend the path to cover us, but remember to remove us again when


@@ -222,36 +326,33 @@ class ConfigSection (object):
     path.append(me.name)
     try:
 
     path.append(me.name)
     try:
 

-      ## If we've been this way before on another pass through then return
-      ## the value we found then.  If we're still thinking about it then
-      ## we've found a cycle.
-      try: threadp, value = map[me.name]
+      ## If we've been this way before on another pass through then return the
+      ## value we found then.  If we're still thinking about it then we've
+      ## found a cycle.
+      try: v, p = me._cache[key]

       except KeyError: pass
       else:
       except KeyError: pass
       else:

-        if threadp: raise InheritanceCycleError(key, path[:])
+        if p is None: raise InheritanceCycleError(key, path[:])
+        else: return v, path + p

 
       ## See whether the answer is ready waiting for us.
       try: v = me._itemmap[key]
       except KeyError: pass
 
       ## See whether the answer is ready waiting for us.
       try: v = me._itemmap[key]
       except KeyError: pass

-      else: return v, path[:]
-
-      ## No, apparently, not.  Find out our list of parents.
-      try:
-        parents = [me._cp.section(p) for p in
-                   me._itemmap['@inherit'].replace(',', ' ').split()]
-      except KeyError:
-        parents = []
+      else:
+        p = path[:]
+        me._cache[key] = v, []
+        return v, p

 
       ## Initially we have no idea.
       value = None
 
       ## Initially we have no idea.
       value = None

-      winner = None
+      winner = []

 
       ## Go through our parents and ask them what they think.
 
       ## Go through our parents and ask them what they think.

-      map[me.name] = True, None
-      for p in parents:
+      me._cache[key] = None, None
+      for p in me._parents():

 
         ## See whether we get an answer.  If not, keep on going.
 
         ## See whether we get an answer.  If not, keep on going.

-        v, pp = p._get(key, map, path)
+        v, pp = p._get(key, path)

         if v is None: continue
 
         ## If we got an answer, check that it matches any previous ones.
         if v is None: continue
 
         ## If we got an answer, check that it matches any previous ones.


@@ -262,7 +363,7 @@ class ConfigSection (object):
           raise AmbiguousOptionError(key, winner, value, pp, v)
 
       ## That's the best we could manage.
           raise AmbiguousOptionError(key, winner, value, pp, v)
 
       ## That's the best we could manage.

-      map[me.name] = False, value
+      me._cache[key] = value, winner[len(path):]

       return value, winner
 
     finally:
       return value, winner
 
     finally:


@@ -277,6 +378,9 @@ class ConfigSection (object):
     ## Special handling for the `name' key.
     if key == 'name':
       value = me._itemmap.get('name', me.name)
     ## Special handling for the `name' key.
     if key == 'name':
       value = me._itemmap.get('name', me.name)

+    elif key == '@inherits':
+      try: return me._itemmap['@inherits']
+      except KeyError: raise MissingKeyException(me.name, key)

     else:
       value, _ = me._get(key)
       if value is None:
     else:
       value, _ = me._get(key)
       if value is None:


@@ -291,23 +395,22 @@ class ConfigSection (object):
     """
 
     ## Initialize for a depth-first walk of the inheritance graph.
     """
 
     ## Initialize for a depth-first walk of the inheritance graph.

-    d = {}
-    visited = {}
-    stack = [me.name]
+    seen = { 'name': True }
+    visiting = { me.name: True }
+    stack = [me]

 
     ## Visit nodes, collecting their keys.  Don't believe the values:
     ## resolving inheritance is too hard to do like this.
     while stack:
 
     ## Visit nodes, collecting their keys.  Don't believe the values:
     ## resolving inheritance is too hard to do like this.
     while stack:

-      sec = me._cp.section(stack.pop())
-      if sec.name in visited: continue
-      visited[sec.name] = True
+      sec = stack.pop()
+      for p in sec._parents():
+        if p.name not in visiting:
+          stack.append(p); visiting[p.name] = True

 
 

-      for key, value in sec._itemmap.iteritems():
-        if key == '@inherit': stack += value.replace(',', ' ').split()
-        else: d[key] = None
+      for key in sec._itemmap.iterkeys(): seen[key] = None

 
     ## And we're done.
 
     ## And we're done.

-    return d.iterkeys()
+    return seen.iterkeys()

 
 class MyConfigParser (object):
   """
 
 class MyConfigParser (object):
   """


@@ -322,8 +425,10 @@ class MyConfigParser (object):
     * It recognizes `$(VAR)' references to configuration variables during
       expansion and processes them correctly.
 
     * It recognizes `$(VAR)' references to configuration variables during
       expansion and processes them correctly.
 

-    * It recognizes `$[HOST]' name-resolver requests and handles them
-      correctly.
+    * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them
+      correctly.  FLAGS consists of characters `4' (IPv4 addresses), `6'
+      (IPv6 addresses), and `*' (all, space-separated, rather than just the
+      first).

 
     * Its parsing behaviour is well-defined.
 
 
     * Its parsing behaviour is well-defined.
 


@@ -436,7 +541,7 @@ class MyConfigParser (object):
       for key in sec.items():
         value = sec.get(key, resolvep = False)
         for match in RX_RESOLVE.finditer(value):
       for key in sec.items():
         value = sec.get(key, resolvep = False)
         for match in RX_RESOLVE.finditer(value):

-          me._resolver.prepare(match.group(1))
+          me._resolver.prepare(match.group(2))

     me._resolver.run()
 
 ###--------------------------------------------------------------------------
     me._resolver.run()
 
 ###--------------------------------------------------------------------------


@@ -519,10 +624,10 @@ def output(conf, cdb):
       try: u = sec.get('user')
       except MissingKeyException: pass
       else: cdb.add('U%s' % u)
       try: u = sec.get('user')
       except MissingKeyException: pass
       else: cdb.add('U%s' % u)

-    url = M.URLEncode(laxp = True, semip = True)
+    url = M.URLEncode(semip = True)

     for key in sorted(sec.items()):
       if not key.startswith('@'):
     for key in sorted(sec.items()):
       if not key.startswith('@'):

-        url.encode(key, ' '.join(M.split(sec.get(key))[0]))
+        url.encode(key, sec.get(key))

     cdb.add(label, url.result)
   cdb.add('%AUTO', ' '.join(auto))
   cdb.finish()
     cdb.add(label, url.result)
   cdb.add('%AUTO', ' '.join(auto))
   cdb.finish()


@@ -534,8 +639,12 @@ def main():
     cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
   else:
     cdb = CDBFake()
     cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
   else:
     cdb = CDBFake()

-  conf = getconf(args[1:])
-  output(conf, cdb)
+  try:
+    conf = getconf(args[1:])
+    output(conf, cdb)
+  except ExpectedError, e:
+    M.moan(str(e))
+    exit(2)

 
 if __name__ == '__main__':
   main()
 
 if __name__ == '__main__':
   main()