3 from hippotatlib
import *
10 import twisted
.internet
11 from twisted
.web
.server
import NOT_DONE_YET
13 import hippotatlib
.ownsource
14 from hippotatlib
.ownsource
import SourceShipmentPreparer
16 #import twisted.web.server import Site
17 #from twisted.web.resource import Resource
25 #---------- "router" ----------
27 def route(packet
, iface
, saddr
, daddr
):
29 log_debug(DBG
.ROUTE
, 'route: %s -> %s: %s' %
(saddr
,daddr
,dest
), d
=packet
)
30 try: dclient
= clients
[daddr
]
31 except KeyError: dclient
= None
32 if dclient
is not None:
34 dclient
.queue_outbound(packet
)
35 elif daddr
== c
.vaddr
or daddr
not in c
.vnetwork
:
37 queue_inbound(ipif
, packet
)
38 elif daddr
== c
.relay
:
40 log_discard(packet
, iface
, saddr
, daddr
, 'relay')
42 lt('discard no-client')
43 log_discard(packet
, iface
, saddr
, daddr
, 'no-client')
45 #---------- client ----------
48 def __init__(self
, ip
, cc
):
49 # instance data members
52 self
._rq
= collections
.deque() # requests
53 self
._pq
= PacketQueue(str(ip
), self
.cc
.max_queue_time
)
55 if ip
not in c
.vnetwork
:
56 raise ValueError('client %s not in vnetwork' % ip
)
59 raise ValueError('multiple client cfg sections for %s' % ip
)
62 self
._log(DBG
.INIT
, 'new')
64 def _log(self
, dflag
, msg
, **kwargs
):
65 log_debug(dflag
, ('client %s: ' % self
._ip
)+msg
, **kwargs
)
67 def process_arriving_data(self
, d
):
68 self
._log(DBG
.FLOW
, "req data (enc'd)", d
=d
)
70 for packet
in slip
.decode(d
):
71 (saddr
, daddr
) = packet_addrs(packet
)
73 raise ValueError('wrong source address %s' % saddr
)
74 route(packet
, self
._ip
, saddr
, daddr
)
76 def _req_cancel(self
, request
):
77 self
._log(DBG
.HTTP_CTRL
, 'cancel', idof
=request
)
80 def _req_error(self
, err
, request
):
81 self
._log(DBG
.HTTP_CTRL
, 'error %s' % err
, idof
=request
)
82 self
._req_cancel(request
)
84 def queue_outbound(self
, packet
):
85 self
._pq
.append(packet
)
86 self
._check_outbound()
88 def _req_fin(self
, dummy
, request
, cl
):
89 self
._log(DBG
.HTTP_CTRL
, '_req_fin ' + repr(dummy
), idof
=request
)
91 except twisted
.internet
.error
.AlreadyCalled
: pass
93 def new_request(self
, request
):
94 request
.setHeader('Content-Type','application/octet-stream')
95 cl
= reactor
.callLater(self
.cc
.http_timeout
, self
._req_cancel
, request
)
96 nf
= request
.notifyFinish()
97 nf
.addErrback(self
._req_error
, request
)
98 nf
.addCallback(self
._req_fin
, request
, cl
)
99 self
._rq
.append(request
)
100 self
._check_outbound()
102 def _req_write(self
, req
, d
):
103 self
._log(DBG
.HTTP
, 'req_write ', idof
=req
, d
=d
)
106 def _check_outbound(self
):
107 log_debug(DBG
.HTTP_CTRL
, 'CHKO')
109 try: request
= self
._rq
[0]
110 except IndexError: request
= None
111 if request
and request
.finished
:
112 self
._log(DBG
.HTTP_CTRL
, 'CHKO req finished, discard', idof
=request
)
116 if not self
._pq
.nonempty():
117 # no packets, oh well
118 self
._log(DBG
.HTTP_CTRL
, 'CHKO no packets, OUT-DONE', idof
=request
)
123 self
._log(DBG
.HTTP_CTRL
, 'CHKO no request, OUT-DONE', idof
=request
)
126 self
._log(DBG
.HTTP_CTRL
, 'CHKO processing', idof
=request
)
127 # request, and also some non-expired packets
128 self
._pq
.process((lambda: request
.sentLength
),
129 (lambda d
: self
._req_write(request
, d
)),
130 self
.cc
.max_batch_down
)
132 assert(request
.sentLength
)
135 self
._log(DBG
.HTTP
, 'complete', idof
=request
)
136 # round again, looking for more to do
138 while len(self
._rq
) > self
.cc
.target_requests_outstanding
:
139 request
= self
._rq
.popleft()
140 self
._log(DBG
.HTTP
, 'CHKO above target, returning empty', idof
=request
)
143 def process_request(request
, desca
):
144 # find client, update config, etc.
145 metadata
= request
.args
[b
'm'][0]
146 metadata
= metadata
.split(b
'\r\n')
147 (ci_s
, pw
, tro
, cto
) = metadata
[0:4]
148 desca
['m[0,2:3]'] = [ci_s
, tro
, cto
]
149 ci_s
= ci_s
.decode('utf-8')
150 tro
= int(tro
); desca
['tro']= tro
151 cto
= int(cto
); desca
['cto']= cto
155 if pw
!= cl
.cc
.password
: raise ValueError('bad password')
158 if tro
!= cl
.cc
.target_requests_outstanding
:
159 raise ValueError('tro must be %d' % cl
.cc
.target_requests_outstanding
)
161 if cto
< cl
.cc
.http_timeout
:
162 raise ValueError('cto must be >= %d' % cl
.cc
.http_timeout
)
165 d
= request
.args
[b
'd'][0]
167 desca
['dlen'] = len(d
)
172 log_http(desca
, 'processing', idof
=id(request
), d
=d
)
174 d
= mime_translate(d
)
176 cl
.process_arriving_data(d
)
177 cl
.new_request(request
)
179 def log_http(desca
, msg
, **kwargs
):
181 kwargs
['d'] = desca
['d']
185 log_debug(DBG
.HTTP
, msg
+ repr(desca
), **kwargs
)
187 class NotStupidResource(twisted
.web
.resource
.Resource
):
188 # why this is not the default is a mystery!
189 def getChild(self
, name
, request
):
190 if name
== b
'': return self
191 else: return twisted
.web
.resource
.Resource
.getChild(name
, request
)
193 class IphttpResource(NotStupidResource
):
194 def render_POST(self
, request
):
195 log_debug(DBG
.HTTP_FULL
,
196 'req recv: ' + repr(request
) + ' ' + repr(request
.args
),
199 try: process_request(request
, desca
)
200 except Exception as e
:
201 emsg
= traceback
.format_exc()
202 log_http(desca
, 'RETURNING EXCEPTION ' + emsg
)
203 request
.setHeader('Content-Type','text/plain; charset="utf-8"')
204 request
.setResponseCode(400)
205 return (emsg
+ ' # ' + repr(desca
) + '\r\n').encode('utf-8')
206 log_debug(DBG
.HTTP_CTRL
, '...', idof
=id(request
))
209 def render_GET(self
, request
):
210 log_debug(DBG
.HTTP
, 'GET request')
211 return b
'<html><body>hippotat</body></html>'
213 class SourceResource(NotStupidResource
):
215 td
= tempfile
.mkdtemp()
218 try: shutil
.rmtree(td
)
219 except FileNotFoundError
: pass
221 cleanups
.append(cleanup
)
223 self
._ssp
= SourceShipmentPreparer(td
)
224 self
._ssp
.logger
= self
.log
228 log_debug(DBG
.OWNSOURCE
, m
)
230 def render_GET(self
, request
):
231 return b
'<html><body>SUBDIR</body></html>'
234 resource
= IphttpResource()
235 resource
.putChild(b
'source',SourceResource())
236 site
= twisted
.web
.server
.Site(resource
)
238 ep
= sa
.make_endpoint()
239 crash_on_defer(ep
.listen(site
))
240 log_debug(DBG
.INIT
, 'listening on %s' % sa
)
242 #---------- config and setup ----------
244 def process_cfg(putative_servers
, putative_clients
):
247 c
.server
= cfg
.get('SERVER','server')
249 cfg_process_common(c
, c
.server
)
250 cfg_process_saddrs(c
, c
.server
)
251 cfg_process_vnetwork(c
, c
.server
)
252 cfg_process_vaddr(c
, c
.server
)
254 for (ci
,cs
) in putative_clients
.items():
256 sections
= cfg_process_client_common(cc
,c
.server
,cs
,ci
)
257 if not sections
: continue
258 cfg_process_client_limited(cc
,c
.server
,sections
, 'max_batch_down')
259 cfg_process_client_limited(cc
,c
.server
,sections
, 'max_queue_time')
263 c
.vrelay
= cfg
.get(c
.server
, 'vrelay')
264 except NoOptionError
:
265 for search
in c
.vnetwork
.hosts():
266 if search
== c
.vaddr
: continue
271 [c
.server
, 'DEFAULT'],
274 ('rnets','vnetwork')))
276 def catch_termination():
278 for cleanup
in cleanups
:
281 atexit
.register(run_cleanups
)
283 def signal_handler(name
, sig
, *args
):
284 signal
.signal(sig
, signal
.SIG_DFL
)
285 print('exiting due to %s' % name
, file=sys
.stderr
)
287 os
.kill(os
.getpid(), sig
)
288 raise RuntimeError('did not die due to signal %s !' % name
)
290 for sig
in (signal
.SIGINT
, signal
.SIGTERM
):
291 signal
.signal(sig
, partial(signal_handler
, sig
.name
))
293 common_startup(process_cfg
)
295 ipif
= start_ipif(c
.ipif_command
, (lambda p
,s
,d
: route(p
,"[ipif]",s
,d
)))