Commit | Line | Data |
---|---|---|
094ee3a2 | 1 | #!/usr/bin/python3 |
0256fc10 IJ |
2 | # |
3 | # Hippotat - Asinine IP Over HTTP program | |
4 | # ./hippotatd - server main program | |
5 | # | |
6 | # Copyright 2017 Ian Jackson | |
7 | # | |
f85d143f | 8 | # AGPLv3+ + CAFv2+ |
0256fc10 | 9 | # |
f85d143f IJ |
10 | # This program is free software: you can redistribute it and/or |
11 | # modify it under the terms of the GNU Affero General Public | |
12 | # License as published by the Free Software Foundation, either | |
13 | # version 3 of the License, or (at your option) any later version, | |
14 | # with the "CAF Login Exception" as published by Ian Jackson | |
15 | # (version 2, or at your option any later version) as an Additional | |
16 | # Permission. | |
0256fc10 | 17 | # |
f85d143f IJ |
18 | # This program is distributed in the hope that it will be useful, |
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
21 | # Affero General Public License for more details. | |
22 | # | |
23 | # You should have received a copy of the GNU Affero General Public | |
24 | # License and the CAF Login Exception along with this program, in | |
25 | # the file AGPLv3+CAFv2. If not, email Ian Jackson | |
26 | # <ijackson@chiark.greenend.org.uk>. | |
0256fc10 | 27 | |
3fba9787 | 28 | |
5a37bac8 | 29 | from hippotatlib import * |
aa663282 | 30 | |
e2d41dc1 | 31 | import os |
4a780703 IJ |
32 | import tempfile |
33 | import atexit | |
34 | import shutil | |
d84dbf4b | 35 | import subprocess |
e2d41dc1 | 36 | |
e2d41dc1 | 37 | import twisted.internet |
e2d41dc1 | 38 | from twisted.web.server import NOT_DONE_YET |
e2d41dc1 | 39 | |
99e6411f | 40 | import twisted.web.static |
ec2c9312 | 41 | import twisted.python.syslog |
99e6411f | 42 | |
4a780703 IJ |
43 | import hippotatlib.ownsource |
44 | from hippotatlib.ownsource import SourceShipmentPreparer | |
45 | ||
5da7763e IJ |
46 | #import twisted.web.server import Site |
47 | #from twisted.web.resource import Resource | |
3fba9787 | 48 | |
c4b6d990 IJ |
49 | import syslog |
50 | ||
4a780703 IJ |
51 | cleanups = [ ] |
52 | ||
b0cfbfce | 53 | clients = { } |
3fba9787 | 54 | |
5da7763e IJ |
55 | #---------- "router" ---------- |
56 | ||
a8827d59 | 57 | def route(packet, iface, saddr, daddr): |
d579a048 | 58 | def lt(dest): |
8c3b6620 | 59 | log_debug(DBG.ROUTE, 'route: %s -> %s: %s' % (saddr,daddr,dest), d=packet) |
84e763c7 | 60 | try: dclient = clients[daddr] |
5da7763e IJ |
61 | except KeyError: dclient = None |
62 | if dclient is not None: | |
d579a048 | 63 | lt('client') |
5da7763e | 64 | dclient.queue_outbound(packet) |
8d374606 | 65 | elif daddr == c.vaddr or daddr not in c.vnetwork: |
d579a048 | 66 | lt('inbound') |
909e0ff3 | 67 | queue_inbound(ipif, packet) |
74934d63 | 68 | elif daddr == c.relay: |
d579a048 | 69 | lt('discard relay') |
a8827d59 | 70 | log_discard(packet, iface, saddr, daddr, 'relay') |
5da7763e | 71 | else: |
d579a048 | 72 | lt('discard no-client') |
a8827d59 | 73 | log_discard(packet, iface, saddr, daddr, 'no-client') |
5da7763e | 74 | |
5da7763e | 75 | #---------- client ---------- |
c4b6d990 | 76 | |
ec88b1f1 | 77 | class Client(): |
8d374606 | 78 | def __init__(self, ip, cc): |
ec88b1f1 IJ |
79 | # instance data members |
80 | self._ip = ip | |
8d374606 | 81 | self.cc = cc |
0ac316c8 | 82 | self._rq = collections.deque() # requests |
74934d63 | 83 | self._pq = PacketQueue(str(ip), self.cc.max_queue_time) |
88487243 | 84 | |
8d374606 | 85 | if ip not in c.vnetwork: |
c7f134ce | 86 | raise ValueError('client %s not in vnetwork' % ip) |
88487243 | 87 | |
88487243 IJ |
88 | if ip in clients: |
89 | raise ValueError('multiple client cfg sections for %s' % ip) | |
90 | clients[ip] = self | |
91 | ||
8c3b6620 IJ |
92 | self._log(DBG.INIT, 'new') |
93 | ||
b68c0739 IJ |
94 | def _log(self, dflag, msg, **kwargs): |
95 | log_debug(dflag, ('client %s: ' % self._ip)+msg, **kwargs) | |
d579a048 | 96 | |
88487243 | 97 | def process_arriving_data(self, d): |
e8fcf3b7 | 98 | self._log(DBG.FLOW, "req data (enc'd)", d=d) |
8718b02c | 99 | if not len(d): return |
88487243 IJ |
100 | for packet in slip.decode(d): |
101 | (saddr, daddr) = packet_addrs(packet) | |
102 | if saddr != self._ip: | |
103 | raise ValueError('wrong source address %s' % saddr) | |
a8827d59 | 104 | route(packet, self._ip, saddr, daddr) |
88487243 IJ |
105 | |
106 | def _req_cancel(self, request): | |
8718b02c | 107 | self._log(DBG.HTTP_CTRL, 'cancel', idof=request) |
88487243 IJ |
108 | request.finish() |
109 | ||
110 | def _req_error(self, err, request): | |
8718b02c | 111 | self._log(DBG.HTTP_CTRL, 'error %s' % err, idof=request) |
88487243 IJ |
112 | self._req_cancel(request) |
113 | ||
114 | def queue_outbound(self, packet): | |
115 | self._pq.append(packet) | |
ca732796 | 116 | self._check_outbound() |
88487243 | 117 | |
7432045d IJ |
118 | def _req_fin(self, dummy, request, cl): |
119 | self._log(DBG.HTTP_CTRL, '_req_fin ' + repr(dummy), idof=request) | |
120 | try: cl.cancel() | |
121 | except twisted.internet.error.AlreadyCalled: pass | |
122 | ||
d579a048 | 123 | def new_request(self, request): |
88487243 | 124 | request.setHeader('Content-Type','application/octet-stream') |
74934d63 | 125 | cl = reactor.callLater(self.cc.http_timeout, self._req_cancel, request) |
7432045d IJ |
126 | nf = request.notifyFinish() |
127 | nf.addErrback(self._req_error, request) | |
128 | nf.addCallback(self._req_fin, request, cl) | |
88487243 IJ |
129 | self._rq.append(request) |
130 | self._check_outbound() | |
131 | ||
3d003cdd IJ |
132 | def _req_write(self, req, d): |
133 | self._log(DBG.HTTP, 'req_write ', idof=req, d=d) | |
134 | req.write(d) | |
135 | ||
88487243 | 136 | def _check_outbound(self): |
8718b02c | 137 | log_debug(DBG.HTTP_CTRL, 'CHKO') |
88487243 IJ |
138 | while True: |
139 | try: request = self._rq[0] | |
140 | except IndexError: request = None | |
141 | if request and request.finished: | |
8c3b6620 | 142 | self._log(DBG.HTTP_CTRL, 'CHKO req finished, discard', idof=request) |
88487243 IJ |
143 | self._rq.popleft() |
144 | continue | |
145 | ||
146 | if not self._pq.nonempty(): | |
147 | # no packets, oh well | |
8c3b6620 | 148 | self._log(DBG.HTTP_CTRL, 'CHKO no packets, OUT-DONE', idof=request) |
d579a048 | 149 | break |
88487243 IJ |
150 | |
151 | if request is None: | |
152 | # no request | |
8c3b6620 | 153 | self._log(DBG.HTTP_CTRL, 'CHKO no request, OUT-DONE', idof=request) |
88487243 IJ |
154 | break |
155 | ||
8c3b6620 | 156 | self._log(DBG.HTTP_CTRL, 'CHKO processing', idof=request) |
88487243 | 157 | # request, and also some non-expired packets |
7b07f0b5 | 158 | self._pq.process((lambda: request.sentLength), |
3d003cdd | 159 | (lambda d: self._req_write(request, d)), |
74934d63 | 160 | self.cc.max_batch_down) |
0ac316c8 | 161 | |
88487243 | 162 | assert(request.sentLength) |
84e763c7 | 163 | self._rq.popleft() |
88487243 | 164 | request.finish() |
8c3b6620 | 165 | self._log(DBG.HTTP, 'complete', idof=request) |
88487243 | 166 | # round again, looking for more to do |
0ac316c8 | 167 | |
74934d63 | 168 | while len(self._rq) > self.cc.target_requests_outstanding: |
88487243 | 169 | request = self._rq.popleft() |
8c3b6620 | 170 | self._log(DBG.HTTP, 'CHKO above target, returning empty', idof=request) |
88487243 | 171 | request.finish() |
650a3251 | 172 | |
d579a048 | 173 | def process_request(request, desca): |
a4e03162 | 174 | # find client, update config, etc. |
5dd3275b | 175 | metadata = request.args[b'm'][0] |
00192d6a | 176 | metadata = metadata.split(b'\r\n') |
ba5630fd IJ |
177 | (ci_s, pw, tro, cto) = metadata[0:4] |
178 | desca['m[0,2:3]'] = [ci_s, tro, cto] | |
a9a369c7 | 179 | ci_s = ci_s.decode('utf-8') |
ba5630fd IJ |
180 | tro = int(tro); desca['tro']= tro |
181 | cto = int(cto); desca['cto']= cto | |
a4e03162 | 182 | ci = ipaddr(ci_s) |
d579a048 | 183 | desca['ci'] = ci |
a4e03162 | 184 | cl = clients[ci] |
74934d63 | 185 | if pw != cl.cc.password: raise ValueError('bad password') |
b68c0739 | 186 | desca['pwok']=True |
1672ded0 | 187 | |
74934d63 IJ |
188 | if tro != cl.cc.target_requests_outstanding: |
189 | raise ValueError('tro must be %d' % cl.cc.target_requests_outstanding) | |
5da7763e | 190 | |
74934d63 IJ |
191 | if cto < cl.cc.http_timeout: |
192 | raise ValueError('cto must be >= %d' % cl.cc.http_timeout) | |
ba5630fd | 193 | |
d579a048 | 194 | try: |
e8fcf3b7 | 195 | d = request.args[b'd'][0] |
d579a048 | 196 | desca['d'] = d |
19f5f9b5 IJ |
197 | desca['dlen'] = len(d) |
198 | except KeyError: | |
199 | d = b'' | |
200 | desca['dlen'] = None | |
201 | ||
202 | log_http(desca, 'processing', idof=id(request), d=d) | |
5da7763e | 203 | |
6f387df3 IJ |
204 | d = mime_translate(d) |
205 | ||
a4e03162 IJ |
206 | cl.process_arriving_data(d) |
207 | cl.new_request(request) | |
5da7763e | 208 | |
19f5f9b5 | 209 | def log_http(desca, msg, **kwargs): |
8c3b6620 | 210 | try: |
19f5f9b5 | 211 | kwargs['d'] = desca['d'] |
8c3b6620 IJ |
212 | del desca['d'] |
213 | except KeyError: | |
19f5f9b5 IJ |
214 | pass |
215 | log_debug(DBG.HTTP, msg + repr(desca), **kwargs) | |
8c3b6620 | 216 | |
eb113b2c IJ |
217 | class NotStupidResource(twisted.web.resource.Resource): |
218 | # why this is not the default is a mystery! | |
219 | def getChild(self, name, request): | |
220 | if name == b'': return self | |
221 | else: return twisted.web.resource.Resource.getChild(name, request) | |
222 | ||
223 | class IphttpResource(NotStupidResource): | |
a4e03162 | 224 | def render_POST(self, request): |
297b3ebf IJ |
225 | log_debug(DBG.HTTP_FULL, |
226 | 'req recv: ' + repr(request) + ' ' + repr(request.args), | |
227 | idof=id(request)) | |
d579a048 IJ |
228 | desca = {'d': None} |
229 | try: process_request(request, desca) | |
0d10f35f | 230 | except Exception as e: |
68afd97b | 231 | emsg = traceback.format_exc() |
6f387df3 | 232 | log_http(desca, 'RETURNING EXCEPTION ' + emsg) |
0d10f35f IJ |
233 | request.setHeader('Content-Type','text/plain; charset="utf-8"') |
234 | request.setResponseCode(400) | |
a9a369c7 | 235 | return (emsg + ' # ' + repr(desca) + '\r\n').encode('utf-8') |
19f5f9b5 | 236 | log_debug(DBG.HTTP_CTRL, '...', idof=id(request)) |
d579a048 | 237 | return NOT_DONE_YET |
84f2d011 | 238 | |
b80a8f5c IJ |
239 | # instantiator should set |
240 | # self.hippotat_sources = (source_names[0], source_names[1]) | |
241 | def __init__(self): | |
242 | self.hippotat_sources = [None, None] | |
243 | super().__init__() | |
244 | ||
8e279651 | 245 | def render_GET(self, request): |
d579a048 | 246 | log_debug(DBG.HTTP, 'GET request') |
3c506501 IJ |
247 | s = '<html><body>hippotat\n' |
248 | (s0,s1) = self.hippotat_sources | |
249 | if s0: | |
250 | s += '<p><a href="%s">source</a>\n' % s0 | |
251 | if self.hippotat_sources[1]: | |
252 | s += ('(and that of dependency <a href="%s">packages</a>)\n' % s1) | |
253 | s += 'available' | |
254 | else: | |
255 | s += 'TESTING' | |
256 | s += '</body></html>' | |
257 | return s.encode('utf-8') | |
99e6411f | 258 | |
5da7763e IJ |
259 | def start_http(): |
260 | resource = IphttpResource() | |
b11c6e7a | 261 | site = twisted.web.server.Site(resource) |
a7d05900 | 262 | |
88487243 IJ |
263 | for sa in c.saddrs: |
264 | ep = sa.make_endpoint() | |
b11c6e7a | 265 | crash_on_defer(ep.listen(site)) |
d579a048 | 266 | log_debug(DBG.INIT, 'listening on %s' % sa) |
a7d05900 IJ |
267 | |
268 | td = tempfile.mkdtemp() | |
269 | ||
270 | def cleanup(): | |
271 | try: shutil.rmtree(td) | |
272 | except FileNotFoundError: pass | |
273 | ||
274 | cleanups.append(cleanup) | |
275 | ||
276 | ssp = SourceShipmentPreparer(td) | |
277 | ssp.logger = partial(log_debug, DBG.OWNSOURCE) | |
ff0fc3fa | 278 | if DBG.OWNSOURCE in debug_set: ssp.stream_debug = sys.stdout |
3c506501 IJ |
279 | ssp.download_packages = opts.ownsource >= 2 |
280 | if opts.ownsource >= 1: ssp.generate() | |
a7d05900 | 281 | |
b80a8f5c IJ |
282 | for ix in (0,1): |
283 | bn = ssp.output_names[ix] | |
284 | op = ssp.output_paths[ix] | |
3c506501 | 285 | if op is None: continue |
b80a8f5c IJ |
286 | resource.hippotat_sources[ix] = bn |
287 | subresource =twisted.web.static.File(op) | |
288 | resource.putChild(bn.encode('utf-8'), subresource) | |
a7d05900 | 289 | |
1e43fae0 | 290 | reactor.callLater(0.1, (lambda: log.info('hippotatd started', dflag=False))) |
5da7763e IJ |
291 | |
292 | #---------- config and setup ---------- | |
4a780703 | 293 | |
1cc6968f IJ |
294 | def process_cfg(_opts, putative_servers, putative_clients): |
295 | global opts | |
296 | opts = _opts | |
297 | ||
8d374606 | 298 | global c |
c7fb640e IJ |
299 | c = ConfigResults() |
300 | c.server = cfg.get('SERVER','server') | |
301 | ||
8d374606 IJ |
302 | cfg_process_common(c, c.server) |
303 | cfg_process_saddrs(c, c.server) | |
304 | cfg_process_vnetwork(c, c.server) | |
305 | cfg_process_vaddr(c, c.server) | |
c7fb640e IJ |
306 | |
307 | for (ci,cs) in putative_clients.items(): | |
308 | cc = ConfigResults() | |
74934d63 | 309 | sections = cfg_process_client_common(cc,c.server,cs,ci) |
c7fb640e IJ |
310 | if not sections: continue |
311 | cfg_process_client_limited(cc,c.server,sections, 'max_batch_down') | |
312 | cfg_process_client_limited(cc,c.server,sections, 'max_queue_time') | |
74934d63 | 313 | Client(ci, cc) |
e75e9c17 IJ |
314 | |
315 | try: | |
74934d63 | 316 | c.vrelay = cfg.get(c.server, 'vrelay') |
e2d41dc1 | 317 | except NoOptionError: |
8d374606 IJ |
318 | for search in c.vnetwork.hosts(): |
319 | if search == c.vaddr: continue | |
74934d63 | 320 | c.vrelay = search |
e75e9c17 | 321 | break |
3fba9787 | 322 | |
8d374606 | 323 | cfg_process_ipif(c, |
c7fb640e IJ |
324 | [c.server, 'DEFAULT'], |
325 | (('local','vaddr'), | |
326 | ('peer', 'vrelay'), | |
327 | ('rnets','vnetwork'))) | |
5bae5ba3 | 328 | |
4a780703 IJ |
329 | def catch_termination(): |
330 | def run_cleanups(): | |
331 | for cleanup in cleanups: | |
332 | cleanup() | |
333 | ||
334 | atexit.register(run_cleanups) | |
335 | ||
336 | def signal_handler(name, sig, *args): | |
337 | signal.signal(sig, signal.SIG_DFL) | |
338 | print('exiting due to %s' % name, file=sys.stderr) | |
339 | run_cleanups() | |
340 | os.kill(os.getpid(), sig) | |
341 | raise RuntimeError('did not die due to signal %s !' % name) | |
342 | ||
343 | for sig in (signal.SIGINT, signal.SIGTERM): | |
344 | signal.signal(sig, partial(signal_handler, sig.name)) | |
345 | ||
ec2c9312 IJ |
346 | def daemonise(): |
347 | global syslogfacility | |
348 | if opts.daemon and opts.syslogfacility is None: | |
349 | opts.syslogfacility = 'daemon' | |
350 | ||
351 | if opts.syslogfacility is not None: | |
352 | facilnum = syslog.__dict__['LOG_' + opts.syslogfacility.upper()] | |
353 | syslog.openlog('hippotatd', | |
354 | facility=facilnum, | |
355 | logoption=syslog.LOG_PID) | |
356 | def emit(event): | |
357 | m = twisted.logger.formatEvent(event) | |
358 | #print(repr(event), m, file=org_stderr) | |
359 | level = event.get('log_level') | |
360 | if event.get('dflag',None) is not None: sl = syslog.LOG_DEBUG | |
361 | elif level == LogLevel.critical : sl = syslog.LOG_CRIT | |
362 | elif level == LogLevel.error : sl = syslog.LOG_ERR | |
363 | elif level == LogLevel.warn : sl = syslog.LOG_WARNING | |
364 | else : sl = syslog.LOG_INFO | |
365 | syslog.syslog(sl,m) | |
366 | glp = twisted.logger.globalLogPublisher | |
367 | glp.addObserver(emit) | |
368 | log_debug(DBG.INIT, 'starting to log to syslog') | |
369 | ||
d84dbf4b IJ |
370 | #log.crit('daemonic hippotatd crashed', dflag=False) |
371 | if opts.daemon: | |
372 | daemonic_reactor = (twisted.internet.interfaces.IReactorDaemonize | |
373 | .providedBy(reactor)) | |
374 | if daemonic_reactor: reactor.beforeDaemonize() | |
375 | rfd, wfd = os.pipe() | |
376 | childpid = os.fork() | |
377 | if childpid: | |
378 | # we are the parent | |
379 | os.close(wfd) | |
380 | st = os.read(rfd, 1) | |
381 | try: | |
382 | st = st[0] | |
383 | except IndexError: | |
384 | st = 127 | |
385 | log.critical('daemonic hippotatd crashed', dflag=False) | |
386 | os._exit(st) | |
387 | os.close(rfd) | |
388 | os.setsid() | |
389 | grandchildpid = os.fork() | |
390 | if grandchildpid: | |
391 | # we are the intermediate child | |
392 | os._exit(0) | |
393 | ||
394 | logger = subprocess.Popen(['logger','-d', | |
395 | '-t','hippotat(stderr)', | |
396 | '--id=%d' % os.getpid(), | |
397 | '-p',opts.syslogfacility + '.err'], | |
398 | stdin=subprocess.PIPE, | |
399 | stdout=subprocess.DEVNULL, | |
400 | stderr=subprocess.DEVNULL, | |
401 | restore_signals=True) | |
402 | ||
403 | nullfd = os.open('/dev/null', os.O_RDWR) | |
404 | os.dup2(nullfd, 0) | |
405 | os.dup2(nullfd, 1) | |
406 | os.dup2(logger.stdin.fileno(), 2) | |
407 | os.close(nullfd) | |
408 | if daemonic_reactor: reactor.afterDaemonize() | |
409 | log_debug(DBG.INIT, 'daemonised') | |
410 | os.write(wfd, b'\0') | |
411 | os.close(wfd) | |
412 | ||
413 | if opts.syslogfacility is not None: | |
ec2c9312 IJ |
414 | glp.removeObserver(hippotatlib.file_log_observer) |
415 | ||
3c506501 IJ |
416 | optparser.add_option('--ownsource', default=2, |
417 | action='store_const', dest='ownsource', const=2, | |
418 | help='source download fully enabled (default)') | |
419 | ||
420 | optparser.add_option('--ownsource-local', | |
421 | action='store_const', dest='ownsource', const=1, | |
422 | help='source download is local source code only') | |
423 | ||
424 | optparser.add_option('--no-ownsource', | |
425 | action='store_const', dest='ownsource', const=0, | |
426 | help='source download disabled (for testing only)') | |
427 | ||
d84dbf4b IJ |
428 | optparser.add_option('--daemon', |
429 | action='store_true', dest='daemon', default=False, | |
430 | help='daemonize (and log to syslog)') | |
431 | ||
ec2c9312 IJ |
432 | optparser.add_option('--syslog-facility', |
433 | nargs=1, type='string',action='store', | |
434 | metavar='FACILITY', dest='syslogfacility', | |
435 | default=None, | |
436 | help='log to syslog, with specified facility') | |
437 | ||
5510890e | 438 | common_startup(process_cfg) |
4a780703 | 439 | catch_termination() |
87a7c0c7 | 440 | start_http() |
ec2c9312 | 441 | daemonise() |
d84dbf4b | 442 | ipif = start_ipif(c.ipif_command, (lambda p,s,d: route(p,"[ipif]",s,d))) |
ae7c7784 | 443 | common_run() |