Commit | Line | Data |
---|---|---|
c55f394e | 1 | #!/usr/bin/python3 |
6b926141 IJ |
2 | # |
3 | # Hippotat - Asinine IP Over HTTP program | |
4 | # ./hippotat - client main program | |
5 | # | |
6 | # Copyright 2017 Ian Jackson | |
7 | # | |
f85d143f | 8 | # GPLv3+ |
6b926141 | 9 | # |
f85d143f IJ |
10 | # This program is free software: you can redistribute it and/or modify |
11 | # it under the terms of the GNU General Public License as published by | |
12 | # the Free Software Foundation, either version 3 of the License, or | |
13 | # (at your option) any later version. | |
6b926141 | 14 | # |
f85d143f IJ |
15 | # This program is distributed in the hope that it will be useful, |
16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
18 | # GNU General Public License for more details. | |
19 | # | |
20 | # You should have received a copy of the GNU General Public License | |
21 | # along with this program, in the file GPLv3. If not, | |
22 | # see <http://www.gnu.org/licenses/>. | |
6b926141 | 23 | |
1d33eef3 | 24 | #@ import sys; sys.path.append('@PYBUILD_INSTALL_DIR@') |
5a37bac8 | 25 | from hippotatlib import * |
c55f394e | 26 | |
c0c90673 IJ |
27 | import twisted.web |
28 | import twisted.web.client | |
29 | ||
dd6665ee IJ |
30 | import io |
31 | ||
0accf0d3 | 32 | class GeneralResponseConsumer(twisted.internet.protocol.Protocol): |
dbf9b0e5 | 33 | def __init__(self, cl, req, resp, desc): |
c7fb640e | 34 | self._cl = cl |
8b62cd2c | 35 | self._req = req |
d4161704 | 36 | self._resp = resp |
0accf0d3 | 37 | self._desc = desc |
14c6d55c IJ |
38 | |
39 | def _log(self, dflag, msg, **kwargs): | |
74934d63 | 40 | self._cl.log(dflag, '%s: %s' % (self._desc, msg), idof=self._req, **kwargs) |
0accf0d3 IJ |
41 | |
42 | def connectionMade(self): | |
43 | self._log(DBG.HTTP_CTRL, 'connectionMade') | |
44 | ||
216519e3 | 45 | def connectionLostOK(self, reason): |
916021af IJ |
46 | return (reason.check(twisted.web.client.ResponseDone) or |
47 | reason.check(twisted.web.client.PotentialDataLoss)) | |
48 | # twisted.web.client.PotentialDataLoss is an entirely daft | |
49 | # exception. It will occur every time if the origin server does | |
50 | # not provide a Content-Length. (hippotatd does, of course, but | |
51 | # the HTTP transaction might be proxied.) | |
216519e3 | 52 | |
0accf0d3 | 53 | class ResponseConsumer(GeneralResponseConsumer): |
4224dc19 | 54 | def __init__(self, cl, req, resp): |
dbf9b0e5 | 55 | super().__init__(cl, req, resp, 'RC') |
0accf0d3 | 56 | ssddesc = '[%s] %s' % (id(req), self._desc) |
909e0ff3 | 57 | self._ssd = SlipStreamDecoder(ssddesc, partial(queue_inbound, cl.ipif)) |
0accf0d3 | 58 | self._log(DBG.HTTP_CTRL, '__init__') |
bd9e77fb | 59 | |
62b51bcf | 60 | def dataReceived(self, data): |
380ed56c | 61 | self._log(DBG.HTTP, 'dataReceived', d=data) |
9b65cdd4 | 62 | try: |
02cdcb52 | 63 | self._ssd.inputdata(data) |
9b65cdd4 | 64 | except Exception as e: |
eedc8b30 | 65 | self._handleexception() |
ccd371b3 | 66 | |
62b51bcf | 67 | def connectionLost(self, reason): |
d5008b7c IJ |
68 | reason_msg = 'connectionLost ' + str(reason) |
69 | self._log(DBG.HTTP_CTRL, reason_msg) | |
216519e3 | 70 | if not self.connectionLostOK(reason): |
d5008b7c | 71 | self._latefailure(reason_msg) |
765aba55 IJ |
72 | return |
73 | try: | |
380ed56c | 74 | self._log(DBG.HTTP, 'ResponseDone') |
765aba55 | 75 | self._ssd.flush() |
74934d63 | 76 | self._cl.req_fin(self._req) |
765aba55 | 77 | except Exception as e: |
eedc8b30 | 78 | self._handleexception() |
909e0ff3 | 79 | self._cl.report_running() |
eedc8b30 IJ |
80 | |
81 | def _handleexception(self): | |
0accf0d3 | 82 | self._latefailure(traceback.format_exc()) |
33932420 | 83 | |
0accf0d3 | 84 | def _latefailure(self, reason): |
380ed56c | 85 | self._log(DBG.HTTP_CTRL, '_latefailure ' + str(reason)) |
74934d63 | 86 | self._cl.req_err(self._req, reason) |
bd9e77fb | 87 | |
74934d63 | 88 | class ErrorResponseConsumer(GeneralResponseConsumer): |
c7fb640e | 89 | def __init__(self, cl, req, resp): |
dbf9b0e5 | 90 | super().__init__(cl, req, resp, 'ERROR-RC') |
0accf0d3 | 91 | self._m = b'' |
6e4af0a2 IJ |
92 | try: |
93 | self._phrase = resp.phrase.decode('utf-8') | |
94 | except Exception: | |
95 | self._phrase = repr(resp.phrase) | |
6e4af0a2 IJ |
96 | self._log(DBG.HTTP_CTRL, '__init__ %d %s' % (resp.code, self._phrase)) |
97 | ||
765aba55 IJ |
98 | def dataReceived(self, data): |
99 | self._log(DBG.HTTP_CTRL, 'dataReceived ' + repr(data)) | |
100 | self._m += data | |
101 | ||
6e4af0a2 IJ |
102 | def connectionLost(self, reason): |
103 | try: | |
104 | mbody = self._m.decode('utf-8') | |
105 | except Exception: | |
106 | mbody = repr(self._m) | |
216519e3 | 107 | if not self.connectionLostOK(reason): |
765aba55 | 108 | mbody += ' || ' + str(reason) |
74934d63 | 109 | self._cl.req_err(self._req, |
765aba55 IJ |
110 | "FAILED %d %s | %s" |
111 | % (self._resp.code, self._phrase, mbody)) | |
6e4af0a2 | 112 | |
c7fb640e IJ |
113 | class Client(): |
114 | def __init__(cl, c,ss,cs): | |
115 | cl.c = c | |
116 | cl.outstanding = { } | |
117 | cl.desc = '[%s %s] ' % (ss,cs) | |
909e0ff3 IJ |
118 | cl.running_reported = False |
119 | cl.log_info('setting up') | |
120 | ||
121 | def log_info(cl, msg): | |
122 | log.info(cl.desc + msg, dflag=False) | |
123 | ||
124 | def report_running(cl): | |
125 | if not cl.running_reported: | |
126 | cl.log_info('running OK') | |
127 | cl.running_reported = True | |
c7fb640e IJ |
128 | |
129 | def log(cl, dflag, msg, **kwargs): | |
130 | log_debug(dflag, cl.desc + msg, **kwargs) | |
131 | ||
132 | def log_outstanding(cl): | |
c7f134ce | 133 | cl.log(DBG.CTRL_DUMP, 'OS %s' % cl.outstanding) |
c7fb640e IJ |
134 | |
135 | def start(cl): | |
8d374606 | 136 | cl.queue = PacketQueue('up', cl.c.max_queue_time) |
c7fb640e | 137 | cl.agent = twisted.web.client.Agent( |
8d374606 | 138 | reactor, connectTimeout = cl.c.http_timeout) |
c7fb640e IJ |
139 | |
140 | def outbound(cl, packet, saddr, daddr): | |
141 | #print('OUT ', saddr, daddr, repr(packet)) | |
142 | cl.queue.append(packet) | |
143 | cl.check_outbound() | |
144 | ||
145 | def req_ok(cl, req, resp): | |
146 | cl.log(DBG.HTTP_CTRL, | |
5dd3275b IJ |
147 | 'req_ok %d %s %s' % (resp.code, repr(resp.phrase), str(resp)), |
148 | idof=req) | |
8d374606 | 149 | if resp.code == 200: |
4224dc19 | 150 | rc = ResponseConsumer(cl, req, resp) |
8d374606 IJ |
151 | else: |
152 | rc = ErrorResponseConsumer(cl, req, resp) | |
5dd3275b | 153 | |
8d374606 IJ |
154 | resp.deliverBody(rc) |
155 | # now rc is responsible for calling req_fin | |
7b07f0b5 | 156 | |
c7fb640e IJ |
157 | def req_err(cl, req, err): |
158 | # called when the Deferred fails, or (if it completes), | |
159 | # later, by ResponsConsumer or ErrorResponsConsumer | |
160 | try: | |
161 | cl.log(DBG.HTTP_CTRL, 'req_err ' + str(err), idof=req) | |
1a691ffd | 162 | cl.running_reported = False |
c7fb640e IJ |
163 | if isinstance(err, twisted.python.failure.Failure): |
164 | err = err.getTraceback() | |
7ec61cc0 IJ |
165 | print('%s[%#x] %s' % (cl.desc, id(req), err.strip('\n').replace('\n',' / ')), |
166 | file=sys.stderr) | |
c7f134ce IJ |
167 | if not isinstance(cl.outstanding[req], int): |
168 | raise RuntimeError('[%#x] previously %s' % | |
169 | (id(req), cl.outstanding[req])) | |
c7fb640e IJ |
170 | cl.outstanding[req] = err |
171 | cl.log_outstanding() | |
8d374606 | 172 | reactor.callLater(cl.c.http_retry, partial(cl.req_fin, req)) |
c7fb640e IJ |
173 | except Exception as e: |
174 | crash(traceback.format_exc() + '\n----- handling -----\n' + err) | |
175 | ||
176 | def req_fin(cl, req): | |
177 | del cl.outstanding[req] | |
c7f134ce | 178 | cl.log(DBG.HTTP_CTRL, 'req_fin OS=%d' % len(cl.outstanding), idof=req) |
c7fb640e IJ |
179 | cl.check_outbound() |
180 | ||
181 | def check_outbound(cl): | |
182 | while True: | |
183 | if len(cl.outstanding) >= cl.c.max_outstanding: | |
184 | break | |
185 | ||
c7f134ce IJ |
186 | if (not cl.queue.nonempty() and |
187 | len(cl.outstanding) >= cl.c.target_requests_outstanding): | |
c7fb640e IJ |
188 | break |
189 | ||
190 | d = b'' | |
191 | def moredata(s): nonlocal d; d += s | |
c7f134ce | 192 | cl.queue.process((lambda: len(d)), |
c7fb640e IJ |
193 | moredata, |
194 | cl.c.max_batch_up) | |
195 | ||
196 | d = mime_translate(d) | |
197 | ||
ef041033 IJ |
198 | token = authtoken_make(cl.c.secret) |
199 | ||
c7fb640e IJ |
200 | crlf = b'\r\n' |
201 | lf = b'\n' | |
202 | mime = (b'--b' + crlf + | |
203 | b'Content-Type: text/plain; charset="utf-8"' + crlf + | |
204 | b'Content-Disposition: form-data; name="m"' + crlf + crlf + | |
205 | str(cl.c.client) .encode('ascii') + crlf + | |
ef041033 | 206 | token + crlf + |
c7f134ce IJ |
207 | str(cl.c.target_requests_outstanding) |
208 | .encode('ascii') + crlf + | |
c7fb640e IJ |
209 | str(cl.c.http_timeout) .encode('ascii') + crlf + |
210 | (( | |
211 | b'--b' + crlf + | |
212 | b'Content-Type: application/octet-stream' + crlf + | |
213 | b'Content-Disposition: form-data; name="d"' + crlf + crlf + | |
214 | d + crlf | |
215 | ) if len(d) else b'') + | |
216 | b'--b--' + crlf) | |
217 | ||
218 | #df = open('data.dump.dbg', mode='wb') | |
219 | #df.write(mime) | |
220 | #df.close() | |
221 | # POST -use -c 'multipart/form-data; boundary="b"' http://localhost:8099/ <data.dump.dbg | |
222 | ||
223 | cl.log(DBG.HTTP_FULL, 'requesting: ' + str(mime)) | |
224 | ||
225 | hh = { 'User-Agent': ['hippotat'], | |
226 | 'Content-Type': ['multipart/form-data; boundary="b"'], | |
227 | 'Content-Length': [str(len(mime))] } | |
228 | ||
229 | bytesreader = io.BytesIO(mime) | |
230 | producer = twisted.web.client.FileBodyProducer(bytesreader) | |
231 | ||
c7f134ce | 232 | req = cl.agent.request(b'POST', |
c7fb640e IJ |
233 | cl.c.url, |
234 | twisted.web.client.Headers(hh), | |
235 | producer) | |
236 | ||
237 | cl.outstanding[req] = len(d) | |
238 | cl.log(DBG.HTTP_CTRL, | |
239 | 'request OS=%d' % len(cl.outstanding), | |
240 | idof=req, d=d) | |
241 | req.addTimeout(cl.c.http_timeout, reactor) | |
242 | req.addCallback(partial(cl.req_ok, req)) | |
243 | req.addErrback(partial(cl.req_err, req)) | |
244 | ||
245 | cl.log_outstanding() | |
246 | ||
247 | clients = [ ] | |
248 | ||
1cc6968f | 249 | def process_cfg(_opts, putative_servers, putative_clients): |
c7fb640e IJ |
250 | global clients |
251 | ||
252 | for ss in putative_servers.values(): | |
253 | for (ci,cs) in putative_clients.items(): | |
254 | c = ConfigResults() | |
255 | ||
8d374606 | 256 | sections = cfg_process_client_common(c,ss,cs,ci) |
c7fb640e IJ |
257 | if not sections: continue |
258 | ||
8c771381 IJ |
259 | log_debug_config('processing client [%s %s]' % (ss, cs)) |
260 | ||
c7fb640e IJ |
261 | def srch(getter,key): return cfg_search(getter,key,sections) |
262 | ||
263 | c.http_timeout += srch(cfg.getint, 'http_timeout_grace') | |
264 | c.max_outstanding = srch(cfg.getint, 'max_requests_outstanding') | |
265 | c.max_batch_up = srch(cfg.getint, 'max_batch_up') | |
266 | c.http_retry = srch(cfg.getint, 'http_retry') | |
f754eec4 | 267 | c.max_queue_time = srch(cfg.getint, 'max_queue_time') |
c7fb640e IJ |
268 | c.vroutes = srch(cfg.get, 'vroutes') |
269 | ||
d72f8360 IJ |
270 | try: c.ifname = srch(cfg_get_raw, 'ifname_client') |
271 | except NoOptionError: pass | |
272 | ||
c7fb640e IJ |
273 | try: c.url = srch(cfg.get,'url') |
274 | except NoOptionError: | |
8d374606 | 275 | cfg_process_saddrs(c, ss) |
c7fb640e IJ |
276 | c.url = c.saddrs[0].url() |
277 | ||
8d374606 IJ |
278 | c.client = ci |
279 | ||
280 | cfg_process_vaddr(c,ss) | |
281 | ||
282 | cfg_process_ipif(c, | |
c7fb640e IJ |
283 | sections, |
284 | (('local','client'), | |
285 | ('peer', 'vaddr'), | |
286 | ('rnets','vroutes'))) | |
287 | ||
288 | clients.append(Client(c,ss,cs)) | |
0accf0d3 | 289 | |
5510890e | 290 | common_startup(process_cfg) |
c7fb640e IJ |
291 | |
292 | for cl in clients: | |
293 | cl.start() | |
909e0ff3 | 294 | cl.ipif = start_ipif(cl.c.ipif_command, cl.outbound) |
c7fb640e IJ |
295 | cl.check_outbound() |
296 | ||
034284c3 | 297 | common_run() |