Commit | Line | Data |
---|---|---|
0256fc10 IJ |
1 | # -*- python -*- |
2 | # | |
3 | # Hippotat - Asinine IP Over HTTP program | |
4 | # hippotatlib/ownsource.py - Automatic source code provision (AGPL compliance) | |
5 | # | |
6 | # Copyright 2017 Ian Jackson | |
7 | # | |
f85d143f | 8 | # AGPLv3+ + CAFv2+ |
0256fc10 | 9 | # |
f85d143f IJ |
10 | # This program is free software: you can redistribute it and/or |
11 | # modify it under the terms of the GNU Affero General Public | |
12 | # License as published by the Free Software Foundation, either | |
13 | # version 3 of the License, or (at your option) any later version, | |
14 | # with the "CAF Login Exception" as published by Ian Jackson | |
15 | # (version 2, or at your option any later version) as an Additional | |
16 | # Permission. | |
0256fc10 | 17 | # |
f85d143f IJ |
18 | # This program is distributed in the hope that it will be useful, |
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
21 | # Affero General Public License for more details. | |
22 | # | |
23 | # You should have received a copy of the GNU Affero General Public | |
24 | # License and the CAF Login Exception along with this program, in | |
25 | # the file AGPLv3+CAFv2. If not, email Ian Jackson | |
26 | # <ijackson@chiark.greenend.org.uk>. | |
0256fc10 | 27 | |
4460af45 | 28 | |
b635cd93 | 29 | import os |
4460af45 IJ |
30 | import sys |
31 | import fnmatch | |
b635cd93 IJ |
32 | import stat |
33 | import subprocess | |
e68fca7c | 34 | import tempfile |
a7d05900 IJ |
35 | import shutil |
36 | ||
37 | try: import debian.deb822 | |
38 | except ImportError: pass | |
4460af45 IJ |
39 | |
40 | class SourceShipmentPreparer(): | |
41 | def __init__(s, destdir): | |
42 | # caller may modify, and should read after calling generate() | |
06e61e3b | 43 | s.output_names = ['srcbomb.tar.gz', 'srcpkgsbomb.tar'] |
a7d05900 | 44 | s.output_paths = [None,None] # alternatively caller may read this |
4460af45 | 45 | # defaults, caller can modify after creation |
c2c0da38 | 46 | s.logger = lambda m: print('SourceShipmentPreparer',m) |
4460af45 | 47 | s.src_filter = s.src_filter_glob |
e68fca7c | 48 | s.src_package_globs = ['!/usr/local/*', '/usr*'] |
e98894bc | 49 | s.src_filter_globs = ['!/etc/*'] |
4460af45 | 50 | s.src_likeparent = s.src_likeparent_git |
28861486 | 51 | s.src_direxcludes = s.src_direxcludes_git |
e98894bc | 52 | s.report_from_packages = s.report_from_packages_debian |
4460af45 | 53 | s.cwd = os.getcwd() |
b635cd93 | 54 | s.find_rune_base = "find -type f -perm -004 \! -path '*/tmp/*'" |
28861486 | 55 | s.ignores = ['*~', '*.bak', '*.tmp', '#*#', '__pycache__', |
eed788f8 | 56 | '[0-9][0-9][0-9][0-9]-src.tar'] |
4460af45 | 57 | s.rune_shell = ['/bin/bash', '-ec'] |
b635cd93 | 58 | s.show_pathnames = True |
a7d05900 | 59 | s.download_packages = True |
ff0fc3fa IJ |
60 | s.stream_stderr = sys.stderr |
61 | s.stream_debug = open('/dev/null','w') | |
b635cd93 | 62 | s.rune_cpio = r''' |
4460af45 IJ |
63 | set -o pipefail |
64 | ( | |
65 | %s | |
66 | # ^ by default, is find ... -print0 | |
67 | ) | ( | |
68 | cpio -Hustar -o --quiet -0 -R 1000:1000 || \ | |
69 | cpio -Hustar -o --quiet -0 | |
70 | ) | |
71 | ''' | |
72 | s.rune_portmanteau = r''' | |
a7d05900 IJ |
73 | GZIP=-1 tar zcf - "$@" |
74 | ''' | |
75 | s.rune_portmanteau_uncompressed = r''' | |
76 | tar cf - "$@" | |
4460af45 IJ |
77 | ''' |
78 | s.manifest_name='0000-MANIFEST.txt' | |
79 | # private | |
80 | s._destdir = destdir | |
b635cd93 | 81 | s._outcounter = 0 |
4460af45 | 82 | s._manifest = [] |
2371516a | 83 | s._dirmap = { } |
e98894bc | 84 | s._package_files = { } # map filename => infol |
a7d05900 IJ |
85 | s._packages_path = os.path.join(s._destdir, 'packages') |
86 | s._package_sources = [] | |
4460af45 | 87 | |
e98894bc IJ |
88 | def thing_matches_globs(s, thing, globs): |
89 | for pat in globs: | |
4460af45 IJ |
90 | negate = pat.startswith('!') |
91 | if negate: pat = pat[1:] | |
e98894bc | 92 | if fnmatch.fnmatch(thing, pat): |
4460af45 IJ |
93 | return not negate |
94 | return negate | |
95 | ||
e98894bc | 96 | def src_filter_glob(s, src): # default s.src_filter |
e68fca7c | 97 | return s.thing_matches_globs(src, s.src_filter_globs) |
e98894bc | 98 | |
28861486 IJ |
99 | def src_direxcludes_git(s, d): |
100 | try: | |
101 | excl = open(os.path.join(d, '.gitignore')) | |
102 | except FileNotFoundError: | |
103 | return [] | |
104 | r = [] | |
105 | for l in excl: | |
063bf41e | 106 | l = l.strip() |
28861486 IJ |
107 | if l.startswith('#'): next |
108 | if not len(l): next | |
063bf41e | 109 | r.append(l) |
28861486 IJ |
110 | return r |
111 | ||
4460af45 IJ |
112 | def src_likeparent_git(s, src): |
113 | try: | |
b635cd93 | 114 | os.stat(os.path.join(src, '.git/.')) |
4460af45 IJ |
115 | except FileNotFoundError: |
116 | return False | |
117 | else: | |
118 | return True | |
119 | ||
120 | def src_parentfinder(s, src, infol): # callers may monkey-patch away | |
121 | for deref in (False,True): | |
122 | xinfo = [] | |
123 | ||
124 | search = src | |
125 | if deref: | |
126 | search = os.path.realpath(search) | |
127 | ||
128 | def ascend(): | |
b635cd93 | 129 | nonlocal search |
4460af45 IJ |
130 | xinfo.append(os.path.basename(search)) |
131 | search = os.path.dirname(search) | |
132 | ||
133 | try: | |
b635cd93 | 134 | stab = os.lstat(search) |
4460af45 IJ |
135 | except FileNotFoundError: |
136 | return | |
137 | if stat.S_ISREG(stab.st_mode): | |
138 | ascend() | |
139 | ||
140 | while not os.path.ismount(search): | |
141 | if s.src_likeparent(search): | |
142 | xinfo.reverse() | |
b635cd93 | 143 | if len(xinfo): infol.append('want=' + os.path.join(*xinfo)) |
4460af45 IJ |
144 | return search |
145 | ||
146 | ascend() | |
147 | ||
148 | # no .git found anywhere | |
b635cd93 | 149 | return src |
4460af45 | 150 | |
e68fca7c | 151 | def path_prenormaliser(s, d, infol): # callers may monkey-patch away |
4460af45 IJ |
152 | return os.path.join(s.cwd, os.path.abspath(d)) |
153 | ||
e98894bc | 154 | def srcdir_find_rune(s, d): |
b635cd93 | 155 | script = s.find_rune_base |
a7d05900 | 156 | ignores = s.ignores + s.output_names + [s.manifest_name] |
28861486 IJ |
157 | ignores += s.src_direxcludes(d) |
158 | for excl in ignores: | |
4460af45 | 159 | assert("'" not in excl) |
28861486 IJ |
160 | script += r" \! -name '%s'" % excl |
161 | script += r" \! -path '*/%s/*'" % excl | |
4460af45 | 162 | script += ' -print0' |
b635cd93 | 163 | return script |
4460af45 | 164 | |
2371516a | 165 | def manifest_append(s, name, infol): |
a53aba4d | 166 | s._manifest.append({ 'file':name, 'info':' '.join(infol) }) |
2371516a | 167 | |
e68fca7c IJ |
168 | def manifest_append_absentfile(s, name, infol): |
169 | s._manifest.append({ 'file_print':name, 'info':' '.join(infol) }) | |
170 | ||
4460af45 | 171 | def new_output_name(s, nametail, infol): |
b635cd93 IJ |
172 | s._outcounter += 1 |
173 | name = '%04d-%s' % (s._outcounter, nametail) | |
2371516a | 174 | s.manifest_append(name, infol) |
4460af45 IJ |
175 | return name |
176 | ||
177 | def open_output_fh(s, name, mode): | |
178 | return open(os.path.join(s._destdir, name), mode) | |
179 | ||
e98894bc | 180 | def src_dir(s, d, infol): |
2371516a IJ |
181 | try: name = s._dirmap[d] |
182 | except KeyError: pass | |
183 | else: | |
184 | s.manifest_append(name, infol) | |
185 | return | |
186 | ||
b635cd93 | 187 | if s.show_pathnames: infol.append(d) |
e98894bc | 188 | find_rune = s.srcdir_find_rune(d) |
4460af45 | 189 | total_rune = s.rune_cpio % find_rune |
2371516a | 190 | |
eed788f8 | 191 | name = s.new_output_name('src.tar', infol) |
2371516a IJ |
192 | s._dirmap[d] = name |
193 | fh = s.open_output_fh(name, 'wb') | |
194 | ||
c2c0da38 IJ |
195 | s.logger('packing up into %s: %s (because %s)' % |
196 | (name, d, ' '.join(infol))) | |
197 | ||
4460af45 | 198 | subprocess.run(s.rune_shell + [total_rune], |
b635cd93 | 199 | cwd=d, |
4460af45 IJ |
200 | stdin=subprocess.DEVNULL, |
201 | stdout=fh, | |
b635cd93 IJ |
202 | restore_signals=True, |
203 | check=True) | |
4460af45 IJ |
204 | fh.close() |
205 | ||
e98894bc | 206 | def src_indir(s, d, infol): |
e68fca7c | 207 | d = s.path_prenormaliser(d, infol) |
4460af45 | 208 | if not s.src_filter(d): return |
e98894bc | 209 | |
4460af45 | 210 | d = s.src_parentfinder(d, infol) |
e68fca7c IJ |
211 | if d is None: return |
212 | s.src_dir(d, infol) | |
e98894bc IJ |
213 | |
214 | def report_from_packages_debian(s, files): | |
e68fca7c | 215 | dpkg_S_in = tempfile.TemporaryFile(mode='w+') |
e98894bc IJ |
216 | for (file, infols) in files.items(): |
217 | assert('\n' not in file) | |
218 | dpkg_S_in.write(file) | |
219 | dpkg_S_in.write('\0') | |
220 | dpkg_S_in.seek(0) | |
221 | cmdl = ['xargs','-0r','dpkg','-S','--'] | |
222 | dpkg_S = subprocess.Popen(cmdl, | |
a53aba4d IJ |
223 | cwd='/', |
224 | stdin=dpkg_S_in, | |
225 | stdout=subprocess.PIPE, | |
226 | stderr=sys.stderr, | |
227 | close_fds=False) | |
e68fca7c | 228 | dpkg_show_in = tempfile.TemporaryFile(mode='w+') |
e98894bc | 229 | pkginfos = { } |
e68fca7c IJ |
230 | for l in dpkg_S.stdout: |
231 | l = l.strip(b'\n').decode('utf-8') | |
e98894bc | 232 | (pkgs, fname) = l.split(': ',1) |
e68fca7c IJ |
233 | pks = pkgs.split(', ') |
234 | for pk in pks: | |
235 | pkginfos.setdefault(pk,{'files':[]})['files'].append(fname) | |
236 | print(pk, file=dpkg_show_in) | |
a53aba4d IJ |
237 | assert(dpkg_S.wait() == 0) |
238 | dpkg_show_in.seek(0) | |
239 | cmdl = ['xargs','-r','dpkg-query', | |
a7d05900 | 240 | r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n', |
a53aba4d IJ |
241 | '--show','--'] |
242 | dpkg_show = subprocess.Popen(cmdl, | |
243 | cwd='/', | |
244 | stdin=dpkg_show_in, | |
245 | stdout=subprocess.PIPE, | |
246 | stderr=sys.stderr, | |
247 | close_fds=False) | |
248 | for l in dpkg_show.stdout: | |
e68fca7c | 249 | l = l.strip(b'\n').decode('utf-8') |
a7d05900 | 250 | (pk,p,a,v,sp,sv,suv) = l.split('\t') |
e68fca7c IJ |
251 | pkginfos[pk]['binary'] = p |
252 | pkginfos[pk]['arch'] = a | |
253 | pkginfos[pk]['version'] = v | |
254 | pkginfos[pk]['source'] = sp | |
255 | pkginfos[pk]['sourceversion'] = sv | |
a7d05900 | 256 | pkginfos[pk]['sourceupstreamversion'] = sv |
a53aba4d | 257 | assert(dpkg_show.wait() == 0) |
e68fca7c IJ |
258 | for pk in sorted(pkginfos.keys()): |
259 | pi = pkginfos[pk] | |
260 | debfname = '%s_%s_%s.deb' % (pi['binary'], pi['version'], pi['arch']) | |
a53aba4d | 261 | dscfname = '%s_%s.dsc' % (pi['source'], pi['sourceversion']) |
e68fca7c | 262 | s.manifest_append_absentfile(dscfname, [debfname]) |
c2c0da38 IJ |
263 | s.logger('mentioning %s and %s because %s' % |
264 | (dscfname, debfname, pi['files'][0])) | |
e68fca7c IJ |
265 | for fname in pi['files']: |
266 | infol = files[fname] | |
56ffddf8 | 267 | if s.show_pathnames: infol = infol + ['loaded='+fname] |
e68fca7c | 268 | s.manifest_append_absentfile(' \t' + debfname, infol) |
e98894bc | 269 | |
a7d05900 IJ |
270 | if s.download_packages: |
271 | try: os.mkdir(s._packages_path) | |
272 | except FileExistsError: pass | |
273 | ||
274 | cmdl = ['apt-get','--download-only','source', | |
275 | '%s=%s' % (pi['source'], pi['sourceversion'])] | |
276 | subprocess.run(cmdl, | |
277 | cwd=s._packages_path, | |
278 | stdin=subprocess.DEVNULL, | |
ff0fc3fa IJ |
279 | stdout=s.stream_debug, |
280 | stderr=s.stream_stderr, | |
a7d05900 IJ |
281 | restore_signals=True, |
282 | check=True) | |
283 | ||
284 | s._package_sources.append(dscfname) | |
285 | dsc = debian.deb822.Dsc(open(s._packages_path + '/' + dscfname)) | |
286 | for indsc in dsc['Files']: | |
287 | s._package_sources.append(indsc['name']) | |
288 | ||
e98894bc IJ |
289 | def thing_ought_packaged(s, fname): |
290 | return s.thing_matches_globs(fname, s.src_package_globs) | |
291 | ||
e68fca7c IJ |
292 | def src_file_packaged(s, fname, infol): |
293 | s._package_files.setdefault(fname,[]).extend(infol) | |
e98894bc IJ |
294 | |
295 | def src_file(s, fname, infol): | |
296 | def fngens(): | |
e68fca7c IJ |
297 | yield (infol, fname) |
298 | infol_copy = infol.copy() | |
299 | yield (infol_copy, s.path_prenormaliser(fname, infol_copy)) | |
300 | yield (infol, os.path.realpath(fname)) | |
301 | ||
302 | for (tinfol, tfname) in fngens(): | |
303 | if s.thing_ought_packaged(tfname): | |
304 | s.src_file_packaged(tfname, tinfol) | |
e98894bc | 305 | return |
4460af45 | 306 | |
e98894bc IJ |
307 | s.src_indir(fname, infol) |
308 | ||
309 | def src_argv0(s, program, infol): | |
e68fca7c | 310 | s.src_file(program, infol) |
e98894bc IJ |
311 | |
312 | def src_syspath(s, fname, infol): | |
56ffddf8 | 313 | if s.thing_ought_packaged(fname): return |
e98894bc IJ |
314 | s.src_indir(fname, infol) |
315 | ||
316 | def src_module(s, m, infol): | |
317 | try: fname = m.__file__ | |
16374080 | 318 | except AttributeError: return |
56ffddf8 | 319 | infol.append('module='+m.__name__) |
16374080 | 320 | |
e98894bc IJ |
321 | if s.thing_ought_packaged(fname): |
322 | s.src_file_packaged(fname, infol) | |
323 | else: | |
e68fca7c | 324 | s.src_indir(fname, infol) |
e98894bc IJ |
325 | |
326 | def srcs_allitems(s, dirs=sys.path): | |
c2c0da38 | 327 | s.logger('allitems') |
e98894bc | 328 | s.src_argv0(sys.argv[0], ['argv[0]']) |
4460af45 | 329 | for d in sys.path: |
e98894bc | 330 | s.src_syspath(d, ['sys.path']) |
16374080 | 331 | for m in sys.modules.values(): |
e98894bc | 332 | s.src_module(m, ['sys.modules']) |
e68fca7c | 333 | s.report_from_packages(s._package_files) |
c2c0da38 | 334 | s.logger('allitems done') |
4460af45 | 335 | |
a7d05900 IJ |
336 | def _mk_portmanteau(s, ix, rune, cwd, files): |
337 | output_name = s.output_names[ix] | |
338 | s.logger('making portmanteau %s' % output_name) | |
339 | output_path = os.path.join(s._destdir, output_name) | |
340 | subprocess.run(s.rune_shell + [ rune, 'x' ] + files, | |
341 | cwd=cwd, | |
342 | stdin=subprocess.DEVNULL, | |
343 | stdout=open(output_path, 'wb'), | |
344 | restore_signals=True, | |
345 | check=True) | |
346 | s.output_paths[ix] = output_path | |
347 | ||
348 | def mk_inner_portmanteau(s): | |
349 | outputs = [s.manifest_name] | |
9693ff43 | 350 | outputs_done = { } |
b635cd93 | 351 | mfh = s.open_output_fh(s.manifest_name,'w') |
a53aba4d IJ |
352 | for me in s._manifest: |
353 | try: fname = me['file'] | |
354 | except KeyError: fname = me.get('file_print','') | |
9693ff43 IJ |
355 | else: |
356 | try: outputs_done[fname] | |
357 | except KeyError: | |
358 | outputs.append(fname) | |
359 | outputs_done[fname] = 1 | |
a53aba4d | 360 | print('%s\t%s' % (fname, me['info']), file=mfh) |
4460af45 | 361 | mfh.close() |
a7d05900 IJ |
362 | |
363 | s._mk_portmanteau(0, s.rune_portmanteau, | |
364 | s._destdir, outputs) | |
365 | ||
366 | def mk_packages_portmanteau(s): | |
94da3ccb | 367 | if not s.download_packages: return |
a7d05900 IJ |
368 | s._mk_portmanteau(1, s.rune_portmanteau_uncompressed, |
369 | s._packages_path, s._package_sources) | |
4460af45 IJ |
370 | |
371 | def generate(s): | |
e98894bc | 372 | s.srcs_allitems() |
a7d05900 IJ |
373 | s.mk_inner_portmanteau() |
374 | s.mk_packages_portmanteau() | |
375 | s.logger('portmanteau ready in %s %s' % tuple(s.output_paths)) |