ownsrc debugging
[hippotat] / hippotatlib / ownsource.py
CommitLineData
0256fc10
IJ
1# -*- python -*-
2#
3# Hippotat - Asinine IP Over HTTP program
4# hippotatlib/ownsource.py - Automatic source code provision (AGPL compliance)
5#
6# Copyright 2017 Ian Jackson
7#
f85d143f 8# AGPLv3+ + CAFv2+
0256fc10 9#
f85d143f
IJ
10# This program is free software: you can redistribute it and/or
11# modify it under the terms of the GNU Affero General Public
12# License as published by the Free Software Foundation, either
13# version 3 of the License, or (at your option) any later version,
14# with the "CAF Login Exception" as published by Ian Jackson
15# (version 2, or at your option any later version) as an Additional
16# Permission.
0256fc10 17#
f85d143f
IJ
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21# Affero General Public License for more details.
22#
23# You should have received a copy of the GNU Affero General Public
24# License and the CAF Login Exception along with this program, in
25# the file AGPLv3+CAFv2. If not, email Ian Jackson
26# <ijackson@chiark.greenend.org.uk>.
0256fc10 27
4460af45 28
b635cd93 29import os
4460af45
IJ
30import sys
31import fnmatch
b635cd93
IJ
32import stat
33import subprocess
e68fca7c 34import tempfile
a7d05900
IJ
35import shutil
36
37try: import debian.deb822
38except ImportError: pass
4460af45
IJ
39
40class SourceShipmentPreparer():
41 def __init__(s, destdir):
42 # caller may modify, and should read after calling generate()
a7d05900
IJ
43 s.output_names = ['srcbomb.tar.gz', 'fullsrcbomb.tar']
44 s.output_paths = [None,None] # alternatively caller may read this
4460af45 45 # defaults, caller can modify after creation
c2c0da38 46 s.logger = lambda m: print('SourceShipmentPreparer',m)
4460af45 47 s.src_filter = s.src_filter_glob
e68fca7c 48 s.src_package_globs = ['!/usr/local/*', '/usr*']
e98894bc 49 s.src_filter_globs = ['!/etc/*']
4460af45 50 s.src_likeparent = s.src_likeparent_git
28861486 51 s.src_direxcludes = s.src_direxcludes_git
e98894bc 52 s.report_from_packages = s.report_from_packages_debian
4460af45 53 s.cwd = os.getcwd()
b635cd93 54 s.find_rune_base = "find -type f -perm -004 \! -path '*/tmp/*'"
28861486 55 s.ignores = ['*~', '*.bak', '*.tmp', '#*#', '__pycache__',
eed788f8 56 '[0-9][0-9][0-9][0-9]-src.tar']
4460af45 57 s.rune_shell = ['/bin/bash', '-ec']
b635cd93 58 s.show_pathnames = True
a7d05900 59 s.download_packages = True
ff0fc3fa
IJ
60 s.stream_stderr = sys.stderr
61 s.stream_debug = open('/dev/null','w')
b635cd93 62 s.rune_cpio = r'''
4460af45
IJ
63 set -o pipefail
64 (
65 %s
66 # ^ by default, is find ... -print0
67 ) | (
68 cpio -Hustar -o --quiet -0 -R 1000:1000 || \
69 cpio -Hustar -o --quiet -0
70 )
71 '''
72 s.rune_portmanteau = r'''
a7d05900
IJ
73 GZIP=-1 tar zcf - "$@"
74 '''
75 s.rune_portmanteau_uncompressed = r'''
76 tar cf - "$@"
4460af45
IJ
77 '''
78 s.manifest_name='0000-MANIFEST.txt'
79 # private
80 s._destdir = destdir
b635cd93 81 s._outcounter = 0
4460af45 82 s._manifest = []
2371516a 83 s._dirmap = { }
e98894bc 84 s._package_files = { } # map filename => infol
a7d05900
IJ
85 s._packages_path = os.path.join(s._destdir, 'packages')
86 s._package_sources = []
4460af45 87
e98894bc
IJ
88 def thing_matches_globs(s, thing, globs):
89 for pat in globs:
4460af45
IJ
90 negate = pat.startswith('!')
91 if negate: pat = pat[1:]
e98894bc 92 if fnmatch.fnmatch(thing, pat):
4460af45
IJ
93 return not negate
94 return negate
95
e98894bc 96 def src_filter_glob(s, src): # default s.src_filter
e68fca7c 97 return s.thing_matches_globs(src, s.src_filter_globs)
e98894bc 98
28861486
IJ
99 def src_direxcludes_git(s, d):
100 try:
101 excl = open(os.path.join(d, '.gitignore'))
102 except FileNotFoundError:
103 return []
104 r = []
105 for l in excl:
106 l.strip
107 if l.startswith('#'): next
108 if not len(l): next
109 r += l
110 return r
111
4460af45
IJ
112 def src_likeparent_git(s, src):
113 try:
b635cd93 114 os.stat(os.path.join(src, '.git/.'))
4460af45
IJ
115 except FileNotFoundError:
116 return False
117 else:
118 return True
119
120 def src_parentfinder(s, src, infol): # callers may monkey-patch away
121 for deref in (False,True):
122 xinfo = []
123
124 search = src
125 if deref:
126 search = os.path.realpath(search)
127
128 def ascend():
b635cd93 129 nonlocal search
4460af45
IJ
130 xinfo.append(os.path.basename(search))
131 search = os.path.dirname(search)
132
133 try:
b635cd93 134 stab = os.lstat(search)
4460af45
IJ
135 except FileNotFoundError:
136 return
137 if stat.S_ISREG(stab.st_mode):
138 ascend()
139
140 while not os.path.ismount(search):
141 if s.src_likeparent(search):
142 xinfo.reverse()
b635cd93 143 if len(xinfo): infol.append('want=' + os.path.join(*xinfo))
4460af45
IJ
144 return search
145
146 ascend()
147
148 # no .git found anywhere
b635cd93 149 return src
4460af45 150
e68fca7c 151 def path_prenormaliser(s, d, infol): # callers may monkey-patch away
4460af45
IJ
152 return os.path.join(s.cwd, os.path.abspath(d))
153
e98894bc 154 def srcdir_find_rune(s, d):
b635cd93 155 script = s.find_rune_base
a7d05900 156 ignores = s.ignores + s.output_names + [s.manifest_name]
28861486
IJ
157 ignores += s.src_direxcludes(d)
158 for excl in ignores:
4460af45 159 assert("'" not in excl)
28861486
IJ
160 script += r" \! -name '%s'" % excl
161 script += r" \! -path '*/%s/*'" % excl
4460af45 162 script += ' -print0'
b635cd93 163 return script
4460af45 164
2371516a 165 def manifest_append(s, name, infol):
a53aba4d 166 s._manifest.append({ 'file':name, 'info':' '.join(infol) })
2371516a 167
e68fca7c
IJ
168 def manifest_append_absentfile(s, name, infol):
169 s._manifest.append({ 'file_print':name, 'info':' '.join(infol) })
170
4460af45 171 def new_output_name(s, nametail, infol):
b635cd93
IJ
172 s._outcounter += 1
173 name = '%04d-%s' % (s._outcounter, nametail)
2371516a 174 s.manifest_append(name, infol)
4460af45
IJ
175 return name
176
177 def open_output_fh(s, name, mode):
178 return open(os.path.join(s._destdir, name), mode)
179
e98894bc 180 def src_dir(s, d, infol):
2371516a
IJ
181 try: name = s._dirmap[d]
182 except KeyError: pass
183 else:
184 s.manifest_append(name, infol)
185 return
186
b635cd93 187 if s.show_pathnames: infol.append(d)
e98894bc 188 find_rune = s.srcdir_find_rune(d)
4460af45 189 total_rune = s.rune_cpio % find_rune
2371516a 190
eed788f8 191 name = s.new_output_name('src.tar', infol)
2371516a
IJ
192 s._dirmap[d] = name
193 fh = s.open_output_fh(name, 'wb')
194
c2c0da38
IJ
195 s.logger('packing up into %s: %s (because %s)' %
196 (name, d, ' '.join(infol)))
197
4460af45 198 subprocess.run(s.rune_shell + [total_rune],
b635cd93 199 cwd=d,
4460af45
IJ
200 stdin=subprocess.DEVNULL,
201 stdout=fh,
b635cd93
IJ
202 restore_signals=True,
203 check=True)
4460af45
IJ
204 fh.close()
205
e98894bc 206 def src_indir(s, d, infol):
e68fca7c 207 d = s.path_prenormaliser(d, infol)
4460af45 208 if not s.src_filter(d): return
e98894bc 209
4460af45 210 d = s.src_parentfinder(d, infol)
e68fca7c
IJ
211 if d is None: return
212 s.src_dir(d, infol)
e98894bc
IJ
213
214 def report_from_packages_debian(s, files):
e68fca7c 215 dpkg_S_in = tempfile.TemporaryFile(mode='w+')
e98894bc
IJ
216 for (file, infols) in files.items():
217 assert('\n' not in file)
218 dpkg_S_in.write(file)
219 dpkg_S_in.write('\0')
220 dpkg_S_in.seek(0)
221 cmdl = ['xargs','-0r','dpkg','-S','--']
222 dpkg_S = subprocess.Popen(cmdl,
a53aba4d
IJ
223 cwd='/',
224 stdin=dpkg_S_in,
225 stdout=subprocess.PIPE,
226 stderr=sys.stderr,
227 close_fds=False)
e68fca7c 228 dpkg_show_in = tempfile.TemporaryFile(mode='w+')
e98894bc 229 pkginfos = { }
e68fca7c
IJ
230 for l in dpkg_S.stdout:
231 l = l.strip(b'\n').decode('utf-8')
e98894bc 232 (pkgs, fname) = l.split(': ',1)
e68fca7c
IJ
233 pks = pkgs.split(', ')
234 for pk in pks:
235 pkginfos.setdefault(pk,{'files':[]})['files'].append(fname)
236 print(pk, file=dpkg_show_in)
a53aba4d
IJ
237 assert(dpkg_S.wait() == 0)
238 dpkg_show_in.seek(0)
239 cmdl = ['xargs','-r','dpkg-query',
a7d05900 240 r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n',
a53aba4d
IJ
241 '--show','--']
242 dpkg_show = subprocess.Popen(cmdl,
243 cwd='/',
244 stdin=dpkg_show_in,
245 stdout=subprocess.PIPE,
246 stderr=sys.stderr,
247 close_fds=False)
248 for l in dpkg_show.stdout:
e68fca7c 249 l = l.strip(b'\n').decode('utf-8')
a7d05900 250 (pk,p,a,v,sp,sv,suv) = l.split('\t')
e68fca7c
IJ
251 pkginfos[pk]['binary'] = p
252 pkginfos[pk]['arch'] = a
253 pkginfos[pk]['version'] = v
254 pkginfos[pk]['source'] = sp
255 pkginfos[pk]['sourceversion'] = sv
a7d05900 256 pkginfos[pk]['sourceupstreamversion'] = sv
a53aba4d 257 assert(dpkg_show.wait() == 0)
e68fca7c
IJ
258 for pk in sorted(pkginfos.keys()):
259 pi = pkginfos[pk]
260 debfname = '%s_%s_%s.deb' % (pi['binary'], pi['version'], pi['arch'])
a53aba4d 261 dscfname = '%s_%s.dsc' % (pi['source'], pi['sourceversion'])
e68fca7c 262 s.manifest_append_absentfile(dscfname, [debfname])
c2c0da38
IJ
263 s.logger('mentioning %s and %s because %s' %
264 (dscfname, debfname, pi['files'][0]))
e68fca7c
IJ
265 for fname in pi['files']:
266 infol = files[fname]
56ffddf8 267 if s.show_pathnames: infol = infol + ['loaded='+fname]
e68fca7c 268 s.manifest_append_absentfile(' \t' + debfname, infol)
e98894bc 269
a7d05900
IJ
270 if s.download_packages:
271 try: os.mkdir(s._packages_path)
272 except FileExistsError: pass
273
274 cmdl = ['apt-get','--download-only','source',
275 '%s=%s' % (pi['source'], pi['sourceversion'])]
276 subprocess.run(cmdl,
277 cwd=s._packages_path,
278 stdin=subprocess.DEVNULL,
ff0fc3fa
IJ
279 stdout=s.stream_debug,
280 stderr=s.stream_stderr,
a7d05900
IJ
281 restore_signals=True,
282 check=True)
283
284 s._package_sources.append(dscfname)
285 dsc = debian.deb822.Dsc(open(s._packages_path + '/' + dscfname))
286 for indsc in dsc['Files']:
287 s._package_sources.append(indsc['name'])
288
e98894bc
IJ
289 def thing_ought_packaged(s, fname):
290 return s.thing_matches_globs(fname, s.src_package_globs)
291
e68fca7c
IJ
292 def src_file_packaged(s, fname, infol):
293 s._package_files.setdefault(fname,[]).extend(infol)
e98894bc
IJ
294
295 def src_file(s, fname, infol):
296 def fngens():
e68fca7c
IJ
297 yield (infol, fname)
298 infol_copy = infol.copy()
299 yield (infol_copy, s.path_prenormaliser(fname, infol_copy))
300 yield (infol, os.path.realpath(fname))
301
302 for (tinfol, tfname) in fngens():
303 if s.thing_ought_packaged(tfname):
304 s.src_file_packaged(tfname, tinfol)
e98894bc 305 return
4460af45 306
e98894bc
IJ
307 s.src_indir(fname, infol)
308
309 def src_argv0(s, program, infol):
e68fca7c 310 s.src_file(program, infol)
e98894bc
IJ
311
312 def src_syspath(s, fname, infol):
56ffddf8 313 if s.thing_ought_packaged(fname): return
e98894bc
IJ
314 s.src_indir(fname, infol)
315
316 def src_module(s, m, infol):
317 try: fname = m.__file__
16374080 318 except AttributeError: return
56ffddf8 319 infol.append('module='+m.__name__)
16374080 320
e98894bc
IJ
321 if s.thing_ought_packaged(fname):
322 s.src_file_packaged(fname, infol)
323 else:
e68fca7c 324 s.src_indir(fname, infol)
e98894bc
IJ
325
326 def srcs_allitems(s, dirs=sys.path):
c2c0da38 327 s.logger('allitems')
e98894bc 328 s.src_argv0(sys.argv[0], ['argv[0]'])
4460af45 329 for d in sys.path:
e98894bc 330 s.src_syspath(d, ['sys.path'])
16374080 331 for m in sys.modules.values():
e98894bc 332 s.src_module(m, ['sys.modules'])
e68fca7c 333 s.report_from_packages(s._package_files)
c2c0da38 334 s.logger('allitems done')
4460af45 335
a7d05900
IJ
336 def _mk_portmanteau(s, ix, rune, cwd, files):
337 output_name = s.output_names[ix]
338 s.logger('making portmanteau %s' % output_name)
339 output_path = os.path.join(s._destdir, output_name)
340 subprocess.run(s.rune_shell + [ rune, 'x' ] + files,
341 cwd=cwd,
342 stdin=subprocess.DEVNULL,
343 stdout=open(output_path, 'wb'),
344 restore_signals=True,
345 check=True)
346 s.output_paths[ix] = output_path
347
348 def mk_inner_portmanteau(s):
349 outputs = [s.manifest_name]
9693ff43 350 outputs_done = { }
b635cd93 351 mfh = s.open_output_fh(s.manifest_name,'w')
a53aba4d
IJ
352 for me in s._manifest:
353 try: fname = me['file']
354 except KeyError: fname = me.get('file_print','')
9693ff43
IJ
355 else:
356 try: outputs_done[fname]
357 except KeyError:
358 outputs.append(fname)
359 outputs_done[fname] = 1
a53aba4d 360 print('%s\t%s' % (fname, me['info']), file=mfh)
4460af45 361 mfh.close()
a7d05900
IJ
362
363 s._mk_portmanteau(0, s.rune_portmanteau,
364 s._destdir, outputs)
365
366 def mk_packages_portmanteau(s):
367 s._mk_portmanteau(1, s.rune_portmanteau_uncompressed,
368 s._packages_path, s._package_sources)
4460af45
IJ
369
370 def generate(s):
e98894bc 371 s.srcs_allitems()
a7d05900
IJ
372 s.mk_inner_portmanteau()
373 s.mk_packages_portmanteau()
374 s.logger('portmanteau ready in %s %s' % tuple(s.output_paths))