Commit | Line | Data |
---|---|---|
4460af45 IJ |
1 | # Automatic source code provision (AGPL compliance) |
2 | ||
b635cd93 | 3 | import os |
4460af45 IJ |
4 | import sys |
5 | import fnmatch | |
b635cd93 IJ |
6 | import stat |
7 | import subprocess | |
e68fca7c | 8 | import tempfile |
4460af45 IJ |
9 | |
10 | class SourceShipmentPreparer(): | |
11 | def __init__(s, destdir): | |
12 | # caller may modify, and should read after calling generate() | |
13 | s.output_name = 'srcbomb.tar.gz' | |
c2c0da38 | 14 | # s.output_path alternatively caller may read this |
4460af45 | 15 | # defaults, caller can modify after creation |
c2c0da38 | 16 | s.logger = lambda m: print('SourceShipmentPreparer',m) |
4460af45 | 17 | s.src_filter = s.src_filter_glob |
e68fca7c | 18 | s.src_package_globs = ['!/usr/local/*', '/usr*'] |
e98894bc | 19 | s.src_filter_globs = ['!/etc/*'] |
4460af45 | 20 | s.src_likeparent = s.src_likeparent_git |
28861486 | 21 | s.src_direxcludes = s.src_direxcludes_git |
e98894bc | 22 | s.report_from_packages = s.report_from_packages_debian |
4460af45 | 23 | s.cwd = os.getcwd() |
b635cd93 | 24 | s.find_rune_base = "find -type f -perm -004 \! -path '*/tmp/*'" |
28861486 | 25 | s.ignores = ['*~', '*.bak', '*.tmp', '#*#', '__pycache__', |
b635cd93 | 26 | '[0-9][0-9][0-9][0-9]-src.cpio'] |
4460af45 | 27 | s.rune_shell = ['/bin/bash', '-ec'] |
b635cd93 IJ |
28 | s.show_pathnames = True |
29 | s.rune_cpio = r''' | |
4460af45 IJ |
30 | set -o pipefail |
31 | ( | |
32 | %s | |
33 | # ^ by default, is find ... -print0 | |
34 | ) | ( | |
35 | cpio -Hustar -o --quiet -0 -R 1000:1000 || \ | |
36 | cpio -Hustar -o --quiet -0 | |
37 | ) | |
38 | ''' | |
39 | s.rune_portmanteau = r''' | |
40 | outfile=$1; shift | |
41 | rm -f "$outfile" | |
16374080 | 42 | GZIP=-1 tar zcf "$outfile" "$@" |
4460af45 IJ |
43 | ''' |
44 | s.manifest_name='0000-MANIFEST.txt' | |
45 | # private | |
46 | s._destdir = destdir | |
b635cd93 | 47 | s._outcounter = 0 |
4460af45 | 48 | s._manifest = [] |
2371516a | 49 | s._dirmap = { } |
e98894bc | 50 | s._package_files = { } # map filename => infol |
4460af45 | 51 | |
e98894bc IJ |
52 | def thing_matches_globs(s, thing, globs): |
53 | for pat in globs: | |
4460af45 IJ |
54 | negate = pat.startswith('!') |
55 | if negate: pat = pat[1:] | |
e98894bc | 56 | if fnmatch.fnmatch(thing, pat): |
4460af45 IJ |
57 | return not negate |
58 | return negate | |
59 | ||
e98894bc | 60 | def src_filter_glob(s, src): # default s.src_filter |
e68fca7c | 61 | return s.thing_matches_globs(src, s.src_filter_globs) |
e98894bc | 62 | |
28861486 IJ |
63 | def src_direxcludes_git(s, d): |
64 | try: | |
65 | excl = open(os.path.join(d, '.gitignore')) | |
66 | except FileNotFoundError: | |
67 | return [] | |
68 | r = [] | |
69 | for l in excl: | |
70 | l.strip | |
71 | if l.startswith('#'): next | |
72 | if not len(l): next | |
73 | r += l | |
74 | return r | |
75 | ||
4460af45 IJ |
76 | def src_likeparent_git(s, src): |
77 | try: | |
b635cd93 | 78 | os.stat(os.path.join(src, '.git/.')) |
4460af45 IJ |
79 | except FileNotFoundError: |
80 | return False | |
81 | else: | |
82 | return True | |
83 | ||
84 | def src_parentfinder(s, src, infol): # callers may monkey-patch away | |
85 | for deref in (False,True): | |
86 | xinfo = [] | |
87 | ||
88 | search = src | |
89 | if deref: | |
90 | search = os.path.realpath(search) | |
91 | ||
92 | def ascend(): | |
b635cd93 | 93 | nonlocal search |
4460af45 IJ |
94 | xinfo.append(os.path.basename(search)) |
95 | search = os.path.dirname(search) | |
96 | ||
97 | try: | |
b635cd93 | 98 | stab = os.lstat(search) |
4460af45 IJ |
99 | except FileNotFoundError: |
100 | return | |
101 | if stat.S_ISREG(stab.st_mode): | |
102 | ascend() | |
103 | ||
104 | while not os.path.ismount(search): | |
105 | if s.src_likeparent(search): | |
106 | xinfo.reverse() | |
b635cd93 | 107 | if len(xinfo): infol.append('want=' + os.path.join(*xinfo)) |
4460af45 IJ |
108 | return search |
109 | ||
110 | ascend() | |
111 | ||
112 | # no .git found anywhere | |
b635cd93 | 113 | return src |
4460af45 | 114 | |
e68fca7c | 115 | def path_prenormaliser(s, d, infol): # callers may monkey-patch away |
4460af45 IJ |
116 | return os.path.join(s.cwd, os.path.abspath(d)) |
117 | ||
e98894bc | 118 | def srcdir_find_rune(s, d): |
b635cd93 | 119 | script = s.find_rune_base |
28861486 IJ |
120 | ignores = s.ignores + [s.output_name, s.manifest_name] |
121 | ignores += s.src_direxcludes(d) | |
122 | for excl in ignores: | |
4460af45 | 123 | assert("'" not in excl) |
28861486 IJ |
124 | script += r" \! -name '%s'" % excl |
125 | script += r" \! -path '*/%s/*'" % excl | |
4460af45 | 126 | script += ' -print0' |
b635cd93 | 127 | return script |
4460af45 | 128 | |
2371516a | 129 | def manifest_append(s, name, infol): |
a53aba4d | 130 | s._manifest.append({ 'file':name, 'info':' '.join(infol) }) |
2371516a | 131 | |
e68fca7c IJ |
132 | def manifest_append_absentfile(s, name, infol): |
133 | s._manifest.append({ 'file_print':name, 'info':' '.join(infol) }) | |
134 | ||
4460af45 | 135 | def new_output_name(s, nametail, infol): |
b635cd93 IJ |
136 | s._outcounter += 1 |
137 | name = '%04d-%s' % (s._outcounter, nametail) | |
2371516a | 138 | s.manifest_append(name, infol) |
4460af45 IJ |
139 | return name |
140 | ||
141 | def open_output_fh(s, name, mode): | |
142 | return open(os.path.join(s._destdir, name), mode) | |
143 | ||
e98894bc | 144 | def src_dir(s, d, infol): |
2371516a IJ |
145 | try: name = s._dirmap[d] |
146 | except KeyError: pass | |
147 | else: | |
148 | s.manifest_append(name, infol) | |
149 | return | |
150 | ||
b635cd93 | 151 | if s.show_pathnames: infol.append(d) |
e98894bc | 152 | find_rune = s.srcdir_find_rune(d) |
4460af45 | 153 | total_rune = s.rune_cpio % find_rune |
2371516a IJ |
154 | |
155 | name = s.new_output_name('src.cpio', infol) | |
156 | s._dirmap[d] = name | |
157 | fh = s.open_output_fh(name, 'wb') | |
158 | ||
c2c0da38 IJ |
159 | s.logger('packing up into %s: %s (because %s)' % |
160 | (name, d, ' '.join(infol))) | |
161 | ||
4460af45 | 162 | subprocess.run(s.rune_shell + [total_rune], |
b635cd93 | 163 | cwd=d, |
4460af45 IJ |
164 | stdin=subprocess.DEVNULL, |
165 | stdout=fh, | |
b635cd93 IJ |
166 | restore_signals=True, |
167 | check=True) | |
4460af45 IJ |
168 | fh.close() |
169 | ||
e98894bc | 170 | def src_indir(s, d, infol): |
e68fca7c | 171 | d = s.path_prenormaliser(d, infol) |
4460af45 | 172 | if not s.src_filter(d): return |
e98894bc | 173 | |
4460af45 | 174 | d = s.src_parentfinder(d, infol) |
e68fca7c IJ |
175 | if d is None: return |
176 | s.src_dir(d, infol) | |
e98894bc IJ |
177 | |
178 | def report_from_packages_debian(s, files): | |
e68fca7c | 179 | dpkg_S_in = tempfile.TemporaryFile(mode='w+') |
e98894bc IJ |
180 | for (file, infols) in files.items(): |
181 | assert('\n' not in file) | |
182 | dpkg_S_in.write(file) | |
183 | dpkg_S_in.write('\0') | |
184 | dpkg_S_in.seek(0) | |
185 | cmdl = ['xargs','-0r','dpkg','-S','--'] | |
186 | dpkg_S = subprocess.Popen(cmdl, | |
a53aba4d IJ |
187 | cwd='/', |
188 | stdin=dpkg_S_in, | |
189 | stdout=subprocess.PIPE, | |
190 | stderr=sys.stderr, | |
191 | close_fds=False) | |
e68fca7c | 192 | dpkg_show_in = tempfile.TemporaryFile(mode='w+') |
e98894bc | 193 | pkginfos = { } |
e68fca7c IJ |
194 | for l in dpkg_S.stdout: |
195 | l = l.strip(b'\n').decode('utf-8') | |
e98894bc | 196 | (pkgs, fname) = l.split(': ',1) |
e68fca7c IJ |
197 | pks = pkgs.split(', ') |
198 | for pk in pks: | |
199 | pkginfos.setdefault(pk,{'files':[]})['files'].append(fname) | |
200 | print(pk, file=dpkg_show_in) | |
a53aba4d IJ |
201 | assert(dpkg_S.wait() == 0) |
202 | dpkg_show_in.seek(0) | |
203 | cmdl = ['xargs','-r','dpkg-query', | |
e68fca7c | 204 | r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\n', |
a53aba4d IJ |
205 | '--show','--'] |
206 | dpkg_show = subprocess.Popen(cmdl, | |
207 | cwd='/', | |
208 | stdin=dpkg_show_in, | |
209 | stdout=subprocess.PIPE, | |
210 | stderr=sys.stderr, | |
211 | close_fds=False) | |
212 | for l in dpkg_show.stdout: | |
e68fca7c IJ |
213 | l = l.strip(b'\n').decode('utf-8') |
214 | (pk,p,a,v,sp,sv) = l.split('\t') | |
215 | pkginfos[pk]['binary'] = p | |
216 | pkginfos[pk]['arch'] = a | |
217 | pkginfos[pk]['version'] = v | |
218 | pkginfos[pk]['source'] = sp | |
219 | pkginfos[pk]['sourceversion'] = sv | |
a53aba4d | 220 | assert(dpkg_show.wait() == 0) |
e68fca7c IJ |
221 | for pk in sorted(pkginfos.keys()): |
222 | pi = pkginfos[pk] | |
223 | debfname = '%s_%s_%s.deb' % (pi['binary'], pi['version'], pi['arch']) | |
a53aba4d | 224 | dscfname = '%s_%s.dsc' % (pi['source'], pi['sourceversion']) |
e68fca7c | 225 | s.manifest_append_absentfile(dscfname, [debfname]) |
c2c0da38 IJ |
226 | s.logger('mentioning %s and %s because %s' % |
227 | (dscfname, debfname, pi['files'][0])) | |
e68fca7c IJ |
228 | for fname in pi['files']: |
229 | infol = files[fname] | |
56ffddf8 | 230 | if s.show_pathnames: infol = infol + ['loaded='+fname] |
e68fca7c | 231 | s.manifest_append_absentfile(' \t' + debfname, infol) |
e98894bc IJ |
232 | |
233 | def thing_ought_packaged(s, fname): | |
234 | return s.thing_matches_globs(fname, s.src_package_globs) | |
235 | ||
e68fca7c IJ |
236 | def src_file_packaged(s, fname, infol): |
237 | s._package_files.setdefault(fname,[]).extend(infol) | |
e98894bc IJ |
238 | |
239 | def src_file(s, fname, infol): | |
240 | def fngens(): | |
e68fca7c IJ |
241 | yield (infol, fname) |
242 | infol_copy = infol.copy() | |
243 | yield (infol_copy, s.path_prenormaliser(fname, infol_copy)) | |
244 | yield (infol, os.path.realpath(fname)) | |
245 | ||
246 | for (tinfol, tfname) in fngens(): | |
247 | if s.thing_ought_packaged(tfname): | |
248 | s.src_file_packaged(tfname, tinfol) | |
e98894bc | 249 | return |
4460af45 | 250 | |
e98894bc IJ |
251 | s.src_indir(fname, infol) |
252 | ||
253 | def src_argv0(s, program, infol): | |
e68fca7c | 254 | s.src_file(program, infol) |
e98894bc IJ |
255 | |
256 | def src_syspath(s, fname, infol): | |
56ffddf8 | 257 | if s.thing_ought_packaged(fname): return |
e98894bc IJ |
258 | s.src_indir(fname, infol) |
259 | ||
260 | def src_module(s, m, infol): | |
261 | try: fname = m.__file__ | |
16374080 | 262 | except AttributeError: return |
56ffddf8 | 263 | infol.append('module='+m.__name__) |
16374080 | 264 | |
e98894bc IJ |
265 | if s.thing_ought_packaged(fname): |
266 | s.src_file_packaged(fname, infol) | |
267 | else: | |
e68fca7c | 268 | s.src_indir(fname, infol) |
e98894bc IJ |
269 | |
270 | def srcs_allitems(s, dirs=sys.path): | |
c2c0da38 | 271 | s.logger('allitems') |
e98894bc | 272 | s.src_argv0(sys.argv[0], ['argv[0]']) |
4460af45 | 273 | for d in sys.path: |
e98894bc | 274 | s.src_syspath(d, ['sys.path']) |
16374080 | 275 | for m in sys.modules.values(): |
e98894bc | 276 | s.src_module(m, ['sys.modules']) |
e68fca7c | 277 | s.report_from_packages(s._package_files) |
c2c0da38 | 278 | s.logger('allitems done') |
4460af45 | 279 | |
b635cd93 | 280 | def mk_portmanteau(s): |
c2c0da38 | 281 | s.logger('making portmanteau') |
4460af45 IJ |
282 | cmdl = s.rune_shell + [ s.rune_portmanteau, 'x', |
283 | s.output_name, s.manifest_name ] | |
b635cd93 | 284 | mfh = s.open_output_fh(s.manifest_name,'w') |
a53aba4d IJ |
285 | for me in s._manifest: |
286 | try: fname = me['file'] | |
287 | except KeyError: fname = me.get('file_print','') | |
e68fca7c | 288 | else: cmdl.append(fname) |
a53aba4d | 289 | print('%s\t%s' % (fname, me['info']), file=mfh) |
4460af45 | 290 | mfh.close() |
2371516a | 291 | subprocess.run(cmdl, |
b635cd93 | 292 | cwd=s._destdir, |
4460af45 IJ |
293 | stdin=subprocess.DEVNULL, |
294 | stdout=sys.stderr, | |
b635cd93 IJ |
295 | restore_signals=True, |
296 | check=True) | |
c2c0da38 IJ |
297 | s.output_path = os.path.join(s._destdir, s.output_name) |
298 | s.logger('portmanteau ready in %s' % s.output_path) | |
4460af45 IJ |
299 | |
300 | def generate(s): | |
e98894bc | 301 | s.srcs_allitems() |
4460af45 | 302 | s.mk_portmanteau() |