3 # Hippotat - Asinine IP Over HTTP program
4 # hippotatlib/ownsource.py - Automatic source code provision (AGPL compliance)
6 # Copyright 2017 Ian Jackson
10 # This program is free software: you can redistribute it and/or
11 # modify it under the terms of the GNU Affero General Public
12 # License as published by the Free Software Foundation, either
13 # version 3 of the License, or (at your option) any later version,
14 # with the "CAF Login Exception" as published by Ian Jackson
15 # (version 2, or at your option any later version) as an Additional
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21 # Affero General Public License for more details.
23 # You should have received a copy of the GNU Affero General Public
24 # License and the CAF Login Exception along with this program, in
25 # the file AGPLv3+CAFv2. If not, email Ian Jackson
26 # <ijackson@chiark.greenend.org.uk>.
37 try: import debian
.deb822
38 except ImportError: pass
40 class SourceShipmentPreparer():
41 def __init__(s
, destdir
):
42 # caller may modify, and should read after calling generate()
43 s
.output_names
= ['srcbomb.tar.gz', 'fullsrcbomb.tar']
44 s
.output_paths
= [None,None] # alternatively caller may read this
45 # defaults, caller can modify after creation
46 s
.logger
= lambda m
: print('SourceShipmentPreparer',m
)
47 s
.src_filter
= s
.src_filter_glob
48 s
.src_package_globs
= ['!/usr/local/*', '/usr*']
49 s
.src_filter_globs
= ['!/etc/*']
50 s
.src_likeparent
= s
.src_likeparent_git
51 s
.src_direxcludes
= s
.src_direxcludes_git
52 s
.report_from_packages
= s
.report_from_packages_debian
54 s
.find_rune_base
= "find -type f -perm -004 \! -path '*/tmp/*'"
55 s
.ignores
= ['*~', '*.bak', '*.tmp', '#*#', '__pycache__',
56 '[0-9][0-9][0-9][0-9]-src.tar']
57 s
.rune_shell
= ['/bin/bash', '-ec']
58 s
.show_pathnames
= True
59 s
.download_packages
= True
60 s
.stream_stderr
= sys
.stderr
61 s
.stream_debug
= open('/dev/null','w')
66 # ^ by default, is find ... -print0
68 cpio -Hustar -o --quiet -0 -R 1000:1000 || \
69 cpio -Hustar -o --quiet -0
72 s
.rune_portmanteau
= r
'''
73 GZIP=-1 tar zcf - "$@"
75 s
.rune_portmanteau_uncompressed
= r
'''
78 s
.manifest_name
='0000-MANIFEST.txt'
84 s
._package_files
= { } # map filename => infol
85 s
._packages_path
= os
.path
.join(s
._destdir
, 'packages')
86 s
._package_sources
= []
88 def thing_matches_globs(s
, thing
, globs
):
90 negate
= pat
.startswith('!')
91 if negate
: pat
= pat
[1:]
92 if fnmatch
.fnmatch(thing
, pat
):
96 def src_filter_glob(s
, src
): # default s.src_filter
97 return s
.thing_matches_globs(src
, s
.src_filter_globs
)
99 def src_direxcludes_git(s
, d
):
101 excl
= open(os
.path
.join(d
, '.gitignore'))
102 except FileNotFoundError
:
107 if l
.startswith('#'): next
112 def src_likeparent_git(s
, src
):
114 os
.stat(os
.path
.join(src
, '.git/.'))
115 except FileNotFoundError
:
120 def src_parentfinder(s
, src
, infol
): # callers may monkey-patch away
121 for deref
in (False,True):
126 search
= os
.path
.realpath(search
)
130 xinfo
.append(os
.path
.basename(search
))
131 search
= os
.path
.dirname(search
)
134 stab
= os
.lstat(search
)
135 except FileNotFoundError
:
137 if stat
.S_ISREG(stab
.st_mode
):
140 while not os
.path
.ismount(search
):
141 if s
.src_likeparent(search
):
143 if len(xinfo
): infol
.append('want=' + os
.path
.join(*xinfo
))
148 # no .git found anywhere
151 def path_prenormaliser(s
, d
, infol
): # callers may monkey-patch away
152 return os
.path
.join(s
.cwd
, os
.path
.abspath(d
))
154 def srcdir_find_rune(s
, d
):
155 script
= s
.find_rune_base
156 ignores
= s
.ignores
+ s
.output_names
+ [s
.manifest_name
]
157 ignores
+= s
.src_direxcludes(d
)
159 assert("'" not in excl
)
160 script
+= r
" \! -name '%s'" % excl
161 script
+= r
" \! -path '*/%s/*'" % excl
165 def manifest_append(s
, name
, infol
):
166 s
._manifest
.append({ 'file':name
, 'info':' '.join(infol
) })
168 def manifest_append_absentfile(s
, name
, infol
):
169 s
._manifest
.append({ 'file_print':name
, 'info':' '.join(infol
) })
171 def new_output_name(s
, nametail
, infol
):
173 name
= '%04d-%s' %
(s
._outcounter
, nametail
)
174 s
.manifest_append(name
, infol
)
177 def open_output_fh(s
, name
, mode
):
178 return open(os
.path
.join(s
._destdir
, name
), mode
)
180 def src_dir(s
, d
, infol
):
181 try: name
= s
._dirmap
[d
]
182 except KeyError: pass
184 s
.manifest_append(name
, infol
)
187 if s
.show_pathnames
: infol
.append(d
)
188 find_rune
= s
.srcdir_find_rune(d
)
189 total_rune
= s
.rune_cpio % find_rune
191 name
= s
.new_output_name('src.tar', infol
)
193 fh
= s
.open_output_fh(name
, 'wb')
195 s
.logger('packing up into %s: %s (because %s)' %
196 (name
, d
, ' '.join(infol
)))
198 subprocess
.run(s
.rune_shell
+ [total_rune
],
200 stdin
=subprocess
.DEVNULL
,
202 restore_signals
=True,
206 def src_indir(s
, d
, infol
):
207 d
= s
.path_prenormaliser(d
, infol
)
208 if not s
.src_filter(d
): return
210 d
= s
.src_parentfinder(d
, infol
)
214 def report_from_packages_debian(s
, files
):
215 dpkg_S_in
= tempfile
.TemporaryFile(mode
='w+')
216 for (file, infols
) in files
.items():
217 assert('\n' not in file)
218 dpkg_S_in
.write(file)
219 dpkg_S_in
.write('\0')
221 cmdl
= ['xargs','-0r','dpkg','-S','--']
222 dpkg_S
= subprocess
.Popen(cmdl
,
225 stdout
=subprocess
.PIPE
,
228 dpkg_show_in
= tempfile
.TemporaryFile(mode
='w+')
230 for l
in dpkg_S
.stdout
:
231 l
= l
.strip(b
'\n').decode('utf-8')
232 (pkgs
, fname
) = l
.split(': ',1)
233 pks
= pkgs
.split(', ')
235 pkginfos
.setdefault(pk
,{'files':[]})['files'].append(fname
)
236 print(pk
, file=dpkg_show_in
)
237 assert(dpkg_S
.wait() == 0)
239 cmdl
= ['xargs','-r','dpkg-query',
240 r
'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n',
242 dpkg_show
= subprocess
.Popen(cmdl
,
245 stdout
=subprocess
.PIPE
,
248 for l
in dpkg_show
.stdout
:
249 l
= l
.strip(b
'\n').decode('utf-8')
250 (pk
,p
,a
,v
,sp
,sv
,suv
) = l
.split('\t')
251 pkginfos
[pk
]['binary'] = p
252 pkginfos
[pk
]['arch'] = a
253 pkginfos
[pk
]['version'] = v
254 pkginfos
[pk
]['source'] = sp
255 pkginfos
[pk
]['sourceversion'] = sv
256 pkginfos
[pk
]['sourceupstreamversion'] = sv
257 assert(dpkg_show
.wait() == 0)
258 for pk
in sorted(pkginfos
.keys()):
260 debfname
= '%s_%s_%s.deb' %
(pi
['binary'], pi
['version'], pi
['arch'])
261 dscfname
= '%s_%s.dsc' %
(pi
['source'], pi
['sourceversion'])
262 s
.manifest_append_absentfile(dscfname
, [debfname
])
263 s
.logger('mentioning %s and %s because %s' %
264 (dscfname
, debfname
, pi
['files'][0]))
265 for fname
in pi
['files']:
267 if s
.show_pathnames
: infol
= infol
+ ['loaded='+fname
]
268 s
.manifest_append_absentfile(' \t' + debfname
, infol
)
270 if s
.download_packages
:
271 try: os
.mkdir(s
._packages_path
)
272 except FileExistsError
: pass
274 cmdl
= ['apt-get','--download-only','source',
275 '%s=%s' %
(pi
['source'], pi
['sourceversion'])]
277 cwd
=s
._packages_path
,
278 stdin
=subprocess
.DEVNULL
,
279 stdout
=s
.stream_debug
,
280 stderr
=s
.stream_stderr
,
281 restore_signals
=True,
284 s
._package_sources
.append(dscfname
)
285 dsc
= debian
.deb822
.Dsc(open(s
._packages_path
+ '/' + dscfname
))
286 for indsc
in dsc
['Files']:
287 s
._package_sources
.append(indsc
['name'])
289 def thing_ought_packaged(s
, fname
):
290 return s
.thing_matches_globs(fname
, s
.src_package_globs
)
292 def src_file_packaged(s
, fname
, infol
):
293 s
._package_files
.setdefault(fname
,[]).extend(infol
)
295 def src_file(s
, fname
, infol
):
298 infol_copy
= infol
.copy()
299 yield (infol_copy
, s
.path_prenormaliser(fname
, infol_copy
))
300 yield (infol
, os
.path
.realpath(fname
))
302 for (tinfol
, tfname
) in fngens():
303 if s
.thing_ought_packaged(tfname
):
304 s
.src_file_packaged(tfname
, tinfol
)
307 s
.src_indir(fname
, infol
)
309 def src_argv0(s
, program
, infol
):
310 s
.src_file(program
, infol
)
312 def src_syspath(s
, fname
, infol
):
313 if s
.thing_ought_packaged(fname
): return
314 s
.src_indir(fname
, infol
)
316 def src_module(s
, m
, infol
):
317 try: fname
= m
.__file__
318 except AttributeError: return
319 infol
.append('module='+m
.__name__
)
321 if s
.thing_ought_packaged(fname
):
322 s
.src_file_packaged(fname
, infol
)
324 s
.src_indir(fname
, infol
)
326 def srcs_allitems(s
, dirs
=sys
.path
):
328 s
.src_argv0(sys
.argv
[0], ['argv[0]'])
330 s
.src_syspath(d
, ['sys.path'])
331 for m
in sys
.modules
.values():
332 s
.src_module(m
, ['sys.modules'])
333 s
.report_from_packages(s
._package_files
)
334 s
.logger('allitems done')
336 def _mk_portmanteau(s
, ix
, rune
, cwd
, files
):
337 output_name
= s
.output_names
[ix
]
338 s
.logger('making portmanteau %s' % output_name
)
339 output_path
= os
.path
.join(s
._destdir
, output_name
)
340 subprocess
.run(s
.rune_shell
+ [ rune
, 'x' ] + files
,
342 stdin
=subprocess
.DEVNULL
,
343 stdout
=open(output_path
, 'wb'),
344 restore_signals
=True,
346 s
.output_paths
[ix
] = output_path
348 def mk_inner_portmanteau(s
):
349 outputs
= [s
.manifest_name
]
351 mfh
= s
.open_output_fh(s
.manifest_name
,'w')
352 for me
in s
._manifest
:
353 try: fname
= me
['file']
354 except KeyError: fname
= me
.get('file_print','')
356 try: outputs_done
[fname
]
358 outputs
.append(fname
)
359 outputs_done
[fname
] = 1
360 print('%s\t%s' %
(fname
, me
['info']), file=mfh
)
363 s
._mk_portmanteau(0, s
.rune_portmanteau
,
366 def mk_packages_portmanteau(s
):
367 s
._mk_portmanteau(1, s
.rune_portmanteau_uncompressed
,
368 s
._packages_path
, s
._package_sources
)
372 s
.mk_inner_portmanteau()
373 s
.mk_packages_portmanteau()
374 s
.logger('portmanteau ready in %s %s' %
tuple(s
.output_paths
))