3 # Hippotat - Asinine IP Over HTTP program
4 # hippotatlib/ownsource.py - Automatic source code provision (AGPL compliance)
6 # Copyright 2017 Ian Jackson
10 # This program is free software: you can redistribute it and/or
11 # modify it under the terms of the GNU Affero General Public
12 # License as published by the Free Software Foundation, either
13 # version 3 of the License, or (at your option) any later version,
14 # with the "CAF Login Exception" as published by Ian Jackson
15 # (version 2, or at your option any later version) as an Additional
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21 # Affero General Public License for more details.
23 # You should have received a copy of the GNU Affero General Public
24 # License and the CAF Login Exception along with this program, in
25 # the file AGPLv3+CAFv2. If not, email Ian Jackson
26 # <ijackson@chiark.greenend.org.uk>.
37 try: import debian
.deb822
38 except ImportError: pass
40 class SourceShipmentPreparer():
41 def __init__(s
, destdir
):
42 # caller may modify, and should read after calling generate()
43 s
.output_names
= ['srcbomb.tar.gz', 'fullsrcbomb.tar']
44 s
.output_paths
= [None,None] # alternatively caller may read this
45 # defaults, caller can modify after creation
46 s
.logger
= lambda m
: print('SourceShipmentPreparer',m
)
47 s
.src_filter
= s
.src_filter_glob
48 s
.src_package_globs
= ['!/usr/local/*', '/usr*']
49 s
.src_filter_globs
= ['!/etc/*']
50 s
.src_likeparent
= s
.src_likeparent_git
51 s
.src_direxcludes
= s
.src_direxcludes_git
52 s
.report_from_packages
= s
.report_from_packages_debian
54 s
.find_rune_base
= "find -type f -perm -004 \! -path '*/tmp/*'"
55 s
.ignores
= ['*~', '*.bak', '*.tmp', '#*#', '__pycache__',
56 '[0-9][0-9][0-9][0-9]-src.tar']
57 s
.rune_shell
= ['/bin/bash', '-ec']
58 s
.show_pathnames
= True
59 s
.download_packages
= True
64 # ^ by default, is find ... -print0
66 cpio -Hustar -o --quiet -0 -R 1000:1000 || \
67 cpio -Hustar -o --quiet -0
70 s
.rune_portmanteau
= r
'''
71 GZIP=-1 tar zcf - "$@"
73 s
.rune_portmanteau_uncompressed
= r
'''
76 s
.manifest_name
='0000-MANIFEST.txt'
82 s
._package_files
= { } # map filename => infol
83 s
._packages_path
= os
.path
.join(s
._destdir
, 'packages')
84 s
._package_sources
= []
86 def thing_matches_globs(s
, thing
, globs
):
88 negate
= pat
.startswith('!')
89 if negate
: pat
= pat
[1:]
90 if fnmatch
.fnmatch(thing
, pat
):
94 def src_filter_glob(s
, src
): # default s.src_filter
95 return s
.thing_matches_globs(src
, s
.src_filter_globs
)
97 def src_direxcludes_git(s
, d
):
99 excl
= open(os
.path
.join(d
, '.gitignore'))
100 except FileNotFoundError
:
105 if l
.startswith('#'): next
110 def src_likeparent_git(s
, src
):
112 os
.stat(os
.path
.join(src
, '.git/.'))
113 except FileNotFoundError
:
118 def src_parentfinder(s
, src
, infol
): # callers may monkey-patch away
119 for deref
in (False,True):
124 search
= os
.path
.realpath(search
)
128 xinfo
.append(os
.path
.basename(search
))
129 search
= os
.path
.dirname(search
)
132 stab
= os
.lstat(search
)
133 except FileNotFoundError
:
135 if stat
.S_ISREG(stab
.st_mode
):
138 while not os
.path
.ismount(search
):
139 if s
.src_likeparent(search
):
141 if len(xinfo
): infol
.append('want=' + os
.path
.join(*xinfo
))
146 # no .git found anywhere
149 def path_prenormaliser(s
, d
, infol
): # callers may monkey-patch away
150 return os
.path
.join(s
.cwd
, os
.path
.abspath(d
))
152 def srcdir_find_rune(s
, d
):
153 script
= s
.find_rune_base
154 ignores
= s
.ignores
+ s
.output_names
+ [s
.manifest_name
]
155 ignores
+= s
.src_direxcludes(d
)
157 assert("'" not in excl
)
158 script
+= r
" \! -name '%s'" % excl
159 script
+= r
" \! -path '*/%s/*'" % excl
163 def manifest_append(s
, name
, infol
):
164 s
._manifest
.append({ 'file':name
, 'info':' '.join(infol
) })
166 def manifest_append_absentfile(s
, name
, infol
):
167 s
._manifest
.append({ 'file_print':name
, 'info':' '.join(infol
) })
169 def new_output_name(s
, nametail
, infol
):
171 name
= '%04d-%s' %
(s
._outcounter
, nametail
)
172 s
.manifest_append(name
, infol
)
175 def open_output_fh(s
, name
, mode
):
176 return open(os
.path
.join(s
._destdir
, name
), mode
)
178 def src_dir(s
, d
, infol
):
179 try: name
= s
._dirmap
[d
]
180 except KeyError: pass
182 s
.manifest_append(name
, infol
)
185 if s
.show_pathnames
: infol
.append(d
)
186 find_rune
= s
.srcdir_find_rune(d
)
187 total_rune
= s
.rune_cpio % find_rune
189 name
= s
.new_output_name('src.tar', infol
)
191 fh
= s
.open_output_fh(name
, 'wb')
193 s
.logger('packing up into %s: %s (because %s)' %
194 (name
, d
, ' '.join(infol
)))
196 subprocess
.run(s
.rune_shell
+ [total_rune
],
198 stdin
=subprocess
.DEVNULL
,
200 restore_signals
=True,
204 def src_indir(s
, d
, infol
):
205 d
= s
.path_prenormaliser(d
, infol
)
206 if not s
.src_filter(d
): return
208 d
= s
.src_parentfinder(d
, infol
)
212 def report_from_packages_debian(s
, files
):
213 dpkg_S_in
= tempfile
.TemporaryFile(mode
='w+')
214 for (file, infols
) in files
.items():
215 assert('\n' not in file)
216 dpkg_S_in
.write(file)
217 dpkg_S_in
.write('\0')
219 cmdl
= ['xargs','-0r','dpkg','-S','--']
220 dpkg_S
= subprocess
.Popen(cmdl
,
223 stdout
=subprocess
.PIPE
,
226 dpkg_show_in
= tempfile
.TemporaryFile(mode
='w+')
228 for l
in dpkg_S
.stdout
:
229 l
= l
.strip(b
'\n').decode('utf-8')
230 (pkgs
, fname
) = l
.split(': ',1)
231 pks
= pkgs
.split(', ')
233 pkginfos
.setdefault(pk
,{'files':[]})['files'].append(fname
)
234 print(pk
, file=dpkg_show_in
)
235 assert(dpkg_S
.wait() == 0)
237 cmdl
= ['xargs','-r','dpkg-query',
238 r
'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n',
240 dpkg_show
= subprocess
.Popen(cmdl
,
243 stdout
=subprocess
.PIPE
,
246 for l
in dpkg_show
.stdout
:
247 l
= l
.strip(b
'\n').decode('utf-8')
248 (pk
,p
,a
,v
,sp
,sv
,suv
) = l
.split('\t')
249 pkginfos
[pk
]['binary'] = p
250 pkginfos
[pk
]['arch'] = a
251 pkginfos
[pk
]['version'] = v
252 pkginfos
[pk
]['source'] = sp
253 pkginfos
[pk
]['sourceversion'] = sv
254 pkginfos
[pk
]['sourceupstreamversion'] = sv
255 assert(dpkg_show
.wait() == 0)
256 for pk
in sorted(pkginfos
.keys()):
258 debfname
= '%s_%s_%s.deb' %
(pi
['binary'], pi
['version'], pi
['arch'])
259 dscfname
= '%s_%s.dsc' %
(pi
['source'], pi
['sourceversion'])
260 s
.manifest_append_absentfile(dscfname
, [debfname
])
261 s
.logger('mentioning %s and %s because %s' %
262 (dscfname
, debfname
, pi
['files'][0]))
263 for fname
in pi
['files']:
265 if s
.show_pathnames
: infol
= infol
+ ['loaded='+fname
]
266 s
.manifest_append_absentfile(' \t' + debfname
, infol
)
268 if s
.download_packages
:
269 try: os
.mkdir(s
._packages_path
)
270 except FileExistsError
: pass
272 cmdl
= ['apt-get','--download-only','source',
273 '%s=%s' %
(pi
['source'], pi
['sourceversion'])]
275 cwd
=s
._packages_path
,
276 stdin
=subprocess
.DEVNULL
,
279 restore_signals
=True,
282 s
._package_sources
.append(dscfname
)
283 dsc
= debian
.deb822
.Dsc(open(s
._packages_path
+ '/' + dscfname
))
284 for indsc
in dsc
['Files']:
285 s
._package_sources
.append(indsc
['name'])
287 def thing_ought_packaged(s
, fname
):
288 return s
.thing_matches_globs(fname
, s
.src_package_globs
)
290 def src_file_packaged(s
, fname
, infol
):
291 s
._package_files
.setdefault(fname
,[]).extend(infol
)
293 def src_file(s
, fname
, infol
):
296 infol_copy
= infol
.copy()
297 yield (infol_copy
, s
.path_prenormaliser(fname
, infol_copy
))
298 yield (infol
, os
.path
.realpath(fname
))
300 for (tinfol
, tfname
) in fngens():
301 if s
.thing_ought_packaged(tfname
):
302 s
.src_file_packaged(tfname
, tinfol
)
305 s
.src_indir(fname
, infol
)
307 def src_argv0(s
, program
, infol
):
308 s
.src_file(program
, infol
)
310 def src_syspath(s
, fname
, infol
):
311 if s
.thing_ought_packaged(fname
): return
312 s
.src_indir(fname
, infol
)
314 def src_module(s
, m
, infol
):
315 try: fname
= m
.__file__
316 except AttributeError: return
317 infol
.append('module='+m
.__name__
)
319 if s
.thing_ought_packaged(fname
):
320 s
.src_file_packaged(fname
, infol
)
322 s
.src_indir(fname
, infol
)
324 def srcs_allitems(s
, dirs
=sys
.path
):
326 s
.src_argv0(sys
.argv
[0], ['argv[0]'])
328 s
.src_syspath(d
, ['sys.path'])
329 for m
in sys
.modules
.values():
330 s
.src_module(m
, ['sys.modules'])
331 s
.report_from_packages(s
._package_files
)
332 s
.logger('allitems done')
334 def _mk_portmanteau(s
, ix
, rune
, cwd
, files
):
335 output_name
= s
.output_names
[ix
]
336 s
.logger('making portmanteau %s' % output_name
)
337 output_path
= os
.path
.join(s
._destdir
, output_name
)
338 subprocess
.run(s
.rune_shell
+ [ rune
, 'x' ] + files
,
340 stdin
=subprocess
.DEVNULL
,
341 stdout
=open(output_path
, 'wb'),
342 restore_signals
=True,
344 s
.output_paths
[ix
] = output_path
346 def mk_inner_portmanteau(s
):
347 outputs
= [s
.manifest_name
]
349 mfh
= s
.open_output_fh(s
.manifest_name
,'w')
350 for me
in s
._manifest
:
351 try: fname
= me
['file']
352 except KeyError: fname
= me
.get('file_print','')
354 try: outputs_done
[fname
]
356 outputs
.append(fname
)
357 outputs_done
[fname
] = 1
358 print('%s\t%s' %
(fname
, me
['info']), file=mfh
)
361 s
._mk_portmanteau(0, s
.rune_portmanteau
,
364 def mk_packages_portmanteau(s
):
365 s
._mk_portmanteau(1, s
.rune_portmanteau_uncompressed
,
366 s
._packages_path
, s
._package_sources
)
370 s
.mk_inner_portmanteau()
371 s
.mk_packages_portmanteau()
372 s
.logger('portmanteau ready in %s %s' %
tuple(s
.output_paths
))