1 # Automatic source code provision (AGPL compliance)
10 class SourceShipmentPreparer():
11 def __init__(s
, destdir
):
12 # caller may modify, and should read after calling generate()
13 s
.output_name
= 'srcbomb.tar.gz'
14 # s.output_path alternatively caller may read this
15 # defaults, caller can modify after creation
16 s
.logger
= lambda m
: print('SourceShipmentPreparer',m
)
17 s
.src_filter
= s
.src_filter_glob
18 s
.src_package_globs
= ['!/usr/local/*', '/usr*']
19 s
.src_filter_globs
= ['!/etc/*']
20 s
.src_likeparent
= s
.src_likeparent_git
21 s
.src_direxcludes
= s
.src_direxcludes_git
22 s
.report_from_packages
= s
.report_from_packages_debian
24 s
.find_rune_base
= "find -type f -perm -004 \! -path '*/tmp/*'"
25 s
.ignores
= ['*~', '*.bak', '*.tmp', '#*#', '__pycache__',
26 '[0-9][0-9][0-9][0-9]-src.cpio']
27 s
.rune_shell
= ['/bin/bash', '-ec']
28 s
.show_pathnames
= True
33 # ^ by default, is find ... -print0
35 cpio -Hustar -o --quiet -0 -R 1000:1000 || \
36 cpio -Hustar -o --quiet -0
39 s
.rune_portmanteau
= r
'''
42 GZIP=-1 tar zcf "$outfile" "$@"
44 s
.manifest_name
='0000-MANIFEST.txt'
50 s
._package_files
= { } # map filename => infol
52 def thing_matches_globs(s
, thing
, globs
):
54 negate
= pat
.startswith('!')
55 if negate
: pat
= pat
[1:]
56 if fnmatch
.fnmatch(thing
, pat
):
60 def src_filter_glob(s
, src
): # default s.src_filter
61 return s
.thing_matches_globs(src
, s
.src_filter_globs
)
63 def src_direxcludes_git(s
, d
):
65 excl
= open(os
.path
.join(d
, '.gitignore'))
66 except FileNotFoundError
:
71 if l
.startswith('#'): next
76 def src_likeparent_git(s
, src
):
78 os
.stat(os
.path
.join(src
, '.git/.'))
79 except FileNotFoundError
:
84 def src_parentfinder(s
, src
, infol
): # callers may monkey-patch away
85 for deref
in (False,True):
90 search
= os
.path
.realpath(search
)
94 xinfo
.append(os
.path
.basename(search
))
95 search
= os
.path
.dirname(search
)
98 stab
= os
.lstat(search
)
99 except FileNotFoundError
:
101 if stat
.S_ISREG(stab
.st_mode
):
104 while not os
.path
.ismount(search
):
105 if s
.src_likeparent(search
):
107 if len(xinfo
): infol
.append('want=' + os
.path
.join(*xinfo
))
112 # no .git found anywhere
115 def path_prenormaliser(s
, d
, infol
): # callers may monkey-patch away
116 return os
.path
.join(s
.cwd
, os
.path
.abspath(d
))
118 def srcdir_find_rune(s
, d
):
119 script
= s
.find_rune_base
120 ignores
= s
.ignores
+ [s
.output_name
, s
.manifest_name
]
121 ignores
+= s
.src_direxcludes(d
)
123 assert("'" not in excl
)
124 script
+= r
" \! -name '%s'" % excl
125 script
+= r
" \! -path '*/%s/*'" % excl
129 def manifest_append(s
, name
, infol
):
130 s
._manifest
.append({ 'file':name
, 'info':' '.join(infol
) })
132 def manifest_append_absentfile(s
, name
, infol
):
133 s
._manifest
.append({ 'file_print':name
, 'info':' '.join(infol
) })
135 def new_output_name(s
, nametail
, infol
):
137 name
= '%04d-%s' %
(s
._outcounter
, nametail
)
138 s
.manifest_append(name
, infol
)
141 def open_output_fh(s
, name
, mode
):
142 return open(os
.path
.join(s
._destdir
, name
), mode
)
144 def src_dir(s
, d
, infol
):
145 try: name
= s
._dirmap
[d
]
146 except KeyError: pass
148 s
.manifest_append(name
, infol
)
151 if s
.show_pathnames
: infol
.append(d
)
152 find_rune
= s
.srcdir_find_rune(d
)
153 total_rune
= s
.rune_cpio % find_rune
155 name
= s
.new_output_name('src.cpio', infol
)
157 fh
= s
.open_output_fh(name
, 'wb')
159 s
.logger('packing up into %s: %s (because %s)' %
160 (name
, d
, ' '.join(infol
)))
162 subprocess
.run(s
.rune_shell
+ [total_rune
],
164 stdin
=subprocess
.DEVNULL
,
166 restore_signals
=True,
170 def src_indir(s
, d
, infol
):
171 d
= s
.path_prenormaliser(d
, infol
)
172 if not s
.src_filter(d
): return
174 d
= s
.src_parentfinder(d
, infol
)
178 def report_from_packages_debian(s
, files
):
179 dpkg_S_in
= tempfile
.TemporaryFile(mode
='w+')
180 for (file, infols
) in files
.items():
181 assert('\n' not in file)
182 dpkg_S_in
.write(file)
183 dpkg_S_in
.write('\0')
185 cmdl
= ['xargs','-0r','dpkg','-S','--']
186 dpkg_S
= subprocess
.Popen(cmdl
,
189 stdout
=subprocess
.PIPE
,
192 dpkg_show_in
= tempfile
.TemporaryFile(mode
='w+')
194 for l
in dpkg_S
.stdout
:
195 l
= l
.strip(b
'\n').decode('utf-8')
196 (pkgs
, fname
) = l
.split(': ',1)
197 pks
= pkgs
.split(', ')
199 pkginfos
.setdefault(pk
,{'files':[]})['files'].append(fname
)
200 print(pk
, file=dpkg_show_in
)
201 assert(dpkg_S
.wait() == 0)
203 cmdl
= ['xargs','-r','dpkg-query',
204 r
'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\n',
206 dpkg_show
= subprocess
.Popen(cmdl
,
209 stdout
=subprocess
.PIPE
,
212 for l
in dpkg_show
.stdout
:
213 l
= l
.strip(b
'\n').decode('utf-8')
214 (pk
,p
,a
,v
,sp
,sv
) = l
.split('\t')
215 pkginfos
[pk
]['binary'] = p
216 pkginfos
[pk
]['arch'] = a
217 pkginfos
[pk
]['version'] = v
218 pkginfos
[pk
]['source'] = sp
219 pkginfos
[pk
]['sourceversion'] = sv
220 assert(dpkg_show
.wait() == 0)
221 for pk
in sorted(pkginfos
.keys()):
223 debfname
= '%s_%s_%s.deb' %
(pi
['binary'], pi
['version'], pi
['arch'])
224 dscfname
= '%s_%s.dsc' %
(pi
['source'], pi
['sourceversion'])
225 s
.manifest_append_absentfile(dscfname
, [debfname
])
226 s
.logger('mentioning %s and %s because %s' %
227 (dscfname
, debfname
, pi
['files'][0]))
228 for fname
in pi
['files']:
230 if s
.show_pathnames
: infol
= infol
+ ['loaded='+fname
]
231 s
.manifest_append_absentfile(' \t' + debfname
, infol
)
233 def thing_ought_packaged(s
, fname
):
234 return s
.thing_matches_globs(fname
, s
.src_package_globs
)
236 def src_file_packaged(s
, fname
, infol
):
237 s
._package_files
.setdefault(fname
,[]).extend(infol
)
239 def src_file(s
, fname
, infol
):
242 infol_copy
= infol
.copy()
243 yield (infol_copy
, s
.path_prenormaliser(fname
, infol_copy
))
244 yield (infol
, os
.path
.realpath(fname
))
246 for (tinfol
, tfname
) in fngens():
247 if s
.thing_ought_packaged(tfname
):
248 s
.src_file_packaged(tfname
, tinfol
)
251 s
.src_indir(fname
, infol
)
253 def src_argv0(s
, program
, infol
):
254 s
.src_file(program
, infol
)
256 def src_syspath(s
, fname
, infol
):
257 if s
.thing_ought_packaged(fname
): return
258 s
.src_indir(fname
, infol
)
260 def src_module(s
, m
, infol
):
261 try: fname
= m
.__file__
262 except AttributeError: return
263 infol
.append('module='+m
.__name__
)
265 if s
.thing_ought_packaged(fname
):
266 s
.src_file_packaged(fname
, infol
)
268 s
.src_indir(fname
, infol
)
270 def srcs_allitems(s
, dirs
=sys
.path
):
272 s
.src_argv0(sys
.argv
[0], ['argv[0]'])
274 s
.src_syspath(d
, ['sys.path'])
275 for m
in sys
.modules
.values():
276 s
.src_module(m
, ['sys.modules'])
277 s
.report_from_packages(s
._package_files
)
278 s
.logger('allitems done')
280 def mk_portmanteau(s
):
281 s
.logger('making portmanteau')
282 cmdl
= s
.rune_shell
+ [ s
.rune_portmanteau
, 'x',
283 s
.output_name
, s
.manifest_name
]
284 mfh
= s
.open_output_fh(s
.manifest_name
,'w')
285 for me
in s
._manifest
:
286 try: fname
= me
['file']
287 except KeyError: fname
= me
.get('file_print','')
288 else: cmdl
.append(fname
)
289 print('%s\t%s' %
(fname
, me
['info']), file=mfh
)
293 stdin
=subprocess
.DEVNULL
,
295 restore_signals
=True,
297 s
.output_path
= os
.path
.join(s
._destdir
, s
.output_name
)
298 s
.logger('portmanteau ready in %s' % s
.output_path
)