1 # Automatic source code provision (AGPL compliance)
10 class SourceShipmentPreparer():
11 def __init__(s
, destdir
):
12 # caller may modify, and should read after calling generate()
13 s
.output_name
= 'srcbomb.tar.gz'
14 # defaults, caller can modify after creation
15 s
.src_filter
= s
.src_filter_glob
16 s
.src_package_globs
= ['!/usr/local/*', '/usr*']
17 s
.src_filter_globs
= ['!/etc/*']
18 s
.src_likeparent
= s
.src_likeparent_git
19 s
.src_direxcludes
= s
.src_direxcludes_git
20 s
.report_from_packages
= s
.report_from_packages_debian
22 s
.find_rune_base
= "find -type f -perm -004 \! -path '*/tmp/*'"
23 s
.ignores
= ['*~', '*.bak', '*.tmp', '#*#', '__pycache__',
24 '[0-9][0-9][0-9][0-9]-src.cpio']
25 s
.rune_shell
= ['/bin/bash', '-ec']
26 s
.show_pathnames
= True
31 # ^ by default, is find ... -print0
33 cpio -Hustar -o --quiet -0 -R 1000:1000 || \
34 cpio -Hustar -o --quiet -0
37 s
.rune_portmanteau
= r
'''
40 GZIP=-1 tar zcf "$outfile" "$@"
42 s
.manifest_name
='0000-MANIFEST.txt'
48 s
._package_files
= { } # map filename => infol
50 def thing_matches_globs(s
, thing
, globs
):
52 negate
= pat
.startswith('!')
53 if negate
: pat
= pat
[1:]
54 if fnmatch
.fnmatch(thing
, pat
):
58 def src_filter_glob(s
, src
): # default s.src_filter
59 return s
.thing_matches_globs(src
, s
.src_filter_globs
)
61 def src_direxcludes_git(s
, d
):
63 excl
= open(os
.path
.join(d
, '.gitignore'))
64 except FileNotFoundError
:
69 if l
.startswith('#'): next
74 def src_likeparent_git(s
, src
):
76 os
.stat(os
.path
.join(src
, '.git/.'))
77 except FileNotFoundError
:
82 def src_parentfinder(s
, src
, infol
): # callers may monkey-patch away
83 for deref
in (False,True):
88 search
= os
.path
.realpath(search
)
92 xinfo
.append(os
.path
.basename(search
))
93 search
= os
.path
.dirname(search
)
96 stab
= os
.lstat(search
)
97 except FileNotFoundError
:
99 if stat
.S_ISREG(stab
.st_mode
):
102 while not os
.path
.ismount(search
):
103 if s
.src_likeparent(search
):
105 if len(xinfo
): infol
.append('want=' + os
.path
.join(*xinfo
))
110 # no .git found anywhere
113 def path_prenormaliser(s
, d
, infol
): # callers may monkey-patch away
114 return os
.path
.join(s
.cwd
, os
.path
.abspath(d
))
116 def srcdir_find_rune(s
, d
):
117 script
= s
.find_rune_base
118 ignores
= s
.ignores
+ [s
.output_name
, s
.manifest_name
]
119 ignores
+= s
.src_direxcludes(d
)
121 assert("'" not in excl
)
122 script
+= r
" \! -name '%s'" % excl
123 script
+= r
" \! -path '*/%s/*'" % excl
127 def manifest_append(s
, name
, infol
):
128 s
._manifest
.append({ 'file':name
, 'info':' '.join(infol
) })
130 def manifest_append_absentfile(s
, name
, infol
):
131 s
._manifest
.append({ 'file_print':name
, 'info':' '.join(infol
) })
133 def new_output_name(s
, nametail
, infol
):
135 name
= '%04d-%s' %
(s
._outcounter
, nametail
)
136 s
.manifest_append(name
, infol
)
139 def open_output_fh(s
, name
, mode
):
140 return open(os
.path
.join(s
._destdir
, name
), mode
)
142 def src_dir(s
, d
, infol
):
143 try: name
= s
._dirmap
[d
]
144 except KeyError: pass
146 s
.manifest_append(name
, infol
)
149 if s
.show_pathnames
: infol
.append(d
)
150 find_rune
= s
.srcdir_find_rune(d
)
151 total_rune
= s
.rune_cpio % find_rune
153 name
= s
.new_output_name('src.cpio', infol
)
155 fh
= s
.open_output_fh(name
, 'wb')
157 subprocess
.run(s
.rune_shell
+ [total_rune
],
159 stdin
=subprocess
.DEVNULL
,
161 restore_signals
=True,
165 def src_indir(s
, d
, infol
):
166 d
= s
.path_prenormaliser(d
, infol
)
167 if not s
.src_filter(d
): return
169 d
= s
.src_parentfinder(d
, infol
)
173 def report_from_packages_debian(s
, files
):
174 dpkg_S_in
= tempfile
.TemporaryFile(mode
='w+')
175 for (file, infols
) in files
.items():
176 assert('\n' not in file)
177 dpkg_S_in
.write(file)
178 dpkg_S_in
.write('\0')
180 cmdl
= ['xargs','-0r','dpkg','-S','--']
181 dpkg_S
= subprocess
.Popen(cmdl
,
184 stdout
=subprocess
.PIPE
,
187 dpkg_show_in
= tempfile
.TemporaryFile(mode
='w+')
189 for l
in dpkg_S
.stdout
:
190 l
= l
.strip(b
'\n').decode('utf-8')
191 (pkgs
, fname
) = l
.split(': ',1)
192 pks
= pkgs
.split(', ')
194 pkginfos
.setdefault(pk
,{'files':[]})['files'].append(fname
)
195 print(pk
, file=dpkg_show_in
)
196 assert(dpkg_S
.wait() == 0)
198 cmdl
= ['xargs','-r','dpkg-query',
199 r
'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\n',
201 dpkg_show
= subprocess
.Popen(cmdl
,
204 stdout
=subprocess
.PIPE
,
207 for l
in dpkg_show
.stdout
:
208 l
= l
.strip(b
'\n').decode('utf-8')
209 (pk
,p
,a
,v
,sp
,sv
) = l
.split('\t')
210 pkginfos
[pk
]['binary'] = p
211 pkginfos
[pk
]['arch'] = a
212 pkginfos
[pk
]['version'] = v
213 pkginfos
[pk
]['source'] = sp
214 pkginfos
[pk
]['sourceversion'] = sv
215 assert(dpkg_show
.wait() == 0)
216 for pk
in sorted(pkginfos
.keys()):
218 debfname
= '%s_%s_%s.deb' %
(pi
['binary'], pi
['version'], pi
['arch'])
219 dscfname
= '%s_%s.dsc' %
(pi
['source'], pi
['sourceversion'])
220 s
.manifest_append_absentfile(dscfname
, [debfname
])
221 for fname
in pi
['files']:
223 if s
.show_pathnames
: infol
= infol
+ ['loaded='+fname
]
224 s
.manifest_append_absentfile(' \t' + debfname
, infol
)
226 def thing_ought_packaged(s
, fname
):
227 return s
.thing_matches_globs(fname
, s
.src_package_globs
)
229 def src_file_packaged(s
, fname
, infol
):
230 s
._package_files
.setdefault(fname
,[]).extend(infol
)
232 def src_file(s
, fname
, infol
):
235 infol_copy
= infol
.copy()
236 yield (infol_copy
, s
.path_prenormaliser(fname
, infol_copy
))
237 yield (infol
, os
.path
.realpath(fname
))
239 for (tinfol
, tfname
) in fngens():
240 if s
.thing_ought_packaged(tfname
):
241 s
.src_file_packaged(tfname
, tinfol
)
244 s
.src_indir(fname
, infol
)
246 def src_argv0(s
, program
, infol
):
247 s
.src_file(program
, infol
)
249 def src_syspath(s
, fname
, infol
):
250 if s
.thing_ought_packaged(fname
): return
251 s
.src_indir(fname
, infol
)
253 def src_module(s
, m
, infol
):
254 try: fname
= m
.__file__
255 except AttributeError: return
256 infol
.append('module='+m
.__name__
)
258 if s
.thing_ought_packaged(fname
):
259 s
.src_file_packaged(fname
, infol
)
261 s
.src_indir(fname
, infol
)
263 def srcs_allitems(s
, dirs
=sys
.path
):
264 s
.src_argv0(sys
.argv
[0], ['argv[0]'])
266 s
.src_syspath(d
, ['sys.path'])
267 for m
in sys
.modules
.values():
268 s
.src_module(m
, ['sys.modules'])
269 s
.report_from_packages(s
._package_files
)
271 def mk_portmanteau(s
):
272 cmdl
= s
.rune_shell
+ [ s
.rune_portmanteau
, 'x',
273 s
.output_name
, s
.manifest_name
]
274 mfh
= s
.open_output_fh(s
.manifest_name
,'w')
275 for me
in s
._manifest
:
276 try: fname
= me
['file']
277 except KeyError: fname
= me
.get('file_print','')
278 else: cmdl
.append(fname
)
279 print('%s\t%s' %
(fname
, me
['info']), file=mfh
)
283 stdin
=subprocess
.DEVNULL
,
285 restore_signals
=True,