ownsource: wip packages

[hippotat] / hippotatlib / ownsource.py

# -*- python -*-
#
# Hippotat - Asinine IP Over HTTP program
# hippotatlib/ownsource.py - Automatic source code provision (AGPL compliance)
#
# Copyright 2017 Ian Jackson
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version, with the "CAF Login
# Exception" as published by Ian Jackson (version 2, or at your option
# any later version) as an Additional Permission.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License and the CAF Login Exception along with this program, in the
# file AGPLv3+CAFv2.  If not, email Ian Jackson
# <ijackson@chiark.greenend.org.uk>.


import os
import sys
import fnmatch
import stat
import subprocess
import tempfile
import shutil

try: import debian.deb822
except ImportError: pass

class SourceShipmentPreparer():
  def __init__(s, destdir):
    # caller may modify, and should read after calling generate()
    s.output_names = ['srcbomb.tar.gz', 'fullsrcbomb.tar']
    s.output_paths = [None,None] # alternatively caller may read this
    # defaults, caller can modify after creation
    s.logger = lambda m: print('SourceShipmentPreparer',m)
    s.src_filter = s.src_filter_glob
    s.src_package_globs = ['!/usr/local/*', '/usr*']
    s.src_filter_globs = ['!/etc/*']
    s.src_likeparent = s.src_likeparent_git
    s.src_direxcludes = s.src_direxcludes_git
    s.report_from_packages = s.report_from_packages_debian
    s.cwd = os.getcwd()
    s.find_rune_base = "find -type f -perm -004 \! -path '*/tmp/*'"
    s.ignores = ['*~', '*.bak', '*.tmp', '#*#', '__pycache__',
                  '[0-9][0-9][0-9][0-9]-src.cpio']
    s.rune_shell = ['/bin/bash', '-ec']
    s.show_pathnames = True
    s.download_packages = True
    s.rune_cpio = r'''
            set -o pipefail
	    (
	     %s
             # ^ by default, is find ... -print0
	    ) | (
	     cpio -Hustar -o --quiet -0 -R 1000:1000 || \
             cpio -Hustar -o --quiet -0
            )
    '''
    s.rune_portmanteau = r'''
            GZIP=-1 tar zcf - "$@"
    '''
    s.rune_portmanteau_uncompressed = r'''
            tar cf - "$@"
    '''
    s.manifest_name='0000-MANIFEST.txt'
    # private
    s._destdir = destdir
    s._outcounter = 0
    s._manifest = []
    s._dirmap = { }
    s._package_files = { } # map filename => infol
    s._packages_path = os.path.join(s._destdir, 'packages')
    s._package_sources = []

  def thing_matches_globs(s, thing, globs):
    for pat in globs:
      negate = pat.startswith('!')
      if negate: pat = pat[1:]
      if fnmatch.fnmatch(thing, pat):
        return not negate
    return negate

  def src_filter_glob(s, src): # default s.src_filter
    return s.thing_matches_globs(src, s.src_filter_globs)

  def src_direxcludes_git(s, d):
    try:
      excl = open(os.path.join(d, '.gitignore'))
    except FileNotFoundError:
      return []
    r = []
    for l in excl:
      l.strip
      if l.startswith('#'): next
      if not len(l): next
      r += l
    return r

  def src_likeparent_git(s, src):
    try:
      os.stat(os.path.join(src, '.git/.'))
    except FileNotFoundError:
      return False
    else:
      return True

  def src_parentfinder(s, src, infol): # callers may monkey-patch away
    for deref in (False,True):
      xinfo = []

      search = src
      if deref:
        search = os.path.realpath(search)

      def ascend():
        nonlocal search
        xinfo.append(os.path.basename(search))
        search = os.path.dirname(search)

      try:
        stab = os.lstat(search)
      except FileNotFoundError:
        return
      if stat.S_ISREG(stab.st_mode):
        ascend()

      while not os.path.ismount(search):
        if s.src_likeparent(search):
          xinfo.reverse()
          if len(xinfo): infol.append('want=' + os.path.join(*xinfo))
          return search

        ascend()

    # no .git found anywhere
    return src

  def path_prenormaliser(s, d, infol): # callers may monkey-patch away
    return os.path.join(s.cwd, os.path.abspath(d))

  def srcdir_find_rune(s, d):
    script = s.find_rune_base
    ignores = s.ignores + s.output_names + [s.manifest_name]
    ignores += s.src_direxcludes(d)
    for excl in ignores:
      assert("'" not in excl)
      script += r" \! -name '%s'"     % excl
      script += r" \! -path '*/%s/*'" % excl
    script += ' -print0'
    return script

  def manifest_append(s, name, infol):
    s._manifest.append({ 'file':name, 'info':' '.join(infol) })

  def manifest_append_absentfile(s, name, infol):
    s._manifest.append({ 'file_print':name, 'info':' '.join(infol) })

  def new_output_name(s, nametail, infol):
    s._outcounter += 1
    name = '%04d-%s' % (s._outcounter, nametail)
    s.manifest_append(name, infol)
    return name

  def open_output_fh(s, name, mode):
    return open(os.path.join(s._destdir, name), mode)

  def src_dir(s, d, infol):
    try: name = s._dirmap[d]
    except KeyError: pass
    else:
      s.manifest_append(name, infol)
      return

    if s.show_pathnames: infol.append(d)
    find_rune = s.srcdir_find_rune(d)
    total_rune = s.rune_cpio % find_rune

    name = s.new_output_name('src.cpio', infol)
    s._dirmap[d] = name
    fh = s.open_output_fh(name, 'wb')

    s.logger('packing up into %s: %s (because %s)' %
             (name, d, ' '.join(infol)))

    subprocess.run(s.rune_shell + [total_rune],
                   cwd=d,
                   stdin=subprocess.DEVNULL,
                   stdout=fh,
                   restore_signals=True,
                   check=True)
    fh.close()

  def src_indir(s, d, infol):
    d = s.path_prenormaliser(d, infol)
    if not s.src_filter(d): return

    d = s.src_parentfinder(d, infol)
    if d is None: return
    s.src_dir(d, infol)

  def report_from_packages_debian(s, files):
    dpkg_S_in = tempfile.TemporaryFile(mode='w+')
    for (file, infols) in files.items():
      assert('\n' not in file)
      dpkg_S_in.write(file)
      dpkg_S_in.write('\0')
    dpkg_S_in.seek(0)
    cmdl = ['xargs','-0r','dpkg','-S','--']
    dpkg_S = subprocess.Popen(cmdl,
                              cwd='/',
                              stdin=dpkg_S_in,
                              stdout=subprocess.PIPE,
                              stderr=sys.stderr,
                              close_fds=False)
    dpkg_show_in = tempfile.TemporaryFile(mode='w+')
    pkginfos = { }
    for l in dpkg_S.stdout:
      l = l.strip(b'\n').decode('utf-8')
      (pkgs, fname) = l.split(': ',1)
      pks = pkgs.split(', ')
      for pk in pks:
        pkginfos.setdefault(pk,{'files':[]})['files'].append(fname)
        print(pk, file=dpkg_show_in)
    assert(dpkg_S.wait() == 0)
    dpkg_show_in.seek(0)
    cmdl = ['xargs','-r','dpkg-query',
            r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n',
            '--show','--']
    dpkg_show = subprocess.Popen(cmdl,
                                 cwd='/',
                                 stdin=dpkg_show_in,
                                 stdout=subprocess.PIPE,
                                 stderr=sys.stderr,
                                 close_fds=False)
    for l in dpkg_show.stdout:
      l = l.strip(b'\n').decode('utf-8')
      (pk,p,a,v,sp,sv,suv) = l.split('\t')
      pkginfos[pk]['binary'] = p
      pkginfos[pk]['arch'] = a
      pkginfos[pk]['version'] = v
      pkginfos[pk]['source'] = sp
      pkginfos[pk]['sourceversion'] = sv
      pkginfos[pk]['sourceupstreamversion'] = sv
    assert(dpkg_show.wait() == 0)
    for pk in sorted(pkginfos.keys()):
      pi = pkginfos[pk]
      debfname = '%s_%s_%s.deb' % (pi['binary'], pi['version'], pi['arch'])
      dscfname = '%s_%s.dsc' % (pi['source'], pi['sourceversion'])
      s.manifest_append_absentfile(dscfname, [debfname])
      s.logger('mentioning %s and %s because %s' %
               (dscfname, debfname, pi['files'][0]))
      for fname in pi['files']:
        infol = files[fname]
        if s.show_pathnames: infol = infol + ['loaded='+fname]
        s.manifest_append_absentfile(' \t' + debfname, infol)

      if s.download_packages:
        try: os.mkdir(s._packages_path)
        except FileExistsError: pass

        cmdl = ['apt-get','--download-only','source',
                '%s=%s' % (pi['source'], pi['sourceversion'])]
        subprocess.run(cmdl,
                       cwd=s._packages_path,
                       stdin=subprocess.DEVNULL,
                       stdout=sys.stdout,
                       stderr=sys.stderr,
                       restore_signals=True,
                       check=True)

        s._package_sources.append(dscfname)
        dsc = debian.deb822.Dsc(open(s._packages_path + '/' + dscfname))
        for indsc in dsc['Files']:
          s._package_sources.append(indsc['name'])

  def thing_ought_packaged(s, fname):
    return s.thing_matches_globs(fname, s.src_package_globs)

  def src_file_packaged(s, fname, infol):
    s._package_files.setdefault(fname,[]).extend(infol)

  def src_file(s, fname, infol):
    def fngens():
      yield (infol, fname)
      infol_copy = infol.copy()
      yield (infol_copy, s.path_prenormaliser(fname, infol_copy))
      yield (infol, os.path.realpath(fname))

    for (tinfol, tfname) in fngens():
      if s.thing_ought_packaged(tfname):
        s.src_file_packaged(tfname, tinfol)
        return

    s.src_indir(fname, infol)

  def src_argv0(s, program, infol):
    s.src_file(program, infol)

  def src_syspath(s, fname, infol):
    if s.thing_ought_packaged(fname): return
    s.src_indir(fname, infol)

  def src_module(s, m, infol):
    try: fname = m.__file__
    except AttributeError: return
    infol.append('module='+m.__name__)

    if s.thing_ought_packaged(fname):
      s.src_file_packaged(fname, infol)
    else:
      s.src_indir(fname, infol)

  def srcs_allitems(s, dirs=sys.path):
    s.logger('allitems')
    s.src_argv0(sys.argv[0], ['argv[0]'])
    for d in sys.path:
      s.src_syspath(d, ['sys.path'])
    for m in sys.modules.values():
      s.src_module(m, ['sys.modules'])
    s.report_from_packages(s._package_files)
    s.logger('allitems done')

  def _mk_portmanteau(s, ix, rune, cwd, files):
    output_name = s.output_names[ix]
    s.logger('making portmanteau %s' % output_name)
    output_path = os.path.join(s._destdir, output_name)
    subprocess.run(s.rune_shell + [ rune, 'x' ] + files,
                   cwd=cwd,
                   stdin=subprocess.DEVNULL,
                   stdout=open(output_path, 'wb'),
                   restore_signals=True,
                   check=True)
    s.output_paths[ix] = output_path

  def mk_inner_portmanteau(s):
    outputs = [s.manifest_name]
    mfh = s.open_output_fh(s.manifest_name,'w')
    for me in s._manifest:
      try: fname = me['file']
      except KeyError: fname = me.get('file_print','')
      else: outputs.append(fname)
      print('%s\t%s' % (fname, me['info']), file=mfh)
    mfh.close()

    s._mk_portmanteau(0, s.rune_portmanteau,
                      s._destdir, outputs)

  def mk_packages_portmanteau(s):
    s._mk_portmanteau(1, s.rune_portmanteau_uncompressed,
                      s._packages_path, s._package_sources)

  def generate(s):
    s.srcs_allitems()
    s.mk_inner_portmanteau()
    s.mk_packages_portmanteau()
    s.logger('portmanteau ready in %s %s' % tuple(s.output_paths))