From 2d24e0586be6f84826ce0043a7449d1dfdfb3556 Mon Sep 17 00:00:00 2001 From: Fredrik Fornwall Date: Sun, 19 Nov 2017 14:29:49 +0100 Subject: [PATCH] Cleanup scripts/buildorder.py --- scripts/buildorder.py | 222 ++++++++++++++++++++++++-------------------------- 1 file changed, 107 insertions(+), 115 deletions(-) diff --git a/scripts/buildorder.py b/scripts/buildorder.py index e371a9d9..02b94979 100755 --- a/scripts/buildorder.py +++ b/scripts/buildorder.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# buildorder.py - script to generate a build order respecting package dependencies +"Script to generate a build order respecting package dependencies." import os import re @@ -8,12 +8,12 @@ import sys from itertools import filterfalse -# https://docs.python.org/3/library/itertools.html#itertools-recipes - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D + """List unique elements, preserving order. Remember all elements ever seen. + See https://docs.python.org/3/library/itertools.html#itertools-recipes + Examples: + unique_everseen('AAAABBBCCDAABBB') --> A B C D + unique_everseen('ABBCcAD', str.lower) --> A B C D""" seen = set() seen_add = seen.add if key is None: @@ -29,50 +29,44 @@ def unique_everseen(iterable, key=None): def die(msg): + "Exit the process with an error message." sys.exit('ERROR: ' + msg) -def rchop(thestring, ending): - if thestring.endswith(ending): - return thestring[:-len(ending)] - return thestring - -class TermuxBuildFile(object): - def __init__(self, path): - self.path = path - - def _get_dependencies(self): - pkg_dep_prefix = 'TERMUX_PKG_DEPENDS=' - pkg_build_dep_prefix = 'TERMUX_PKG_BUILD_DEPENDS=' - subpkg_dep_prefix = 'TERMUX_SUBPKG_DEPENDS=' - comma_deps = '' - - with open(self.path, encoding="utf-8") as f: - prefix = None - for line in f: - if line.startswith(pkg_dep_prefix): - prefix = pkg_dep_prefix - elif line.startswith(pkg_build_dep_prefix): - prefix = pkg_build_dep_prefix - elif line.startswith(subpkg_dep_prefix): - prefix = subpkg_dep_prefix - else: - continue - comma_deps += line[len(prefix):].replace('"', '').replace("'", '').replace("\n", ",") +def parse_build_file_dependencies(path): + "Extract the dependencies of a build.sh or *.subpackage.sh file." + pkg_dep_prefix = 'TERMUX_PKG_DEPENDS=' + pkg_build_dep_prefix = 'TERMUX_PKG_BUILD_DEPENDS=' + subpkg_dep_prefix = 'TERMUX_SUBPKG_DEPENDS=' + dependencies = [] + + with open(path, encoding="utf-8") as build_script: + prefix = None + for line in build_script: + if line.startswith(pkg_dep_prefix): + prefix = pkg_dep_prefix + elif line.startswith(pkg_build_dep_prefix): + prefix = pkg_build_dep_prefix + elif line.startswith(subpkg_dep_prefix): + prefix = subpkg_dep_prefix + else: + continue + + dependencies_string = line[len(prefix):] + for char in "\"'\n": + dependencies_string = dependencies_string.replace(char, '') - # Remove trailing ',' that is otherwise replacing the final newline - comma_deps = comma_deps[:-1] - if not comma_deps: - # no deps found - return set() + for dependency_value in dependencies_string.split(','): + # Replace parenthesis to ignore version qualifiers as in "gcc (>= 5.0)": + dependency_value = re.sub(r'\(.*?\)', '', dependency_value).strip() + dependency_value = re.sub('-dev$', '', dependency_value) + dependencies.append(dependency_value) - return set([ - # Replace parenthesis to handle version qualifiers, as in "gcc (>= 5.0)": - rchop(re.sub(r'\(.*?\)', '', dep).strip(), '-dev') for dep in comma_deps.split(',') - ]) + return set(dependencies) class TermuxPackage(object): + "A main package definition represented by a directory with a build.sh file." def __init__(self, dir_path): self.dir = dir_path self.name = os.path.basename(self.dir) @@ -82,8 +76,7 @@ class TermuxPackage(object): if not os.path.isfile(build_sh_path): raise Exception("build.sh not found for package '" + self.name + "'") - self.buildfile = TermuxBuildFile(build_sh_path) - self.deps = self.buildfile._get_dependencies() + self.deps = parse_build_file_dependencies(build_sh_path) if 'libandroid-support' not in self.deps and self.name != 'libandroid-support': # Every package may depend on libandroid-support without declaring it: self.deps.add('libandroid-support') @@ -92,7 +85,8 @@ class TermuxPackage(object): self.subpkgs = [] for filename in os.listdir(self.dir): - if not filename.endswith('.subpackage.sh'): continue + if not filename.endswith('.subpackage.sh'): + continue subpkg = TermuxSubPackage(self.dir + '/' + filename, self) self.subpkgs.append(subpkg) @@ -103,71 +97,72 @@ class TermuxPackage(object): # Do not depend on any sub package self.deps.difference_update([subpkg.name for subpkg in self.subpkgs]) - self.needed_by = set() # to be completed outside, reverse of deps + self.needed_by = set() # Populated outside constructor, reverse of deps. def __repr__(self): return "<{} '{}'>".format(self.__class__.__name__, self.name) + def recursive_dependencies(self, pkgs_map): + "All the dependencies of the package, both direct and indirect." + result = [] + for dependency_name in sorted(self.deps): + dependency_package = pkgs_map[dependency_name] + result += dependency_package.recursive_dependencies(pkgs_map) + result += [dependency_package] + return unique_everseen(result) -class TermuxSubPackage(TermuxPackage): + +class TermuxSubPackage: + "A sub-package represented by a ${PACKAGE_NAME}.subpackage.sh file." def __init__(self, subpackage_file_path, parent): if parent is None: raise Exception("SubPackages should have a parent") - self.buildfile = TermuxBuildFile(subpackage_file_path) self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0] self.parent = parent - self.deps = self.buildfile._get_dependencies() + self.deps = parse_build_file_dependencies(subpackage_file_path) def __repr__(self): return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent) -# Tracks non-visited deps for each package -remaining_deps = {} - -# Mapping from package name to TermuxPackage -# (if subpackage, mapping from subpackage name to parent package) -pkgs_map = {} - -# Reverse dependencies -pkg_depends_on = {} - -PACKAGES_DIRS = ['packages'] - - -def populate(): +def read_packages_from_directories(directories): + """Construct a map from package name to TermuxPackage. + For subpackages this maps from the subpackage name to the parent package.""" + pkgs_map = {} all_packages = [] - for package_dir in PACKAGES_DIRS: + + for package_dir in directories: for pkgdir_name in sorted(os.listdir(package_dir)): dir_path = package_dir + '/' + pkgdir_name if os.path.isfile(dir_path + '/build.sh'): - all_packages.append(TermuxPackage(package_dir + '/' + pkgdir_name)) - - for pkg in all_packages: - if pkg.name in pkgs_map: die('Duplicated package: ' + pkg.name) - else: pkgs_map[pkg.name] = pkg - - for subpkg in pkg.subpkgs: - pkgs_map[subpkg.name] = pkg - remaining_deps[subpkg.name] = set(subpkg.deps) - - remaining_deps[pkg.name] = set(pkg.deps) + new_package = TermuxPackage(package_dir + '/' + pkgdir_name) - all_pkg_names = set(pkgs_map.keys()) + if new_package.name in pkgs_map: + die('Duplicated package: ' + new_package.name) + else: + pkgs_map[new_package.name] = new_package + all_packages.append(new_package) - for name, pkg in pkgs_map.items(): - for dep_name in remaining_deps[name]: - if dep_name not in all_pkg_names: - die('Package %s depends on non-existing package "%s"' % ( - name, dep_name - )) + for subpkg in new_package.subpkgs: + if subpkg.name in pkgs_map: + die('Duplicated package: ' + subpkg.name) + else: + pkgs_map[subpkg.name] = new_package + all_packages.append(subpkg) - dep_pkg = pkgs_map[dep_name] - dep_pkg.needed_by.add(pkg) + for pkg in all_packages: + for dependency_name in pkg.deps: + if dependency_name not in pkgs_map: + die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name)) + dep_pkg = pkgs_map[dependency_name] + if not isinstance(pkg, TermuxSubPackage): + dep_pkg.needed_by.add(pkg) + return pkgs_map -def generate_full_buildorder(): +def generate_full_buildorder(pkgs_map): + "Generate a build order for building all packages." build_order = [] # List of all TermuxPackages without dependencies @@ -182,6 +177,13 @@ def generate_full_buildorder(): # Topological sorting visited = set() + # Tracks non-visited deps for each package + remaining_deps = {} + for name, pkg in pkgs_map.items(): + remaining_deps[name] = set(pkg.deps) + for subpkg in pkg.subpkgs: + remaining_deps[subpkg.name] = set(subpkg.deps) + while pkg_queue: pkg = pkg_queue.pop(0) if pkg.name in visited: @@ -212,50 +214,40 @@ def generate_full_buildorder(): return build_order -def deps(pkg): - l = [] - for dep in sorted(pkg.deps): - l += deps_then_me(pkgs_map[dep]) - return l - -def deps_then_me(pkg): - l = [] - for dep in sorted(pkg.deps): - l += deps_then_me(pkgs_map[dep]) - l += [pkg] - return l - - -def generate_targets_buildorder(target_paths): - buildorder = [] - for target_path in target_paths: - if target_path.endswith('/'): target_path = target_path[:-1] - pkgname = os.path.basename(target_path) - if not pkgname in pkgs_map: - die('Dependencies for ' + pkgname + ' could not be calculated (skip dependency check with -s)') - buildorder += deps(pkgs_map[pkgname]) +def generate_target_buildorder(target_path, pkgs_map): + "Generate a build order for building the dependencies of the specified package." + if target_path.endswith('/'): + target_path = target_path[:-1] - return unique_everseen(buildorder) + package_name = os.path.basename(target_path) + package = pkgs_map[package_name] + return package.recursive_dependencies(pkgs_map) -if __name__ == '__main__': +def main(): + "Generate the build order either for all packages or a specific one." + packages_directories = ['packages'] full_buildorder = len(sys.argv) == 1 if not full_buildorder: packages_real_path = os.path.realpath('packages') for path in sys.argv[1:]: if not os.path.isdir(path): die('Not a directory: ' + path) - if path.endswith('/'): path = path[:-1] + if path.endswith('/'): + path = path[:-1] parent_path = os.path.dirname(path) if packages_real_path != os.path.realpath(parent_path): - PACKAGES_DIRS.append(parent_path) + packages_directories.append(parent_path) - populate() + pkgs_map = read_packages_from_directories(packages_directories) if full_buildorder: - bo = generate_full_buildorder() + build_order = generate_full_buildorder(pkgs_map) else: - bo = generate_targets_buildorder(sys.argv[1:]) + build_order = generate_target_buildorder(sys.argv[1], pkgs_map) - for pkg in bo: + for pkg in build_order: print(pkg.dir) + +if __name__ == '__main__': + main() -- 2.11.0