From 5312f51e315cc147ca901d499aa3c3cc64aa3340 Mon Sep 17 00:00:00 2001 From: Noam Postavsky Date: Tue, 14 Apr 2020 12:27:34 -0400 Subject: [PATCH] Optimize admin/nt dependency computation admin/nt/dist-build/build-dep-zips.py (immediate_deps) (extract_deps): Gather package dependency info in batches, rather than one at a time. This reduces the number of invocations of 'pacman -Si ...' to the depth of the dependency tree, rather than the number of dependent packages. (top-level): Don't call 'extract_deps' when given the '-l' option. --- admin/nt/dist-build/build-dep-zips.py | 45 ++++++++++++++------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/admin/nt/dist-build/build-dep-zips.py b/admin/nt/dist-build/build-dep-zips.py index b0345a42cf3..dd7dc5e99dc 100755 --- a/admin/nt/dist-build/build-dep-zips.py +++ b/admin/nt/dist-build/build-dep-zips.py @@ -22,6 +22,8 @@ import os import shutil import re +import functools +import operator from subprocess import check_output @@ -47,7 +49,7 @@ ## Packages to fiddle with ## Source for gcc-libs is part of gcc SKIP_SRC_PKGS=["mingw-w64-gcc-libs"] -SKIP_DEP_PKGS=["mingw-w64-x86_64-glib2"] +SKIP_DEP_PKGS=frozenset(["mingw-w64-x86_64-glib2"]) MUNGE_SRC_PKGS={"mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git"} MUNGE_DEP_PKGS={ "mingw-w64-i686-libwinpthread":"mingw-w64-i686-libwinpthread-git", @@ -68,16 +70,14 @@ def check_output_maybe(*args,**kwargs): else: return check_output(*args,**kwargs) -def immediate_deps(pkg): - package_info = check_output(["pacman", "-Si", pkg]).decode("utf-8").split("\n") +def immediate_deps(pkgs): + package_info = check_output(["pacman", "-Si"] + pkgs).decode("utf-8").splitlines() - ## Extract the "Depends On" line - depends_on = [x for x in package_info if x.startswith("Depends On")][0] - ## Remove "Depends On" prefix - dependencies = depends_on.split(":")[1] - - ## Split into dependencies - dependencies = dependencies.strip().split(" ") + ## Extract the packages listed for "Depends On:" lines. + dependencies = [line.split(":")[1].split() for line in package_info + if line.startswith("Depends On")] + ## Flatten dependency lists from multiple packages into one list. + dependencies = functools.reduce(operator.iconcat, dependencies, []) ## Remove > signs TODO can we get any other punctuation here? dependencies = [d.split(">")[0] for d in dependencies if d] @@ -92,18 +92,19 @@ def extract_deps(): print( "Extracting deps" ) # Get a list of all dependencies needed for packages mentioned above. - pkgs = PKG_REQ[:] + pkgs = set(PKG_REQ) print("Initial pkgs", pkgs) - n = 0 - while n < len(pkgs): - subdeps = immediate_deps(pkgs[n]) - for p in subdeps: - if not (p in pkgs or p in SKIP_DEP_PKGS): - print("adding", p) - pkgs.append(p) - n = n + 1 - - return sorted(pkgs) + newdeps = pkgs + print("adding...") + while True: + subdeps = frozenset(immediate_deps(list(newdeps))) + newdeps = subdeps - SKIP_DEP_PKGS - pkgs + if not newdeps: + break + print('\n'.join(newdeps)) + pkgs |= newdeps + + return list(pkgs) def gather_deps(deps, arch, directory): @@ -261,7 +262,7 @@ def clean(): if( args.l ): print("List of dependencies") - print( extract_deps() ) + print( deps ) exit(0) if args.s: -- 2.23.0.windows.1