summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorMichael Olbrich <m.olbrich@pengutronix.de>2019-10-06 19:26:17 +0200
committerMichael Olbrich <m.olbrich@pengutronix.de>2019-10-06 20:02:55 +0200
commitec29df1274f596712f375745526f9de27847aa36 (patch)
tree2531a9408014b7deb16bb2e216f04a06b376f8c8 /scripts
parent683521e253aa231e42d9f7ef93ec1251b3dd03c9 (diff)
downloadptxdist-ec29df1274f596712f375745526f9de27847aa36.tar.gz
ptxdist-ec29df1274f596712f375745526f9de27847aa36.tar.xz
ptxd_lib_dgen: speed up hash generation
To create the hash for patch files, ptxdist executes several processes for each package with patches. With many packages with patches, this creates quite a lot of startup overhead. To avoid this, collect all patch directories first and handle them with an extra awk script and one find call. Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
Diffstat (limited to 'scripts')
-rw-r--r--scripts/lib/ptxd_lib_dgen.awk16
-rwxr-xr-xscripts/lib/ptxd_make_pkghash.awk68
2 files changed, 77 insertions, 7 deletions
diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
index 024a3dc48..eeb34fbdb 100644
--- a/scripts/lib/ptxd_lib_dgen.awk
+++ b/scripts/lib/ptxd_lib_dgen.awk
@@ -366,18 +366,15 @@ function write_deps_pkg_active_cfghash(this_PKG, this_pkg) {
print "ifeq ($(" this_PKG "_PATCH_DIR),)" > DGEN_DEPS_POST;
print "undefine " this_PKG "_PATCH_DIR" > DGEN_DEPS_POST;
print "else" > DGEN_DEPS_POST;
- print "ifeq ($(wildcard " PTXDIST_TEMPDIR "/pkghash-" this_PKG "_EXTRACT.done),)" > DGEN_DEPS_POST;
- print "$(call ptx/force-sh, find '$(" this_PKG "_PATCH_DIR)' -type f ! -name '.*' | sort | xargs cat | tee " \
- PTXDIST_TEMPDIR "/pkghash-" this_PKG "_EXTRACT >> " PTXDIST_TEMPDIR "/pkghash-" this_PKG \
- " && touch " PTXDIST_TEMPDIR "/pkghash-" this_PKG "_EXTRACT.done )" > DGEN_DEPS_POST;
+ print "ifeq ($(PTXDIST_PKGHASH_MAKE),)" > DGEN_DEPS_POST;
+ print "$(file >>" PTXDIST_TEMPDIR "/pkghash.list,PATCHES: " this_PKG " $(" this_PKG "_PATCH_DIR))" > DGEN_DEPS_POST;
print "endif" > DGEN_DEPS_POST;
print "endif" > DGEN_DEPS_POST;
print "endif" > DGEN_DEPS_POST;
+ print "ifeq ($(PTXDIST_PKGHASH_MAKE),)" > DGEN_DEPS_POST;
print "ifneq ($(filter /%,$(" this_PKG "_CONFIG)),)" > DGEN_DEPS_POST;
print "ifneq ($(wildcard $(" this_PKG "_CONFIG)),)" > DGEN_DEPS_POST;
- print "ifeq ($(wildcard " PTXDIST_TEMPDIR "/pkghash-" this_PKG ".done),)" > DGEN_DEPS_POST;
- print "$(call ptx/force-sh, cat '$(" this_PKG "_CONFIG)' >> " PTXDIST_TEMPDIR "/pkghash-" this_PKG \
- " && touch " PTXDIST_TEMPDIR "/pkghash-" this_PKG ".done )" > DGEN_DEPS_POST;
+ print "$(file >>" PTXDIST_TEMPDIR "/pkghash.list,CONFIG: " this_PKG " $(" this_PKG "_CONFIG))" > DGEN_DEPS_POST;
print "endif" > DGEN_DEPS_POST;
print "endif" > DGEN_DEPS_POST;
print "endif" > DGEN_DEPS_POST;
@@ -550,10 +547,15 @@ function write_deps_pkg_active_image(this_PKG, this_pkg, prefix) {
}
END {
+ print "PTXDIST_PKGHASH_MAKE := $(wildcard " PTXDIST_TEMPDIR "/pkghash.make)" > DGEN_DEPS_POST;
# extend pkghash files fist
for (this_PKG in active_PKG_to_pkg)
write_deps_pkg_active_cfghash(this_PKG, this_pkg)
+ print "ifeq ($(PTXDIST_PKGHASH_MAKE),)" > DGEN_DEPS_POST;
+ print "$(call ptx/force-sh, $(PTXDIST_LIB_DIR)/ptxd_make_pkghash.awk " PTXDIST_TEMPDIR "/pkghash.list)" > DGEN_DEPS_POST;
+ print "endif" > DGEN_DEPS_POST;
+
print "$(call ptx/force-sh, md5sum " PTXDIST_TEMPDIR "/pkghash-* | " \
"sed 's;^\\([a-z0-9]*\\).*pkghash-\\(.*\\)$$;\\2_CFGHASH := \\1;' > " \
PTXDIST_TEMPDIR "/pkghash.make)" > DGEN_DEPS_POST;
diff --git a/scripts/lib/ptxd_make_pkghash.awk b/scripts/lib/ptxd_make_pkghash.awk
new file mode 100755
index 000000000..5f90923a9
--- /dev/null
+++ b/scripts/lib/ptxd_make_pkghash.awk
@@ -0,0 +1,68 @@
+#!/usr/bin/gawk -f
+#
+# Copyright (C) 2019 by Michael Olbrich <m.olbrich@pengutronix.de>
+#
+# For further information about the PTXdist project and license conditions
+# see the README file.
+#
+
+BEGIN {
+ PTXDIST_TEMPDIR = ENVIRON["PTXDIST_TEMPDIR"];
+ dirs = ""
+}
+
+$1 == "PATCHES:" {
+ pkg = $2
+ patch_dir = $3
+ pkgs[patch_dir] = pkgs[patch_dir] " " pkg
+ dirs = dirs " " patch_dir
+}
+
+$1 == "CONFIG:" {
+ pkg = $2
+ config = $3
+ configs[pkg] = configs[pkg] " " config
+}
+
+function dump_file(src, dst, tmp) {
+ if (!src)
+ return
+
+ old_RS = RS
+ RS = "^$"
+ getline tmp < src
+ printf "%s", tmp >> dst
+ RS = old_RS
+ close(src)
+ close(dst)
+}
+
+END {
+ for (pkg in configs) {
+ config = configs[pkg]
+ f1 = PTXDIST_TEMPDIR "/pkghash-" pkg
+ split(configs[pkg], cfgs)
+ asort(cfgs, cfgs)
+ for (config in cfgs)
+ dump_file(cfgs[config], f1)
+ }
+ command = "find " dirs " -type f ! -name '.*' -printf '%H %P\\n'"
+ while (command | getline)
+ files[$1] = files[$1] " " $2
+ close(command)
+ for (dir in pkgs) {
+ split(pkgs[dir], list, " ")
+ split(files[dir], file_list, " ")
+ asort(file_list, file_list)
+ for (pkg in list) {
+ pkg = list[pkg]
+ f1 = PTXDIST_TEMPDIR "/pkghash-" pkg
+ f2 = PTXDIST_TEMPDIR "/pkghash-" pkg "_EXTRACT"
+ for (file in file_list) {
+ file = dir "/" file_list[file]
+ dump_file(file, f1)
+ dump_file(file, f2)
+ }
+ }
+ }
+}