From ec29df1274f596712f375745526f9de27847aa36 Mon Sep 17 00:00:00 2001
From: Michael Olbrich <m.olbrich@pengutronix.de>
Date: Sun, 6 Oct 2019 19:26:17 +0200
Subject: ptxd_lib_dgen: speed up hash generation

To create the hash for patch files, ptxdist executes several processes for
each package with patches. With many packages with patches, this creates
quite a lot of startup overhead.
To avoid this, collect all patch directories first and handle them with an
extra awk script and one find call.

Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
---
 scripts/lib/ptxd_lib_dgen.awk     | 16 +++++----
 scripts/lib/ptxd_make_pkghash.awk | 68 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 7 deletions(-)
 create mode 100755 scripts/lib/ptxd_make_pkghash.awk

diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
index 024a3dc48..eeb34fbdb 100644
--- a/scripts/lib/ptxd_lib_dgen.awk
+++ b/scripts/lib/ptxd_lib_dgen.awk
@@ -366,18 +366,15 @@ function write_deps_pkg_active_cfghash(this_PKG, this_pkg) {
 	print "ifeq ($(" this_PKG "_PATCH_DIR),)"								> DGEN_DEPS_POST;
 	print "undefine " this_PKG "_PATCH_DIR"									> DGEN_DEPS_POST;
 	print "else"												> DGEN_DEPS_POST;
-	print "ifeq ($(wildcard " PTXDIST_TEMPDIR "/pkghash-" this_PKG "_EXTRACT.done),)"			> DGEN_DEPS_POST;
-	print "$(call ptx/force-sh, find '$(" this_PKG "_PATCH_DIR)' -type f ! -name '.*' | sort | xargs cat | tee " \
-		PTXDIST_TEMPDIR "/pkghash-" this_PKG "_EXTRACT >> " PTXDIST_TEMPDIR "/pkghash-" this_PKG \
-		" && touch " PTXDIST_TEMPDIR "/pkghash-" this_PKG "_EXTRACT.done )"				> DGEN_DEPS_POST;
+	print "ifeq ($(PTXDIST_PKGHASH_MAKE),)"									> DGEN_DEPS_POST;
+	print "$(file >>" PTXDIST_TEMPDIR "/pkghash.list,PATCHES: " this_PKG " $(" this_PKG "_PATCH_DIR))"	> DGEN_DEPS_POST;
 	print "endif"												> DGEN_DEPS_POST;
 	print "endif"												> DGEN_DEPS_POST;
 	print "endif"												> DGEN_DEPS_POST;
+	print "ifeq ($(PTXDIST_PKGHASH_MAKE),)"									> DGEN_DEPS_POST;
 	print "ifneq ($(filter /%,$(" this_PKG "_CONFIG)),)"							> DGEN_DEPS_POST;
 	print "ifneq ($(wildcard $(" this_PKG "_CONFIG)),)"							> DGEN_DEPS_POST;
-	print "ifeq ($(wildcard " PTXDIST_TEMPDIR "/pkghash-" this_PKG ".done),)"				> DGEN_DEPS_POST;
-	print "$(call ptx/force-sh, cat '$(" this_PKG "_CONFIG)' >> " PTXDIST_TEMPDIR "/pkghash-" this_PKG \
-		" && touch " PTXDIST_TEMPDIR "/pkghash-" this_PKG ".done )"					> DGEN_DEPS_POST;
+	print "$(file >>" PTXDIST_TEMPDIR "/pkghash.list,CONFIG: " this_PKG " $(" this_PKG "_CONFIG))"		> DGEN_DEPS_POST;
 	print "endif"												> DGEN_DEPS_POST;
 	print "endif"												> DGEN_DEPS_POST;
 	print "endif"												> DGEN_DEPS_POST;
@@ -550,10 +547,15 @@ function write_deps_pkg_active_image(this_PKG, this_pkg, prefix) {
 }
 
 END {
+	print "PTXDIST_PKGHASH_MAKE := $(wildcard " PTXDIST_TEMPDIR "/pkghash.make)"				> DGEN_DEPS_POST;
 	# extend pkghash files fist
 	for (this_PKG in active_PKG_to_pkg)
 		write_deps_pkg_active_cfghash(this_PKG, this_pkg)
 
+	print "ifeq ($(PTXDIST_PKGHASH_MAKE),)"									> DGEN_DEPS_POST;
+	print "$(call ptx/force-sh, $(PTXDIST_LIB_DIR)/ptxd_make_pkghash.awk " PTXDIST_TEMPDIR "/pkghash.list)"	> DGEN_DEPS_POST;
+	print "endif"												> DGEN_DEPS_POST;
+
 	print "$(call ptx/force-sh, md5sum " PTXDIST_TEMPDIR "/pkghash-* | " \
 		"sed 's;^\\([a-z0-9]*\\).*pkghash-\\(.*\\)$$;\\2_CFGHASH := \\1;' > " \
 			PTXDIST_TEMPDIR "/pkghash.make)"							> DGEN_DEPS_POST;
diff --git a/scripts/lib/ptxd_make_pkghash.awk b/scripts/lib/ptxd_make_pkghash.awk
new file mode 100755
index 000000000..5f90923a9
--- /dev/null
+++ b/scripts/lib/ptxd_make_pkghash.awk
@@ -0,0 +1,68 @@
+#!/usr/bin/gawk -f
+#
+# Copyright (C) 2019 by Michael Olbrich <m.olbrich@pengutronix.de>
+#
+# For further information about the PTXdist project and license conditions
+# see the README file.
+#
+
+BEGIN {
+	PTXDIST_TEMPDIR		= ENVIRON["PTXDIST_TEMPDIR"];
+	dirs = ""
+}
+
+$1 == "PATCHES:" {
+	pkg = $2
+	patch_dir = $3
+	pkgs[patch_dir]  = pkgs[patch_dir] " " pkg
+	dirs = dirs " " patch_dir
+}
+
+$1 == "CONFIG:" {
+	pkg = $2
+	config = $3
+	configs[pkg] = configs[pkg] " " config
+}
+
+function dump_file(src, dst, tmp) {
+	if (!src)
+		return
+
+	old_RS = RS
+	RS = "^$"
+	getline tmp < src
+	printf "%s", tmp >> dst
+	RS = old_RS
+	close(src)
+	close(dst)
+}
+
+END {
+	for (pkg in configs) {
+		config = configs[pkg]
+		f1 = PTXDIST_TEMPDIR "/pkghash-" pkg
+		split(configs[pkg], cfgs)
+		asort(cfgs, cfgs)
+		for (config in cfgs)
+			dump_file(cfgs[config], f1)
+	}
+	command = "find " dirs " -type f ! -name '.*' -printf '%H %P\\n'"
+	while (command | getline)
+		files[$1] = files[$1] " " $2
+	close(command)
+	for (dir in pkgs) {
+		split(pkgs[dir], list, " ")
+		split(files[dir], file_list, " ")
+		asort(file_list, file_list)
+		for (pkg in list) {
+			pkg = list[pkg]
+			f1 = PTXDIST_TEMPDIR "/pkghash-" pkg
+			f2 = PTXDIST_TEMPDIR "/pkghash-" pkg "_EXTRACT"
+			for (file in file_list) {
+				file = dir "/" file_list[file]
+				dump_file(file, f1)
+				dump_file(file, f2)
+			}
+		}
+	}
+}
-- 
cgit v1.2.3