[openwrt/openwrt] build: make find_md5 reproducible with AUTOREMOVE

[LEDE Commits]

ansuel pushed a commit to openwrt/openwrt.git, branch master:
https://git.openwrt.org/53a08e37437972ba0a8fbf953a93a70a6b784ef4

commit 53a08e37437972ba0a8fbf953a93a70a6b784ef4
Author: Christian Marangi <ansuelsmth at gmail.com>
AuthorDate: Tue Sep 6 16:48:37 2022 +0200

    build: make find_md5 reproducible with AUTOREMOVE
    
    While experimenting with the AUTOREMOVE option in search of a way to use
    prebuilt host tools in different buildroot, it was discovered that the
    md5 generated by find_md5 in depends.mk is not reproducible.
    
    Currently the hash is generated by the path of the file in addition to
    the file mod time. Out of confusion, probably, there was an idea that
    such command was used on the package build_dir. Reality is that this
    command is run on the package files. (Makefile, patches, src)
    
    This is problematic because the package Makefile (for example) change at
    each git clone and base the hash on the Makefile mtime doesn't really
    reflect if the Makefile actually changes across a buildroot or not.
    
    A better approach is to generate an hash of each file and then generate
    an hash on the sort hash list. This way we remove the problem of git
    clone setting a wrong mtime while keeping the integrity of checking if a
    file changed for the package as any change will result in a different
    hash.
    
    Introduce a new kind of find_md5 function, find_md5_reproducible that
    apply this new logic and limit it only with AUTOREMOVE option set to
    prevent any kind of slowdown due to additional hash generation.
    
    Signed-off-by: Christian Marangi <ansuelsmth at gmail.com>
---
 include/depends.mk      | 1 +
 include/host-build.mk   | 2 +-
 include/kernel-build.mk | 2 +-
 include/package.mk      | 2 +-
 4 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/include/depends.mk b/include/depends.mk
index 073fb03632..a858d4b033 100644
--- a/include/depends.mk
+++ b/include/depends.mk
@@ -12,6 +12,7 @@
 DEP_FINDPARAMS := -x "*/.svn*" -x ".*" -x "*:*" -x "*\!*" -x "* *" -x "*\\\#*" -x "*/.*_check" -x "*/.*.swp" -x "*/.pkgdir*"
 
 find_md5=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -printf "%p%T@\n" | sort | $(MKHASH) md5
+find_md5_reproducible=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -print0 | xargs -0 $(MKHASH) md5 | sort | $(MKHASH) md5
 
 define rdep
   .PRECIOUS: $(2)
diff --git a/include/host-build.mk b/include/host-build.mk
index f06dd80230..22fcc31f15 100644
--- a/include/host-build.mk
+++ b/include/host-build.mk
@@ -21,7 +21,7 @@ include $(INCLUDE_DIR)/depends.mk
 include $(INCLUDE_DIR)/quilt.mk
 
 BUILD_TYPES += host
-HOST_STAMP_PREPARED=$(HOST_BUILD_DIR)/.prepared$(if $(HOST_QUILT)$(DUMP),,$(shell $(call find_md5,${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(HOST_PREPARED_DEPENDS)))
+HOST_STAMP_PREPARED=$(HOST_BUILD_DIR)/.prepared$(if $(HOST_QUILT)$(DUMP),,$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(HOST_PREPARED_DEPENDS)))
 HOST_STAMP_CONFIGURED:=$(HOST_BUILD_DIR)/.configured
 HOST_STAMP_BUILT:=$(HOST_BUILD_DIR)/.built
 HOST_BUILD_PREFIX?=$(if $(IS_PACKAGE_BUILD),$(STAGING_DIR_HOSTPKG),$(STAGING_DIR_HOST))
diff --git a/include/kernel-build.mk b/include/kernel-build.mk
index cc651f29b1..5b332258f8 100644
--- a/include/kernel-build.mk
+++ b/include/kernel-build.mk
@@ -10,7 +10,7 @@ ifneq ($(DUMP),1)
 endif
 
 KERNEL_FILE_DEPENDS=$(GENERIC_BACKPORT_DIR) $(GENERIC_PATCH_DIR) $(GENERIC_HACK_DIR) $(PATCH_DIR) $(GENERIC_FILES_DIR) $(FILES_DIR)
-STAMP_PREPARED=$(LINUX_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call find_md5,$(KERNEL_FILE_DEPENDS),)))
+STAMP_PREPARED=$(LINUX_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),$(KERNEL_FILE_DEPENDS),)))
 STAMP_CONFIGURED:=$(LINUX_DIR)/.configured
 include $(INCLUDE_DIR)/download.mk
 include $(INCLUDE_DIR)/quilt.mk
diff --git a/include/package.mk b/include/package.mk
index 5861533bf5..368bf0d7ca 100644
--- a/include/package.mk
+++ b/include/package.mk
@@ -85,7 +85,7 @@ ifneq ($(PREV_STAMP_PREPARED),)
   STAMP_PREPARED:=$(PREV_STAMP_PREPARED)
   CONFIG_AUTOREBUILD:=
 else
-  STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call find_md5,${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(PKG_PREPARED_DEPENDS)))
+  STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(PKG_PREPARED_DEPENDS)))
 endif
 STAMP_CONFIGURED=$(PKG_BUILD_DIR)/.configured$(if $(DUMP),,_$(call confvar,$(PKG_CONFIG_DEPENDS)))
 STAMP_CONFIGURED_WILDCARD=$(PKG_BUILD_DIR)/.configured_*