[buildbot] pahse1: merge sha256sums with remote kmods entry if present

[LEDE Commits]

ynezz pushed a commit to buildbot.git, branch main:
https://git.openwrt.org/9f22657ef41bb599558aaeffda451353eaf35199

commit 9f22657ef41bb599558aaeffda451353eaf35199
Author: Christian Marangi <ansuelsmth at gmail.com>
AuthorDate: Fri Nov 15 15:30:09 2024 +0100

    pahse1: merge sha256sums with remote kmods entry if present
    
    Add additional logic to merge the sha256sums with the remote kmods entry
    (in remote sha256sums) already present.
    
    This is to produce a more consistent sha256sums that actually reflect
    what is present in the remote server by including also the sha of the
    kmods for each kernel version supported.
    
    Signed-off-by: Christian Marangi <ansuelsmth at gmail.com>
---
 phase1/master.cfg | 107 ++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 76 insertions(+), 31 deletions(-)

diff --git a/phase1/master.cfg b/phase1/master.cfg
index 380338b..765852b 100644
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -588,6 +588,10 @@ def IsKmodArchiveAndRsyncEnabled(step):
     return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))
 
 
+def IsRemoteShaSumsAvailable(step):
+    return step.getProperty("have_remote_shasums")
+
+
 def GetBaseVersion(branch):
     if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
         return branch.split("-")[1]
@@ -1383,6 +1387,78 @@ def prepareFactory(target):
         )
     )
 
+    # download remote sha256sums to 'target-sha256sums'
+    factory.addStep(
+        ShellCommandAndSetProperty(
+            name="target-sha256sums",
+            description="Fetching remote sha256sums for target",
+            descriptionDone="Remote sha256sums for target fetched",
+            command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
+            + rsync_defopts
+            + [
+                Interpolate(
+                    "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
+                    url=GetRsyncParams.withArgs("bin", "url"),
+                    target=target,
+                    subtarget=subtarget,
+                    prefix=GetVersionPrefix,
+                ),
+                "target-sha256sums",
+            ],
+            env={
+                "RSYNC_PASSWORD": Interpolate(
+                    "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
+                )
+            },
+            property="have_remote_shasums",
+            logEnviron=False,
+            haltOnFailure=False,
+            flunkOnFailure=False,
+            warnOnFailure=False,
+            doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
+        )
+    )
+
+    factory.addStep(
+        ShellCommand(
+            name="target-sha256sums_kmodsparse",
+            description="Extract kmods from remote sha256sums",
+            descriptionDone="Kmods extracted",
+            command="sed \"/ \\*kmods\\//! d\" target-sha256sums | tee target-sha256sums-kmods",
+            haltOnFailure=False,
+            doStepIf=IsRemoteShaSumsAvailable,
+        )
+    )
+
+    factory.addStep(
+        ShellCommand(
+            name="mergesha256sum",
+            description="Merge sha256sums kmods with sha256sums",
+            descriptionDone="Sha256sums merged",
+            command=[
+                "sort",
+                "-t", " ",
+                "-k", 2,
+                "-u",
+                Interpolate(
+                    "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
+                    target=target,
+                    subtarget=subtarget,
+                ),
+                "target-sha256sums-kmods",
+                "-o",
+                Interpolate(
+                    "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
+                    target=target,
+                    subtarget=subtarget,
+                ),
+            ],
+            env={"LC_ALL": "C"},
+            haltOnFailure=False,
+            doStepIf=IsRemoteShaSumsAvailable,
+        )
+    )
+
     # sign
     factory.addStep(
         MasterShellCommand(
@@ -1541,37 +1617,6 @@ def prepareFactory(target):
         )
     )
 
-    # download remote sha256sums to 'target-sha256sums'
-    factory.addStep(
-        ShellCommand(
-            name="target-sha256sums",
-            description="Fetching remote sha256sums for target",
-            descriptionDone="Remote sha256sums for target fetched",
-            command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
-            + rsync_defopts
-            + [
-                Interpolate(
-                    "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
-                    url=GetRsyncParams.withArgs("bin", "url"),
-                    target=target,
-                    subtarget=subtarget,
-                    prefix=GetVersionPrefix,
-                ),
-                "target-sha256sums",
-            ],
-            env={
-                "RSYNC_PASSWORD": Interpolate(
-                    "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
-                )
-            },
-            logEnviron=False,
-            haltOnFailure=False,
-            flunkOnFailure=False,
-            warnOnFailure=False,
-            doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
-        )
-    )
-
     # build list of files to upload
     factory.addStep(
         FileDownload(