[buildbot] phase1: make 'rsync -4' worker-configurable

[LEDE Commits]

ynezz pushed a commit to buildbot.git, branch master:
https://git.openwrt.org/54e80d5cedcd61455d2e297ef52db71e470ddf5f

commit 54e80d5cedcd61455d2e297ef52db71e470ddf5f
Author: Thibaut VARÈNE <hacks at slashdirt.org>
AuthorDate: Tue Nov 15 10:44:56 2022 +0100

    phase1: make 'rsync -4' worker-configurable
    
    We set '-4' in rsync_defopts, asking rsync to "prefer" ipv4 whenever
    possible on _all_ workers. This was historically done because some
    workers had flaky ipv6 connectivity, however in the future the reverse
    may be true, with worker having worse networking over ipv4 than ipv6.
    
    This change introduces an 'rsync_ipv4' worker configuration option:
    when set to a true value, the old rsync behavior (adding '-4') is used,
    when unset or set to false, the extra rsync argument is not used.
    
    Signed-off-by: Thibaut VARÈNE <hacks at slashdirt.org>
---
 phase1/config.ini.example |  2 ++
 phase1/master.cfg         | 19 +++++++++++--------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/phase1/config.ini.example b/phase1/config.ini.example
index 5ae45fa..d7262bd 100644
--- a/phase1/config.ini.example
+++ b/phase1/config.ini.example
@@ -68,3 +68,5 @@ ul_lock = host1
 dl_lock = host1
 # tag_only workers will only build forced tag buildrequests (i.e. release builds)
 tag_only = yes
+# if rsync operations must prefer ipv4 ('rsync -4'), set the following
+rsync_ipv4 = yes
diff --git a/phase1/master.cfg b/phase1/master.cfg
index 15112ce..f511e03 100644
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -62,7 +62,7 @@ scripts_dir = os.path.abspath("../scripts")
 
 repo_url = ini['repo'].get("url")
 
-rsync_defopts = ["-v", "-4", "--timeout=120"]
+rsync_defopts = ["-v", "--timeout=120"]
 
 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
 #	rsync_bin_defopts += ["--contimeout=20"]
@@ -136,6 +136,7 @@ def ini_parse_workers(section):
 	password = section.get("password")
 	phase = section.getint("phase")
 	tagonly = section.getboolean("tag_only")
+	rsyncipv4 = section.getboolean("rsync_ipv4")
 
 	if not name or not password or not phase == 1:
 		log.msg("invalid worker configuration ignored: {}".format(repr(section)))
@@ -152,6 +153,8 @@ def ini_parse_workers(section):
 		sl_props['ul_lock'] = lockname
 		if lockname not in NetLocks:
 			NetLocks[lockname] = locks.MasterLock(lockname)
+	if rsyncipv4:
+		sl_props['rsync_ipv4'] = True	# only set prop if required, we use '+' Interpolate substitution
 
 	log.msg("Configured worker: {}".format(name))
 	# NB: phase1 build factory requires workers to be single-build only
@@ -1120,7 +1123,7 @@ for target in targets:
 		name = "dirupload",
 		description = "Uploading directory structure",
 		descriptionDone = "Directory structure uploaded",
-		command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
+		command = ["rsync", "-az"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), "tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
 		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
 		haltOnFailure = True,
 		logEnviron = False,
@@ -1133,7 +1136,7 @@ for target in targets:
 		name = "target-sha256sums",
 		description = "Fetching remote sha256sums for target",
 		descriptionDone = "Remote sha256sums for target fetched",
-		command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
+		command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
 		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
 		logEnviron = False,
 		haltOnFailure = False,
@@ -1171,7 +1174,7 @@ for target in targets:
 		description = "Uploading target files",
 		descriptionDone = "Target files uploaded",
 		command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
-			["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+			["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
 			Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
 		haltOnFailure = True,
@@ -1185,7 +1188,7 @@ for target in targets:
 		description = "Pruning target files",
 		descriptionDone = "Target files pruned",
 		command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
-			["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+			["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
 			Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
 		haltOnFailure = True,
@@ -1199,7 +1202,7 @@ for target in targets:
 		description = "Uploading kmod archive",
 		descriptionDone = "Kmod archive uploaded",
 		command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
-			["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
+			["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
 			Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
 		haltOnFailure = True,
@@ -1220,8 +1223,8 @@ for target in targets:
 		name = "sourceupload",
 		description = "Uploading source archives",
 		descriptionDone = "Source archives uploaded",
-		command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
-			[Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
+		command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"),
+			Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
 		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
 		haltOnFailure = True,
 		logEnviron = False,