[buildbot] phase1: add support for per-slave cleanup
LEDE Commits
lede-commits at lists.infradead.org
Sat Dec 9 06:02:29 PST 2017
jow pushed a commit to buildbot.git, branch master:
https://git.lede-project.org/0339b05397f5199aa0992aa84871ef8d3c1cb4b4
commit 0339b05397f5199aa0992aa84871ef8d3c1cb4b4
Author: Jo-Philipp Wich <jo at mein.io>
AuthorDate: Wed Dec 6 16:25:21 2017 +0100
phase1: add support for per-slave cleanup
Signed-off-by: Jo-Philipp Wich <jo at mein.io>
---
phase1/cleanup.sh | 99 +++++++++++++++++++++++++++++++++++++++++++++++
phase1/config.ini.example | 1 +
phase1/master.cfg | 73 ++++++++++++++++++++++++----------
3 files changed, 153 insertions(+), 20 deletions(-)
diff --git a/phase1/cleanup.sh b/phase1/cleanup.sh
new file mode 100755
index 0000000..f2d0bcc
--- /dev/null
+++ b/phase1/cleanup.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+export LC_ALL=C
+
+master_url="$1"
+current_slave="$2"
+current_builder="$3"
+current_mode="$4"
+
+running_builders="$(wget -qO- "${master_url%/}/json/slaves/$current_slave?as_text=1" | sed -ne 's,^.*"builderName": "\(.*\)".*$,\1,p')"
+
+is_running() {
+ local running_builder
+ for running_builder in $running_builders; do
+ if [ "${running_builder//\//_}" = "${1//\//_}" ]; then
+ return 0
+ fi
+ done
+ return 1
+}
+
+do_cleanup() {
+ printf "Cleaning up '$current_builder' work directory"
+
+ if [ -d .git ]; then
+ echo " using git"
+ git reset --hard HEAD
+ git clean -f -d -x
+ else
+ find . -mindepth 1 -maxdepth 1 | while read entry; do
+ rm -vrf "$entry" | while read entry2; do
+ case "$entry2" in *directory:*)
+ printf "."
+ esac
+ done
+ done
+ fi
+
+ echo ""
+}
+
+#
+# Sanity check, current builder should be in running builders list
+#
+
+if ! is_running "$current_builder"; then
+ echo "Current builder '$current_builder' not found in current builders list, aborting cleanup."
+ exit 1
+fi
+
+
+#
+# Clean up leftovers
+#
+
+if [ "$current_mode" = full ]; then
+(
+ if ! flock -x -w 2700 200; then
+ echo "Unable to obtain exclusive lock, aborting cleanup."
+ exit 1
+ fi
+
+ for build_dir in ../../*; do
+
+ build_dir="$(readlink -f "$build_dir")"
+
+ if [ -z "$build_dir" ] || [ ! -d "$build_dir/build/build_dir" ]; then
+ continue
+ fi
+
+ current_builder="${build_dir##*/}"
+
+ if is_running "$current_builder"; then
+ echo "Skipping currently active '$current_builder' work directory."
+ continue
+ fi
+
+ (
+ cd "$build_dir/build"
+
+ if [ -d build_dir ]; then
+ do_cleanup
+ else
+ echo "Skipping clean '$current_builder' work directory."
+ fi
+ )
+ done
+
+) 200>../../cleanup.lock
+
+#
+# Clean up current build
+#
+
+else
+ do_cleanup
+fi
+
+exit 0
diff --git a/phase1/config.ini.example b/phase1/config.ini.example
index bfbe837..47e454d 100644
--- a/phase1/config.ini.example
+++ b/phase1/config.ini.example
@@ -44,3 +44,4 @@ builds = 3
name = example-slave-2
password = example2
builds = 1
+cleanup = 1
diff --git a/phase1/master.cfg b/phase1/master.cfg
index a68d067..8f53f9d 100644
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -18,6 +18,23 @@ ini.read("./config.ini")
# a shorter alias to save typing.
c = BuildmasterConfig = {}
+####### PROJECT IDENTITY
+
+# the 'title' string will appear at the top of this buildbot
+# installation's html.WebStatus home page (linked to the
+# 'titleURL') and is embedded in the title of the waterfall HTML page.
+
+c['title'] = ini.get("general", "title")
+c['titleURL'] = ini.get("general", "title_url")
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server (usually the html.WebStatus page) is visible. This
+# typically uses the port number set in the Waterfall 'status' entry, but
+# with an externally-visible host name which the buildbot cannot figure out
+# without some help.
+
+c['buildbotURL'] = ini.get("general", "buildbot_url")
+
####### BUILDSLAVES
# The 'slaves' list defines the set of recognized buildslaves. Each element is
@@ -32,6 +49,7 @@ if ini.has_option("general", "port"):
c['slaves'] = []
max_builds = dict()
+do_cleanup = dict()
for section in ini.sections():
if section.startswith("slave "):
@@ -39,8 +57,11 @@ for section in ini.sections():
name = ini.get(section, "name")
password = ini.get(section, "password")
max_builds[name] = 1
+ do_cleanup[name] = False
if ini.has_option(section, "builds"):
max_builds[name] = ini.getint(section, "builds")
+ if ini.has_option(section, "cleanup"):
+ do_cleanup[name] = ini.getboolean(section, "cleanup")
c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
@@ -182,7 +203,7 @@ CleanTargetMap = [
[ "dist", "distclean" ]
]
-def IsCleanRequested(pattern):
+def IsMakeCleanRequested(pattern):
def CheckCleanProperty(step):
val = step.getProperty("clean")
if val and re.match(pattern, val):
@@ -192,6 +213,13 @@ def IsCleanRequested(pattern):
return CheckCleanProperty
+def IsCleanupRequested(step):
+ val = step.getProperty("slavename")
+ if val and do_cleanup[val]:
+ return True
+ else:
+ return False
+
def IsTaggingRequested(step):
val = step.getProperty("tag")
if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
@@ -373,6 +401,29 @@ for target in targets:
haltOnFailure = True,
timeout = 2400))
+ # cleanup.sh if needed
+ factory.addStep(FileDownload(
+ mastersrc = "cleanup.sh",
+ slavedest = "cleanup.sh",
+ mode = 0755,
+ doStepIf = IsCleanupRequested))
+
+ factory.addStep(ShellCommand(
+ name = "cleanold",
+ description = "Cleaning previous builds",
+ command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
+ haltOnFailure = True,
+ doStepIf = IsCleanupRequested,
+ timeout = 2400))
+
+ factory.addStep(ShellCommand(
+ name = "cleanup",
+ description = "Cleaning work area",
+ command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
+ haltOnFailure = True,
+ doStepIf = IsCleanupRequested,
+ timeout = 2400))
+
# user-requested clean targets
for tuple in CleanTargetMap:
factory.addStep(ShellCommand(
@@ -380,7 +431,7 @@ for target in targets:
description = 'User-requested "make %s"' % tuple[1],
command = ["make", tuple[1], "V=s"],
env = MakeEnv(),
- doStepIf = IsCleanRequested(tuple[0])
+ doStepIf = IsMakeCleanRequested(tuple[0])
))
# switch to branch
@@ -808,24 +859,6 @@ if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.h
c['status'].append(irc)
-
-####### PROJECT IDENTITY
-
-# the 'title' string will appear at the top of this buildbot
-# installation's html.WebStatus home page (linked to the
-# 'titleURL') and is embedded in the title of the waterfall HTML page.
-
-c['title'] = ini.get("general", "title")
-c['titleURL'] = ini.get("general", "title_url")
-
-# the 'buildbotURL' string should point to the location where the buildbot's
-# internal web server (usually the html.WebStatus page) is visible. This
-# typically uses the port number set in the Waterfall 'status' entry, but
-# with an externally-visible host name which the buildbot cannot figure out
-# without some help.
-
-c['buildbotURL'] = ini.get("general", "buildbot_url")
-
####### DB URL
c['db'] = {
More information about the lede-commits
mailing list