[buildbot] phase1, phase2: implement force scheduler logic
LEDE Commits
lede-commits at lists.infradead.org
Thu Jul 30 06:14:23 EDT 2020
jow pushed a commit to buildbot.git, branch master:
https://git.openwrt.org/b2508bcba5e90a136a0cb3119e01519baeac8f79
commit b2508bcba5e90a136a0cb3119e01519baeac8f79
Author: Jo-Philipp Wich <jo at mein.io>
AuthorDate: Mon Feb 3 16:50:06 2020 +0100
phase1, phase2: implement force scheduler logic
Signed-off-by: Jo-Philipp Wich <jo at mein.io>
---
phase1/master.cfg | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++----
phase2/master.cfg | 90 ++++++++++++++++++++++++++++----
2 files changed, 222 insertions(+), 20 deletions(-)
diff --git a/phase1/master.cfg b/phase1/master.cfg
index 24262ad..83d9e8b 100644
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -205,17 +205,107 @@ c['change_source'].append(GitPoller(
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.schedulers.forcesched import BaseParameter
+from buildbot.schedulers.forcesched import ValidationError
+from buildbot.plugins import schedulers
+from buildbot.plugins import util
from buildbot.changes import filter
+
+class TagChoiceParameter(BaseParameter):
+ spec_attributes = ["strict", "choices"]
+ type = "list"
+ strict = True
+
+ def __init__(self, name, label=None, **kw):
+ super().__init__(name, label, **kw)
+ self._choice_list = []
+
+ @property
+ def choices(self):
+ taglist = []
+ basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
+
+ if basever:
+ findtags = subprocess.Popen(
+ ['git', 'ls-remote', '--tags', repo_url],
+ stdout = subprocess.PIPE)
+
+ while True:
+ line = findtags.stdout.readline()
+
+ if not line:
+ break
+
+ tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
+
+ if tagver and tagver[1].find(basever[1]) == 0:
+ taglist.append(tagver[1])
+
+ taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
+ taglist.insert(0, '')
+
+ self._choice_list = taglist
+
+ return self._choice_list
+
+ def parse_from_arg(self, s):
+ if self.strict and s not in self._choice_list:
+ raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
+ return s
+
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
- name="all",
- change_filter=filter.ChangeFilter(branch=repo_branch),
- treeStableTimer=60,
- builderNames=targets))
+ name = "all",
+ change_filter = filter.ChangeFilter(branch=repo_branch),
+ treeStableTimer = 60,
+ builderNames = targets))
c['schedulers'].append(ForceScheduler(
- name="force",
- builderNames=targets))
+ name = "force",
+ buttonName = "Force builds",
+ label = "Force build details",
+ builderNames = [ "00_force_build" ],
+
+ codebases = [
+ util.CodebaseParameter(
+ "",
+ label = "Repository",
+ branch = util.FixedParameter(name = "branch", default = ""),
+ revision = util.FixedParameter(name = "revision", default = ""),
+ repository = util.FixedParameter(name = "repository", default = ""),
+ project = util.FixedParameter(name = "project", default = "")
+ )
+ ],
+
+ reason = util.StringParameter(
+ name = "reason",
+ label = "Reason",
+ default = "Trigger build",
+ required = True,
+ size = 80
+ ),
+
+ properties = [
+ util.NestedParameter(
+ name="options",
+ label="Build Options",
+ layout="vertical",
+ fields=[
+ util.ChoiceStringParameter(
+ name = "target",
+ label = "Build target",
+ default = "all",
+ choices = [ "all" ] + targets
+ ),
+ TagChoiceParameter(
+ name = "tag",
+ label = "Build tag",
+ default = ""
+ )
+ ]
+ )
+ ]
+))
####### BUILDERS
@@ -231,8 +321,11 @@ from buildbot.steps.transfer import FileUpload
from buildbot.steps.transfer import FileDownload
from buildbot.steps.transfer import StringDownload
from buildbot.steps.master import MasterShellCommand
+from buildbot.plugins import steps
from buildbot.process.properties import Interpolate
+from buildbot.process.properties import Property
from buildbot.process import properties
+from buildbot.config import BuilderConfig
CleanTargetMap = [
@@ -283,7 +376,7 @@ def IsGitCleanRequested(step):
def IsTaggingRequested(step):
val = step.getProperty("tag")
- if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
+ if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
return True
else:
return False
@@ -295,7 +388,7 @@ def IsNoMasterBuild(step):
return repo_branch != "master"
def GetBaseVersion():
- if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
+ if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
return repo_branch.split('-')[1]
else:
return "master"
@@ -303,7 +396,7 @@ def GetBaseVersion():
@properties.renderer
def GetVersionPrefix(props):
basever = GetBaseVersion()
- if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
+ if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
return "%s/" % props["tag"]
elif basever != "master":
return "%s-SNAPSHOT/" % basever
@@ -390,6 +483,29 @@ def NetLockUl(props):
else:
return []
+ at util.renderer
+def TagPropertyValue(props):
+ if props.hasProperty("options"):
+ options = props.getProperty("options")
+ if type(options) is dict:
+ return options.get("tag")
+ return None
+
+def IsTargetSelected(target):
+ def CheckTargetProperty(step):
+ try:
+ options = step.getProperty("options")
+ if type(options) is dict:
+ selected_target = options.get("target", "all")
+ if selected_target != "all" and selected_target != target:
+ return False
+ except KeyError:
+ pass
+
+ return True
+
+ return CheckTargetProperty
+
def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
try:
seckey = base64.b64decode(seckey)
@@ -462,6 +578,13 @@ slaveNames = [ ]
for slave in c['workers']:
slaveNames.append(slave.workername)
+force_factory = BuildFactory()
+
+c['builders'].append(BuilderConfig(
+ name = "00_force_build",
+ workernames = slaveNames,
+ factory = force_factory))
+
for target in targets:
ts = target.split('/')
@@ -1167,10 +1290,17 @@ for target in targets:
alwaysRun = True,
))
- from buildbot.config import BuilderConfig
-
c['builders'].append(BuilderConfig(name=target, workernames=slaveNames, factory=factory, nextBuild=GetNextBuild))
+ c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
+ force_factory.addStep(steps.Trigger(
+ name = "trigger_%s" % target,
+ description = "Triggering %s build" % target,
+ schedulerNames = [ "trigger_%s" % target ],
+ set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
+ doStepIf = IsTargetSelected(target)
+ ))
+
####### STATUS TARGETS
diff --git a/phase2/master.cfg b/phase2/master.cfg
index f5be23e..d3a7614 100644
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -195,17 +195,59 @@ def branch_change_filter(change):
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.plugins import schedulers
+from buildbot.plugins import util
+from buildbot.plugins import steps
from buildbot.changes import filter
+
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
- name="all",
- change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
- treeStableTimer=60,
- builderNames=archnames))
+ name = "all",
+ change_filter = filter.ChangeFilter(filter_fn=branch_change_filter),
+ treeStableTimer = 60,
+ builderNames = archnames))
c['schedulers'].append(ForceScheduler(
- name="force",
- builderNames=archnames))
+ name = "force",
+ buttonName = "Force builds",
+ label = "Force build details",
+ builderNames = [ "00_force_build" ],
+
+ codebases = [
+ util.CodebaseParameter(
+ "",
+ label = "Repository",
+ branch = util.FixedParameter(name = "branch", default = ""),
+ revision = util.FixedParameter(name = "revision", default = ""),
+ repository = util.FixedParameter(name = "repository", default = ""),
+ project = util.FixedParameter(name = "project", default = "")
+ )
+ ],
+
+ reason = util.StringParameter(
+ name = "reason",
+ label = "Reason",
+ default = "Trigger build",
+ required = True,
+ size = 80
+ ),
+
+ properties = [
+ util.NestedParameter(
+ name="options",
+ label="Build Options",
+ layout="vertical",
+ fields=[
+ util.ChoiceStringParameter(
+ name = "architecture",
+ label = "Build architecture",
+ default = "all",
+ choices = [ "all" ] + archnames
+ )
+ ]
+ )
+ ]
+))
####### BUILDERS
@@ -221,10 +263,12 @@ from buildbot.steps.transfer import FileDownload
from buildbot.steps.transfer import StringDownload
from buildbot.steps.master import MasterShellCommand
from buildbot.process.properties import WithProperties
+from buildbot.process.properties import Property
+from buildbot.config import BuilderConfig
def GetDirectorySuffix(props):
- verpat = re.compile('^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
+ verpat = re.compile(r'^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
if props.hasProperty("release_version"):
m = verpat.match(props["release_version"])
if m is not None:
@@ -245,6 +289,21 @@ def GetCwd(props):
else:
return "/"
+def IsArchitectureSelected(target):
+ def CheckArchitectureProperty(step):
+ try:
+ options = step.getProperty("options")
+ if type(options) is dict:
+ selected_arch = options.get("architecture", "all")
+ if selected_arch != "all" and selected_arch != target:
+ return False
+ except KeyError:
+ pass
+
+ return True
+
+ return CheckArchitectureProperty
+
def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
try:
seckey = base64.b64decode(seckey)
@@ -267,6 +326,13 @@ slaveNames = [ ]
for slave in c['workers']:
slaveNames.append(slave.workername)
+force_factory = BuildFactory()
+
+c['builders'].append(BuilderConfig(
+ name = "00_force_build",
+ workernames = slaveNames,
+ factory = force_factory))
+
for arch in arches:
ts = arch[1].split('/')
@@ -604,10 +670,16 @@ for arch in arches:
alwaysRun = True
))
- from buildbot.config import BuilderConfig
-
c['builders'].append(BuilderConfig(name=arch[0], workernames=slaveNames, factory=factory))
+ c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ]))
+ force_factory.addStep(steps.Trigger(
+ name = "trigger_%s" % arch[0],
+ description = "Triggering %s build" % arch[0],
+ schedulerNames = [ "trigger_%s" % arch[0] ],
+ set_properties = { "reason": Property("reason") },
+ doStepIf = IsArchitectureSelected(arch[0])
+ ))
####### STATUS arches
More information about the lede-commits
mailing list