[buildbot] phase1,phase2: implement round robin builds

[LEDE Commits]

ynezz pushed a commit to buildbot.git, branch master:
https://git.openwrt.org/27f0b7fe926d70c4f3e9c5bb1b297b76d6e1f704

commit 27f0b7fe926d70c4f3e9c5bb1b297b76d6e1f704
Author: Petr Štetiar <ynezz at true.cz>
AuthorDate: Tue Mar 16 18:21:15 2021 +0100

    phase1,phase2: implement round robin builds
    
    Gather newest complete and not skipped build timestamps, reverse sort
    them and use that as builder priority, so the newest built targets are
    at the bottom.
    
    Signed-off-by: Petr Štetiar <ynezz at true.cz>
---
 phase1/master.cfg | 81 +++++++++++++++++++++++++++++++++++++++++++++++++------
 phase2/master.cfg | 71 +++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 140 insertions(+), 12 deletions(-)

diff --git a/phase1/master.cfg b/phase1/master.cfg
index 2ccac8e..53d1812 100644
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -4,13 +4,17 @@
 import os
 import re
 import base64
-import random
 import subprocess
 import configparser
 
-from datetime import timedelta
+from dateutil.tz import tzutc
+from datetime import datetime, timedelta
+
+from twisted.internet import defer
+from twisted.python import log
 
 from buildbot import locks
+from buildbot.data import resultspec
 from buildbot.changes import filter
 from buildbot.changes.gitpoller import GitPoller
 from buildbot.config import BuilderConfig
@@ -19,6 +23,7 @@ from buildbot.plugins import schedulers
 from buildbot.plugins import steps
 from buildbot.plugins import util
 from buildbot.process import properties
+from buildbot.process import results
 from buildbot.process.factory import BuildFactory
 from buildbot.process.properties import Interpolate
 from buildbot.process.properties import Property
@@ -123,6 +128,69 @@ c['configurators'] = [util.JanitorConfigurator(
     hour=6,
 )]
 
+ at defer.inlineCallbacks
+def getNewestCompleteTime(bldr):
+	"""Returns the complete_at of the latest completed and not SKIPPED
+	build request for this builder, or None if there are no such build
+	requests. We need to filter out SKIPPED requests because we're
+	using collapseRequests=True which is unfortunately marking all
+	previous requests as complete when new buildset is created.
+
+	@returns: datetime instance or None, via Deferred
+	"""
+
+	bldrid = yield bldr.getBuilderId()
+	completed = yield bldr.master.data.get(
+			('builders', bldrid, 'buildrequests'),
+			[
+				resultspec.Filter('complete', 'eq', [True]),
+				resultspec.Filter('results', 'ne', [results.SKIPPED]),
+			],
+			order=['-complete_at'], limit=1)
+	if not completed:
+		return
+
+	return completed[0]['complete_at']
+
+ at defer.inlineCallbacks
+def prioritizeBuilders(master, builders):
+	"""Returns sorted list of builders by their last timestamp of completed and
+	not skipped build.
+
+	@returns: list of sorted builders
+	"""
+
+	def is_building(bldr):
+		return bool(bldr.building) or bool(bldr.old_building)
+
+	def bldr_info(bldr):
+		d = defer.maybeDeferred(getNewestCompleteTime, bldr)
+		d.addCallback(lambda complete_at: (complete_at, bldr))
+		return d
+
+	def bldr_sort(item):
+		(complete_at, bldr) = item
+
+		if not complete_at:
+			date = datetime.min
+			complete_at = date.replace(tzinfo=tzutc())
+
+		if is_building(bldr):
+			date = datetime.max
+			complete_at = date.replace(tzinfo=tzutc())
+
+		return (complete_at, bldr.name)
+
+	results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
+	results.sort(key=bldr_sort)
+
+	for r in results:
+		log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
+
+	return [r[1] for r in results]
+
+c['prioritizeBuilders'] = prioritizeBuilders
+
 ####### CHANGESOURCES
 
 work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
@@ -456,13 +524,10 @@ def GetNextBuild(builder, requests):
 	for r in requests:
 		if r.properties and r.properties.hasProperty("tag"):
 			return r
-	return requests[0]
 
-def prioritizeBuilders(buildmaster, builders):
-	random.shuffle(builders)
-	return builders
-
-c['prioritizeBuilders'] = prioritizeBuilders
+	r = requests[0]
+	log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
+	return r
 
 def MakeEnv(overrides=None, tryccache=False):
 	env = {
diff --git a/phase2/master.cfg b/phase2/master.cfg
index ac44fcd..a54c191 100644
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -8,15 +8,21 @@ import random
 import subprocess
 import configparser
 
-from datetime import timedelta
+from dateutil.tz import tzutc
+from datetime import datetime, timedelta
+
+from twisted.internet import defer
+from twisted.python import log
 
 from buildbot import locks
+from buildbot.data import resultspec
 from buildbot.changes import filter
 from buildbot.changes.gitpoller import GitPoller
 from buildbot.config import BuilderConfig
 from buildbot.plugins import schedulers
 from buildbot.plugins import steps
 from buildbot.plugins import util
+from buildbot.process import results
 from buildbot.process.factory import BuildFactory
 from buildbot.process.properties import Property
 from buildbot.process.properties import WithProperties
@@ -319,9 +325,66 @@ def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
 def IsSharedWorkdir(step):
 	return bool(step.getProperty("shared_wd"))
 
-def prioritizeBuilders(buildmaster, builders):
-	random.shuffle(builders)
-	return builders
+ at defer.inlineCallbacks
+def getNewestCompleteTime(bldr):
+	"""Returns the complete_at of the latest completed and not SKIPPED
+	build request for this builder, or None if there are no such build
+	requests. We need to filter out SKIPPED requests because we're
+	using collapseRequests=True which is unfortunately marking all
+	previous requests as complete when new buildset is created.
+
+	@returns: datetime instance or None, via Deferred
+	"""
+
+	bldrid = yield bldr.getBuilderId()
+	completed = yield bldr.master.data.get(
+			('builders', bldrid, 'buildrequests'),
+			[
+				resultspec.Filter('complete', 'eq', [True]),
+				resultspec.Filter('results', 'ne', [results.SKIPPED]),
+			],
+			order=['-complete_at'], limit=1)
+	if not completed:
+		return
+
+	return completed[0]['complete_at']
+
+ at defer.inlineCallbacks
+def prioritizeBuilders(master, builders):
+	"""Returns sorted list of builders by their last timestamp of completed and
+	not skipped build.
+
+	@returns: list of sorted builders
+	"""
+
+	def is_building(bldr):
+		return bool(bldr.building) or bool(bldr.old_building)
+
+	def bldr_info(bldr):
+		d = defer.maybeDeferred(getNewestCompleteTime, bldr)
+		d.addCallback(lambda complete_at: (complete_at, bldr))
+		return d
+
+	def bldr_sort(item):
+		(complete_at, bldr) = item
+
+		if not complete_at:
+			date = datetime.min
+			complete_at = date.replace(tzinfo=tzutc())
+
+		if is_building(bldr):
+			date = datetime.max
+			complete_at = date.replace(tzinfo=tzutc())
+
+		return (complete_at, bldr.name)
+
+	results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
+	results.sort(key=bldr_sort)
+
+	for r in results:
+		log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
+
+	return [r[1] for r in results]
 
 c['prioritizeBuilders'] = prioritizeBuilders
 c['builders'] = []