[RFC PATCH kvmtool 7/7] Don't apply --vcpu-affinity to threads spawned from VCPUs

[Alexandru Elisei]

Threads spawned with pthread_create() inherit the parent's affinity. For
kvmtool, threads spawned from the VCPU threads, like the virtio threads,
will inherit the affinity specified with --vcpu-affinity, instead of
inheriting the main process' affinity. Make kvm_create_thread() use the
main thread's affinity when spawning new threads.

A command like this:

$ taskset -c 0-5 ./lkvm run .. --vcpu-affinity 6-11

will work as expected and all kvmtool threads will be pinned to CPUs 0-5,
instead of some of them being pinned on the same CPUs as the VCPUs.

Signed-off-by: Alexandru Elisei <alexandru.elisei at arm.com>
---
 arm64/pmu.c       | 19 +++----------------
 include/kvm/kvm.h |  1 +
 kvm.c             | 15 +++++++++++++++
 util/util.c       | 33 +++++++++++++++++----------------
 4 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/arm64/pmu.c b/arm64/pmu.c
index bd3f225842be..10d1641b42cc 100644
--- a/arm64/pmu.c
+++ b/arm64/pmu.c
@@ -200,20 +200,10 @@ static int find_pmu(struct kvm *kvm)
 
 	setsize = CPU_ALLOC_SIZE(NR_CPUS);
 
-	if (kvm->vcpu_affinity) {
+	if (kvm->vcpu_affinity)
 		affinity = kvm->vcpu_affinity;
-	} else {
-		affinity = CPU_ALLOC(NR_CPUS);
-		if (!affinity)
-			die_perror("CPU_ALLOC");
-		CPU_ZERO_S(setsize, affinity);
-
-		ret = sched_getaffinity(0, setsize, affinity);
-		if (ret < 0) {
-			ret = -errno;
-			goto out_free;
-		}
-	}
+	else
+		affinity = kvm->main_affinity;
 
 	for (i = 0; i < NR_CPUS; i ++) {
 		if (CPU_ISSET_S(i, setsize, affinity))
@@ -222,10 +212,7 @@ static int find_pmu(struct kvm *kvm)
 
 	ret = find_pmu_cpumask(kvm, cpumask);
 
-out_free:
 	free(cpumask);
-	if (!kvm->vcpu_affinity)
-		CPU_FREE(affinity);
 	return ret;
 }
 
diff --git a/include/kvm/kvm.h b/include/kvm/kvm.h
index 01c4f5fa952d..91a30bbbbfd0 100644
--- a/include/kvm/kvm.h
+++ b/include/kvm/kvm.h
@@ -102,6 +102,7 @@ struct kvm {
 	int                     nr_disks;
 
 	int			vm_state;
+	cpu_set_t		*main_affinity;
 	cpu_set_t		*vcpu_affinity;
 
 #ifdef KVM_BRLOCK_DEBUG
diff --git a/kvm.c b/kvm.c
index e29b73d27f33..7bc91cce97b0 100644
--- a/kvm.c
+++ b/kvm.c
@@ -156,6 +156,9 @@ static int kvm__check_extensions(struct kvm *kvm)
 struct kvm *kvm__new(void)
 {
 	struct kvm *kvm = calloc(1, sizeof(*kvm));
+	size_t setsize;
+	int ret;
+
 	if (!kvm)
 		return ERR_PTR(-ENOMEM);
 
@@ -168,6 +171,17 @@ struct kvm *kvm__new(void)
 	kvm->brlock_sem = (pthread_rwlock_t) PTHREAD_RWLOCK_INITIALIZER;
 #endif
 
+	kvm->main_affinity = CPU_ALLOC(NR_CPUS);
+	if (!kvm->main_affinity)
+		die_perror("CPU_ALLOC");
+
+	setsize = CPU_ALLOC_SIZE(NR_CPUS);
+	CPU_ZERO_S(setsize, kvm->main_affinity);
+
+	ret = sched_getaffinity(0, setsize, kvm->main_affinity);
+	if (ret)
+		die_perror("sched_getaffinity");
+
 	return kvm;
 }
 
@@ -182,6 +196,7 @@ int kvm__exit(struct kvm *kvm)
 		free(bank);
 	}
 
+	CPU_FREE(kvm->main_affinity);
 	if (kvm->vcpu_affinity)
 		CPU_FREE(kvm->vcpu_affinity);
 
diff --git a/util/util.c b/util/util.c
index 7238fe57bace..a779eb59817c 100644
--- a/util/util.c
+++ b/util/util.c
@@ -207,28 +207,15 @@ void *mmap_guest_memfd(struct kvm *kvm, u64 size)
 	return addr;
 }
 
-int kvm_create_thread(struct kvm *kvm, pthread_t *thread,
-		      void *(*start_routine)(void *), void *arg)
-{
-	int ret;
-
-	ret = pthread_create(thread, NULL, start_routine, arg);
-	if (ret)
-		errno = ret;
-
-	return -ret;
-}
-
-int kvm_create_vcpu_thread(struct kvm *kvm, pthread_t *thread,
-			   void *(*start_routine)(void *), void *arg)
+static int __kvm_create_thread(struct kvm *kvm, cpu_set_t *affinity,
+			       pthread_t *thread, void *(*start_routine)(void *),
+			       void *arg)
 {
 	pthread_attr_t attr;
-	cpu_set_t *affinity;
 	int ret;
 
 	pthread_attr_init(&attr);
 
-	affinity = kvm->vcpu_affinity;
 	if (affinity) {
 		ret = pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), affinity);
 		if (ret) {
@@ -247,3 +234,17 @@ int kvm_create_vcpu_thread(struct kvm *kvm, pthread_t *thread,
 
 	return -ret;
 }
+
+int kvm_create_thread(struct kvm *kvm, pthread_t *thread,
+		      void *(*start_routine)(void *), void *arg)
+{
+	return __kvm_create_thread(kvm, kvm->main_affinity, thread,
+				   start_routine, arg);
+}
+
+int kvm_create_vcpu_thread(struct kvm *kvm, pthread_t *thread,
+			   void *(*start_routine)(void *), void *arg)
+{
+	return __kvm_create_thread(kvm, kvm->vcpu_affinity, thread,
+				   start_routine, arg);
+}
-- 
2.55.0