[PATCH 25/33] KVM: arm64: Add AP-list overflow split/splice

[Marc Zyngier]

When EOImode==0, the deactivation of an interrupt outside of the LRs
results in an increment of EOIcount, but not much else.

In order to figure out what interrupts did not make it into the LRs,
split the ap_list to make it easy to find the interrupts EOIcount
applies to. Also provide a bit of documentation for how things are
expected to work.

Signed-off-by: Marc Zyngier <maz at kernel.org>
---
 arch/arm64/kvm/vgic/vgic-init.c |  2 ++
 arch/arm64/kvm/vgic/vgic.c      | 48 ++++++++++++++++++++++++++++++++-
 include/kvm/arm_vgic.h          |  6 +++++
 3 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c
index 1796b1a22a72a..f03cbf0ad154a 100644
--- a/arch/arm64/kvm/vgic/vgic-init.c
+++ b/arch/arm64/kvm/vgic/vgic-init.c
@@ -331,6 +331,7 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
 	vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
 
 	INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
+	INIT_LIST_HEAD(&vgic_cpu->overflow_ap_list_head);
 	raw_spin_lock_init(&vgic_cpu->ap_list_lock);
 	atomic_set(&vgic_cpu->vgic_v3.its_vpe.vlpi_count, 0);
 
@@ -455,6 +456,7 @@ static void __kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
 	vgic_flush_pending_lpis(vcpu);
 
 	INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
+	INIT_LIST_HEAD(&vgic_cpu->overflow_ap_list_head);
 	kfree(vgic_cpu->private_irqs);
 	vgic_cpu->private_irqs = NULL;
 
diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c
index 5c9204d18b27d..bd77365331530 100644
--- a/arch/arm64/kvm/vgic/vgic.c
+++ b/arch/arm64/kvm/vgic/vgic.c
@@ -687,6 +687,15 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
 retry:
 	raw_spin_lock(&vgic_cpu->ap_list_lock);
 
+	/*
+	 * Replug the overflow list into the ap_list so that we can walk the
+	 * whole thing in one go. Note that we only replug it once,
+	 * irrespective of how many tries we perform.
+	 */
+	if (!list_empty(&vgic_cpu->overflow_ap_list_head))
+		list_splice_tail_init(&vgic_cpu->overflow_ap_list_head,
+				      &vgic_cpu->ap_list_head);
+
 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
 		struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB;
 		bool target_vcpu_needs_kick = false;
@@ -914,12 +923,33 @@ static void summarize_ap_list(struct kvm_vcpu *vcpu,
  *   if they were made pending sequentially. This may mean that we don't
  *   always present the HPPI if other interrupts with lower priority are
  *   pending in the LRs. Big deal.
+ *
+ * Additional complexity comes from dealing with these overflow interrupts,
+ * as they are not easy to locate on exit (the ap_list isn't immutable while
+ * the vcpu is running, and new interrupts can be added).
+ *
+ * To deal with this, we play some games with the ap_list:
+ *
+ * - On entering the guest, interrupts that haven't made it onto the LRs are
+ *   placed on an overflow list. These entries are still notionally part of
+ *   the ap_list (the vcpu field still points to the owner).
+ *
+ * - On exiting the guest, the overflow list is used to handle the
+ *   deactivations signaled by EOIcount, by walking the list and
+ *   deactivating EOIcount interrupts from the overflow list.
+ *
+ * - The overflow list is then spliced back with the rest of the ap_list,
+ *   before pruning of idle interrupts.
+ *
+ * - Interrupts that are made pending while the vcpu is running are added to
+ *   the ap_list itself, never to the overflow list. This ensures that these
+ *   new interrupts are not evaluated for deactivation when the vcpu exits.
  */
 static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
 {
 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 	struct ap_list_summary als;
-	struct vgic_irq *irq;
+	struct vgic_irq *irq, *last = NULL;
 	int count = 0;
 
 	lockdep_assert_held(&vgic_cpu->ap_list_lock);
@@ -933,6 +963,7 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
 		scoped_guard(raw_spinlock,  &irq->irq_lock) {
 			if (likely(vgic_target_oracle(irq) == vcpu)) {
 				vgic_populate_lr(vcpu, irq, count++);
+				last = irq;
 			}
 		}
 
@@ -951,6 +982,21 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
 		vcpu->arch.vgic_cpu.vgic_v3.used_lrs = count;
 		vgic_v3_configure_hcr(vcpu, &als);
 	}
+
+	/*
+	 * Move the end of the list to the overflow list, unless:
+	 *
+	 * - either we didn't inject anything at all
+	 * - or we injected everything there was to inject
+	 */
+	if (!count ||
+	    (last && list_is_last(&last->ap_list, &vgic_cpu->ap_list_head))) {
+		INIT_LIST_HEAD(&vgic_cpu->overflow_ap_list_head);
+		return;
+	}
+
+	vgic_cpu->overflow_ap_list_head = vgic_cpu->ap_list_head;
+	list_cut_position(&vgic_cpu->ap_list_head, &vgic_cpu->overflow_ap_list_head, &last->ap_list);
 }
 
 static inline bool can_access_vgic_from_kernel(void)
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index ec349c5a4a8b6..1d700850f6ea7 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -359,6 +359,12 @@ struct vgic_cpu {
 	 */
 	struct list_head ap_list_head;
 
+	/*
+	 * List of IRQs that have not made it onto an LR, but still
+	 * notionally par of the AP list
+	 */
+	struct list_head overflow_ap_list_head;
+
 	/*
 	 * Members below are used with GICv3 emulation only and represent
 	 * parts of the redistributor.
-- 
2.47.3