[PATCH 38/38] irqchip/gic-v5: Add NMI support
Vladimir Murzin
vladimir.murzin at arm.com
Mon Sep 14 03:21:02 PDT 2026
Implement NMI support for PPIs, SPIs, LPIs and IPIs by toggling their
priority between the superpriority and regular interrupt levels.
Signed-off-by: Vladimir Murzin <vladimir.murzin at arm.com>
---
drivers/irqchip/irq-gic-v5.c | 244 +++++++++++++++++++++++++++++++++--
1 file changed, 234 insertions(+), 10 deletions(-)
diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c
index 5f2551cf077d..7e7d313e77bc 100644
--- a/drivers/irqchip/irq-gic-v5.c
+++ b/drivers/irqchip/irq-gic-v5.c
@@ -22,6 +22,7 @@
static u8 pri_bits __ro_after_init = 5;
#define GICV5_IRQ_PRI_MASK 0x1f
+#define GICV5_IRQ_PRI_NMI 0
#define GICV5_IRQ_PRI_MI (GICV5_IRQ_PRI_MASK & GENMASK(4, 5 - pri_bits))
#define PPI_NR 128
@@ -97,6 +98,56 @@ static void gicv5_hwirq_init(irq_hw_number_t hwirq, u8 priority, u8 hwirq_type)
}
}
+static void gicv5_ppi_irq_set_priority(struct irq_data *d, u8 priority)
+{
+ u64 hwirq_id_offset = d->hwirq / 8;
+ u64 hwirq_id_index = d->hwirq % 8;
+ u64 hwirq_id_clear = (u64)GICV5_IRQ_PRI_MASK << (hwirq_id_index * 8);
+ u64 hwirq_id_set = (u64)priority << (hwirq_id_index * 8);
+
+#define SET_ICC_PPI_PRIORITYRn_EL1(cnt) \
+ case cnt: sysreg_clear_set_s(SYS_ICC_PPI_PRIORITYR ## cnt ## _EL1, hwirq_id_clear, hwirq_id_set); break
+
+ switch (hwirq_id_offset) {
+ SET_ICC_PPI_PRIORITYRn_EL1(0);
+ SET_ICC_PPI_PRIORITYRn_EL1(1);
+ SET_ICC_PPI_PRIORITYRn_EL1(2);
+ SET_ICC_PPI_PRIORITYRn_EL1(3);
+ SET_ICC_PPI_PRIORITYRn_EL1(4);
+ SET_ICC_PPI_PRIORITYRn_EL1(5);
+ SET_ICC_PPI_PRIORITYRn_EL1(6);
+ SET_ICC_PPI_PRIORITYRn_EL1(7);
+ SET_ICC_PPI_PRIORITYRn_EL1(8);
+ SET_ICC_PPI_PRIORITYRn_EL1(9);
+ SET_ICC_PPI_PRIORITYRn_EL1(10);
+ SET_ICC_PPI_PRIORITYRn_EL1(11);
+ SET_ICC_PPI_PRIORITYRn_EL1(12);
+ SET_ICC_PPI_PRIORITYRn_EL1(13);
+ SET_ICC_PPI_PRIORITYRn_EL1(14);
+ SET_ICC_PPI_PRIORITYRn_EL1(15);
+#undef SET_ICC_PPI_PRIORITYRn_EL1
+ default:
+ WARN_ON(1);
+ }
+ /*
+ * Context syncronization required to make sure system register writes
+ * effects are synchronised.
+ */
+ isb();
+}
+
+static void gicv5_hwirq_set_priority(irq_hw_number_t hwirq, u8 priority, u8 hwirq_type) {
+ u64 cdpri;
+
+ cdpri = FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority) |
+ FIELD_PREP(GICV5_GIC_CDPRI_TYPE_MASK, hwirq_type) |
+ FIELD_PREP(GICV5_GIC_CDPRI_ID_MASK, hwirq);
+
+ gic_insn(cdpri, CDPRI);
+
+ return;
+}
+
static void gicv5_ppi_irq_mask(struct irq_data *d)
{
u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64);
@@ -512,7 +563,152 @@ static int gicv5_ppi_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
return 0;
}
-static const struct irq_chip gicv5_ppi_irq_chip = {
+static int gicv5_hwirq_peek_irq(irq_hw_number_t hwirq, u8 hwirq_type)
+{
+ u64 icsr, cdrcfg;
+
+ cdrcfg = hwirq | FIELD_PREP(GICV5_GIC_CDRCFG_TYPE_MASK, hwirq_type);
+
+ gic_insn(cdrcfg, CDRCFG);
+ isb();
+ icsr = read_sysreg_s(SYS_ICC_ICSR_EL1);
+
+ if (FIELD_GET(ICC_ICSR_EL1_F, icsr)) {
+ pr_err("ICSR_EL1 is invalid\n");
+ return -EINVAL;
+ }
+
+ return !!(FIELD_GET(ICC_ICSR_EL1_Enabled, icsr));
+
+}
+
+static int gicv5_ppi_irq_nmi_setup(struct irq_data *d)
+{
+ unsigned long hwirq_id_bit = d->hwirq % 64;
+ unsigned long enable;
+
+ if (!system_uses_nmi())
+ return -EINVAL;
+
+ if (d->hwirq < 64)
+ enable = read_sysreg_s(SYS_ICC_PPI_ENABLER0_EL1);
+ else
+ enable = read_sysreg_s(SYS_ICC_PPI_ENABLER1_EL1);
+
+ if (test_bit(hwirq_id_bit, &enable))
+ return -EINVAL;
+
+ gicv5_ppi_irq_set_priority(d, GICV5_IRQ_PRI_NMI);
+
+ return 0;
+}
+
+static void gicv5_ppi_irq_nmi_teardown(struct irq_data *d)
+{
+ unsigned long hwirq_id_bit = d->hwirq % 64;
+ unsigned long enable;
+
+ if (WARN_ON(!system_uses_nmi()))
+ return;
+
+ if (d->hwirq < 64)
+ enable = read_sysreg_s(SYS_ICC_PPI_ENABLER0_EL1);
+ else
+ enable = read_sysreg_s(SYS_ICC_PPI_ENABLER1_EL1);
+
+ if (test_bit(hwirq_id_bit, &enable))
+ return;
+
+ gicv5_ppi_irq_set_priority(d, GICV5_IRQ_PRI_MI);
+
+ return;
+}
+
+static int gicv5_hwirq_irq_nmi_setup(irq_hw_number_t hwirq, u8 hwirq_type)
+{
+ if (!system_uses_nmi())
+ return -EINVAL;
+
+ if (WARN_ON(hwirq_type != GICV5_HWIRQ_TYPE_LPI && hwirq_type != GICV5_HWIRQ_TYPE_SPI))
+ return -EINVAL;
+
+ if (gicv5_hwirq_peek_irq(hwirq, hwirq_type))
+ return -EINVAL;
+
+ gicv5_hwirq_set_priority(hwirq, GICV5_IRQ_PRI_NMI, hwirq_type);
+
+ return 0;
+}
+
+static void gicv5_hwirq_irq_nmi_teardown(irq_hw_number_t hwirq, u8 hwirq_type)
+{
+ if (WARN_ON(!system_uses_nmi()))
+ return;
+
+ if (WARN_ON(hwirq_type != GICV5_HWIRQ_TYPE_LPI && hwirq_type != GICV5_HWIRQ_TYPE_SPI))
+ return;
+
+ if (gicv5_hwirq_peek_irq(hwirq, hwirq_type))
+ return;
+
+ gicv5_hwirq_set_priority(hwirq, GICV5_IRQ_PRI_MI, hwirq_type);
+
+ return;
+}
+
+static int gicv5_lpi_irq_nmi_setup(struct irq_data *d)
+{
+ int ret = gicv5_hwirq_irq_nmi_setup(d->hwirq, GICV5_HWIRQ_TYPE_LPI);
+
+ if (!ret)
+ irq_to_desc(d->irq)->handle_irq = handle_fasteoi_nmi;
+
+ return ret;
+}
+
+static void gicv5_lpi_irq_nmi_teardown(struct irq_data *d)
+{
+ gicv5_hwirq_irq_nmi_teardown(d->hwirq, GICV5_HWIRQ_TYPE_LPI);
+ irq_to_desc(d->irq)->handle_irq = handle_fasteoi_irq;
+}
+
+static int gicv5_spi_irq_nmi_setup(struct irq_data *d)
+{
+ int ret = gicv5_hwirq_irq_nmi_setup(d->hwirq, GICV5_HWIRQ_TYPE_SPI);
+
+ if (!ret)
+ irq_to_desc(d->irq)->handle_irq = handle_fasteoi_nmi;
+
+ return ret;
+}
+
+static void gicv5_spi_irq_nmi_teardown(struct irq_data *d)
+{
+ gicv5_hwirq_irq_nmi_teardown(d->hwirq, GICV5_HWIRQ_TYPE_SPI);
+ irq_to_desc(d->irq)->handle_irq = handle_fasteoi_irq;
+}
+
+static int gicv5_ipi_irq_nmi_setup(struct irq_data *d)
+{
+ if (WARN_ON(!d->parent_data))
+ return -EINVAL;
+
+ d = d->parent_data;
+
+ return gicv5_hwirq_irq_nmi_setup(d->hwirq, GICV5_HWIRQ_TYPE_LPI);
+}
+
+static void gicv5_ipi_irq_nmi_teardown(struct irq_data *d)
+{
+ if (WARN_ON(!d->parent_data))
+ return;
+
+ d = d->parent_data;
+
+ gicv5_hwirq_irq_nmi_teardown(d->hwirq, GICV5_HWIRQ_TYPE_LPI);
+}
+
+static struct irq_chip gicv5_ppi_irq_chip __ro_after_init = {
.name = "GICv5-PPI",
.irq_mask = gicv5_ppi_irq_mask,
.irq_unmask = gicv5_ppi_irq_unmask,
@@ -521,11 +717,13 @@ static const struct irq_chip gicv5_ppi_irq_chip = {
.irq_get_irqchip_state = gicv5_ppi_irq_get_irqchip_state,
.irq_set_irqchip_state = gicv5_ppi_irq_set_irqchip_state,
.irq_set_vcpu_affinity = gicv5_ppi_irq_set_vcpu_affinity,
+ .irq_nmi_setup = gicv5_ppi_irq_nmi_setup,
+ .irq_nmi_teardown = gicv5_ppi_irq_nmi_teardown,
.flags = IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
};
-static const struct irq_chip gicv5_spi_irq_chip = {
+static struct irq_chip gicv5_spi_irq_chip __ro_after_init = {
.name = "GICv5-SPI",
.irq_mask = gicv5_spi_irq_mask,
.irq_unmask = gicv5_spi_irq_unmask,
@@ -535,13 +733,15 @@ static const struct irq_chip gicv5_spi_irq_chip = {
.irq_retrigger = gicv5_spi_irq_retrigger,
.irq_get_irqchip_state = gicv5_spi_irq_get_irqchip_state,
.irq_set_irqchip_state = gicv5_spi_irq_set_irqchip_state,
+ .irq_nmi_setup = gicv5_spi_irq_nmi_setup,
+ .irq_nmi_teardown = gicv5_spi_irq_nmi_teardown,
.flags = IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_AFFINITY_PRE_STARTUP |
IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
};
-static const struct irq_chip gicv5_lpi_irq_chip = {
+static struct irq_chip gicv5_lpi_irq_chip __ro_after_init = {
.name = "GICv5-LPI",
.irq_mask = gicv5_lpi_irq_mask,
.irq_unmask = gicv5_lpi_irq_unmask,
@@ -550,12 +750,14 @@ static const struct irq_chip gicv5_lpi_irq_chip = {
.irq_retrigger = gicv5_lpi_irq_retrigger,
.irq_get_irqchip_state = gicv5_lpi_irq_get_irqchip_state,
.irq_set_irqchip_state = gicv5_lpi_irq_set_irqchip_state,
+ .irq_nmi_setup = gicv5_lpi_irq_nmi_setup,
+ .irq_nmi_teardown = gicv5_lpi_irq_nmi_teardown,
.flags = IRQCHIP_AFFINITY_PRE_STARTUP |
IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
};
-static const struct irq_chip gicv5_ipi_irq_chip = {
+static struct irq_chip gicv5_ipi_irq_chip __ro_after_init = {
.name = "GICv5-IPI",
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
@@ -563,6 +765,8 @@ static const struct irq_chip gicv5_ipi_irq_chip = {
.irq_set_affinity = irq_chip_set_affinity_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
+ .irq_nmi_setup = gicv5_ipi_irq_nmi_setup,
+ .irq_nmi_teardown = gicv5_ipi_irq_nmi_teardown,
.ipi_send_single = gicv5_ipi_send_single,
.flags = IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
@@ -907,11 +1111,12 @@ static const struct irq_domain_ops gicv5_irq_ipi_domain_ops = {
.free = gicv5_irq_ipi_domain_free,
};
-static void handle_irq_per_domain(u32 hwirq)
+static void handle_irq_per_domain(u32 hwirq, bool is_nmi)
{
u8 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, hwirq);
u32 hwirq_id = FIELD_GET(GICV5_HWIRQ_ID, hwirq);
struct irq_domain *domain;
+ int ret;
switch (hwirq_type) {
case GICV5_HWIRQ_TYPE_PPI:
@@ -928,26 +1133,32 @@ static void handle_irq_per_domain(u32 hwirq)
return;
}
- if (generic_handle_domain_irq(domain, hwirq_id)) {
- pr_err_once("Could not handle, hwirq = 0x%x", hwirq_id);
+ if (is_nmi)
+ ret = generic_handle_domain_nmi(domain, hwirq_id);
+ else
+ ret = generic_handle_domain_irq(domain, hwirq_id);
+
+ if (ret) {
+ pr_err_once("Could not handle, hwirq = 0x%x (%d)", hwirq_id, ret);
gicv5_hwirq_eoi(hwirq_id, hwirq_type);
}
}
static void __exception_irq_entry gicv5_handle_irq(struct pt_regs *regs)
{
+ bool is_nmi = in_nmi();
bool valid;
u32 hwirq;
u64 ia;
- ia = gicr_insn(CDIA);
+ ia = is_nmi ? gicr_insn(CDNMIA) : gicr_insn(CDIA);
valid = GICV5_GICR_CDIA_VALID(ia);
if (!valid)
return;
/*
- * Ensure that the CDIA instruction effects (ie IRQ activation) are
+ * Ensure that the CD{NM}IA instruction effects (ie IRQ activation) are
* completed before handling the interrupt.
*/
gsb_ack();
@@ -967,7 +1178,7 @@ static void __exception_irq_entry gicv5_handle_irq(struct pt_regs *regs)
hwirq = FIELD_GET(GICV5_HWIRQ_INTID, ia);
- handle_irq_per_domain(hwirq);
+ handle_irq_per_domain(hwirq, is_nmi);
}
static void gicv5_cpu_disable_interrupts(void)
@@ -1010,6 +1221,17 @@ static int gicv5_starting_cpu(unsigned int cpu)
return gicv5_irs_register_cpu(cpu);
}
+static void gicv5_enable_nmi_support(void)
+{
+ if (!system_uses_nmi())
+ return;
+
+ gicv5_ppi_irq_chip.flags |= IRQCHIP_SUPPORTS_NMI;
+ gicv5_spi_irq_chip.flags |= IRQCHIP_SUPPORTS_NMI;
+ gicv5_lpi_irq_chip.flags |= IRQCHIP_SUPPORTS_NMI;
+ gicv5_ipi_irq_chip.flags |= IRQCHIP_SUPPORTS_NMI;
+}
+
static void __init gicv5_smp_init(void)
{
unsigned int num_ipis = GICV5_IPIS_PER_CPU * nr_cpu_ids;
@@ -1176,6 +1398,8 @@ static int __init gicv5_init_common(struct fwnode_handle *parent_domain)
if (ret)
goto out_handle;
+ gicv5_enable_nmi_support();
+
gicv5_smp_init();
gicv5_irs_its_probe();
--
2.34.1
More information about the linux-arm-kernel
mailing list