[PATCH 03/10] lib: sbi: Add Virtual IRQ (VIRQ) subsystem
Raymond Mao
raymondmaoca at gmail.com
Thu May 14 15:57:49 PDT 2026
From: Raymond Mao <raymond.mao at riscstar.com>
VIRQ is an abstraction framework providing per-MPXY-channel
HWIRQ<->VIRQ mapping, per-(domain,hart) VIRQ routing and couriering.
It notifies S-mode payload via irqchip SEIP helper when a VIRQ is
mapped/routed/enqueued and provides ecall extension for a S-mode
payload to pop/complete a pending VIRQ.
VIRQ layer is composed of three major parts:
1. VIRQ mapping and allocation
- Provides a stable per-MPXY-channel mapping between a host
physical interrupt endpoint (chip_uid, hwirq) and a VIRQ
number.
- VIRQ number allocation uses a scalable bitmap.
2. HWIRQ->(Domain,hart) routing rules
- Routing rules are derived from sysirq nodes by
interrupts-extended property, for example:
interrupts-extended =
<&aplic HWIRQx IRQ_TYPE>, // virq 0
<&aplic HWIRQy IRQ_TYPE>, // virq 1
...;
- VIRQ numbers are allocated from zero, implicit from the order
of the entries in the interrupts-extended property.
- Each entry is cached as a routing rule.
- Default behavior: if an asserted HWIRQ does not match any
routing rule, it will be routed to the root domain
(channel 0) as a fallback.
3. Per-(domain,hart) pending queue couriering
- Each domain maintains a per-hart ring buffer queue of pending
VIRQs. A courier handler enqueues VIRQs on HWIRQ assertion.
- The couriering is domain-aware. It switches to the target domain
when it is not the same as the current one, and sends request for
returning to the previous domain after SEIP completion.
- S-mode notification is edge-triggered based on the irqchip notify
state, and is cleared only when the queue becomes empty.
Signed-off-by: Raymond Mao <raymond.mao at riscstar.com>
---
include/sbi/sbi_domain.h | 2 +
include/sbi/sbi_virq.h | 492 +++++++++++++++++
lib/sbi/objects.mk | 1 +
lib/sbi/sbi_domain.c | 10 +
lib/sbi/sbi_virq.c | 1136 ++++++++++++++++++++++++++++++++++++++
5 files changed, 1641 insertions(+)
create mode 100644 include/sbi/sbi_virq.h
create mode 100644 lib/sbi/sbi_virq.c
diff --git a/include/sbi/sbi_domain.h b/include/sbi/sbi_domain.h
index c507023c..7e288cd8 100644
--- a/include/sbi/sbi_domain.h
+++ b/include/sbi/sbi_domain.h
@@ -219,6 +219,8 @@ struct sbi_domain {
bool fw_region_inited;
/** Whether to notify S-mode for VIRQ couriering */
bool virq_seip_notify;
+ /** per-domain wired-IRQ courier state */
+ void *virq_priv;
};
/** The root domain instance */
diff --git a/include/sbi/sbi_virq.h b/include/sbi/sbi_virq.h
new file mode 100644
index 00000000..566ae827
--- /dev/null
+++ b/include/sbi/sbi_virq.h
@@ -0,0 +1,492 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+/*
+ * Virtual IRQ (VIRQ) courier/routing layer for OpenSBI.
+ *
+ * This header defines:
+ * 1) VIRQ number allocation and (chip_uid,hwirq) <-> VIRQ mapping
+ * 2) HWIRQ -> Domain routing rules (from DeviceTree "opensbi,mpxy-sysirq")
+ * 3) Per-(domain,hart) pending queue (push in M-mode, pop/complete in S-mode)
+ *
+ * High-level design intent:
+ * - All physical host IRQs are handled in M-mode by host irqchip drivers.
+ * - For each incoming HWIRQ, OpenSBI determines the destination domain using
+ * DT-defined routing rules and enqueues a VIRQ into the per-(domain,hart)
+ * pending queue.
+ * - S-mode payload consumes pending VIRQs via pop(), and completes them via
+ * complete(), which unmasks the corresponding host HWIRQ line.
+ * - M-mode notifies S-mode via the irqchip notification mechanism.
+ *
+ * Notes:
+ * - "opensbi,mpxy-sysirq" routing is derived from the sysirq node's
+ * "interrupts-extended" entries. It does not encode privilege level
+ * delivery. Hardware delivery (MEI vs SEI) is determined by platform IRQ
+ * topology and interrupt-parent.
+ *
+ * Copyright (c) 2026 RISCstar Solutions Corporation.
+ *
+ * Author: Raymond Mao <raymond.mao at riscstar.com>
+ */
+
+#ifndef __SBI_VIRQ_H__
+#define __SBI_VIRQ_H__
+
+#include <sbi/sbi_domain.h>
+#include <sbi/sbi_irqchip.h>
+#include <sbi/riscv_locks.h>
+#include <sbi/sbi_types.h>
+
+/*
+ * Current implementation behavior when queue overflows:
+ * - Drop the incoming VIRQ
+ * - Return SBI_ENOMEM
+ */
+#define VIRQ_QSIZE 64
+
+/*
+ * Reverse mapping table is chunked to avoid a single large static array.
+ * VIRQ is used as an index into a chunk; chunks are allocated on demand.
+ */
+#define VIRQ_CHUNK_SHIFT 6U
+#define VIRQ_CHUNK_SIZE (1U << VIRQ_CHUNK_SHIFT)
+#define VIRQ_CHUNK_MASK (VIRQ_CHUNK_SIZE - 1U)
+
+/* Minimum growth step for forward mapping vector and related metadata. */
+#define VEC_GROW_MIN 16U
+
+/* Returned by pop when no pending VIRQ is available. */
+#define VIRQ_INVALID 0xffffffffU
+
+/*
+ * VIRQ allocator and (chip_uid,hwirq) <-> VIRQ mapping
+ */
+
+/*
+ * VIRQ mapping model:
+ * - Forward mapping: (chip_uid,hwirq) -> VIRQ
+ * Implementation: dynamic vector of entries (linear search).
+ *
+ * - Reverse mapping: VIRQ -> (chip_uid,hwirq)
+ * Implementation: chunked table allocated on demand, O(1) lookup.
+ *
+ * - VIRQ number allocation:
+ * Implementation: growable bitmap; capacity expands as needed.
+ *
+ * Memory usage scales with the number of installed mappings.
+ */
+
+/* Entry of reverse mapping table: represents (chip_uid,hwirq) endpoint */
+struct virq_entry {
+ u32 chip_uid;
+ u32 hwirq;
+};
+
+/* Chunked reverse mapping table: VIRQ -> (chip_uid,hwirq) */
+struct virq_chunk {
+ struct virq_entry e[VIRQ_CHUNK_SIZE];
+};
+
+/*
+ * HWIRQ -> Domain routing rules
+ */
+
+/*
+ * A routing rule maps a single HWIRQ to a domain.
+ *
+ * Rules are populated once during cold boot while parsing the DT
+ * opensbi-domains configuration (sysirq node "opensbi,mpxy-sysirq").
+ *
+ * DT encodes mapping via "interrupts-extended"; the index within this array
+ * becomes the VIRQ number for the given MPXY channel.
+ *
+ * Policy notes:
+ * - Duplicate HWIRQ entries are rejected and return SBI_EALREADY.
+ * - If no rule matches, routing falls back to the root domain (&root).
+ */
+struct sbi_virq_route_rule {
+ u32 hwirq;
+ struct sbi_domain *dom; /* owner domain */
+ u32 channel_id; /* VIRQ space/channel */
+};
+
+/*
+ * Courier context passed as 'opaque' to sbi_virq_courier_handler(), created
+ * per host irqchip.
+ *
+ * The courier handler needs to:
+ * - map (chip_uid,hwirq) -> VIRQ
+ * - mask/unmask HWIRQ using the correct irqchip device
+ * Therefore the irqchip device pointer is carried here.
+ */
+struct sbi_virq_courier_ctx {
+ struct sbi_irqchip_device *chip;
+};
+
+/*
+ * Per-(domain,hart) pending VIRQ state and queue management
+ */
+
+/*
+ * Per-(domain,hart) VIRQ state.
+ *
+ * Locking:
+ * - lock protects head/tail and q[].
+ *
+ * Queue semantics:
+ * - q[] stores VIRQs pending handling for this (domain,hart).
+ * - enqueue is performed by M-mode (courier handler) according to route rule
+ * populated from DT.
+ * - pop/complete is performed by S-mode payload running in the destination
+ * domain on the current hart.
+ * - chip caches the irqchip device for unmasking on complete().
+ */
+struct sbi_domain_virq_state {
+ spinlock_t lock;
+ u32 head;
+ u32 tail;
+
+ /* Pending VIRQ ring buffer. */
+ struct {
+ u32 virq;
+ u32 channel_id;
+ struct sbi_irqchip_device *chip;
+ } q[VIRQ_QSIZE];
+
+ /* Last popped entry for completion. */
+ u32 last_pop_virq;
+ u32 last_pop_channel_id;
+ struct sbi_irqchip_device *last_pop_chip;
+
+ /* Return to previous domain after VIRQ completion. */
+ bool return_to_prev;
+};
+
+/*
+ * Per-domain private VIRQ context.
+ *
+ * Attached to struct sbi_domain and contains per-hart states.
+ */
+struct sbi_domain_virq_priv {
+ /* number of platform harts */
+ u32 nharts;
+
+ /* number of allocated per-hart states */
+ u32 st_count;
+
+ /* per-hart VIRQ state pointer array (indexed by hart index) */
+ struct sbi_domain_virq_state *st_by_hart[];
+};
+
+/* Courier binding used when enqueuing a VIRQ. */
+struct sbi_virq_courier_binding {
+ /* destination domain */
+ struct sbi_domain *dom;
+
+ /* irqchip device that asserted the HWIRQ */
+ struct sbi_irqchip_device *chip;
+
+ /* VIRQ space/channel ID */
+ u32 channel_id;
+
+ /* VIRQ number to enqueue */
+ u32 virq;
+};
+
+/*
+ * Public APIs
+ */
+
+/*
+ * Initialize a per-channel VIRQ map.
+ *
+ * @channel_id:
+ * VIRQ space/channel ID (0 is the default channel).
+ *
+ * @init_virq_cap:
+ * Initial capacity in VIRQ bits (e.g., 256). Implementation may grow beyond.
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_ENOMEM on allocation failure
+ */
+int sbi_virq_map_init(u32 channel_id, u32 init_virq_cap);
+
+/*
+ * Create or get a stable mapping for (channel_id, chip_uid, hwirq) -> VIRQ.
+ *
+ * @channel_id:
+ * Paravirt channel ID; VIRQ numbering is local to each channel.
+ *
+ * @chip_uid:
+ * Unique 32-bit ID of the host irqchip device.
+ *
+ * @hwirq:
+ * Host HWIRQ number as produced by the irqchip driver (e.g. APLIC claim ID).
+ *
+ * @allow_identity:
+ * If true, allocator may attempt VIRQ == hwirq for small ranges.
+ *
+ * @identity_limit:
+ * Upper bound (exclusive) for identity mapping trial: hwirq < identity_limit.
+ *
+ * @out_virq:
+ * Output pointer receiving the mapped/allocated VIRQ (0 is valid).
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_ENOMEM on allocation failure
+ * SBI_ENOSPC if allocator cannot allocate
+ * SBI_EINVAL on invalid parameters
+ */
+int sbi_virq_map_one(u32 channel_id, u32 chip_uid, u32 hwirq,
+ bool allow_identity, u32 identity_limit, u32 *out_virq);
+
+/*
+ * Force a mapping for (channel_id, chip_uid, hwirq) -> VIRQ.
+ *
+ * @channel_id:
+ * Paravirt channel ID; VIRQ numbering is local to each channel.
+ *
+ * @chip_uid:
+ * Unique 32-bit ID of the host irqchip device.
+ *
+ * @hwirq:
+ * Host HWIRQ number as produced by the irqchip driver.
+ *
+ * @virq:
+ * VIRQ number to assign (0 is valid).
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_ENOMEM on allocation failure
+ * SBI_EINVAL on invalid parameters
+ * SBI_EALREADY if a different mapping already exists
+ */
+int sbi_virq_map_set(u32 channel_id, u32 chip_uid, u32 hwirq, u32 virq);
+
+/*
+ * Ensure VIRQ map capacity for a given channel.
+ *
+ * @channel_id:
+ * Paravirt channel ID.
+ *
+ * @min_virq_cap:
+ * Minimum VIRQ bitmap capacity in bits (will be rounded up).
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EINVAL if the map is not initialized (channel 0)
+ * SBI_ENOMEM on allocation failure
+ */
+int sbi_virq_map_ensure_cap(u32 channel_id, u32 min_virq_cap);
+
+/*
+ * Lookup existing mapping: (channel_id, chip_uid, hwirq) -> VIRQ.
+ *
+ * @channel_id:
+ * Paravirt channel ID; VIRQ numbering is local to each channel.
+ *
+ * @chip_uid:
+ * Irqchip unique id.
+ *
+ * @hwirq:
+ * Host hwirq number.
+ *
+ * @out_virq:
+ * Output VIRQ (0 is valid).
+ *
+ * Return:
+ * SBI_OK if found
+ * SBI_ENOENT if not mapped
+ * SBI_EINVAL on invalid input
+ */
+int sbi_virq_hwirq2virq(u32 channel_id, u32 chip_uid, u32 hwirq,
+ u32 *out_virq);
+
+/*
+ * Reverse lookup: (channel_id, VIRQ) -> (chip_uid, hwirq).
+ *
+ * @channel_id:
+ * Paravirt channel ID; VIRQ numbering is local to each channel.
+ *
+ * @virq:
+ * VIRQ number to look up.
+ *
+ * @out_chip_uid:
+ * Output pointer receiving irqchip unique id.
+ *
+ * @out_hwirq:
+ * Output pointer receiving host hwirq number.
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EINVAL if virq is VIRQ_INVALID, out of range, not allocated, or
+ * reverse entry missing
+ */
+int sbi_virq_virq2hwirq(u32 channel_id, u32 virq,
+ u32 *out_chip_uid, u32 *out_hwirq);
+
+/*
+ * Unmap a single VIRQ mapping and free the VIRQ number.
+ *
+ * @virq:
+ * VIRQ number to unmap.
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EINVAL if virq is invalid or state is inconsistent
+ */
+int sbi_virq_unmap_one(u32 virq);
+
+/*
+ * Uninitialize the VIRQ mapping allocator and free all resources.
+ *
+ * Notes:
+ * - This frees bitmap, forward vector, and reverse chunks.
+ */
+void sbi_virq_map_uninit(void);
+
+/*
+ * Reset all HWIRQ->Domain routing rules (frees the rule array).
+ *
+ * Typical usage:
+ * - Called once at cold boot during init before parsing DT domains.
+ */
+void sbi_virq_route_reset(void);
+
+/*
+ * Add a routing rule: hwirq -> dom with channel_id.
+ *
+ * @dom:
+ * Target domain that should receive HWIRQs in this range.
+ *
+ * @hwirq:
+ * HWIRQ number to route.
+ *
+ * @channel_id:
+ * Paravirt channel ID for VIRQ mapping (MPXY channel).
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EINVAL on invalid parameters
+ * SBI_ENOMEM on allocation failure
+ * SBI_EALREADY if the HWIRQ already has a rule
+ */
+int sbi_virq_route_add(struct sbi_domain *dom, u32 hwirq, u32 channel_id);
+
+/*
+ * Lookup destination domain for a given HWIRQ.
+ *
+ * @hwirq:
+ * Incoming host HWIRQ number.
+ *
+ * @out_dom:
+ * Output pointer receiving destination domain. If no rule matches, &root
+ * is returned.
+ *
+ * @out_channel_id:
+ * Output pointer receiving channel id if non-NULL.
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EINVAL on invalid parameters
+ */
+int sbi_virq_route_lookup(u32 hwirq, struct sbi_domain **out_dom,
+ u32 *out_channel_id);
+
+/*
+ * Enqueue a VIRQ for the destination domain on the current hart.
+ *
+ * @c:
+ * Courier binding containing:
+ * - c->dom : destination domain
+ * - c->chip : irqchip device pointer
+ * - c->virq : VIRQ number
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EINVAL on invalid parameters
+ * SBI_ENODEV if per-(domain,hart) state is not available
+ * SBI_ENOMEM if queue is full
+ */
+int sbi_virq_enqueue(struct sbi_virq_courier_binding *c);
+
+/*
+ * Pop the next pending VIRQ for the current domain on the current hart.
+ *
+ * Return:
+ * VIRQ_INVALID if none pending or state not available
+ * otherwise a VIRQ number (zero is legal)
+ */
+u32 sbi_virq_pop_thishart(void);
+
+/*
+ * Complete a previously couriered VIRQ for the current domain/hart.
+ *
+ * @virq:
+ * VIRQ to complete.
+ */
+void sbi_virq_complete_thishart(u32 virq);
+
+/* Return to previous domain if a VIRQ-driven switch is pending. */
+void sbi_virq_return_to_prev_if_needed(void);
+
+
+/*
+ * Courier handler intended to be registered by host irqchip driver.
+ *
+ * @hwirq:
+ * Incoming host HWIRQ number asserted on the irqchip.
+ *
+ * @opaque:
+ * Point to a valid struct sbi_virq_courier_ctx, which provides the
+ * irqchip device pointer used for mapping and mask/unmask.
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EINVAL on invalid parameters
+ * Other SBI_E* propagated from mapping or enqueue
+ */
+int sbi_virq_courier_handler(u32 hwirq, void *opaque);
+
+/*
+ * Initialize per-domain VIRQ state.
+ *
+ * @dom:
+ * Domain to initialize.
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EINVAL on invalid parameters
+ * SBI_ENOMEM on allocation failure
+ */
+int sbi_virq_domain_init(struct sbi_domain *dom);
+
+/*
+ * Free per-domain VIRQ state.
+ *
+ * @dom:
+ * Free the per-domain VIRQ state.
+ */
+void sbi_virq_domain_exit(struct sbi_domain *dom);
+
+/*
+ * Initialize VIRQ subsystem (mapping allocator + route rules).
+ * Must be called once before parsing sysirq DT nodes.
+ *
+ * @init_virq_cap:
+ * Initial VIRQ bitmap capacity in bits
+ *
+ * Return:
+ * SBI_OK on success
+ * SBI_EALREADY if called more than once
+ * SBI_ENOMEM on allocation failure
+ * Other SBI_E* error codes propagated from mapping init
+ */
+int sbi_virq_init(u32 init_virq_cap);
+
+/*
+ * Query whether the VIRQ subsystem is initialized.
+ */
+bool sbi_virq_is_inited(void);
+
+#endif
diff --git a/lib/sbi/objects.mk b/lib/sbi/objects.mk
index 07d13229..184bf173 100644
--- a/lib/sbi/objects.mk
+++ b/lib/sbi/objects.mk
@@ -86,6 +86,7 @@ libsbi-objs-y += sbi_illegal_insn.o
libsbi-objs-y += sbi_init.o
libsbi-objs-y += sbi_ipi.o
libsbi-objs-y += sbi_irqchip.o
+libsbi-objs-y += sbi_virq.o
libsbi-objs-y += sbi_platform.o
libsbi-objs-y += sbi_pmu.o
libsbi-objs-y += sbi_dbtr.o
diff --git a/lib/sbi/sbi_domain.c b/lib/sbi/sbi_domain.c
index 7030848d..2a846eea 100644
--- a/lib/sbi/sbi_domain.c
+++ b/lib/sbi/sbi_domain.c
@@ -18,6 +18,7 @@
#include <sbi/sbi_platform.h>
#include <sbi/sbi_scratch.h>
#include <sbi/sbi_string.h>
+#include <sbi/sbi_virq.h>
SBI_LIST_HEAD(domain_list);
@@ -693,6 +694,15 @@ int sbi_domain_register(struct sbi_domain *dom,
return rc;
}
+ /* Init per-domain wired-IRQ courier state */
+ rc = sbi_virq_domain_init(dom);
+ if (rc) {
+ sbi_printf("%s: virq init failed for %s (error %d)\n",
+ __func__, dom->name, rc);
+ sbi_list_del(&dom->node);
+ return rc;
+ }
+
return 0;
}
diff --git a/lib/sbi/sbi_virq.c b/lib/sbi/sbi_virq.c
new file mode 100644
index 00000000..fcd83369
--- /dev/null
+++ b/lib/sbi/sbi_virq.c
@@ -0,0 +1,1136 @@
+// SPDX-License-Identifier: BSD-2-Clause
+/*
+ * Copyright (c) 2026 RISCstar Solutions.
+ *
+ * Author: Raymond Mao <raymond.mao at riscstar.com>
+ */
+
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_heap.h>
+#include <sbi/sbi_irqchip.h>
+#include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_platform.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_string.h>
+#include <sbi/sbi_virq.h>
+#include <sbi/sbi_domain.h>
+#include <sbi/sbi_domain_context.h>
+#include <sbi/riscv_asm.h>
+#include <sbi/riscv_locks.h>
+
+struct map_node {
+ u32 chip_uid;
+ u32 hwirq;
+ u32 virq;
+};
+
+struct sbi_virq_map {
+ spinlock_t lock;
+
+ /* allocator bitmap */
+ unsigned long *bmap;
+ u32 bmap_nbits; /* virq range: [0..nbits-1] */
+
+ /* reverse table: virq -> endpoint */
+ struct virq_chunk **chunks;
+ u32 chunks_cap; /* number of chunk pointers */
+
+ /* forward table: vector of mappings, linear search */
+ struct map_node *nodes;
+ u32 nodes_cnt;
+ u32 nodes_cap;
+};
+
+struct sbi_virq_map_list {
+ u32 channel_id;
+ struct sbi_virq_map map;
+};
+
+/*
+ * HWIRQ -> Domain routing rules
+ */
+
+struct sbi_virq_router {
+ spinlock_t lock;
+ struct sbi_virq_route_rule *rules;
+ u32 cnt;
+ u32 cap;
+};
+
+static struct sbi_virq_map g_virq_map; /* channel 0 */
+static struct sbi_virq_map_list *g_virq_maps;
+static u32 g_virq_maps_cnt;
+static u32 g_virq_maps_cap;
+static spinlock_t g_virq_maps_lock;
+static struct sbi_virq_router g_router;
+static bool g_virq_inited;
+
+void sbi_virq_route_reset(void)
+{
+ spin_lock(&g_router.lock);
+ if (g_router.rules) {
+ sbi_free(g_router.rules);
+ g_router.rules = NULL;
+ }
+ g_router.cnt = 0;
+ g_router.cap = 0;
+ spin_unlock(&g_router.lock);
+}
+
+static int router_ensure_cap(u32 need)
+{
+ struct sbi_virq_route_rule *newp;
+ u32 newcap;
+
+ if (g_router.cap >= need)
+ return 0;
+
+ newcap = g_router.cap ? (g_router.cap << 1) : 8;
+ while (newcap < need)
+ newcap <<= 1;
+
+ newp = sbi_zalloc((size_t)newcap * sizeof(*newp));
+ if (!newp)
+ return SBI_ENOMEM;
+
+ if (g_router.rules) {
+ sbi_memcpy(newp, g_router.rules,
+ (size_t)g_router.cnt * sizeof(*newp));
+ sbi_free(g_router.rules);
+ }
+
+ g_router.rules = newp;
+ g_router.cap = newcap;
+
+ return SBI_OK;
+}
+
+int sbi_virq_route_add(struct sbi_domain *dom, u32 hwirq, u32 channel_id)
+{
+ int rc;
+
+ if (!dom)
+ return SBI_EINVAL;
+
+ spin_lock(&g_router.lock);
+
+ /* Reject duplicates to keep routing unambiguous */
+ for (u32 i = 0; i < g_router.cnt; i++) {
+ if (g_router.rules[i].hwirq == hwirq) {
+ spin_unlock(&g_router.lock);
+ return SBI_EALREADY;
+ }
+ }
+
+ rc = router_ensure_cap(g_router.cnt + 1);
+ if (rc) {
+ spin_unlock(&g_router.lock);
+ return rc;
+ }
+
+ g_router.rules[g_router.cnt].hwirq = hwirq;
+ g_router.rules[g_router.cnt].dom = dom;
+ g_router.rules[g_router.cnt].channel_id = channel_id;
+ g_router.cnt++;
+
+ spin_unlock(&g_router.lock);
+
+ return SBI_OK;
+}
+
+int sbi_virq_route_lookup(u32 hwirq, struct sbi_domain **out_dom,
+ u32 *out_channel_id)
+{
+ /* Fast path: no rules */
+ if (!g_router.cnt) {
+ if (out_dom)
+ *out_dom = &root;
+ if (out_channel_id)
+ *out_channel_id = 0;
+ return SBI_OK;
+ }
+
+ spin_lock(&g_router.lock);
+ for (u32 i = 0; i < g_router.cnt; i++) {
+ if (hwirq == g_router.rules[i].hwirq) {
+ struct sbi_domain *d = g_router.rules[i].dom;
+ u32 cid = g_router.rules[i].channel_id;
+
+ spin_unlock(&g_router.lock);
+ if (out_dom)
+ *out_dom = d ? d : &root;
+ if (out_channel_id)
+ *out_channel_id = cid;
+ return SBI_OK;
+ }
+ }
+ spin_unlock(&g_router.lock);
+
+ if (out_dom)
+ *out_dom = &root;
+ if (out_channel_id)
+ *out_channel_id = 0;
+ return SBI_OK;
+}
+
+static inline void virq_state_init(struct sbi_domain_virq_state *st)
+{
+ SPIN_LOCK_INIT(st->lock);
+ st->head = 0;
+ st->tail = 0;
+ st->return_to_prev = false;
+}
+
+static inline
+struct sbi_domain_virq_state *domain_virq_thishart(struct sbi_domain *dom)
+{
+ unsigned long hartidx = sbi_hartid_to_hartindex(current_hartid());
+ struct sbi_domain_virq_priv *p;
+
+ p = (struct sbi_domain_virq_priv *)dom->virq_priv;
+ if (!p || hartidx >= p->nharts)
+ return NULL;
+
+ return p->st_by_hart[hartidx];
+}
+
+static inline bool q_full(struct sbi_domain_virq_state *st)
+{
+ return ((st->tail + 1) % VIRQ_QSIZE) == st->head;
+}
+
+static inline bool q_empty(struct sbi_domain_virq_state *st)
+{
+ return st->head == st->tail;
+}
+
+static inline void virq_set_domain_return_flag(struct sbi_domain *dom,
+ bool return_to_prev)
+{
+ struct sbi_domain_virq_state *st = domain_virq_thishart(dom);
+
+ if (!st)
+ return;
+
+ spin_lock(&st->lock);
+ st->return_to_prev = return_to_prev;
+ spin_unlock(&st->lock);
+}
+
+static u32 sbi_virq_platform_hart_count(void)
+{
+ struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+ const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+
+ return sbi_platform_hart_count(plat);
+}
+
+static int bmap_alloc_one(struct sbi_virq_map *m, u32 *out_virq)
+{
+ u32 v;
+
+ for (v = 0; v < m->bmap_nbits; v++) {
+ if (!bitmap_test(m->bmap, (int)v)) {
+ bitmap_set(m->bmap, (int)v, 1);
+ *out_virq = v;
+ return 0;
+ }
+ }
+
+ return SBI_ENOSPC;
+}
+
+static int bmap_alloc_specific(struct sbi_virq_map *m, u32 virq)
+{
+ if (virq >= m->bmap_nbits)
+ return SBI_EINVAL;
+ if (bitmap_test(m->bmap, (int)virq))
+ return SBI_EALREADY;
+ bitmap_set(m->bmap, (int)virq, 1);
+
+ return 0;
+}
+
+static void bmap_free_one(struct sbi_virq_map *m, u32 virq)
+{
+ if (virq < m->bmap_nbits)
+ bitmap_clear(m->bmap, (int)virq, 1);
+}
+
+static int chunks_ensure_cap(struct sbi_virq_map *m, u32 new_bmap_nbits)
+{
+ u32 new_chunks_cap =
+ (new_bmap_nbits + VIRQ_CHUNK_SIZE - 1U) >> VIRQ_CHUNK_SHIFT;
+ struct virq_chunk **newp;
+
+ if (new_chunks_cap <= m->chunks_cap)
+ return 0;
+
+ newp = sbi_zalloc((size_t)new_chunks_cap * sizeof(*newp));
+ if (!newp)
+ return SBI_ENOMEM;
+
+ if (m->chunks) {
+ sbi_memcpy(newp, m->chunks,
+ (size_t)m->chunks_cap * sizeof(*newp));
+ sbi_free(m->chunks);
+ }
+
+ m->chunks = newp;
+ m->chunks_cap = new_chunks_cap;
+
+ return 0;
+}
+
+static int bmap_grow(struct sbi_virq_map *m, u32 new_nbits)
+{
+ unsigned long *newmap;
+
+ if (new_nbits <= m->bmap_nbits)
+ return 0;
+
+ newmap = sbi_zalloc(bitmap_estimate_size((int)new_nbits));
+ if (!newmap)
+ return SBI_ENOMEM;
+
+ bitmap_zero(newmap, (int)new_nbits);
+ bitmap_copy(newmap, m->bmap, (int)m->bmap_nbits);
+
+ sbi_free(m->bmap);
+ m->bmap = newmap;
+ m->bmap_nbits = new_nbits;
+
+ return chunks_ensure_cap(m, new_nbits);
+}
+
+static struct virq_entry *rev_get_or_alloc(struct sbi_virq_map *m, u32 virq)
+{
+ u32 ci = virq >> VIRQ_CHUNK_SHIFT;
+ u32 off = virq & VIRQ_CHUNK_MASK;
+
+ if (ci >= m->chunks_cap)
+ return NULL;
+
+ if (!m->chunks[ci]) {
+ m->chunks[ci] = sbi_zalloc(sizeof(struct virq_chunk));
+ if (!m->chunks[ci])
+ return NULL;
+ }
+ return &m->chunks[ci]->e[off];
+}
+
+static struct virq_entry *rev_get_existing(struct sbi_virq_map *m, u32 virq)
+{
+ u32 ci = virq >> VIRQ_CHUNK_SHIFT;
+ u32 off = virq & VIRQ_CHUNK_MASK;
+
+ if (ci >= m->chunks_cap || !m->chunks[ci])
+ return NULL;
+ return &m->chunks[ci]->e[off];
+}
+
+static void rev_clear(struct sbi_virq_map *m, u32 virq)
+{
+ struct virq_entry *e = rev_get_existing(m, virq);
+
+ if (e) {
+ e->chip_uid = 0;
+ e->hwirq = 0;
+ }
+}
+
+static int vec_ensure_cap(struct sbi_virq_map *m, u32 need_cnt)
+{
+ struct map_node *newp;
+ u32 newcap;
+
+ if (m->nodes_cap >= need_cnt)
+ return 0;
+
+ newcap = m->nodes_cap ? (m->nodes_cap << 1) :
+ VEC_GROW_MIN;
+ while (newcap < need_cnt)
+ newcap <<= 1;
+
+ newp = sbi_zalloc((size_t)newcap * sizeof(*newp));
+ if (!newp)
+ return SBI_ENOMEM;
+
+ if (m->nodes) {
+ sbi_memcpy(newp, m->nodes,
+ (size_t)m->nodes_cnt * sizeof(*newp));
+ sbi_free(m->nodes);
+ }
+
+ m->nodes = newp;
+ m->nodes_cap = newcap;
+
+ return 0;
+}
+
+static int forward_find_idx(struct sbi_virq_map *m,
+ u32 chip_uid, u32 hwirq, u32 *out_idx)
+{
+ u32 i;
+
+ for (i = 0; i < m->nodes_cnt; i++) {
+ if (m->nodes[i].chip_uid == chip_uid &&
+ m->nodes[i].hwirq == hwirq) {
+ *out_idx = i;
+ return 0;
+ }
+ }
+
+ return SBI_ENOENT;
+}
+
+static int virq_map_init_one(struct sbi_virq_map *m, u32 init_virq_cap)
+{
+ int rc;
+
+ sbi_memset(m, 0, sizeof(*m));
+ SPIN_LOCK_INIT(m->lock);
+
+ if (init_virq_cap < 8U)
+ init_virq_cap = 8U;
+
+ m->bmap_nbits = init_virq_cap;
+ m->bmap =
+ sbi_zalloc(bitmap_estimate_size((int)m->bmap_nbits));
+ if (!m->bmap)
+ return SBI_ENOMEM;
+
+ bitmap_zero(m->bmap, (int)m->bmap_nbits);
+
+ rc = chunks_ensure_cap(m, m->bmap_nbits);
+ if (rc)
+ return rc;
+
+ return SBI_OK;
+}
+
+static struct sbi_virq_map *virq_map_get(u32 channel_id, bool create,
+ u32 init_virq_cap)
+{
+ u32 i;
+ struct sbi_virq_map_list *newp;
+
+ if (channel_id == 0)
+ return &g_virq_map;
+
+ spin_lock(&g_virq_maps_lock);
+ for (i = 0; i < g_virq_maps_cnt; i++) {
+ if (g_virq_maps[i].channel_id == channel_id) {
+ spin_unlock(&g_virq_maps_lock);
+ return &g_virq_maps[i].map;
+ }
+ }
+ if (!create) {
+ spin_unlock(&g_virq_maps_lock);
+ return NULL;
+ }
+
+ if (g_virq_maps_cnt == g_virq_maps_cap) {
+ u32 newcap = g_virq_maps_cap ? (g_virq_maps_cap << 1) : 4;
+
+ newp = sbi_zalloc((size_t)newcap * sizeof(*newp));
+ if (!newp) {
+ spin_unlock(&g_virq_maps_lock);
+ return NULL;
+ }
+ if (g_virq_maps) {
+ sbi_memcpy(newp, g_virq_maps,
+ (size_t)g_virq_maps_cnt * sizeof(*newp));
+ sbi_free(g_virq_maps);
+ }
+ g_virq_maps = newp;
+ g_virq_maps_cap = newcap;
+ }
+
+ g_virq_maps[g_virq_maps_cnt].channel_id = channel_id;
+ if (virq_map_init_one(&g_virq_maps[g_virq_maps_cnt].map,
+ init_virq_cap)) {
+ spin_unlock(&g_virq_maps_lock);
+ return NULL;
+ }
+ g_virq_maps_cnt++;
+ spin_unlock(&g_virq_maps_lock);
+
+ return &g_virq_maps[g_virq_maps_cnt - 1].map;
+}
+
+int sbi_virq_map_init(u32 channel_id, u32 init_virq_cap)
+{
+ if (channel_id == 0)
+ return virq_map_init_one(&g_virq_map, init_virq_cap);
+ SPIN_LOCK_INIT(g_virq_maps_lock);
+ return virq_map_get(channel_id, true, init_virq_cap) ?
+ SBI_OK : SBI_ENOMEM;
+}
+
+int sbi_virq_map_one(u32 channel_id, u32 chip_uid, u32 hwirq,
+ bool allow_identity, u32 identity_limit,
+ u32 *out_virq)
+{
+ u32 idx, virq = 0;
+ int rc;
+ struct sbi_virq_map *m;
+
+ m = virq_map_get(channel_id, true, 0);
+ if (!m)
+ return SBI_ENOMEM;
+
+ spin_lock(&m->lock);
+ /* already mapped? */
+ rc = forward_find_idx(m, chip_uid, hwirq, &idx);
+ if (!rc) {
+ *out_virq = m->nodes[idx].virq;
+ spin_unlock(&m->lock);
+ return 0;
+ }
+
+ /* ensure vector capacity for new node */
+ rc = vec_ensure_cap(m, m->nodes_cnt + 1U);
+ if (rc) {
+ spin_unlock(&m->lock);
+ return rc;
+ }
+
+ /* optional identity */
+ if (allow_identity && hwirq < identity_limit) {
+ /* ensure bitmap covers this virq */
+ if (hwirq >= m->bmap_nbits) {
+ u32 new_nbits = m->bmap_nbits;
+
+ while (new_nbits <= hwirq)
+ new_nbits <<= 1;
+ rc = bmap_grow(m, new_nbits);
+ if (rc) {
+ spin_unlock(&m->lock);
+ return rc;
+ }
+ }
+
+ rc = bmap_alloc_specific(m, hwirq);
+ if (!rc)
+ virq = hwirq;
+ else if (rc != SBI_EALREADY) {
+ spin_unlock(&m->lock);
+ return rc;
+ }
+ }
+
+ /* allocate new virq if identity not taken */
+ if (!virq) {
+ rc = bmap_alloc_one(m, &virq);
+ if (rc == SBI_ENOSPC) {
+ rc = bmap_grow(m, m->bmap_nbits << 1);
+ if (rc) {
+ spin_unlock(&m->lock);
+ return rc;
+ }
+ rc = bmap_alloc_one(m, &virq);
+ }
+ if (rc) {
+ spin_unlock(&m->lock);
+ return rc;
+ }
+ }
+
+ /* install reverse mapping */
+ {
+ struct virq_entry *e = rev_get_or_alloc(m, virq);
+
+ if (!e) {
+ bmap_free_one(m, virq);
+ spin_unlock(&m->lock);
+ return SBI_ENOMEM;
+ }
+ e->chip_uid = chip_uid;
+ e->hwirq = hwirq;
+ }
+
+ /* append forward node */
+ m->nodes[m->nodes_cnt].chip_uid = chip_uid;
+ m->nodes[m->nodes_cnt].hwirq = hwirq;
+ m->nodes[m->nodes_cnt].virq = virq;
+ m->nodes_cnt++;
+
+ *out_virq = virq;
+ spin_unlock(&m->lock);
+
+ return SBI_OK;
+}
+
+int sbi_virq_map_set(u32 channel_id, u32 chip_uid, u32 hwirq, u32 virq)
+{
+ struct sbi_virq_map *m;
+ u32 idx;
+ int rc;
+
+ m = virq_map_get(channel_id, true, virq + 1U);
+ if (!m)
+ return SBI_ENOMEM;
+
+ spin_lock(&m->lock);
+ rc = forward_find_idx(m, chip_uid, hwirq, &idx);
+ if (!rc) {
+ spin_unlock(&m->lock);
+ return (m->nodes[idx].virq == virq) ? SBI_OK : SBI_EALREADY;
+ }
+
+ if (virq >= m->bmap_nbits) {
+ u32 new_nbits = m->bmap_nbits;
+
+ while (new_nbits <= virq)
+ new_nbits <<= 1;
+ rc = bmap_grow(m, new_nbits);
+ if (rc) {
+ spin_unlock(&m->lock);
+ return rc;
+ }
+ }
+
+ rc = bmap_alloc_specific(m, virq);
+ if (rc == SBI_EALREADY) {
+ struct virq_entry *e = rev_get_existing(m, virq);
+
+ if (!e || e->chip_uid != chip_uid || e->hwirq != hwirq) {
+ spin_unlock(&m->lock);
+ return SBI_EALREADY;
+ }
+
+ spin_unlock(&m->lock);
+ return SBI_OK;
+ } else if (rc) {
+ spin_unlock(&m->lock);
+ return rc;
+ }
+
+ rc = vec_ensure_cap(m, m->nodes_cnt + 1U);
+ if (rc) {
+ spin_unlock(&m->lock);
+ return rc;
+ }
+
+ {
+ struct virq_entry *e = rev_get_or_alloc(m, virq);
+
+ if (!e) {
+ bmap_free_one(m, virq);
+ spin_unlock(&m->lock);
+ return SBI_ENOMEM;
+ }
+ e->chip_uid = chip_uid;
+ e->hwirq = hwirq;
+ }
+
+ m->nodes[m->nodes_cnt].chip_uid = chip_uid;
+ m->nodes[m->nodes_cnt].hwirq = hwirq;
+ m->nodes[m->nodes_cnt].virq = virq;
+ m->nodes_cnt++;
+ spin_unlock(&m->lock);
+
+ return SBI_OK;
+}
+
+int sbi_virq_map_ensure_cap(u32 channel_id, u32 min_virq_cap)
+{
+ struct sbi_virq_map *m;
+ u32 new_nbits;
+ int rc = SBI_OK;
+
+ if (min_virq_cap < 8U)
+ min_virq_cap = 8U;
+
+ if (channel_id == 0) {
+ m = &g_virq_map;
+ if (!m->bmap)
+ return SBI_EINVAL;
+ } else {
+ m = virq_map_get(channel_id, true, min_virq_cap);
+ if (!m)
+ return SBI_ENOMEM;
+ }
+
+ if (m->bmap_nbits >= min_virq_cap)
+ return SBI_OK;
+
+ spin_lock(&m->lock);
+ new_nbits = m->bmap_nbits ? m->bmap_nbits : 8U;
+ while (new_nbits < min_virq_cap)
+ new_nbits <<= 1;
+ rc = bmap_grow(m, new_nbits);
+ spin_unlock(&m->lock);
+
+ return rc;
+}
+
+int sbi_virq_hwirq2virq(u32 channel_id, u32 chip_uid, u32 hwirq,
+ u32 *out_virq)
+{
+ u32 idx;
+ int rc;
+ struct sbi_virq_map *m;
+
+ m = virq_map_get(channel_id, false, 0);
+ if (!m)
+ return SBI_ENOENT;
+
+ spin_lock(&m->lock);
+ rc = forward_find_idx(m, chip_uid, hwirq, &idx);
+ if (!rc)
+ *out_virq = m->nodes[idx].virq;
+ spin_unlock(&m->lock);
+
+ return rc;
+}
+
+int sbi_virq_virq2hwirq(u32 channel_id, u32 virq,
+ u32 *out_chip_uid, u32 *out_hwirq)
+{
+ struct virq_entry *e;
+ struct sbi_virq_map *m;
+
+ m = virq_map_get(channel_id, false, 0);
+ if (!m)
+ return SBI_EINVAL;
+
+ spin_lock(&m->lock);
+
+ if (virq >= m->bmap_nbits ||
+ !bitmap_test(m->bmap, (int)virq)) {
+ spin_unlock(&m->lock);
+ return SBI_EINVAL;
+ }
+
+ e = rev_get_existing(m, virq);
+ if (!e) {
+ spin_unlock(&m->lock);
+ return SBI_EINVAL;
+ }
+
+ *out_chip_uid = e->chip_uid;
+ *out_hwirq = e->hwirq;
+
+ spin_unlock(&m->lock);
+
+ return SBI_OK;
+}
+
+int sbi_virq_unmap_one(u32 virq)
+{
+ struct virq_entry *e;
+ u32 idx, last;
+ int rc;
+ struct sbi_virq_map *m = &g_virq_map;
+
+ spin_lock(&m->lock);
+
+ if (virq >= m->bmap_nbits ||
+ !bitmap_test(m->bmap, (int)virq)) {
+ spin_unlock(&m->lock);
+ return SBI_EINVAL;
+ }
+
+ e = rev_get_existing(m, virq);
+ if (!e) {
+ spin_unlock(&m->lock);
+ return SBI_EINVAL;
+ }
+
+ /* find forward node corresponding to this virq (linear) */
+ rc = SBI_ENOENT;
+ for (idx = 0; idx < m->nodes_cnt; idx++) {
+ if (m->nodes[idx].virq == virq) {
+ /* optionally also check endpoint matches e */
+ rc = 0;
+ break;
+ }
+ }
+ if (rc) {
+ /* inconsistent state */
+ spin_unlock(&m->lock);
+ return SBI_EINVAL;
+ }
+
+ /* remove node: swap with last */
+ last = m->nodes_cnt - 1U;
+ if (idx != last)
+ m->nodes[idx] = m->nodes[last];
+ m->nodes_cnt--;
+
+ /* clear reverse + free virq id */
+ rev_clear(m, virq);
+ bmap_free_one(m, virq);
+
+ spin_unlock(&m->lock);
+
+ return SBI_OK;
+}
+
+static void virq_map_uninit_one(struct sbi_virq_map *m)
+{
+ u32 i;
+
+ spin_lock(&m->lock);
+
+ /* free reverse chunks */
+ if (m->chunks) {
+ for (i = 0; i < m->chunks_cap; i++) {
+ if (m->chunks[i])
+ sbi_free(m->chunks[i]);
+ }
+ sbi_free(m->chunks);
+ m->chunks = NULL;
+ m->chunks_cap = 0;
+ }
+
+ /* free forward vector */
+ if (m->nodes) {
+ sbi_free(m->nodes);
+ m->nodes = NULL;
+ m->nodes_cnt = 0;
+ m->nodes_cap = 0;
+ }
+
+ /* free bitmap */
+ if (m->bmap) {
+ sbi_free(m->bmap);
+ m->bmap = NULL;
+ m->bmap_nbits = 0;
+ }
+
+ spin_unlock(&m->lock);
+}
+
+void sbi_virq_map_uninit(void)
+{
+ u32 i;
+
+ virq_map_uninit_one(&g_virq_map);
+
+ spin_lock(&g_virq_maps_lock);
+ for (i = 0; i < g_virq_maps_cnt; i++)
+ virq_map_uninit_one(&g_virq_maps[i].map);
+ if (g_virq_maps) {
+ sbi_free(g_virq_maps);
+ g_virq_maps = NULL;
+ g_virq_maps_cnt = 0;
+ g_virq_maps_cap = 0;
+ }
+ spin_unlock(&g_virq_maps_lock);
+}
+
+int sbi_virq_enqueue(struct sbi_virq_courier_binding *c)
+{
+ struct sbi_domain_virq_state *st;
+
+ if (!c->dom || c->virq == VIRQ_INVALID)
+ return SBI_EINVAL;
+
+ st = domain_virq_thishart(c->dom);
+ if (!st)
+ return SBI_ENODEV;
+
+ spin_lock(&st->lock);
+ if (q_full(st)) {
+ spin_unlock(&st->lock);
+ return SBI_ENOSPC;
+ }
+
+ st->q[st->tail].virq = c->virq;
+ st->q[st->tail].channel_id = c->channel_id;
+ st->q[st->tail].chip = c->chip;
+ st->tail = (st->tail + 1) % VIRQ_QSIZE;
+ spin_unlock(&st->lock);
+
+ return SBI_OK;
+}
+
+u32 sbi_virq_pop_thishart(void)
+{
+ struct sbi_domain *dom = sbi_domain_thishart_ptr();
+ struct sbi_domain_virq_state *st;
+ u32 virq = VIRQ_INVALID;
+
+ if (!dom)
+ return VIRQ_INVALID;
+
+ st = domain_virq_thishart(dom);
+ if (!st)
+ return VIRQ_INVALID;
+
+ spin_lock(&st->lock);
+ if (!q_empty(st)) {
+ virq = st->q[st->head].virq;
+ st->last_pop_virq = virq;
+ st->last_pop_channel_id = st->q[st->head].channel_id;
+ st->last_pop_chip = st->q[st->head].chip;
+ st->head = (st->head + 1) % VIRQ_QSIZE;
+ } else
+ virq = VIRQ_INVALID;
+ spin_unlock(&st->lock);
+
+ if (virq == VIRQ_INVALID) {
+ if (sbi_irqchip_notify_smode_get())
+ sbi_irqchip_notify_smode_clear();
+ }
+
+ return virq;
+}
+
+void sbi_virq_complete_thishart(u32 virq)
+{
+ struct sbi_domain *dom = sbi_domain_thishart_ptr();
+ struct sbi_domain_virq_state *st;
+ u32 hwirq;
+ u32 chip_uid;
+ u32 channel_id;
+ struct sbi_irqchip_device *chip;
+ bool drained = false;
+
+ if (virq == VIRQ_INVALID)
+ return;
+
+ if (!dom)
+ return;
+
+ st = domain_virq_thishart(dom);
+ if (!st)
+ return;
+
+ spin_lock(&st->lock);
+ channel_id = st->last_pop_channel_id;
+ chip = st->last_pop_chip;
+ if (st->last_pop_virq == virq) {
+ st->last_pop_virq = 0;
+ st->last_pop_channel_id = 0;
+ st->last_pop_chip = NULL;
+ }
+ drained = q_empty(st);
+ spin_unlock(&st->lock);
+
+ if (!chip)
+ return;
+
+ sbi_virq_virq2hwirq(channel_id, virq, &chip_uid, &hwirq);
+ (void)chip_uid;
+ if (chip->hwirq_eoi)
+ chip->hwirq_eoi(chip, hwirq);
+ sbi_irqchip_unmask_hwirq(chip, hwirq);
+
+ if (drained) {
+ if (sbi_irqchip_notify_smode_get())
+ sbi_irqchip_notify_smode_clear();
+ sbi_virq_return_to_prev_if_needed();
+ }
+}
+
+void sbi_virq_return_to_prev_if_needed(void)
+{
+ struct sbi_domain *dom = sbi_domain_thishart_ptr();
+ struct sbi_domain_virq_state *st;
+ bool do_return = false;
+
+ if (!dom)
+ return;
+
+ st = domain_virq_thishart(dom);
+ if (!st)
+ return;
+
+ spin_lock(&st->lock);
+ if (st->return_to_prev && q_empty(st)) {
+ st->return_to_prev = false;
+ do_return = true;
+ }
+ spin_unlock(&st->lock);
+
+ if (!do_return)
+ return;
+ sbi_domain_context_request_return_to_prev();
+}
+
+int sbi_virq_courier_handler(u32 hwirq, void *opaque)
+{
+ struct sbi_virq_courier_ctx *ctx =
+ (struct sbi_virq_courier_ctx *)opaque;
+ struct sbi_domain *dom;
+ struct sbi_virq_courier_binding courier;
+ u32 channel_id = 0;
+ u32 virq = 0;
+ int rc;
+ struct sbi_domain *curr_dom;
+
+ if (!ctx || !ctx->chip)
+ return SBI_EINVAL;
+
+ /* Route purely by HWIRQ -> Domain/channel rules (from FDT). */
+ rc = sbi_virq_route_lookup(hwirq, &dom, &channel_id);
+ if (rc || !dom)
+ return SBI_EINVAL;
+
+ curr_dom = sbi_domain_thishart_ptr();
+
+ /* Allocate/Get a stable VIRQ for (chip_uid, hwirq). */
+ rc = sbi_virq_map_one(channel_id, ctx->chip->id, hwirq,
+ false, 0, &virq);
+ if (rc)
+ return rc;
+
+ /*
+ * Mask to avoid level-trigger storm before S-mode clears device source.
+ * S-mode will call sbi_virq_complete_thishart(virq) to unmask.
+ */
+ sbi_irqchip_mask_hwirq(ctx->chip, hwirq);
+
+ courier.dom = dom;
+ courier.chip = ctx->chip;
+ courier.channel_id = channel_id;
+ courier.virq = virq;
+
+ rc = sbi_virq_enqueue(&courier);
+ if (rc) {
+ /* enqueue failed; re-enable to avoid deadlock */
+ sbi_irqchip_unmask_hwirq(ctx->chip, hwirq);
+ return rc;
+ }
+
+ /*
+ * Notify S-mode on notification rising edge.
+ *
+ * If the target is the current domain, operate on the live CSR.
+ * Otherwise, set the pending bit in the target domain context
+ * before switching (covers first-entry). After switching, set the
+ * live CSR only if needed (covers already-initialized targets).
+ */
+ if (dom != curr_dom) {
+ (void)sbi_domain_context_pending_notify_smode(
+ dom, current_hartindex());
+
+ /* Mark return_to_prev for VIRQ-driven domain switch. */
+ virq_set_domain_return_flag(dom, true);
+ rc = sbi_domain_context_enter(dom);
+ if (rc) {
+ /* Switch failed; do not defer EOI */
+ sbi_irqchip_unmask_hwirq(ctx->chip, hwirq);
+ if (ctx->chip->hwirq_eoi)
+ ctx->chip->hwirq_eoi(ctx->chip, hwirq);
+ return SBI_OK;
+ }
+
+ /*
+ * If the domain was already initialized,
+ * sbi_domain_context_enter() returns and CSR_SIP reflect
+ * dom_ctx->sip. For robustness, set the live notify bit if it
+ * is still clear.
+ */
+ if (!sbi_irqchip_notify_smode_get()) {
+ rc = sbi_irqchip_notify_smode_set();
+ if (rc) {
+ /*
+ * notification failed; re-enable to avoid
+ * deadlock
+ */
+ sbi_irqchip_unmask_hwirq(ctx->chip, hwirq);
+ return rc;
+ }
+ }
+ } else if (!sbi_irqchip_notify_smode_get()) {
+ rc = sbi_irqchip_notify_smode_set();
+ if (rc) {
+ /* notification failed; re-enable to avoid deadlock */
+ sbi_irqchip_unmask_hwirq(ctx->chip, hwirq);
+ return rc;
+ }
+ }
+
+ /*
+ * Return SBI_EALREADY to defer EOI until VIRQ COMPLETE so S-mode
+ * notification can be delivered to the target domain.
+ */
+ return SBI_EALREADY;
+}
+
+int sbi_virq_domain_init(struct sbi_domain *dom)
+{
+ struct sbi_domain_virq_priv *p;
+ u32 i, k, nharts, st_count;
+ struct sbi_domain_virq_state *st_base;
+ size_t alloc_size;
+
+ if (!dom)
+ return SBI_EINVAL;
+
+ if (dom->virq_priv)
+ return SBI_OK;
+
+ nharts = sbi_virq_platform_hart_count();
+ st_count = dom->possible_harts ?
+ (u32)sbi_hartmask_weight(dom->possible_harts) : nharts;
+
+ alloc_size = sizeof(*p) +
+ nharts * sizeof(p->st_by_hart[0]) +
+ st_count * sizeof(struct sbi_domain_virq_state);
+ p = sbi_zalloc(alloc_size);
+ if (!p)
+ return SBI_ENOMEM;
+
+ p->nharts = nharts;
+ p->st_count = st_count;
+ st_base = (struct sbi_domain_virq_state *)(p->st_by_hart + nharts);
+
+ if (!dom->possible_harts) {
+ for (i = 0; i < nharts; i++) {
+ p->st_by_hart[i] = &st_base[i];
+ virq_state_init(p->st_by_hart[i]);
+ }
+ } else {
+ for (i = 0; i < nharts; i++)
+ p->st_by_hart[i] = NULL;
+ k = 0;
+ sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
+ if (k >= st_count)
+ break;
+ p->st_by_hart[i] = &st_base[k++];
+ virq_state_init(p->st_by_hart[i]);
+ }
+ }
+ dom->virq_priv = p;
+
+ return SBI_OK;
+}
+
+void sbi_virq_domain_exit(struct sbi_domain *dom)
+{
+ if (!dom || !dom->virq_priv)
+ return;
+
+ sbi_free(dom->virq_priv);
+ dom->virq_priv = NULL;
+}
+
+int sbi_virq_init(u32 init_virq_cap)
+{
+ int rc = SBI_OK;
+
+ if (g_virq_inited)
+ return SBI_EALREADY;
+
+ rc = sbi_virq_map_init(0, init_virq_cap);
+ if (rc)
+ return rc;
+
+ SPIN_LOCK_INIT(g_virq_maps_lock);
+ SPIN_LOCK_INIT(g_router.lock);
+ sbi_virq_route_reset();
+ g_virq_inited = true;
+ return rc;
+}
+
+bool sbi_virq_is_inited(void)
+{
+ return g_virq_inited;
+}
--
2.25.1
More information about the opensbi
mailing list