[PATCH 3/3] PCI: endpoint: pci-epf-vntb: Honor MSI-X selection
Frank Li
Frank.li at oss.nxp.com
Thu Aug 27 12:23:49 PDT 2026
On Fri, Aug 28, 2026 at 03:20:12AM +0900, Koichiro Den wrote:
> ntb_hw_epf tries MSI-X first and falls back to MSI. It reports the
> result in COMMAND_CONFIGURE_DOORBELL. pci-epf-vntb ignores MSIX_ENABLE,
> configures only MSI, and always raises peer doorbells with PCI_IRQ_MSI.
> When MSI-X is selected, the host does not program MSI, so raising it can
> issue a write to an invalid address. This was observed with an IOMMU
> enabled on the RC.
>
> Configure MSI-X when supported and use the selected type for peer
> doorbells. Use the hardware-owned layout when available. Otherwise
> allocate an EPF-owned Table and PBA in the config BAR. Configure db_count
> entries to cover the link event, the reserved slot, and the doorbell
> slots.
>
> Fixes: e35f56bb0330 ("PCI: endpoint: Support NTB transfer between RC and EP")
> Signed-off-by: Koichiro Den <den at valinux.co.jp>
> ---
Reviewed-by: Frank Li <Frank.Li at nxp.com>
> drivers/pci/endpoint/functions/pci-epf-vntb.c | 67 ++++++++++++++-----
> 1 file changed, 51 insertions(+), 16 deletions(-)
>
> diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
> index c3caec927d74..d9622a5d4710 100644
> --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
> +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
> @@ -148,6 +148,7 @@ struct epf_ntb {
> u16 vntb_vid;
>
> bool linkup;
> + bool peer_msix;
>
> /*
> * True when doorbells are interrupt-driven (MSI or embedded), false
> @@ -155,6 +156,7 @@ struct epf_ntb {
> */
> bool msi_doorbell;
> u32 spad_size;
> + struct pci_epc_msix_layout msix_layout;
>
> enum pci_barno epf_ntb_bar[VNTB_BAR_NUM];
>
> @@ -303,6 +305,7 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
>
> switch (command) {
> case COMMAND_CONFIGURE_DOORBELL:
> + WRITE_ONCE(ntb->peer_msix, argument & MSIX_ENABLE);
> ctrl->command_status = COMMAND_STATUS_OK;
> break;
> case COMMAND_TEARDOWN_DOORBELL:
> @@ -439,9 +442,9 @@ static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
> * region
> * @ntb: NTB device that facilitates communication between HOST and VHOST
> *
> - * Allocate the Local Memory mentioned in the above diagram. The size of
> - * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
> - * is obtained from "spad-count" configfs entry.
> + * Allocate the control and scratchpad regions described in the above diagram.
> + * If the EPC does not provide a hardware-owned MSI-X table and PBA, allocate
> + * space for them between the control and scratchpad regions.
> *
> * Returns: Zero for success, or an error code in case of failure
> */
> @@ -454,7 +457,7 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
> struct device *dev = &epf->dev;
> u32 spad_count;
> void *base;
> - int i;
> + int i, ret;
> const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc,
> epf->func_no,
> epf->vfunc_no);
> @@ -462,6 +465,29 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
> spad_count = ntb->spad_count;
>
> ctrl_size = ALIGN(sizeof(struct epf_ntb_ctrl), sizeof(u32));
> + if (epc_features->msix_capable) {
> + ret = pci_epc_get_hw_msix_layout(epc_features,
> + &ntb->msix_layout);
> + if (ret && ret != -ENOENT) {
> + dev_err(dev, "Invalid hardware-owned MSI-X layout\n");
> + return ret;
> + }
> +
> + if (ret == -ENOENT) {
> + ntb->msix_layout.table_bar = barno;
> + ntb->msix_layout.table_offset = ALIGN(ctrl_size, 8);
> + ntb->msix_layout.table_size =
> + ntb->db_count * PCI_MSIX_ENTRY_SIZE;
> + ntb->msix_layout.pba_bar = barno;
> + ntb->msix_layout.pba_offset =
> + ntb->msix_layout.table_offset +
> + ntb->msix_layout.table_size;
> + ntb->msix_layout.pba_size =
> + BITS_TO_U64(ntb->db_count) * sizeof(u64);
> + ctrl_size = ntb->msix_layout.pba_offset +
> + ntb->msix_layout.pba_size;
> + }
> + }
> spad_size = 2 * spad_count * sizeof(u32);
>
> base = pci_epf_alloc_space(epf, ctrl_size + spad_size,
> @@ -502,6 +528,7 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
> static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
> {
> const struct pci_epc_features *epc_features;
> + struct pci_epf *epf = ntb->epf;
> struct device *dev;
> int ret;
>
> @@ -521,16 +548,22 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
> }
>
> if (epc_features->msi_capable) {
> - ret = pci_epc_set_msi(ntb->epf->epc,
> - ntb->epf->func_no,
> - ntb->epf->vfunc_no,
> - 16);
> + ret = pci_epc_set_msi(epf->epc, epf->func_no, epf->vfunc_no, 16);
> if (ret) {
> dev_err(dev, "MSI configuration failed\n");
> return ret;
> }
> }
>
> + if (epc_features->msix_capable) {
> + ret = pci_epc_set_msix(epf->epc, epf->func_no, epf->vfunc_no,
> + ntb->db_count, &ntb->msix_layout);
> + if (ret) {
> + dev_err(dev, "MSI-X configuration failed\n");
> + return ret;
> + }
> + }
> +
> return 0;
> }
>
> @@ -1512,6 +1545,7 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
> struct epf_ntb *ntb = container_of(work, struct epf_ntb, peer_db_work);
> struct pci_epf *epf = ntb->epf;
> unsigned int budget = VNTB_PEER_DB_WORK_BUDGET;
> + unsigned int irq_type;
> u8 func_no, vfunc_no;
> unsigned int db_bit;
> u32 interrupt_num;
> @@ -1523,6 +1557,7 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
>
> func_no = epf->func_no;
> vfunc_no = epf->vfunc_no;
> + irq_type = READ_ONCE(ntb->peer_msix) ? PCI_IRQ_MSIX : PCI_IRQ_MSI;
>
> /*
> * Drain doorbells from peer_db_pending in snapshots (atomic64_xchg()).
> @@ -1536,16 +1571,16 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
>
> while (db_bits) {
> /*
> - * pci_epc_raise_irq() for MSI expects a 1-based
> - * interrupt number. The first usable doorbell starts
> - * at EPF_IRQ_DB_START in the legacy slot layout.
> + * pci_epc_raise_irq() expects a 1-based interrupt
> + * number for MSI and MSI-X. The first usable doorbell
> + * starts at EPF_IRQ_DB_START in the legacy slot layout.
> *
> * Legacy mapping (kept for compatibility):
> *
> - * MSI #1 : link event (reserved)
> - * MSI #2 : unused (historical offset)
> - * MSI #3 : doorbell bit 0 (DB#0)
> - * MSI #4 : doorbell bit 1 (DB#1)
> + * IRQ #1 : link event (reserved)
> + * IRQ #2 : unused (historical offset)
> + * IRQ #3 : doorbell bit 0 (DB#0)
> + * IRQ #4 : doorbell bit 1 (DB#1)
> * ...
> *
> * Do not change this mapping to avoid breaking
> @@ -1556,7 +1591,7 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
> db_bits &= ~BIT_ULL(db_bit);
>
> ret = pci_epc_raise_irq(epf->epc, func_no, vfunc_no,
> - PCI_IRQ_MSI, interrupt_num);
> + irq_type, interrupt_num);
> if (ret)
> dev_err(&ntb->ntb.dev,
> "Failed to raise IRQ for interrupt_num %u: %d\n",
> --
> 2.51.0
>
More information about the linux-arm-kernel
mailing list