[RESEND RFC PATCH v2] nvme-pci: add adaptive interrupt polling
changfengnan
changfengnan at bytedance.com
Mon Aug 10 19:32:35 PDT 2026
> From: "Keith Busch"<kbusch at kernel.org>
> Date: Tue, Aug 11, 2026, 04:55
> Subject: Re: [RESEND RFC PATCH v2] nvme-pci: add adaptive interrupt polling
> To: "Fengnan Chang"<changfengnan at bytedance.com>
> Cc: <axboe at kernel.dk>, <hch at lst.de>, <sagi at grimberg.me>, <linux-nvme at lists.infradead.org>, <linux-kernel at vger.kernel.org>, "Guzebing"<guzebing at bytedance.com>
> On Thu, Aug 06, 2026 at 11:10:58AM +0800, Fengnan Chang wrote:
> > The idea behind this approach is: Let each I/O queue switch itself between
> > interrupt and poll mode based on its own recent completion rate.
> >
> > This version is still in the testing phase, and there are still some issues
> > with the code implementation. I releasing it now to see if the approach
> > is generally acceptable. If the approach looks good, I´ll continue to
> > refine it and conduct more extensive testing. The main implementation
> > logic is in `nvme_adaptive_sample` and `nvme_adaptive_irq_poll`; you should
> > focus on reviewing the implementation of these two functions.
>
> Can we subscribe to the dynamic interrupt moderation (dim) library? I
> know it's generally used in conjuction with a hardware interrupt
> coalescing feature, but we can just do pure software with it too. The
> library provides a hill climb to adapt the policy at run time.
>
> This is a quick PoC I put together. I haven't tested on fast devices, so
> I'm not sure if I've dialed in the profiles, but it's start of what I
> had in mind.
The code looks much cleaner when using dim, I'll see if I can replace the
sample-and-poll logic I wrote myself with the dim library.
I ran a quick test on the POC patch and didn't see any performance
improvements; in fact, there were quite a few regressions. Maybe some
parameters need to be adjusted.
>
> ---
> diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
> index 31974c7dd20c9..9fad1c7b1f678 100644
> --- a/drivers/nvme/host/Kconfig
> +++ b/drivers/nvme/host/Kconfig
> @@ -6,6 +6,7 @@ config BLK_DEV_NVME
> tristate "NVM Express block device"
> depends on PCI && BLOCK
> select NVME_CORE
> + select DIMLIB
> help
> The NVM Express driver is for solid state drives directly
> connected to the PCI or PCI Express bus. If you know you
> diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
> index 8438c904ec496..ff0c8d74fae08 100644
> --- a/drivers/nvme/host/pci.c
> +++ b/drivers/nvme/host/pci.c
> @@ -9,6 +9,8 @@
> #include <linux/blkdev.h>
> #include <linux/blk-mq-dma.h>
> #include <linux/blk-integrity.h>
> +#include <linux/delay.h>
> +#include <linux/dim.h>
> #include <linux/dmi.h>
> #include <linux/init.h>
> #include <linux/interrupt.h>
> @@ -82,6 +84,45 @@ struct quirk_entry {
> static int use_threaded_interrupts;
> module_param(use_threaded_interrupts, int, 0444);
>
> +static unsigned int irq_poll_thresh = 7;
> +module_param(irq_poll_thresh, uint, 0644);
> +MODULE_PARM_DESC(irq_poll_thresh,
> + "outstanding depth after which a queue switches to threaded polling");
> +
> +static unsigned int irq_poll_spin = 7;
> +module_param(irq_poll_spin, uint, 0644);
> +MODULE_PARM_DESC(irq_poll_spin,
> + "poll iterations to spin (cpu_relax) before sleeping in the poll loop (zero based)");
> +
> +static unsigned int irq_poll_sleep_us = 20;
> +static unsigned int irq_poll_idle_us = 80;
> +static unsigned int irq_poll_idle_rounds = DIV_ROUND_UP(80, 20);
> +
> +static int irq_poll_us_set(const char *val, const struct kernel_param *kp)
> +{
> + int ret = param_set_uint(val, kp);
> +
> + if (ret)
> + return ret;
> +
> + irq_poll_idle_rounds = irq_poll_sleep_us ?
> + DIV_ROUND_UP(irq_poll_idle_us, irq_poll_sleep_us) : 1;
> + return 0;
> +}
> +
> +static const struct kernel_param_ops irq_poll_us_ops = {
> + .set = irq_poll_us_set,
> + .get = param_get_uint,
> +};
> +
> +module_param_cb(irq_poll_sleep_us, &irq_poll_us_ops, &irq_poll_sleep_us, 0644);
> +MODULE_PARM_DESC(irq_poll_sleep_us,
> + "microseconds to sleep between poll bursts (<=10 busy-delays, does not yield)");
> +
> +module_param_cb(irq_poll_idle_us, &irq_poll_us_ops, &irq_poll_idle_us, 0644);
> +MODULE_PARM_DESC(irq_poll_idle_us,
> + "microseconds to wait for stragglers before handing a queue back to the IRQ path");
> +
> static bool use_cmb_sqes = true;
> module_param(use_cmb_sqes, bool, 0444);
> MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
> @@ -381,11 +422,21 @@ struct nvme_queue {
> u16 qid;
> u8 cq_phase;
> u8 sqes;
> + /* Adaptive polling knobs, seeded from the irq_poll_* module params. */
> + unsigned int poll_thresh;
> + unsigned int poll_sleep_us;
> + unsigned int poll_idle_rounds;
> + unsigned int poll_budget;
> + /* Adaptive interrupt moderation (DIM) sampling state. */
> + struct dim dim;
> + u16 dim_events; /* cumulative interrupts (BIT_GAP-safe) */
> + u32 dim_comps; /* cumulative completions */
> unsigned long flags;
> #define NVMEQ_ENABLED 0
> #define NVMEQ_SQ_CMB 1
> #define NVMEQ_DELETE_ERROR 2
> #define NVMEQ_POLLED 3
> +#define NVMEQ_POLLING 4
> __le32 *dbbuf_sq_db;
> __le32 *dbbuf_cq_db;
> __le32 *dbbuf_sq_ei;
> @@ -1606,13 +1657,13 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
> }
> }
>
> -static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
> - struct io_comp_batch *iob)
> +static inline int __nvme_poll_cq(struct nvme_queue *nvmeq,
> + struct io_comp_batch *iob, int budget)
> {
> - bool found = false;
> + int found = 0;
>
> while (nvme_cqe_pending(nvmeq)) {
> - found = true;
> + found++;
> /*
> * load-load control dependency between phase and the rest of
> * the cqe requires a full read memory barrier
> @@ -1620,6 +1671,8 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
> dma_rmb();
> nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
> nvme_update_cq_head(nvmeq);
> + if (budget && found == budget)
> + break;
> }
>
> if (found)
> @@ -1627,26 +1680,253 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
> return found;
> }
>
> +/* budget == 0 means reap the whole queue. */
> +static int nvme_poll_cq(struct nvme_queue *nvmeq, int budget)
> +{
> + DEFINE_IO_COMP_BATCH(iob);
> + int found = __nvme_poll_cq(nvmeq, &iob, budget);
> +
> + if (found && !rq_list_empty(&iob.req_list))
> + nvme_pci_complete_batch(&iob);
> + return found;
> +}
> +
> +static inline unsigned int nvmeq_outstanding(struct nvme_queue *nvmeq)
> +{
> + u16 sq_tail = READ_ONCE(nvmeq->last_sq_tail);
> + u16 cq_head = nvmeq->cq_head;
> +
> + if (sq_tail >= cq_head)
> + return sq_tail - cq_head;
> + return nvmeq->q_depth - cq_head + sq_tail;
> +}
> +
> +static inline bool nvmeq_cq_continue(struct nvme_queue *nvmeq)
> +{
> + return nvmeq_outstanding(nvmeq) > nvmeq->poll_thresh;
> +}
> +
> +static inline bool nvmeq_cq_idle(struct nvme_queue *nvmeq)
> +{
> + return nvmeq->cq_head == READ_ONCE(nvmeq->last_sq_tail);
> +}
> +
> +/*
> + * Interrupt-moderation profiles, ordered from least moderation (index 0: enter
> + * polling late, short sleep, brief linger -> lowest latency, most interrupts)
> + * to most moderation (enter early, long sleep, long linger -> fewest
> + * interrupts). The hill-climb walks this table to keep completions-per-
> + * interrupt high, which is what bounds the interrupt rate.
> + */
> +static const struct nvme_poll_prof {
> + u16 thresh;
> + u16 sleep_us;
> + u16 idle_rounds;
> + u16 budget; /* hard-IRQ initial poll cap; 0 == reap all */
> +} nvme_poll_prof[] = {
> + { 31, 0, 1, 0 }, /* drain inline: lowest latency, most interrupts */
> + { 15, 10, 2, 16 },
> + { 7, 20, 4, 8 }, /* start profile; matches irq_poll_* defaults */
> + { 3, 40, 8, 4 },
> + { 1, 80, 16, 2 }, /* reap a little, hand the bulk to the thread */
> +};
> +
> +#define NVME_DIM_START_PROFILE 2
> +
> +static void nvme_apply_profile(struct nvme_queue *nvmeq)
> +{
> + const struct nvme_poll_prof *p = &nvme_poll_prof[nvmeq->dim.profile_ix];
> +
> + nvmeq->poll_thresh = p->thresh;
> + nvmeq->poll_sleep_us = p->sleep_us;
> + nvmeq->poll_idle_rounds = p->idle_rounds;
> + nvmeq->poll_budget = p->budget;
> +}
> +
> +/* Mirror of rdma_dim_step() bounded to our profile table. */
> +static int nvme_dim_step(struct dim *dim)
> +{
> + if (dim->tune_state == DIM_GOING_RIGHT) {
> + if (dim->profile_ix == ARRAY_SIZE(nvme_poll_prof) - 1)
> + return DIM_ON_EDGE;
> + dim->profile_ix++;
> + dim->steps_right++;
> + }
> + if (dim->tune_state == DIM_GOING_LEFT) {
> + if (dim->profile_ix == 0)
> + return DIM_ON_EDGE;
> + dim->profile_ix--;
> + dim->steps_left++;
> + }
> + return DIM_STEPPED;
> +}
> +
> +/* Mirror of rdma_dim_stats_compare(): completion rate first, then batching. */
> +static int nvme_dim_stats_compare(struct dim_stats *curr, struct dim_stats *prev)
> +{
> + if (!prev->cpms)
> + return DIM_STATS_SAME;
> +
> + if (IS_SIGNIFICANT_DIFF(curr->cpms, prev->cpms))
> + return curr->cpms > prev->cpms ? DIM_STATS_BETTER :
> + DIM_STATS_WORSE;
> +
> + if (IS_SIGNIFICANT_DIFF(curr->cpe_ratio, prev->cpe_ratio))
> + return curr->cpe_ratio > prev->cpe_ratio ? DIM_STATS_BETTER :
> + DIM_STATS_WORSE;
> +
> + return DIM_STATS_SAME;
> +}
> +
> +/* Mirror of rdma_dim_decision(); returns true if the profile changed. */
> +static bool nvme_dim_decision(struct dim_stats *curr, struct dim *dim)
> +{
> + int prev_ix = dim->profile_ix;
> + int stats_res;
> +
> + stats_res = nvme_dim_stats_compare(curr, &dim->prev_stats);
> + switch (stats_res) {
> + case DIM_STATS_SAME:
> + if (curr->cpe_ratio <= 50 * prev_ix)
> + dim->profile_ix = 0;
> + break;
> + case DIM_STATS_WORSE:
> + dim_turn(dim);
> + fallthrough;
> + case DIM_STATS_BETTER:
> + if (nvme_dim_step(dim) == DIM_ON_EDGE)
> + dim_turn(dim);
> + break;
> + }
> +
> + dim->prev_stats = *curr;
> + return dim->profile_ix != prev_ix;
> +}
> +
> +/*
> + * Sample the completion/interrupt counters and, once per DIM_NEVENTS
> + * interrupts, compute the load stats. Called only from the interrupt owner
> + * (never concurrently with the poll thread), so the counters have a single
> + * writer. ktime_get() is taken only at a window boundary, not per interrupt.
> + *
> + * On each completed window the hill-climb picks a profile and, if it changed,
> + * applies it to nvmeq->poll_*.
> + */
> +static void nvme_dim(struct nvme_queue *nvmeq)
> +{
> + struct dim *dim = &nvmeq->dim;
> + struct dim_sample end;
> + struct dim_stats stats;
> +
> + if (dim->state == DIM_START_MEASURE) {
> + dim_update_sample_with_comps(nvmeq->dim_events, 0, 0,
> + nvmeq->dim_comps, &dim->start_sample);
> + dim->state = DIM_MEASURE_IN_PROGRESS;
> + return;
> + }
> +
> + /* Cheap gate: only recompute once a full window of events accrues. */
> + if ((u16)(nvmeq->dim_events - dim->start_sample.event_ctr) < DIM_NEVENTS)
> + return;
> +
> + dim_update_sample_with_comps(nvmeq->dim_events, 0, 0, nvmeq->dim_comps,
> + &end);
> + if (dim_calc_stats(&dim->start_sample, &end, &stats)) {
> + if (nvme_dim_decision(&stats, dim))
> + nvme_apply_profile(nvmeq);
> + trace_nvme_dim(nvmeq->qid, stats.cpms, stats.epms,
> + stats.cpe_ratio, dim->profile_ix);
> + }
> + dim->start_sample = end;
> +}
> +
> static irqreturn_t nvme_irq(int irq, void *data)
> {
> struct nvme_queue *nvmeq = data;
> - DEFINE_IO_COMP_BATCH(iob);
> + struct nvme_dev *dev = nvmeq->dev;
> + struct pci_dev *pdev = to_pci_dev(dev->dev);
> + irqreturn_t ret = IRQ_NONE;
> + unsigned int idle = 0;
>
> - if (nvme_poll_cq(nvmeq, &iob)) {
> - if (!rq_list_empty(&iob.req_list))
> - nvme_pci_complete_batch(&iob);
> - return IRQ_HANDLED;
> + for (;;) {
> + bool worked = false;
> + unsigned int i;
> +
> + for (i = 0; i <= irq_poll_spin; i++) {
> + int n = nvme_poll_cq(nvmeq, 0);
> +
> + if (n) {
> + nvmeq->dim_comps += n;
> + ret = IRQ_HANDLED;
> + worked = true;
> + idle = 0;
> + }
> +
> + if (nvmeq_cq_idle(nvmeq))
> + goto done;
> + else if (need_resched())
> + cond_resched();
> + else
> + cpu_relax();
> + }
> +
> + if (worked || ++idle < nvmeq->poll_idle_rounds) {
> + fsleep(nvmeq->poll_sleep_us);
> + continue;
> + }
> +done:
> + clear_bit(NVMEQ_POLLING, &nvmeq->flags);
> + if (!nvme_cqe_pending(nvmeq) ||
> + test_and_set_bit(NVMEQ_POLLING, &nvmeq->flags))
> + break;
> +
> + if (need_resched())
> + cond_resched();
> + else
> + cpu_relax();
> + idle = 0;
> }
> - return IRQ_NONE;
> +
> + if (pdev->msi_enabled)
> + writel(BIT(nvmeq->cq_vector), dev->bar + NVME_REG_INTMC);
> + return ret;
> }
>
> static irqreturn_t nvme_irq_check(int irq, void *data)
> {
> struct nvme_queue *nvmeq = data;
> + int found;
> +
> + nvmeq->dim_events++;
>
> - if (nvme_cqe_pending(nvmeq))
> + if (test_and_set_bit(NVMEQ_POLLING, &nvmeq->flags))
> + return IRQ_HANDLED;
> +
> + found = nvme_poll_cq(nvmeq, nvmeq->poll_budget);
> + nvmeq->dim_comps += found;
> + nvme_dim(nvmeq);
> +
> + if (!found) {
> + clear_bit(NVMEQ_POLLING, &nvmeq->flags);
> + return IRQ_NONE;
> + }
> +
> + /*
> + * Hand off to the poll thread when the queue is still deep, or when the
> + * budgeted initial poll left CQEs behind: their interrupt is already
> + * spent, so releasing here would strand them.
> + */
> + if (nvme_cqe_pending(nvmeq) || nvmeq_cq_continue(nvmeq)) {
> + struct nvme_dev *dev = nvmeq->dev;
> + struct pci_dev *pdev = to_pci_dev(dev->dev);
> +
> + if (pdev->msi_enabled)
> + writel(BIT(nvmeq->cq_vector), dev->bar + NVME_REG_INTMS);
> return IRQ_WAKE_THREAD;
> - return IRQ_NONE;
> + }
> +
> + clear_bit(NVMEQ_POLLING, &nvmeq->flags);
> + return IRQ_HANDLED;
> }
>
> /*
> @@ -1663,7 +1943,7 @@ static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
> irq = pci_irq_vector(pdev, nvmeq->cq_vector);
> disable_irq(irq);
> spin_lock(&nvmeq->cq_poll_lock);
> - nvme_poll_cq(nvmeq, NULL);
> + __nvme_poll_cq(nvmeq, NULL, 0);
> spin_unlock(&nvmeq->cq_poll_lock);
> enable_irq(irq);
> }
> @@ -1671,14 +1951,14 @@ static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
> static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
> {
> struct nvme_queue *nvmeq = hctx->driver_data;
> - bool found;
> + int found;
>
> if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) ||
> !nvme_cqe_pending(nvmeq))
> return 0;
>
> spin_lock(&nvmeq->cq_poll_lock);
> - found = nvme_poll_cq(nvmeq, iob);
> + found = __nvme_poll_cq(nvmeq, iob, 0);
> spin_unlock(&nvmeq->cq_poll_lock);
>
> return found;
> @@ -2075,7 +2355,7 @@ static void nvme_reap_pending_cqes(struct nvme_dev *dev)
>
> for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
> spin_lock(&dev->queues[i].cq_poll_lock);
> - nvme_poll_cq(&dev->queues[i], NULL);
> + __nvme_poll_cq(&dev->queues[i], NULL, 0);
> spin_unlock(&dev->queues[i].cq_poll_lock);
> }
> }
> @@ -2171,13 +2451,8 @@ static int queue_request_irq(struct nvme_queue *nvmeq)
> struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
> int nr = nvmeq->dev->ctrl.instance;
>
> - if (use_threaded_interrupts) {
> - return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
> - nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
> - } else {
> - return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
> - NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
> - }
> + return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
> + nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
> }
>
> static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
> @@ -2188,6 +2463,17 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
> nvmeq->last_sq_tail = 0;
> nvmeq->cq_head = 0;
> nvmeq->cq_phase = 1;
> + nvmeq->poll_thresh = irq_poll_thresh;
> + nvmeq->poll_sleep_us = irq_poll_sleep_us;
> + nvmeq->poll_idle_rounds = irq_poll_idle_rounds;
> + nvmeq->poll_budget = nvme_poll_prof[NVME_DIM_START_PROFILE].budget;
> + memset(&nvmeq->dim, 0, sizeof(nvmeq->dim));
> + nvmeq->dim.priv = nvmeq;
> + nvmeq->dim.profile_ix = NVME_DIM_START_PROFILE;
> + nvmeq->dim.tune_state = DIM_GOING_RIGHT;
> + nvmeq->dim_events = 0;
> + nvmeq->dim_comps = 0;
> + clear_bit(NVMEQ_POLLING, &nvmeq->flags);
> nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
> memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
> nvme_dbbuf_init(dev, nvmeq, qid);
> diff --git a/drivers/nvme/host/trace.c b/drivers/nvme/host/trace.c
> index ad25ad1e40412..4bdd5abc60880 100644
> --- a/drivers/nvme/host/trace.c
> +++ b/drivers/nvme/host/trace.c
> @@ -498,3 +498,4 @@ const char *nvme_trace_disk_name(struct trace_seq *p, char *name)
> }
>
> EXPORT_TRACEPOINT_SYMBOL_GPL(nvme_sq);
> +EXPORT_TRACEPOINT_SYMBOL_GPL(nvme_dim);
> diff --git a/drivers/nvme/host/trace.h b/drivers/nvme/host/trace.h
> index 4fb5922ffdac5..b7099b3026079 100644
> --- a/drivers/nvme/host/trace.h
> +++ b/drivers/nvme/host/trace.h
> @@ -161,6 +161,29 @@ TRACE_EVENT(nvme_sq,
> )
> );
>
> +TRACE_EVENT(nvme_dim,
> + TP_PROTO(u16 qid, int cpms, int epms, int cpe_ratio, u8 profile),
> + TP_ARGS(qid, cpms, epms, cpe_ratio, profile),
> + TP_STRUCT__entry(
> + __field(u16, qid)
> + __field(int, cpms)
> + __field(int, epms)
> + __field(int, cpe_ratio)
> + __field(u8, profile)
> + ),
> + TP_fast_assign(
> + __entry->qid = qid;
> + __entry->cpms = cpms;
> + __entry->epms = epms;
> + __entry->cpe_ratio = cpe_ratio;
> + __entry->profile = profile;
> + ),
> + TP_printk("qid=%u cpms=%d epms=%d cpe_ratio=%d profile=%u",
> + __entry->qid, __entry->cpms, __entry->epms,
> + __entry->cpe_ratio, __entry->profile
> + )
> +);
> +
> #endif /* _TRACE_NVME_H */
>
> #undef TRACE_INCLUDE_PATH
> --
>
More information about the Linux-nvme
mailing list