[PATCH] nvme-pci: add adaptive interrupt polling

[Anuj Gupta]

> +/*
> + * Drain CQEs from IRQ_POLL_SOFTIRQ and compare completion progress with the
> + * IRQ baseline.  Re-arm while within the allowed lag; leave poll mode on lag
> + * or teardown, and start another window only after a faster full window.
> + */
> +static int nvme_adaptive_irq_poll(struct irq_poll *iop, int budget)
> +{
> +	struct nvme_adaptive_poll *adaptive = container_of(iop,
> +					struct nvme_adaptive_poll, iopoll);
> +	struct nvme_queue *nvmeq = adaptive->nvmeq;
> +	unsigned int completions, limit;
> +	unsigned long flags;
> +	u64 deadline, elapsed, now;
> +	DEFINE_IO_COMP_BATCH(iob);
> +
> +	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
> +	if (unlikely(!test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags))) {
> +		completions = 0;
> +		irq_poll_complete(iop);
> +		goto out;
> +	}
> +	if (!test_bit(NVMEQ_ENABLED, &nvmeq->flags)) {
> +		completions = 0;
> +		irq_poll_complete(iop);
> +		nvme_adaptive_poll_end(nvmeq, false);
> +		goto out;
> +	}
> +
> +	limit = min_t(unsigned int,
> +		      budget,
> +		      NVME_ADAPTIVE_EPISODE_CQES - adaptive->completions);
> +	completions = nvme_poll_cq_bounded(nvmeq, &iob, limit);
> +	adaptive->completions += completions;
> +
> +	if (completions >= budget &&
> +	    adaptive->completions < NVME_ADAPTIVE_EPISODE_CQES)
> +		goto out;
> +	irq_poll_complete(iop);

If the last drain reaches NVME_ADAPTIVE_EPISODE_CQES with
completions == budget, this calls irq_poll_complete() but still returns
budget. irq_poll_softirq() will then call list_move_tail() on an entry
that has already been removed.
Should the full-budget case always goto out and let the next invocation
complete and evaluate the episode?