[PATCH 5/7] nvme-pci: use a better encoding for small prp pool allocations
Caleb Sander Mateos
csander at purestorage.com
Wed May 14 10:47:48 PDT 2025
On Tue, May 13, 2025 at 12:00 AM Christoph Hellwig <hch at lst.de> wrote:
>
> Add a separate flag to encode that the transfer is using the small
> page sized pool, and use a normal 0..n count for the number of
> descriptors.
>
> Contains improvements and suggestions from Kanchan Joshi
> <joshi.k at samsung.com> and Leon Romanovsky <leonro at nvidia.com>.
>
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
> drivers/nvme/host/pci.c | 82 ++++++++++++++++++++---------------------
> 1 file changed, 39 insertions(+), 43 deletions(-)
>
> diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
> index 773a2e625657..16e2ce25da83 100644
> --- a/drivers/nvme/host/pci.c
> +++ b/drivers/nvme/host/pci.c
> @@ -229,6 +229,9 @@ struct nvme_queue {
> enum nvme_iod_flags {
> /* this command has been aborted by the timeout handler */
> IOD_ABORTED = 1U << 0,
> +
> + /* uses the small descriptor pool */
> + IOD_SMALL_DESCRIPTOR = 1U << 1,
> };
>
> /*
> @@ -238,7 +241,7 @@ struct nvme_iod {
> struct nvme_request req;
> struct nvme_command cmd;
> u8 flags;
> - s8 nr_descriptors;
> + u8 nr_descriptors;
> unsigned int dma_len; /* length of single DMA segment mapping */
> dma_addr_t first_dma;
> dma_addr_t meta_dma;
> @@ -589,13 +592,27 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req,
> return true;
> }
>
> -static void nvme_free_prps(struct nvme_queue *nvmeq, struct request *req)
> +static inline struct dma_pool *nvme_dma_pool(struct nvme_queue *nvmeq,
> + struct nvme_iod *iod)
> +{
> + if (iod->flags & IOD_SMALL_DESCRIPTOR)
> + return nvmeq->descriptor_pools.small;
> + return nvmeq->descriptor_pools.large;
> +}
> +
> +static void nvme_free_descriptors(struct nvme_queue *nvmeq, struct request *req)
Feels like this name change from "prps" to "descriptors" belongs
better in the prior commit "nvme-pci: rename the descriptor pools"?
Best,
Caleb
> {
> const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
> struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
> dma_addr_t dma_addr = iod->first_dma;
> int i;
>
> + if (iod->nr_descriptors == 1) {
> + dma_pool_free(nvme_dma_pool(nvmeq, iod), iod->descriptors[0],
> + dma_addr);
> + return;
> + }
> +
> for (i = 0; i < iod->nr_descriptors; i++) {
> __le64 *prp_list = iod->descriptors[i];
> dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
> @@ -620,15 +637,7 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_queue *nvmeq,
> WARN_ON_ONCE(!iod->sgt.nents);
>
> dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
> -
> - if (iod->nr_descriptors == 0)
> - dma_pool_free(nvmeq->descriptor_pools.small,
> - iod->descriptors[0], iod->first_dma);
> - else if (iod->nr_descriptors == 1)
> - dma_pool_free(nvmeq->descriptor_pools.large,
> - iod->descriptors[0], iod->first_dma);
> - else
> - nvme_free_prps(nvmeq, req);
> + nvme_free_descriptors(nvmeq, req);
> mempool_free(iod->sgt.sgl, dev->iod_mempool);
> }
>
> @@ -650,7 +659,6 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_queue *nvmeq,
> struct request *req, struct nvme_rw_command *cmnd)
> {
> struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
> - struct dma_pool *pool;
> int length = blk_rq_payload_bytes(req);
> struct scatterlist *sg = iod->sgt.sgl;
> int dma_len = sg_dma_len(sg);
> @@ -658,7 +666,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_queue *nvmeq,
> int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);
> __le64 *prp_list;
> dma_addr_t prp_dma;
> - int nprps, i;
> + int i;
>
> length -= (NVME_CTRL_PAGE_SIZE - offset);
> if (length <= 0) {
> @@ -680,27 +688,23 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_queue *nvmeq,
> goto done;
> }
>
> - nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE);
> - if (nprps <= (256 / 8)) {
> - pool = nvmeq->descriptor_pools.small;
> - iod->nr_descriptors = 0;
> - } else {
> - pool = nvmeq->descriptor_pools.large;
> - iod->nr_descriptors = 1;
> - }
> + if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) <=
> + 256 / sizeof(__le64))
> + iod->flags |= IOD_SMALL_DESCRIPTOR;
>
> - prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
> - if (!prp_list) {
> - iod->nr_descriptors = -1;
> + prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
> + &prp_dma);
> + if (!prp_list)
> return BLK_STS_RESOURCE;
> - }
> - iod->descriptors[0] = prp_list;
> + iod->descriptors[iod->nr_descriptors++] = prp_list;
This is slightly more expensive than the existing code because the
compiler can't tell that iod->nr_descriptors is initially 0. How about
keeping this as iod->descriptors[0] = prp_list and setting
iod->nr_descriptors = 1 on the next line?
> iod->first_dma = prp_dma;
> i = 0;
> for (;;) {
> if (i == NVME_CTRL_PAGE_SIZE >> 3) {
> __le64 *old_prp_list = prp_list;
> - prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
> +
> + prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod),
> + GFP_ATOMIC, &prp_dma);
> if (!prp_list)
> goto free_prps;
> iod->descriptors[iod->nr_descriptors++] = prp_list;
> @@ -727,7 +731,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_queue *nvmeq,
> cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
> return BLK_STS_OK;
> free_prps:
> - nvme_free_prps(nvmeq, req);
> + nvme_free_descriptors(nvmeq, req);
> return BLK_STS_RESOURCE;
> bad_sgl:
> WARN(DO_ONCE(nvme_print_sgl, iod->sgt.sgl, iod->sgt.nents),
> @@ -756,7 +760,6 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_queue *nvmeq,
> struct request *req, struct nvme_rw_command *cmd)
> {
> struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
> - struct dma_pool *pool;
> struct nvme_sgl_desc *sg_list;
> struct scatterlist *sg = iod->sgt.sgl;
> unsigned int entries = iod->sgt.nents;
> @@ -771,21 +774,14 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_queue *nvmeq,
> return BLK_STS_OK;
> }
>
> - if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
> - pool = nvmeq->descriptor_pools.small;
> - iod->nr_descriptors = 0;
> - } else {
> - pool = nvmeq->descriptor_pools.large;
> - iod->nr_descriptors = 1;
> - }
> + if (entries <= 256 / sizeof(*sg_list))
> + iod->flags |= IOD_SMALL_DESCRIPTOR;
>
> - sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
> - if (!sg_list) {
> - iod->nr_descriptors = -1;
> + sg_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
> + &sgl_dma);
> + if (!sg_list)
> return BLK_STS_RESOURCE;
> - }
> -
> - iod->descriptors[0] = sg_list;
> + iod->descriptors[iod->nr_descriptors++] = sg_list;
> iod->first_dma = sgl_dma;
>
> nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries);
> @@ -982,7 +978,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)
> blk_status_t ret;
>
> iod->flags = 0;
> - iod->nr_descriptors = -1;
> + iod->nr_descriptors = 0;
> iod->sgt.nents = 0;
> iod->meta_sgt.nents = 0;
>
> --
> 2.47.2
>
More information about the Linux-nvme
mailing list