fix atomic limits check
alan.adamson at oracle.com
alan.adamson at oracle.com
Mon Jun 23 10:24:50 PDT 2025
On 6/23/25 6:33 AM, Christoph Hellwig wrote:
> On Wed, Jun 18, 2025 at 11:30:20AM -0700, alan.adamson at oracle.com wrote:
>>> Can you double check this really is your CTLR2? At least for me
>>> the async probing reorders nvme devices quite a lot with qemu.
>>>
>> CTRL 0 (nvme0) - AWUN=31 AWUPF=15 nvme1n2
>> CTRL 1 (nvme1) - AWUN=31 AWUPF=31 nvme1n1
>> CTRL 2 (nvme2) - AWUN=15 AWUPF=7 nvme1n3
>> CTRL 3 (nvme3) - AWUN=15 AWUPF=15
> Do you mean AWUN or AWUPF? Because AWUN is totally irrelevant and
> not even parsed by Linux.
The above configuration is the HW/QEMU configuration. Both AWUN and
AWUPF are setup.
>
>> I rebooted and it probed differently. Now CTRL2 is nvme1n3. We know
>> CTRL2 is the same as nvme1n3 because awun and awupf from nvme id-ctrl
>> matches the qemu config (atomic.awun=15,atomic.awupf=7) for CTRL2 (nvme2).
>>
>> # nvme id-ctrl /dev/nvme1n3 | grep awun
>> awun : 15
>> # nvme id-ctrl /dev/nvme1n3 | grep awupf
>> awupf : 7
>> # cat /sys/block/nvme1n3/queue/atomic_write_max_bytes
>> 8192
> The code in nvme_configure_atomic_write calculates the values pretty
> much directly from (N)AWUPF, so I don't see how it would get things
> wrong. Can you run your patched qemu setup with this debug printk
> patch?
>
> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
> index 3da5ac71a9b0..245397b217f4 100644
> --- a/drivers/nvme/host/core.c
> +++ b/drivers/nvme/host/core.c
> @@ -2033,6 +2033,8 @@ static u32 nvme_configure_atomic_write(struct nvme_ns *ns,
> atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
> if (id->nabspf)
> boundary = (le16_to_cpu(id->nabspf) + 1) * bs;
> + dev_info(ns->ctrl->device, "NAWUPF: %u, atomic_bs: %u\n",
> + le16_to_cpu(id->nabspf), atomic_bs);
> } else {
> /*
> * Use the controller wide atomic write unit. This sucks
> @@ -2042,6 +2044,8 @@ static u32 nvme_configure_atomic_write(struct nvme_ns *ns,
> * values for different controllers in the subsystem.
> */
> atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
> + dev_info(ns->ctrl->device, "AWUPF: %u, atomic_bs: %u\n",
> + ns->ctrl->subsys->awupf, atomic_bs);
> }
>
> lim->atomic_write_hw_max = atomic_bs;
Here you go:
CTRL 0 (nvme0) - AWUN=31 AWUPF=15 nvme0n1
CTRL 1 (nvme1) - AWUN=31 AWUPF=31 nvme0n3
CTRL 2 (nvme2) - AWUN=15 AWUPF=7 nvme0n2
CTRL 3 (nvme3) - AWUN=15 AWUPF=15
NS - NAWUN=31 NAWUPF=15 nvme0n4
NS - NAWUN=127 NAWUPF=63 nvme0n5
[root at localhost ~]# nvme id-ctrl /dev/nvme0n1 | grep awupf
awupf : 15
[root at localhost ~]# cat /sys/block/nvme0n1/queue/atomic_write_max_bytes
8192
[root at localhost ~]# nvme id-ctrl /dev/nvme0n2 | grep awupf
awupf : 7
[root at localhost ~]# cat /sys/block/nvme0n2/queue/atomic_write_max_bytes
8192
[root at localhost ~]# nvme id-ctrl /dev/nvme0n3 | grep awupf
awupf : 31
[root at localhost ~]# cat /sys/block/nvme0n3/queue/atomic_write_max_bytes
8192
[root at localhost ~]# nvme id-ns /dev/nvme0n4 | grep nawupf
nawupf : 15
[root at localhost ~]# cat /sys/block/nvme0n4/queue/atomic_write_max_bytes
8192
[root at localhost ~]# nvme id-ns /dev/nvme0n5 | grep nawupf
nawupf : 63
[root at localhost ~]# cat /sys/block/nvme0n5/queue/atomic_write_max_bytes
32768
[root at localhost ~]#
[root at localhost ~]# dmesg | grep nvme | grep atomic_bs
[ 2.831882] nvme nvme0: AWUPF: 15, atomic_bs: 8192
[ 2.840480] nvme nvme2: AWUPF: 15, atomic_bs: 8192
[ 2.842418] nvme nvme1: AWUPF: 15, atomic_bs: 8192
[ 2.861119] nvme nvme3: NAWUPF: 0, atomic_bs: 8192
[ 2.862427] nvme nvme3: NAWUPF: 0, atomic_bs: 32768
[root at localhost ~]#
More information about the Linux-nvme
mailing list