[PATCH v8 1/2] nvme: add per-queue sysfs directories
Saravanan D
saravanand at crusoe.ai
Sat Nov 7 01:53:36 PST 2026
Per I/O queue transport state, such as the nvme-tcp io_cpu, has
nowhere to live in sysfs, since the core has no directory for
individual queues.
Add helpers that create the controller's queues directory and add or
remove one directory per queue
/sys/class/nvme/nvmeX/queues/<qid>/
carrying the attributes a transport sets in the new queue_attr_groups
member of struct nvme_ctrl_ops. For such transports nvme_add_ctrl()
creates the queues directory. It holds a reference on the controller
device, so nvme_uninit_ctrl() deletes it before deleting the device,
after the transport has removed the queue directories.
The transport adds a queue's directory when it sets the queue up and
removes it when it frees the queue, so the directories follow the
queues across reconnects and changes in the queue count. The core
allocates each queue directory and its release frees only that
allocation. Removal waits for running attribute callbacks, so the
transport may tear the queue down once it returns.
Suggested-by: Sagi Grimberg <sagi at grimberg.me>
Link: https://lore.kernel.org/linux-nvme/13c160d0-0657-47ce-9a16-58d348267ab0@grimberg.me/
Assisted-by: Claude:claude-opus-5-5 [Claude Code]
Signed-off-by: Saravanan D <saravanand at crusoe.ai>
---
drivers/nvme/host/core.c | 12 ++++++
drivers/nvme/host/nvme.h | 25 +++++++++++
drivers/nvme/host/sysfs.c | 90 +++++++++++++++++++++++++++++++++++++++
3 files changed, 127 insertions(+)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index cd0df9b03806..66e1e2c614e8 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -5208,6 +5208,8 @@ void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
*/
cancel_work_sync(&ctrl->async_event_work);
nvme_hwmon_exit(ctrl);
+ /* transports remove their queue directories before this point */
+ nvme_queue_dir_destroy(ctrl);
nvme_fault_inject_fini(&ctrl->fault_inject);
dev_pm_qos_hide_latency_tolerance(ctrl->device);
cdev_device_del(&ctrl->cdev, ctrl->device);
@@ -5352,6 +5354,12 @@ int nvme_add_ctrl(struct nvme_ctrl *ctrl)
if (ret)
return ret;
+ if (ctrl->ops->queue_attr_groups) {
+ ret = nvme_queue_dir_create(ctrl);
+ if (ret)
+ goto out_del_cdev;
+ }
+
/*
* Initialize latency tolerance controls. The sysfs files won't
* be visible to userspace unless the device actually supports APST.
@@ -5364,6 +5372,10 @@ int nvme_add_ctrl(struct nvme_ctrl *ctrl)
nvme_get_ctrl(ctrl);
return 0;
+
+out_del_cdev:
+ cdev_device_del(&ctrl->cdev, ctrl->device);
+ return ret;
}
EXPORT_SYMBOL_GPL(nvme_add_ctrl);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 2543e52d6358..95b598c71b4a 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -356,6 +356,28 @@ enum nvme_ctrl_flags {
NVME_CTRL_FROZEN = 6,
};
+/*
+ * Per-queue directory /sys/class/nvme/nvmeX/queues/<qid>/, allocated by the
+ * core and carrying attributes provided by the transport.
+ */
+struct nvme_queue_info {
+ struct kobject kobj;
+ void *data;
+};
+
+static inline void *nvme_queue_info_data(struct kobject *kobj)
+{
+ return container_of(kobj, struct nvme_queue_info, kobj)->data;
+}
+
+struct nvme_ctrl;
+
+int nvme_queue_dir_create(struct nvme_ctrl *ctrl);
+void nvme_queue_dir_destroy(struct nvme_ctrl *ctrl);
+struct nvme_queue_info *
+nvme_add_queue_info(struct nvme_ctrl *ctrl, unsigned int qid, void *data);
+void nvme_remove_queue_info(struct nvme_queue_info *qinfo);
+
struct nvme_ctrl {
bool comp_seen;
bool identified;
@@ -377,6 +399,7 @@ struct nvme_ctrl {
struct srcu_struct srcu;
struct device ctrl_device;
struct device *device; /* char device */
+ struct kobject *queues_kobj;
#ifdef CONFIG_NVME_HWMON
struct device *hwmon_device;
#endif
@@ -675,6 +698,8 @@ struct nvme_ctrl_ops {
#define NVME_F_BLOCKING (1 << 2)
const struct attribute_group **dev_attr_groups;
+ /* attributes of each /sys/class/nvme/nvmeX/queues/<qid>/ directory */
+ const struct attribute_group **queue_attr_groups;
int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index 3724b5ce03a0..ff81030f161c 100644
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -1311,3 +1311,93 @@ const struct attribute_group *nvme_subsys_attrs_groups[] = {
&nvme_subsys_attrs_group,
NULL,
};
+
+static void nvme_queue_info_release(struct kobject *kobj)
+{
+ kfree(container_of(kobj, struct nvme_queue_info, kobj));
+}
+
+static const struct kobj_type nvme_queue_info_ktype = {
+ .sysfs_ops = &kobj_sysfs_ops,
+ .release = nvme_queue_info_release,
+};
+
+/*
+ * nvme_add_ctrl() creates /sys/class/nvme/nvmeX/queues/ for transports that
+ * provide queue attributes, and nvme_uninit_ctrl() destroys it.
+ */
+int nvme_queue_dir_create(struct nvme_ctrl *ctrl)
+{
+ ctrl->queues_kobj = kobject_create_and_add("queues",
+ &ctrl->device->kobj);
+ if (!ctrl->queues_kobj)
+ return -ENOMEM;
+ return 0;
+}
+
+void nvme_queue_dir_destroy(struct nvme_ctrl *ctrl)
+{
+ if (!ctrl->queues_kobj)
+ return;
+
+ kobject_del(ctrl->queues_kobj);
+ kobject_put(ctrl->queues_kobj);
+ ctrl->queues_kobj = NULL;
+}
+
+/**
+ * nvme_add_queue_info - add a directory for one queue
+ * @ctrl: controller, whose queues directory must exist
+ * @qid: queue id, used as the directory name
+ * @data: transport data, returned by nvme_queue_info_data()
+ *
+ * Adds /sys/class/nvme/nvmeX/queues/<qid>/ carrying the transport's
+ * queue_attr_groups. Calls for different queues of
+ * the same controller may run concurrently. Returns the queue info, to be
+ * passed to nvme_remove_queue_info(), or an ERR_PTR().
+ */
+struct nvme_queue_info *
+nvme_add_queue_info(struct nvme_ctrl *ctrl, unsigned int qid, void *data)
+{
+ struct nvme_queue_info *qinfo;
+ int ret;
+
+ if (WARN_ON_ONCE(!ctrl->queues_kobj))
+ return ERR_PTR(-EINVAL);
+
+ qinfo = kzalloc_obj(*qinfo);
+ if (!qinfo)
+ return ERR_PTR(-ENOMEM);
+ qinfo->data = data;
+
+ ret = kobject_init_and_add(&qinfo->kobj, &nvme_queue_info_ktype,
+ ctrl->queues_kobj, "%u", qid);
+ if (ret)
+ goto out_put;
+
+ ret = sysfs_create_groups(&qinfo->kobj, ctrl->ops->queue_attr_groups);
+ if (ret)
+ goto out_del;
+ return qinfo;
+
+out_del:
+ kobject_del(&qinfo->kobj);
+out_put:
+ kobject_put(&qinfo->kobj);
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(nvme_add_queue_info);
+
+/**
+ * nvme_remove_queue_info - remove a queue directory
+ * @qinfo: queue info returned by nvme_add_queue_info()
+ *
+ * Waits for running attribute callbacks, so none run once this returns
+ * and the transport data may be torn down.
+ */
+void nvme_remove_queue_info(struct nvme_queue_info *qinfo)
+{
+ kobject_del(&qinfo->kobj);
+ kobject_put(&qinfo->kobj);
+}
+EXPORT_SYMBOL_GPL(nvme_remove_queue_info);
--
2.56.0
More information about the Linux-nvme
mailing list