[PATCH v5 2/2] nvme-tcp: allow setting per-queue io_cpu through sysfs
Saravanan D
saravanand at crusoe.ai
Sat Oct 17 18:00:59 PDT 2026
nvme_tcp_set_queue_io_cpu() picks each queue's io_cpu at connect time
as the least loaded CPU in the queue's blk-mq map group, and all socket
work runs there for the connection's lifetime. This decision falls
short when the host partitions its CPUs after connect time. On a 384
cpu multi tenant host with 128 queue controllers, blk-mq folds three
CPUs into every map group, some groups straddle two tenants' cpusets,
and 9% of nvme_tcp_io_work executions ran outside the submitting VM's
cpuset, seen by the neighbor as steal time.
Expose the io_cpu in each I/O queue's directory
/sys/class/nvme/nvmeX/queues/<qid>/io_cpu
so a control plane that owns CPU placement can set it directly instead
of relying on the driver's heuristic. The attribute accepts a decimal
cpu number, -1 or "unbound". A user assignment, unbound included, is
marked in the queue flags, persists across reconnects and is never
reselected by the driver. The read-only managed attribute reports 1
while io_cpu is driver managed and 0 once the user assigned it. The
accounting of nvme_tcp_cpu_queues applies regardless of who set the
io_cpu.
A queue's directory is added when the queue is allocated and removed
when it is freed, before the queue is torn down, so a sysfs store
never runs on a freed queue. The queues directory is created before
the I/O queues are set up in parallel. All io_cpu writes, from sysfs
and from the driver, serialize with the accounting under
nvme_tcp_cpu_queues_lock, while the I/O paths read io_cpu locklessly.
The queue stop now clears NVME_TCP_Q_IO_CPU_SET under that lock, since
a sysfs store may move io_cpu in between.
nvme_tcp_free_queue() now clears NVME_TCP_Q_IO_CPU_SET and drops its
count, because a queue can still be counted when it is freed. A sysfs
store may have run after the queue stop, or the connect may have
failed, and the setup error path frees such a queue without calling
nvme_tcp_stop_queue(), where the count is normally dropped.
Suggested-by: Sagi Grimberg <sagi at grimberg.me>
Link: https://lore.kernel.org/linux-nvme/13c160d0-0657-47ce-9a16-58d348267ab0@grimberg.me/
Assisted-by: Claude:claude-opus-5-5 [Claude Code]
Signed-off-by: Saravanan D <saravanand at crusoe.ai>
---
Documentation/ABI/stable/sysfs-nvme | 29 +++++
drivers/nvme/host/tcp.c | 173 ++++++++++++++++++++++++----
2 files changed, 182 insertions(+), 20 deletions(-)
diff --git a/Documentation/ABI/stable/sysfs-nvme b/Documentation/ABI/stable/sysfs-nvme
index 9291595f8102..4160589f7524 100644
--- a/Documentation/ABI/stable/sysfs-nvme
+++ b/Documentation/ABI/stable/sysfs-nvme
@@ -503,3 +503,32 @@ Contact: Hannes Reinecke <hare at suse.de>
Description:
Shows the subsystem type. Possible values: "discovery",
"nvm", "reserved".
+
+What: /sys/class/nvme/nvmeX/queues/<qid>/io_cpu
+What: /sys/class/nvme/nvmeX/queues/<qid>/managed
+Date: October 2026
+KernelVersion: 7.4
+Contact: Saravanan D <saravanand at crusoe.ai>
+Description:
+ Per I/O queue directories, populated by the transport
+ backing the controller. A directory exists while its
+ queue is allocated, so the directories follow the queue
+ count and are absent while the controller reconnects.
+ NVMe over TCP exposes:
+
+ io_cpu: (RW) The CPU that runs the socket work for I/O
+ queue <qid>, selected by the driver at connect time.
+ Writing a decimal online CPU number overrides the
+ selection, and writing -1 or "unbound" leaves the socket
+ work unbound. A written value persists across reconnects
+ for the life of the controller, also when its CPU goes
+ offline. Unbound socket work runs on the CPU that queued
+ it. When the nvme_tcp module is loaded with wq_unbound
+ set, the socket work runs on an unbound workqueue and
+ how closely it follows io_cpu depends on the affinity
+ scope of the nvme_tcp_wq workqueue. Reads return the
+ current CPU, or -1 when the queue is unbound.
+
+ managed: (RO) Shows who owns the CPU placement. 1 while
+ io_cpu is driver managed, 0 once the user assigned it,
+ for the life of the controller.
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index ec678ed07af8..699997c0e8e2 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -95,6 +95,7 @@ enum nvme_tcp_queue_flags {
NVME_TCP_Q_LIVE = 1,
NVME_TCP_Q_POLLING = 2,
NVME_TCP_Q_IO_CPU_SET = 3,
+ NVME_TCP_Q_IO_CPU_USER = 4,
};
enum nvme_tcp_recv_state {
@@ -142,6 +143,7 @@ struct nvme_tcp_queue {
struct completion tls_complete;
int tls_err;
struct page_frag_cache pf_cache;
+ struct nvme_queue_info *qinfo;
void (*state_change)(struct sock *);
void (*data_ready)(struct sock *);
@@ -230,6 +232,19 @@ static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
return queue - queue->ctrl->queues;
}
+/* the I/O paths read io_cpu locklessly, writers hold the accounting lock */
+static inline int nvme_tcp_queue_io_cpu(struct nvme_tcp_queue *queue)
+{
+ return READ_ONCE(queue->io_cpu);
+}
+
+static inline void nvme_tcp_queue_change_io_cpu(struct nvme_tcp_queue *queue,
+ int cpu)
+ __must_hold(&nvme_tcp_cpu_queues_lock)
+{
+ WRITE_ONCE(queue->io_cpu, cpu);
+}
+
static inline bool nvme_tcp_recv_pdu_supported(enum nvme_tcp_pdu_type type)
{
switch (type) {
@@ -427,7 +442,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
* lockdep reports circular locking via elevator_lock. Defer TLS
* sends to the io workqueue instead of inline from this path.
*/
- if (queue->io_cpu == raw_smp_processor_id() &&
+ if (nvme_tcp_queue_io_cpu(queue) == raw_smp_processor_id() &&
!nvme_tcp_queue_tls(queue) &&
empty && mutex_trylock(&queue->send_mutex)) {
nvme_tcp_send_all(queue);
@@ -435,7 +450,8 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
}
if (last && nvme_tcp_queue_has_pending(queue))
- queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
+ queue_work_on(nvme_tcp_queue_io_cpu(queue), nvme_tcp_wq,
+ &queue->io_work);
}
static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
@@ -844,7 +860,8 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
nvme_tcp_setup_h2c_data_pdu(req);
llist_add(&req->lentry, &queue->req_list);
- queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
+ queue_work_on(nvme_tcp_queue_io_cpu(queue), nvme_tcp_wq,
+ &queue->io_work);
return 0;
}
@@ -1131,7 +1148,8 @@ static void nvme_tcp_data_ready(struct sock *sk)
queue = sk->sk_user_data;
if (likely(queue && queue->rd_enabled) &&
!test_bit(NVME_TCP_Q_POLLING, &queue->flags))
- queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
+ queue_work_on(nvme_tcp_queue_io_cpu(queue), nvme_tcp_wq,
+ &queue->io_work);
read_unlock_bh(&sk->sk_callback_lock);
}
@@ -1146,7 +1164,8 @@ static void nvme_tcp_write_space(struct sock *sk)
/* Ensure pending TLS partial records are retried */
if (nvme_tcp_queue_tls(queue))
queue->write_space(sk);
- queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
+ queue_work_on(nvme_tcp_queue_io_cpu(queue), nvme_tcp_wq,
+ &queue->io_work);
}
read_unlock_bh(&sk->sk_callback_lock);
}
@@ -1469,7 +1488,8 @@ static void nvme_tcp_io_work(struct work_struct *w)
} while (!time_after(jiffies, deadline)); /* quota is exhausted */
- queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
+ queue_work_on(nvme_tcp_queue_io_cpu(queue), nvme_tcp_wq,
+ &queue->io_work);
}
static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
@@ -1503,9 +1523,22 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
struct nvme_tcp_queue *queue = &ctrl->queues[qid];
unsigned int noio_flag;
- if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
+ if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
return;
+ /* no io_cpu store runs once the directory is removed */
+ if (queue->qinfo) {
+ nvme_remove_queue_info(queue->qinfo);
+ queue->qinfo = NULL;
+ }
+ clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
+
+ /* a failed connect or a store after the stop left it counted */
+ spin_lock(&nvme_tcp_cpu_queues_lock);
+ if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
+ nvme_tcp_cpu_queues[nvme_tcp_queue_io_cpu(queue)]--;
+ spin_unlock(&nvme_tcp_cpu_queues_lock);
+
page_frag_cache_drain(&queue->pf_cache);
/**
@@ -1725,8 +1758,18 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
unsigned int *mq_map = NULL;
int cpu, min_queues = INT_MAX, io_cpu;
+ spin_lock(&nvme_tcp_cpu_queues_lock);
+ /* a user assigned io_cpu is kept across reconnects */
+ if (test_bit(NVME_TCP_Q_IO_CPU_USER, &queue->flags)) {
+ io_cpu = nvme_tcp_queue_io_cpu(queue);
+ if (io_cpu != WORK_CPU_UNBOUND &&
+ !test_and_set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
+ nvme_tcp_cpu_queues[io_cpu]++;
+ goto out_unlock;
+ }
+
if (wq_unbound)
- goto out;
+ goto out_unlock;
if (nvme_tcp_default_queue(queue))
mq_map = set->map[HCTX_TYPE_DEFAULT].mq_map;
@@ -1736,10 +1779,9 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
mq_map = set->map[HCTX_TYPE_POLL].mq_map;
if (WARN_ON(!mq_map))
- goto out;
+ goto out_unlock;
/* Search for the least used cpu from the mq_map */
- spin_lock(&nvme_tcp_cpu_queues_lock);
io_cpu = WORK_CPU_UNBOUND;
for_each_online_cpu(cpu) {
int num_queues = nvme_tcp_cpu_queues[cpu];
@@ -1752,14 +1794,14 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
}
}
if (io_cpu != WORK_CPU_UNBOUND) {
- queue->io_cpu = io_cpu;
+ nvme_tcp_queue_change_io_cpu(queue, io_cpu);
nvme_tcp_cpu_queues[io_cpu]++;
set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags);
}
+out_unlock:
spin_unlock(&nvme_tcp_cpu_queues_lock);
-out:
dev_dbg(ctrl->ctrl.device, "queue %d: using cpu %d\n",
- qid, queue->io_cpu);
+ qid, nvme_tcp_queue_io_cpu(queue));
}
static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid)
@@ -1847,6 +1889,85 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
return ret;
}
+static ssize_t io_cpu_show(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf)
+{
+ struct nvme_tcp_queue *queue = nvme_queue_info_data(kobj);
+ int io_cpu = nvme_tcp_queue_io_cpu(queue);
+
+ return sysfs_emit(buf, "%d\n",
+ io_cpu == WORK_CPU_UNBOUND ? -1 : io_cpu);
+}
+
+static ssize_t io_cpu_store(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct nvme_tcp_queue *queue = nvme_queue_info_data(kobj);
+ int cpu, old, ret;
+
+ if (sysfs_streq(buf, "unbound")) {
+ cpu = WORK_CPU_UNBOUND;
+ } else {
+ ret = kstrtoint(buf, 10, &cpu);
+ if (ret)
+ return ret;
+ if (cpu == -1)
+ cpu = WORK_CPU_UNBOUND;
+ else if ((unsigned int)cpu >= nr_cpu_ids || !cpu_online(cpu))
+ return -EINVAL;
+ }
+
+ spin_lock(&nvme_tcp_cpu_queues_lock);
+ old = nvme_tcp_queue_io_cpu(queue);
+ nvme_tcp_queue_change_io_cpu(queue, cpu);
+ set_bit(NVME_TCP_Q_IO_CPU_USER, &queue->flags);
+ if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
+ nvme_tcp_cpu_queues[old]--;
+ if (cpu != WORK_CPU_UNBOUND) {
+ nvme_tcp_cpu_queues[cpu]++;
+ set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags);
+ }
+ spin_unlock(&nvme_tcp_cpu_queues_lock);
+
+ return count;
+}
+
+static ssize_t managed_show(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf)
+{
+ struct nvme_tcp_queue *queue = nvme_queue_info_data(kobj);
+
+ return sysfs_emit(buf, "%d\n",
+ !test_bit(NVME_TCP_Q_IO_CPU_USER, &queue->flags));
+}
+
+static struct kobj_attribute nvme_tcp_io_cpu_attr =
+ __ATTR(io_cpu, 0644, io_cpu_show, io_cpu_store);
+static struct kobj_attribute nvme_tcp_managed_attr =
+ __ATTR_RO(managed);
+
+static struct attribute *nvme_tcp_queue_attrs[] = {
+ &nvme_tcp_io_cpu_attr.attr,
+ &nvme_tcp_managed_attr.attr,
+ NULL,
+};
+ATTRIBUTE_GROUPS(nvme_tcp_queue);
+
+static void nvme_tcp_add_queue_sysfs(struct nvme_tcp_queue *queue)
+{
+ struct nvme_ctrl *nctrl = &queue->ctrl->ctrl;
+ struct nvme_queue_info *qinfo;
+
+ qinfo = nvme_add_queue_info(nctrl, nvme_tcp_queue_id(queue),
+ nvme_tcp_queue_groups, queue);
+ if (IS_ERR(qinfo)) {
+ dev_warn(nctrl->device, "queue %d: no sysfs directory: %ld\n",
+ nvme_tcp_queue_id(queue), PTR_ERR(qinfo));
+ return;
+ }
+ queue->qinfo = qinfo;
+}
+
static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
key_serial_t pskid)
{
@@ -1917,7 +2038,11 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
queue->sock->sk->sk_allocation = GFP_ATOMIC;
queue->sock->sk->sk_use_task_frag = false;
- queue->io_cpu = WORK_CPU_UNBOUND;
+ /* no store races here, the lock is for the write helper */
+ spin_lock(&nvme_tcp_cpu_queues_lock);
+ if (!test_bit(NVME_TCP_Q_IO_CPU_USER, &queue->flags))
+ nvme_tcp_queue_change_io_cpu(queue, WORK_CPU_UNBOUND);
+ spin_unlock(&nvme_tcp_cpu_queues_lock);
queue->request = NULL;
queue->data_remaining = 0;
queue->ddgst_remaining = 0;
@@ -1985,6 +2110,9 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
+ if (qid)
+ nvme_tcp_add_queue_sysfs(queue);
+
return 0;
err_init_connect:
@@ -2033,11 +2161,10 @@ static void nvme_tcp_stop_queue_nowait(struct nvme_ctrl *nctrl, int qid)
if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
return;
- if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags)) {
- spin_lock(&nvme_tcp_cpu_queues_lock);
- nvme_tcp_cpu_queues[queue->io_cpu]--;
- spin_unlock(&nvme_tcp_cpu_queues_lock);
- }
+ spin_lock(&nvme_tcp_cpu_queues_lock);
+ if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
+ nvme_tcp_cpu_queues[nvme_tcp_queue_io_cpu(queue)]--;
+ spin_unlock(&nvme_tcp_cpu_queues_lock);
mutex_lock(&queue->queue_lock);
if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
@@ -2306,6 +2433,11 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
if (ret)
return ret;
+ /* the parallel queue setup below adds the queue directories */
+ ret = nvme_queue_dir_create(ctrl);
+ if (ret)
+ return ret;
+
if (new) {
ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set,
&nvme_tcp_mq_ops,
@@ -2893,7 +3025,8 @@ static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
struct nvme_tcp_queue *queue = hctx->driver_data;
if (!llist_empty(&queue->req_list))
- queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
+ queue_work_on(nvme_tcp_queue_io_cpu(queue), nvme_tcp_wq,
+ &queue->io_work);
}
static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
--
2.55.0
More information about the Linux-nvme
mailing list