[PATCH 3/5] nvmet_tcp: assign queue to group

[Wunderlich, Mark]

nvmet_tcp: assign queue to group

Each newly created nvmf target queue connection is
assigned to a specific poll group.

The queue group assignment is based on the queue's socket
so_incoming_cpu value.  Since there is a pre-allocated
poll group for every available active CPU, this policy
will provide fast direct queue to poll group assignment.

The unique napi_id of the new queue socket is added to a
group list.  Multiple napi_ids are maintained per group
to support multi-port configurations where the number of
socket connections could exceed the number of active CPUs.

The group napi_ids are maintained for future patch for
polling when the group is found to be idle.

Signed-off-by: Mark Wunderlich <mark.wunderlich at intel.com>
---
 drivers/nvme/target/tcp.c |   78 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 1 deletion(-)

diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
index 3e3c217c77d4..84dd5b300a1d 100644
--- a/drivers/nvme/target/tcp.c
+++ b/drivers/nvme/target/tcp.c
@@ -128,6 +128,7 @@ struct nvmet_tcp_queue {
 
 	int			idx;
 	struct list_head	queue_list;
+	struct nvmet_tcp_queue_group *group;
 
 	struct nvmet_tcp_cmd	connect;
 
@@ -169,6 +170,7 @@ static struct workqueue_struct *nvmet_tcp_wq;
 static const struct nvmet_fabrics_ops nvmet_tcp_ops;
 static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
 static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd);
+static void nvmet_tcp_remove_from_group(struct nvmet_tcp_queue *queue);
 
 static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue,
 		struct nvmet_tcp_cmd *cmd)
@@ -1367,6 +1369,8 @@ static void nvmet_tcp_release_queue_work(struct work_struct *w)
 	struct nvmet_tcp_queue *queue =
 		container_of(w, struct nvmet_tcp_queue, release_work);
 
+	nvmet_tcp_remove_from_group(queue);
+
 	mutex_lock(&nvmet_tcp_queue_mutex);
 	list_del_init(&queue->queue_list);
 	mutex_unlock(&nvmet_tcp_queue_mutex);
@@ -1487,6 +1491,73 @@ static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue)
 	return 0;
 }
 
+static inline bool nvmet_tcp_add_group_napi_id(struct nvmet_tcp_queue_group *group,
+						unsigned int napi_id)
+{
+	int i, free_index = NVMET_TCP_GROUP_NAPI_LIMIT;
+
+	for (i = 0; i < NVMET_TCP_GROUP_NAPI_LIMIT; i++) {
+		if (group->napi[i].napi_id == napi_id) {
+			group->napi[i].napi_cnt++;
+			return true;
+		} else if (!group->napi[i].napi_id && free_index == NVMET_TCP_GROUP_NAPI_LIMIT) {
+			free_index = i;
+		}
+	}
+
+	if (free_index < NVMET_TCP_GROUP_NAPI_LIMIT) {
+		group->napi[free_index].napi_id = napi_id;
+		group->napi[free_index].napi_cnt++;
+		return true;
+	}
+	return false;
+}
+
+static bool nvmet_tcp_add_to_group(struct nvmet_tcp_queue *queue)
+{
+	struct nvmet_tcp_queue_group *group;
+	bool assigned = true;
+	int i, nr_grps = num_possible_cpus();
+
+	group = nvmet_tcp_queue_groups + queue->sock->sk->sk_incoming_cpu;
+	if (!nvmet_tcp_add_group_napi_id(group, queue->sock->sk->sk_napi_id)) {
+		/*
+		 * attempt assignment to another group with free napi id slot
+		 */
+		assigned = false;
+		for (i = 0; i < nr_grps; i++) {
+			group = nvmet_tcp_queue_groups + i;
+			if (nvmet_tcp_add_group_napi_id(group, queue->sock->sk->sk_napi_id)) {
+				assigned = true;
+				break;
+			}
+		}
+	}
+	if (!assigned)
+		return false;
+
+	queue->group = group;
+	return true;
+}
+
+static void nvmet_tcp_remove_from_group(struct nvmet_tcp_queue *queue)
+{
+	struct nvmet_tcp_queue_group *group = queue->group;
+	int i;
+
+	if (unlikely(!group))
+		return;
+
+	queue->group = NULL;
+
+	for (i = 0; i < NVMET_TCP_GROUP_NAPI_LIMIT; i++) {
+		if (group->napi[i].napi_id == queue->sock->sk->sk_napi_id) {
+			if (--group->napi[i].napi_cnt == 0)
+				group->napi[i].napi_id = 0;
+		}
+	}
+}
+
 static int nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port,
 		struct socket *newsock)
 {
@@ -1528,13 +1599,18 @@ static int nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port,
 	list_add_tail(&queue->queue_list, &nvmet_tcp_queue_list);
 	mutex_unlock(&nvmet_tcp_queue_mutex);
 
+	if (!nvmet_tcp_add_to_group(queue))
+		goto out_destroy_sq;
+
 	ret = nvmet_tcp_set_queue_sock(queue);
 	if (ret)
-		goto out_destroy_sq;
+		goto out_remove_from_group;
 
 	queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work);
 
 	return 0;
+out_remove_from_group:
+	nvmet_tcp_remove_from_group(queue);
 out_destroy_sq:
 	mutex_lock(&nvmet_tcp_queue_mutex);
 	list_del_init(&queue->queue_list);