[PATCH 3/3] fabrics: determine --nr-io-queues when not explicitly specified

[Nilay Shroff]

When performing an NVMe/TCP connect operation, if the user does not
explicitly specify --nr-io-queues, determine the default value using:

	min(nr_hw_queues, num_online_cpus)

Here, nr_hw_queues represents the number of hardware queues currently
configured on the NIC used for the NVMe/TCP connection. Use the
shr_route_get_egress_iface() and shr_netdev_get_hw_queues() helpers to
determine the egress netdev and retrieve its hardware queue count.
Use get_nprocs() to determine the number of online CPUs.

Apply the same logic when connecting to the target using a config INI
file.

Signed-off-by: Nilay Shroff <nilay at linux.ibm.com>
---
 src/fabrics.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 74 insertions(+), 3 deletions(-)

diff --git a/src/fabrics.c b/src/fabrics.c
index ad98c0e38..a640929b6 100644
--- a/src/fabrics.c
+++ b/src/fabrics.c
@@ -19,17 +19,20 @@
  * Fabrics specification standard.
  */
 
+#include <ccan/minmax/minmax.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <getopt.h>
 #include <inttypes.h>
 #include <libgen.h>
+#include <net/if.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <sys/stat.h>
+#include <sys/sysinfo.h>
 #include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
@@ -49,6 +52,7 @@
 #include <ccan/endian/endian.h>
 #include <ccan/str/str.h>
 #include <shared/io-util.h>
+#include <shared/net-util.h>
 #include <shared/sig-util.h>
 
 #include "cleanup.h"
@@ -424,6 +428,48 @@ static int build_conn_tid(const struct libnvmf_config_conn *conn,
 			hostnqn, hostid, tid);
 }
 
+/*
+ * On success returns num of I/O queues and returns 0 on failure or when it's
+ * not possible to determine the I/O queues.
+ */
+static int fabrics_connect_nr_io_queues(const char *transport,
+					const char *traddr,
+					const char *host_traddr,
+					const char *host_iface)
+{
+	uint32_t combined_count, tx_count, rx_count;
+	char ifname[IF_NAMESIZE] = {};
+	int nr_cpus, nr_hw_queues;
+
+	if (strcmp(transport, "tcp"))
+		return 0;
+
+	nr_cpus = get_nprocs();
+	if (nr_cpus <= 0)
+		return 0;
+
+	if (host_iface) {
+		strncpy(ifname, host_iface, IF_NAMESIZE - 1);
+	} else {
+		if (shr_route_get_egress_iface(host_traddr, traddr,
+				ifname, IF_NAMESIZE))
+			return 0;
+	}
+
+	if (shr_netdev_get_hw_queues(ifname, &combined_count,
+			&tx_count, &rx_count))
+		return 0;
+
+	if (combined_count)
+		nr_hw_queues = combined_count;
+	else if (tx_count && rx_count)
+		nr_hw_queues = min(tx_count, rx_count);
+	else
+		return 0;
+
+	return min(nr_cpus, nr_hw_queues);
+}
+
 /* libnvmf_config_conn_for_each() callback: settle addressing/identity,
  * check exclusion, then discover or connect.
  */
@@ -435,6 +481,8 @@ static void consume_conn(const struct libnvmf_config_conn *conn,
 	struct hook_fabrics_data hfd = { .flags = st->flags, .raw = st->raw };
 	__cleanup_nvmf_context struct libnvmf_context *fctx = NULL;
 	__cleanup_nvmf_tid struct libnvmf_tid *tid = NULL;
+	const struct libnvmf_params *params;
+	const char *key = "nr-io-queues";
 	int err;
 
 	if (st->mode == CONSUME_ROLE_BASED && !is_dc && !st->connect)
@@ -460,9 +508,29 @@ static void consume_conn(const struct libnvmf_config_conn *conn,
 		goto record_err;
 
 	err = libnvmf_context_set_connection_from_tid(fctx, tid);
-	if (!err)
-		err = libnvmf_context_apply_params(fctx,
-				libnvmf_config_conn_get_params(conn));
+	if (err)
+		goto record_err;
+
+	params = libnvmf_config_conn_get_params(conn);
+	if (!libnvmf_params_get(params, key)) {
+		int nr_io_queues;
+
+		nr_io_queues = fabrics_connect_nr_io_queues(
+				libnvmf_config_conn_get_transport(conn),
+				libnvmf_config_conn_get_traddr(conn),
+				libnvmf_config_conn_get_host_traddr(conn),
+				libnvmf_config_conn_get_host_iface(conn));
+
+		if (nr_io_queues) {
+			char val[32];
+
+			snprintf(val, sizeof(val), "%d", nr_io_queues);
+			libnvmf_params_set((struct libnvmf_params *)params,
+					key, val);
+		}
+	}
+
+	err = libnvmf_context_apply_params(fctx, params);
 	if (err)
 		goto record_err;
 
@@ -1059,6 +1127,9 @@ int fabrics_connect(const char *desc, int argc, char **argv)
 	if (ret)
 		return ret;
 
+	if (!fa.nr_io_queues)
+		fa.nr_io_queues = fabrics_connect_nr_io_queues(fa.transport,
+				      fa.traddr, fa.host_traddr, fa.host_iface);
 do_connect:
 	ret = nvme_create_global_ctx_hostnqn(&ctx,
 		fa.hostnqn, fa.hostid, &hnqn, &hid);
-- 
2.53.0