Enabling RoCEv2 and connecting to IPv6 targets

Sagi Grimberg sagi at lightbits.io
Thu Dec 22 05:03:53 PST 2016



On 15/12/16 21:13, Narayan Ayalasomayajula wrote:
> All,
>
> I am attempting to enable connecting to IPv6 NVMe-oF targets (not the Linux NVMe-oF target) as well as enable RoCEv2 and have a couple of questions:
>
> 1.  With regards to IPv6, I tried the following syntax in the "echo" command (executed on a Ubuntu Host running the 4.8.0-rc22 version of the kernel) and got the following error message:
>
> 	echo "traddr=fe80::0c6:11ff:fe41:0516,transport=rdma,trsvcid=4420,nqn=nqn.kazannet.0" > /dev/nvme-fabrics
> 	bash: echo: write error: Invalid argument
>
> 	dmesg log indicates:
>
> 	nvme_rdma: malformed IP address passed: fe80::0c6:11ff:fe41:0516
>
> Is the support for connecting to IPv6 NVMe-oF targets not available in  the 4.8 version of the kernel?
>
> 2.  With regards to RoCEv2, I need to select/configure the appropriate GID on the Host NVMe-oF network interface but have not found a way to do so. I see the following example for IB at (https://community.mellanox.com/docs/DOC-2415):
>
> 	ib_send_bw -x 1
>
> How do I do this for NVMe-oF?
>
> Sorry if I am missing something very obvious - any guidance is appreciated.

Hi Narayan,

You're not missing anything obvious.

We actually had this discussion and proposed some patches to add ipv6
support, see thread at:
http://lists.infradead.org/pipermail/linux-nvme/2016-July/thread.html#5572

starts with "[PATCH 2/2] nvme-rdma: Add handling for connecting to IPv6 
targets".

I recall that the conclusion was to come up with a generic addition.

CC'ing Roland...

Can you try with the below three patches applied?

[1] host side:
--
From: Roland Dreier <roland at purestorage.com>

If a target address does not parse as IPv4, try parsing it as IPv6
(including handling '%<scope-id>' suffixes for link-local addresses).

Signed-off-by: Roland Dreier <roland at purestorage.com>
---
  drivers/nvme/host/rdma.c | 55 
++++++++++++++++++++++++++++++++++++++----------
  1 file changed, 44 insertions(+), 11 deletions(-)

diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index b96b88369871..dd4aa54cd709 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -138,6 +138,7 @@ struct nvme_rdma_ctrl {
  	union {
  		struct sockaddr addr;
  		struct sockaddr_in addr_in;
+		struct sockaddr_in6 addr_in6;
  	};

  	struct nvme_ctrl	ctrl;
@@ -1847,19 +1848,51 @@ out_free_io_queues:
  	return ret;
  }

-static int nvme_rdma_parse_ipaddr(struct sockaddr_in *in_addr, char *p)
+static int nvme_rdma_parse_ipaddr(struct sockaddr *addr, char *p)
  {
-	u8 *addr = (u8 *)&in_addr->sin_addr.s_addr;
  	size_t buflen = strlen(p);

-	/* XXX: handle IPv6 addresses */
+	if (buflen <= INET_ADDRSTRLEN) {
+		struct sockaddr_in *addr4 = (struct sockaddr_in *) addr;
+		if (in4_pton(p, buflen, (u8 *) &addr4->sin_addr.s_addr,
+			     '\0', NULL) > 0) {
+			addr4->sin_family = AF_INET;
+			return 0;
+		}
+	}

-	if (buflen > INET_ADDRSTRLEN)
-		return -EINVAL;
-	if (in4_pton(p, buflen, addr, '\0', NULL) == 0)
-		return -EINVAL;
-	in_addr->sin_family = AF_INET;
-	return 0;
+	if (buflen <= INET6_ADDRSTRLEN) {
+		struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *) addr;
+		const char *scope_delim;
+
+		if (in6_pton(p, buflen, (u8 *) &addr6->sin6_addr.s6_addr,
+			     '%', &scope_delim) == 0)
+			return -EINVAL;
+		addr6->sin6_family = AF_INET6;
+
+		if (ipv6_addr_type(&addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL &&
+		    p + buflen != scope_delim && *scope_delim == '%') {
+			char scope_id[16];
+			size_t scope_len = min_t(size_t, sizeof scope_id,
+						 p + buflen - scope_delim - 1);
+			struct net_device *dev;
+
+			memcpy(scope_id, scope_delim + 1, scope_len);
+			scope_id[scope_len] = '\0';
+
+			/* XXX: what network namespace should we use? */
+			dev = dev_get_by_name(&init_net, scope_id);
+			if (dev) {
+				addr6->sin6_scope_id = dev->ifindex;
+				dev_put(dev);
+			} else if (kstrtouint(scope_id, 0, &addr6->sin6_scope_id))
+				return -EINVAL;
+		}
+
+		return 0;
+	}
+
+	return -EINVAL;
  }

  static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
@@ -1875,7 +1908,7 @@ static struct nvme_ctrl 
*nvme_rdma_create_ctrl(struct device *dev,
  	ctrl->ctrl.opts = opts;
  	INIT_LIST_HEAD(&ctrl->list);

-	ret = nvme_rdma_parse_ipaddr(&ctrl->addr_in, opts->traddr);
+	ret = nvme_rdma_parse_ipaddr(&ctrl->addr, opts->traddr);
  	if (ret) {
  		pr_err("malformed IP address passed: %s\n", opts->traddr);
  		goto out_free_ctrl;
@@ -1949,7 +1982,7 @@ static struct nvme_ctrl 
*nvme_rdma_create_ctrl(struct device *dev,
  	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
  	WARN_ON_ONCE(!changed);

-	dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
+	dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n",
  		ctrl->ctrl.opts->subsysnqn, &ctrl->addr);

  	kref_get(&ctrl->ctrl.kref);
-- 

[2] target side:
--
commit 068d8e511d9a6927721bddfd545ad4976196297d
Author: Sagi Grimberg <sagi at grimberg.me>
Date:   Sun Jul 31 13:45:44 2016 +0300

      nvmet-rdma: Add ipv6 support

      Allow the nvme-rdma target accept connections using
      ipv6 address resolution.

      Signed-off-by: Sagi Grimberg <sagi at grimberg.me>

diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index 4e83d92d6bdd..347cc6d37dad 100644
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -1336,29 +1336,65 @@ restart:
          mutex_unlock(&nvmet_rdma_queue_mutex);
   }

-static int nvmet_rdma_add_port(struct nvmet_port *port)
+static int nvmet_rdma_set_addr_port(int family, char *ipaddr,
+               char *ipport, struct sockaddr_storage *addr)
   {
-       struct rdma_cm_id *cm_id;
-       struct sockaddr_in addr_in;
-       u16 port_in;
-       int ret;
+       size_t buflen = strlen(ipaddr);
+       struct sockaddr_in *addr4;
+       struct sockaddr_in6 *addr6;
+       u16 port;
+
+       if (kstrtou16(ipport, 0, &port))
+               return -EINVAL;

-       switch (port->disc_addr.adrfam) {
+       switch (family) {
          case NVMF_ADDR_FAMILY_IP4:
+               if (buflen > INET_ADDRSTRLEN)
+                       return -EINVAL;
+
+               addr4 = (struct sockaddr_in *)addr;
+
+               if (in4_pton(ipaddr, buflen, (u8 *)&addr4->sin_addr.s_addr,
+                            '\n', NULL) == 0)
+                       return -EINVAL;
+
+               addr4->sin_family = AF_INET;
+               addr4->sin_port = htons(port);
+               break;
+       case NVMF_ADDR_FAMILY_IP6:
+               if (buflen > INET6_ADDRSTRLEN)
+                       return -EINVAL;
+
+               addr6 = (struct sockaddr_in6 *) addr;
+
+               if (in6_pton(ipaddr, buflen, (u8
*)&addr6->sin6_addr.s6_addr,
+                            '\n', NULL) == 0)
+                       return -EINVAL;
+
+               addr6->sin6_family = AF_INET6;
+               addr6->sin6_port = htons(port);
                  break;
          default:
-               pr_err("address family %d not supported\n",
-                               port->disc_addr.adrfam);
+               pr_err("address family %d not supported\n", family);
                  return -EINVAL;
          }

-       ret = kstrtou16(port->disc_addr.trsvcid, 0, &port_in);
-       if (ret)
-               return ret;
+       return 0;
+}

-       addr_in.sin_family = AF_INET;
-       addr_in.sin_addr.s_addr = in_aton(port->disc_addr.traddr);
-       addr_in.sin_port = htons(port_in);
+static int nvmet_rdma_add_port(struct nvmet_port *port)
+{
+       struct rdma_cm_id *cm_id;
+       struct sockaddr_storage addr = { };
+       int ret;
+
+       ret = nvmet_rdma_set_addr_port(port->disc_addr.adrfam,
+               port->disc_addr.traddr, port->disc_addr.trsvcid, &addr);
+       if (ret) {
+               pr_err("failed setting addr %s:%s\n",
+               port->disc_addr.traddr, port->disc_addr.trsvcid);
+               return ret;
+       }

          cm_id = rdma_create_id(&init_net, nvmet_rdma_cm_handler, port,
                          RDMA_PS_TCP, IB_QPT_RC);
@@ -1367,20 +1403,33 @@ static int nvmet_rdma_add_port(struct nvmet_port
*port)
                  return PTR_ERR(cm_id);
          }

-       ret = rdma_bind_addr(cm_id, (struct sockaddr *)&addr_in);
+       /*
+        * Allow both IPv4 and IPv6 sockets to bind a single port
+        * at the same time.
+        */
+       ret = rdma_set_afonly(cm_id, 1);
+       if (ret) {
+               pr_err("rdma_set_afonly failed (%d)\n", ret);
+               goto out_destroy_id;
+       }
+
+       ret = rdma_bind_addr(cm_id, (struct sockaddr *)&addr);
          if (ret) {
-               pr_err("binding CM ID to %pISpc failed (%d)\n",
&addr_in, ret);
+               pr_err("binding CM ID to %pISpcs failed (%d)\n",
+                       (struct sockaddr *)&addr, ret);
                  goto out_destroy_id;
          }

          ret = rdma_listen(cm_id, 128);
          if (ret) {
-               pr_err("listening to %pISpc failed (%d)\n", &addr_in, ret);
+               pr_err("listening to %pISpcs failed (%d)\n",
+                       (struct sockaddr *)&addr, ret);
                  goto out_destroy_id;
          }

-       pr_info("enabling port %d (%pISpc)\n",
-               le16_to_cpu(port->disc_addr.portid), &addr_in);
+       pr_info("enabling port %d (%pISpcs)\n",
+               le16_to_cpu(port->disc_addr.portid),
+               (struct sockaddr *)&addr);
          port->priv = cm_id;
          return 0;
--

[3] nvme-cli:
--
commit 6d962c76580680e9cd0f5a56f47bcd30b5a39dd7
Author: Sagi Grimberg <sagi at grimberg.me>
Date:   Sun Jul 31 13:56:58 2016 +0300

      fabrics: Allow ipv6 address resolution

      Signed-off-by: Sagi Grimberg <sagi at grimberg.me>

diff --git a/fabrics.c b/fabrics.c
index 4bf557b5e672..221e34e5e39b 100644
--- a/fabrics.c
+++ b/fabrics.c
@@ -390,7 +390,8 @@ static int connect_ctrl(struct
nvmf_disc_rsp_page_entry *e)
                  /* we can safely ignore the rest of the entries */
                  break;
          case NVMF_TRTYPE_RDMA:
-               if (e->adrfam != NVMF_ADDR_FAMILY_IP4) {
+               if (e->adrfam != NVMF_ADDR_FAMILY_IP4 &&
+                   e->adrfam != NVMF_ADDR_FAMILY_IP6) {
                          fprintf(stderr, "skipping unsupported adrfam\n");
                          return -EINVAL;
                  }
--



More information about the Linux-nvme mailing list