[PATCH v6 01/23] net: Introduce direct data placement tcp offload
Aurelien Aptel
aaptel at nvidia.com
Thu Oct 20 03:18:16 PDT 2022
From: Boris Pismenny <borisp at nvidia.com>
This commit introduces direct data placement (DDP) offload for TCP.
The motivation is saving compute resources/cycles that are spent
to copy data from SKBs to the block layer buffers and CRC
calculation/verification for received PDUs (Protocol Data Units).
The DDP capability is accompanied by new net_device operations that
configure hardware contexts.
There is a context per socket, and a context per DDP operation.
Additionally, a resynchronization routine is used to assist
hardware handle TCP OOO, and continue the offload. Furthermore,
we let the offloading driver advertise what is the max hw
sectors/segments.
The interface includes five net-device ddp operations:
1. sk_add - add offload for the queue represented by socket+config pair
2. sk_del - remove the offload for the socket/queue
3. ddp_setup - request copy offload for buffers associated with an IO
4. ddp_teardown - release offload resources for that IO
5. limits - query NIC driver for quirks and limitations (e.g.
max number of scatter gather entries per IO)
Using this interface, the NIC hardware will scatter TCP payload
directly to the BIO pages according to the command_id.
To maintain the correctness of the network stack, the driver is
expected to construct SKBs that point to the BIO pages.
The SKB passed to the network stack from the driver represents
data as it is on the wire, while it is pointing directly to data
in destination buffers.
As a result, data from page frags should not be copied out to
the linear part. To avoid needless copies, such as when using
skb_condense, we mark the skb->ddp bit.
In addition, the skb->crc will be used by the upper layers to
determine if CRC re-calculation is required. The two separated skb
indications are needed to avoid false positives GRO flushing events.
Follow-up patches will use this interface for DDP in NVMe-TCP.
Signed-off-by: Boris Pismenny <borisp at nvidia.com>
Signed-off-by: Ben Ben-Ishay <benishay at nvidia.com>
Signed-off-by: Or Gerlitz <ogerlitz at nvidia.com>
Signed-off-by: Yoray Zack <yorayz at nvidia.com>
Signed-off-by: Shai Malin <smalin at nvidia.com>
Signed-off-by: Aurelien Aptel <aaptel at nvidia.com>
---
include/linux/netdev_features.h | 3 +-
include/linux/netdevice.h | 5 +
include/linux/skbuff.h | 11 ++
include/net/inet_connection_sock.h | 4 +
include/net/ulp_ddp.h | 171 +++++++++++++++++++++++++++++
net/Kconfig | 10 ++
net/core/skbuff.c | 3 +-
net/ethtool/common.c | 1 +
net/ipv4/tcp_input.c | 8 ++
net/ipv4/tcp_ipv4.c | 3 +
net/ipv4/tcp_offload.c | 3 +
11 files changed, 220 insertions(+), 2 deletions(-)
create mode 100644 include/net/ulp_ddp.h
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
index 7c2d77d75a88..bf7391aa04c7 100644
--- a/include/linux/netdev_features.h
+++ b/include/linux/netdev_features.h
@@ -14,7 +14,7 @@ typedef u64 netdev_features_t;
enum {
NETIF_F_SG_BIT, /* Scatter/gather IO. */
NETIF_F_IP_CSUM_BIT, /* Can checksum TCP/UDP over IPv4. */
- __UNUSED_NETIF_F_1,
+ NETIF_F_HW_ULP_DDP_BIT, /* ULP direct data placement offload */
NETIF_F_HW_CSUM_BIT, /* Can checksum all the packets. */
NETIF_F_IPV6_CSUM_BIT, /* Can checksum TCP/UDP over IPV6 */
NETIF_F_HIGHDMA_BIT, /* Can DMA to high memory. */
@@ -168,6 +168,7 @@ enum {
#define NETIF_F_HW_HSR_TAG_RM __NETIF_F(HW_HSR_TAG_RM)
#define NETIF_F_HW_HSR_FWD __NETIF_F(HW_HSR_FWD)
#define NETIF_F_HW_HSR_DUP __NETIF_F(HW_HSR_DUP)
+#define NETIF_F_HW_ULP_DDP __NETIF_F(HW_ULP_DDP)
/* Finds the next feature with the highest number of the range of start-1 till 0.
*/
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index a36edb0ec199..ff6f4978723a 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1043,6 +1043,7 @@ struct dev_ifalias {
struct devlink;
struct tlsdev_ops;
+struct ulp_ddp_dev_ops;
struct netdev_net_notifier {
struct list_head list;
@@ -2096,6 +2097,10 @@ struct net_device {
const struct tlsdev_ops *tlsdev_ops;
#endif
+#if IS_ENABLED(CONFIG_ULP_DDP)
+ const struct ulp_ddp_dev_ops *ulp_ddp_ops;
+#endif
+
const struct header_ops *header_ops;
unsigned char operstate;
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 9fcf534f2d92..5b6e27d281a6 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -810,6 +810,8 @@ typedef unsigned char *sk_buff_data_t;
* delivery_time in mono clock base (i.e. EDT). Otherwise, the
* skb->tstamp has the (rcv) timestamp at ingress and
* delivery_time at egress.
+ * @ulp_ddp: DDP offloaded
+ * @ulp_crc: CRC offloaded
* @napi_id: id of the NAPI struct this skb came from
* @sender_cpu: (aka @napi_id) source CPU in XPS
* @alloc_cpu: CPU which did the skb allocation.
@@ -982,6 +984,15 @@ struct sk_buff {
#endif
__u8 slow_gro:1;
__u8 csum_not_inet:1;
+#ifdef CONFIG_ULP_DDP
+ __u8 ulp_ddp:1;
+ __u8 ulp_crc:1;
+#define IS_ULP_DDP(skb) ((skb)->ulp_ddp)
+#define IS_ULP_CRC(skb) ((skb)->ulp_crc)
+#else
+#define IS_ULP_DDP(skb) (0)
+#define IS_ULP_CRC(skb) (0)
+#endif
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h
index c2b15f7e5516..2ba73167b3bb 100644
--- a/include/net/inet_connection_sock.h
+++ b/include/net/inet_connection_sock.h
@@ -68,6 +68,8 @@ struct inet_connection_sock_af_ops {
* @icsk_ulp_ops Pluggable ULP control hook
* @icsk_ulp_data ULP private data
* @icsk_clean_acked Clean acked data hook
+ * @icsk_ulp_ddp_ops Pluggable ULP direct data placement control hook
+ * @icsk_ulp_ddp_data ULP direct data placement private data
* @icsk_ca_state: Congestion control state
* @icsk_retransmits: Number of unrecovered [RTO] timeouts
* @icsk_pending: Scheduled timer event
@@ -98,6 +100,8 @@ struct inet_connection_sock {
const struct tcp_ulp_ops *icsk_ulp_ops;
void __rcu *icsk_ulp_data;
void (*icsk_clean_acked)(struct sock *sk, u32 acked_seq);
+ const struct ulp_ddp_ulp_ops *icsk_ulp_ddp_ops;
+ void __rcu *icsk_ulp_ddp_data;
unsigned int (*icsk_sync_mss)(struct sock *sk, u32 pmtu);
__u8 icsk_ca_state:5,
icsk_ca_initialized:1,
diff --git a/include/net/ulp_ddp.h b/include/net/ulp_ddp.h
new file mode 100644
index 000000000000..57cc4ab22e18
--- /dev/null
+++ b/include/net/ulp_ddp.h
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * ulp_ddp.h
+ * Author: Boris Pismenny <borisp at nvidia.com>
+ * Copyright (C) 2022 NVIDIA CORPORATION & AFFILIATES.
+ */
+#ifndef _ULP_DDP_H
+#define _ULP_DDP_H
+
+#include <linux/netdevice.h>
+#include <net/inet_connection_sock.h>
+#include <net/sock.h>
+
+enum ulp_ddp_type {
+ ULP_DDP_NVME = 1,
+};
+
+enum ulp_ddp_offload_capabilities {
+ ULP_DDP_C_NVME_TCP = 1,
+ ULP_DDP_C_NVME_TCP_DDGST_RX = 2,
+};
+
+/**
+ * struct ulp_ddp_limits - Generic ulp ddp limits: tcp ddp
+ * protocol limits.
+ * Protocol implementations must use this as the first member.
+ * Add new instances of ulp_ddp_limits below (nvme-tcp, etc.).
+ *
+ * @max_ddp_sgl_len: maximum sgl size supported (zero means no limit)
+ * @io_threshold: minimum payload size required to offload
+ */
+struct ulp_ddp_limits {
+ enum ulp_ddp_type type;
+ u64 offload_capabilities;
+ int max_ddp_sgl_len;
+ int io_threshold;
+ unsigned char buf[];
+};
+
+/**
+ * struct nvme_tcp_ddp_limits - nvme tcp driver limitations
+ *
+ * @full_ccid_range: true if the driver supports the full CID range
+ */
+struct nvme_tcp_ddp_limits {
+ struct ulp_ddp_limits lmt;
+
+ bool full_ccid_range;
+};
+
+/**
+ * struct ulp_ddp_config - Generic ulp ddp configuration: tcp ddp IO queue
+ * config implementations must use this as the first member.
+ * Add new instances of ulp_ddp_config below (nvme-tcp, etc.).
+ */
+struct ulp_ddp_config {
+ enum ulp_ddp_type type;
+ unsigned char buf[];
+};
+
+/**
+ * struct nvme_tcp_ddp_config - nvme tcp ddp configuration for an IO queue
+ *
+ * @pfv: pdu version (e.g., NVME_TCP_PFV_1_0)
+ * @cpda: controller pdu data alignment (dwords, 0's based)
+ * @dgst: digest types enabled (header or data, see enum nvme_tcp_digest_option).
+ * The netdev will offload crc if it is supported.
+ * @queue_size: number of nvme-tcp IO queue elements
+ * @queue_id: queue identifier
+ * @cpu_io: cpu core running the IO thread for this queue
+ */
+struct nvme_tcp_ddp_config {
+ struct ulp_ddp_config cfg;
+
+ u16 pfv;
+ u8 cpda;
+ u8 dgst;
+ int queue_size;
+ int queue_id;
+ int io_cpu;
+};
+
+/**
+ * struct ulp_ddp_io - ulp ddp configuration for an IO request.
+ *
+ * @command_id: identifier on the wire associated with these buffers
+ * @nents: number of entries in the sg_table
+ * @sg_table: describing the buffers for this IO request
+ * @first_sgl: first SGL in sg_table
+ */
+struct ulp_ddp_io {
+ u32 command_id;
+ int nents;
+ struct sg_table sg_table;
+ struct scatterlist first_sgl[SG_CHUNK_SIZE];
+};
+
+/* struct ulp_ddp_dev_ops - operations used by an upper layer protocol
+ * to configure ddp offload
+ *
+ * @ulp_ddp_limits: query ulp driver limitations and quirks.
+ * @ulp_ddp_sk_add: add offload for the queue represented by socket+config
+ * pair. this function is used to configure either copy, crc
+ * or both offloads.
+ * @ulp_ddp_sk_del: remove offload from the socket, and release any device
+ * related resources.
+ * @ulp_ddp_setup: request copy offload for buffers associated with a
+ * command_id in ulp_ddp_io.
+ * @ulp_ddp_teardown: release offload resources association between buffers
+ * and command_id in ulp_ddp_io.
+ * @ulp_ddp_resync: respond to the driver's resync_request. Called only if
+ * resync is successful.
+ */
+struct ulp_ddp_dev_ops {
+ int (*ulp_ddp_limits)(struct net_device *netdev,
+ struct ulp_ddp_limits *limits);
+ int (*ulp_ddp_sk_add)(struct net_device *netdev,
+ struct sock *sk,
+ struct ulp_ddp_config *config);
+ void (*ulp_ddp_sk_del)(struct net_device *netdev,
+ struct sock *sk);
+ int (*ulp_ddp_setup)(struct net_device *netdev,
+ struct sock *sk,
+ struct ulp_ddp_io *io);
+ int (*ulp_ddp_teardown)(struct net_device *netdev,
+ struct sock *sk,
+ struct ulp_ddp_io *io,
+ void *ddp_ctx);
+ void (*ulp_ddp_resync)(struct net_device *netdev,
+ struct sock *sk, u32 seq);
+};
+
+#define ULP_DDP_RESYNC_PENDING BIT(0)
+
+/**
+ * struct ulp_ddp_ulp_ops - Interface to register uppper layer
+ * Direct Data Placement (DDP) TCP offload.
+ * @resync_request: NIC requests ulp to indicate if @seq is the start
+ * of a message.
+ * @ddp_teardown_done: NIC driver informs the ulp that teardown is done,
+ * used for async completions.
+ */
+struct ulp_ddp_ulp_ops {
+ bool (*resync_request)(struct sock *sk, u32 seq, u32 flags);
+ void (*ddp_teardown_done)(void *ddp_ctx);
+};
+
+/**
+ * struct ulp_ddp_ctx - Generic ulp ddp context: device driver per queue contexts must
+ * use this as the first member.
+ */
+struct ulp_ddp_ctx {
+ enum ulp_ddp_type type;
+ unsigned char buf[];
+};
+
+static inline struct ulp_ddp_ctx *ulp_ddp_get_ctx(const struct sock *sk)
+{
+ struct inet_connection_sock *icsk = inet_csk(sk);
+
+ return (__force struct ulp_ddp_ctx *)icsk->icsk_ulp_ddp_data;
+}
+
+static inline void ulp_ddp_set_ctx(struct sock *sk, void *ctx)
+{
+ struct inet_connection_sock *icsk = inet_csk(sk);
+
+ rcu_assign_pointer(icsk->icsk_ulp_ddp_data, ctx);
+}
+
+#endif /* _ULP_DDP_H */
diff --git a/net/Kconfig b/net/Kconfig
index 48c33c222199..cd59be2d6c6e 100644
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -471,4 +471,14 @@ config NETDEV_ADDR_LIST_TEST
default KUNIT_ALL_TESTS
depends on KUNIT
+config ULP_DDP
+ bool "ULP direct data placement offload"
+ default n
+ help
+ Direct Data Placement (DDP) offload enables ULP, such as
+ NVMe-TCP, to request the NIC to place ULP payload data
+ of a command response directly into kernel pages while
+ calculate/verify the data digest on ULP PDU as they go through
+ the NIC. Thus avoiding the costly per-byte overhead.
+
endif # if NET
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 1d9719e72f9d..a7e92547924f 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -72,6 +72,7 @@
#include <net/mptcp.h>
#include <net/mctp.h>
#include <net/page_pool.h>
+#include <net/ulp_ddp.h>
#include <linux/uaccess.h>
#include <trace/events/skb.h>
@@ -6409,7 +6410,7 @@ void skb_condense(struct sk_buff *skb)
{
if (skb->data_len) {
if (skb->data_len > skb->end - skb->tail ||
- skb_cloned(skb))
+ skb_cloned(skb) || IS_ULP_DDP(skb))
return;
/* Nice, we can free page frag(s) right now */
diff --git a/net/ethtool/common.c b/net/ethtool/common.c
index 566adf85e658..da125a8357df 100644
--- a/net/ethtool/common.c
+++ b/net/ethtool/common.c
@@ -74,6 +74,7 @@ const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = {
[NETIF_F_HW_HSR_TAG_RM_BIT] = "hsr-tag-rm-offload",
[NETIF_F_HW_HSR_FWD_BIT] = "hsr-fwd-offload",
[NETIF_F_HW_HSR_DUP_BIT] = "hsr-dup-offload",
+ [NETIF_F_HW_ULP_DDP_BIT] = "ulp-ddp-offload",
};
const char
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index bc2ea12221f9..17d50cf63fa1 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -5232,6 +5232,10 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
#ifdef CONFIG_TLS_DEVICE
nskb->decrypted = skb->decrypted;
+#endif
+#ifdef CONFIG_ULP_DDP
+ nskb->ulp_ddp = skb->ulp_ddp;
+ nskb->ulp_crc = skb->ulp_crc;
#endif
TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start;
if (list)
@@ -5265,6 +5269,10 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
#ifdef CONFIG_TLS_DEVICE
if (skb->decrypted != nskb->decrypted)
goto end;
+#endif
+#ifdef CONFIG_ULP_DDP
+ if (IS_ULP_CRC(skb) != IS_ULP_CRC(nskb))
+ goto end;
#endif
}
}
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 6376ad915765..81333dcb4349 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -1821,6 +1821,9 @@ bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
#ifdef CONFIG_TLS_DEVICE
tail->decrypted != skb->decrypted ||
+#endif
+#ifdef CONFIG_ULP_DDP
+ IS_ULP_CRC(tail) != IS_ULP_CRC(skb) ||
#endif
thtail->doff != th->doff ||
memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c
index 45dda7889387..5b8b4cbc50c1 100644
--- a/net/ipv4/tcp_offload.c
+++ b/net/ipv4/tcp_offload.c
@@ -268,6 +268,9 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)
#ifdef CONFIG_TLS_DEVICE
flush |= p->decrypted ^ skb->decrypted;
#endif
+#ifdef CONFIG_ULP_DDP
+ flush |= IS_ULP_CRC(p) ^ IS_ULP_CRC(skb);
+#endif
if (flush || skb_gro_receive(p, skb)) {
mss = 1;
--
2.31.1
More information about the Linux-nvme
mailing list