[PATCH v9 18/25] net/mlx5e: NVMEoTCP, offload initialization
Aurelien Aptel
aaptel at nvidia.com
Tue Jan 17 07:35:28 PST 2023
From: Ben Ben-Ishay <benishay at nvidia.com>
This commit introduces the driver structures and initialization blocks
for NVMEoTCP offload. The mlx5 nvmeotcp structures are:
- queue (mlx5e_nvmeotcp_queue) - pairs 1:1 with nvmeotcp driver queues and
deals with the offloading parts. The mlx5e queue is accessed in the ddp
ops: initialized on sk_add, used in ddp setup,teardown,resync and in the
fast path when dealing with packets, destroyed in the sk_del op.
- queue entry (nvmeotcp_queue_entry) - pairs 1:1 with offloaded IO from
that queue. Keeps pointers to the SG elements describing the buffers
used for the IO and the ddp context of it.
- queue handler (mlx5e_nvmeotcp_queue_handler) - we use icosq per NVME-TCP
queue for UMR mapping as part of the ddp offload. Those dedicated SQs are
unique in the sense that they are driven directly by the NVME-TCP layer
to submit and invalidate ddp requests.
Since the life-cycle of these icosqs is not tied to the channels, we
create dedicated napi contexts for polling them such that channels can be
re-created during offloading. The queue handler has pointer to the cq
associated with the queue's sq and napi context.
- main offload context (mlx5e_nvmeotcp) - has ida and hash table instances.
Each offloaded queue gets an ID from the ida instance and the <id, queue>
pairs are kept in the hash table. The id is programmed as flow tag to be
set by HW on the completion (cqe) of all packets related to this queue
(by 5-tuple steering). The fast path which deals with packets uses the
flow tag to access the hash table and retrieve the queue for the
processing.
We query nvmeotcp capabilities to see if the offload can be supported and
use 128B CQEs when this happens. By default, the offload is off but can
be enabled with `ethtool --ulp-ddp <device> nvme-tcp-ddp on`.
Signed-off-by: Ben Ben-Ishay <benishay at nvidia.com>
Signed-off-by: Boris Pismenny <borisp at nvidia.com>
Signed-off-by: Or Gerlitz <ogerlitz at nvidia.com>
Signed-off-by: Yoray Zack <yorayz at nvidia.com>
Signed-off-by: Shai Malin <smalin at nvidia.com>
Signed-off-by: Aurelien Aptel <aaptel at nvidia.com>
Reviewed-by: Tariq Toukan <tariqt at nvidia.com>
---
.../net/ethernet/mellanox/mlx5/core/Kconfig | 11 ++
.../net/ethernet/mellanox/mlx5/core/Makefile | 2 +
drivers/net/ethernet/mellanox/mlx5/core/en.h | 4 +
.../net/ethernet/mellanox/mlx5/core/en/fs.h | 4 +-
.../ethernet/mellanox/mlx5/core/en/params.c | 12 +-
.../ethernet/mellanox/mlx5/core/en/params.h | 3 +
.../mellanox/mlx5/core/en_accel/en_accel.h | 3 +
.../mellanox/mlx5/core/en_accel/fs_tcp.h | 2 +-
.../mellanox/mlx5/core/en_accel/nvmeotcp.c | 168 ++++++++++++++++++
.../mellanox/mlx5/core/en_accel/nvmeotcp.h | 121 +++++++++++++
.../ethernet/mellanox/mlx5/core/en_ethtool.c | 59 +++++-
.../net/ethernet/mellanox/mlx5/core/en_fs.c | 4 +-
.../net/ethernet/mellanox/mlx5/core/en_main.c | 18 ++
.../net/ethernet/mellanox/mlx5/core/main.c | 1 +
14 files changed, 402 insertions(+), 10 deletions(-)
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.h
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
index 26685fd0fdaa..0c790952fdf7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
@@ -163,6 +163,17 @@ config MLX5_EN_TLS
help
Build support for TLS cryptography-offload acceleration in the NIC.
+config MLX5_EN_NVMEOTCP
+ bool "NVMEoTCP acceleration"
+ depends on ULP_DDP
+ depends on MLX5_CORE_EN
+ default y
+ help
+ Build support for NVMEoTCP acceleration in the NIC.
+ This includes Direct Data Placement and CRC offload.
+ Note: Support for hardware with this capability needs to be selected
+ for this option to become available.
+
config MLX5_SW_STEERING
bool "Mellanox Technologies software-managed steering"
depends on MLX5_CORE_EN && MLX5_ESWITCH
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index cd4a1ab0ea78..9df9999047d1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -103,6 +103,8 @@ mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/ktls_stats.o \
en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \
en_accel/ktls_tx.o en_accel/ktls_rx.o
+mlx5_core-$(CONFIG_MLX5_EN_NVMEOTCP) += en_accel/fs_tcp.o en_accel/nvmeotcp.o
+
mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o \
steering/dr_matcher.o steering/dr_rule.o \
steering/dr_icm_pool.o steering/dr_buddy.o \
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 17ccad42555c..136eb608ecaf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -328,6 +328,7 @@ struct mlx5e_params {
unsigned int sw_mtu;
int hard_mtu;
bool ptp_rx;
+ bool nvmeotcp;
};
static inline u8 mlx5e_get_dcb_num_tc(struct mlx5e_params *params)
@@ -959,6 +960,9 @@ struct mlx5e_priv {
#endif
#ifdef CONFIG_MLX5_EN_TLS
struct mlx5e_tls *tls;
+#endif
+#ifdef CONFIG_MLX5_EN_NVMEOTCP
+ struct mlx5e_nvmeotcp *nvmeotcp;
#endif
struct devlink_health_reporter *tx_reporter;
struct devlink_health_reporter *rx_reporter;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
index 5233d4daca41..c3598c2e05d7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
@@ -77,7 +77,7 @@ enum {
MLX5E_INNER_TTC_FT_LEVEL,
MLX5E_FS_TT_UDP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1,
MLX5E_FS_TT_ANY_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1,
-#ifdef CONFIG_MLX5_EN_TLS
+#if defined(CONFIG_MLX5_EN_TLS) || defined(CONFIG_MLX5_EN_NVMEOTCP)
MLX5E_ACCEL_FS_TCP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1,
#endif
#ifdef CONFIG_MLX5_EN_ARFS
@@ -169,7 +169,7 @@ struct mlx5e_fs_any *mlx5e_fs_get_any(struct mlx5e_flow_steering *fs);
void mlx5e_fs_set_any(struct mlx5e_flow_steering *fs, struct mlx5e_fs_any *any);
struct mlx5e_fs_udp *mlx5e_fs_get_udp(struct mlx5e_flow_steering *fs);
void mlx5e_fs_set_udp(struct mlx5e_flow_steering *fs, struct mlx5e_fs_udp *udp);
-#ifdef CONFIG_MLX5_EN_TLS
+#if defined(CONFIG_MLX5_EN_TLS) || defined(CONFIG_MLX5_EN_NVMEOTCP)
struct mlx5e_accel_fs_tcp *mlx5e_fs_get_accel_tcp(struct mlx5e_flow_steering *fs);
void mlx5e_fs_set_accel_tcp(struct mlx5e_flow_steering *fs, struct mlx5e_accel_fs_tcp *accel_tcp);
#endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
index 585bdc8383ee..36f53251a32b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -797,7 +797,8 @@ static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev,
void *cqc = param->cqc;
MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
- if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
+ if (MLX5_CAP_GEN(mdev, cqe_128_always) &&
+ (cache_line_size() >= 128 || param->force_cqe128))
MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);
}
@@ -827,6 +828,9 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
void *cqc = param->cqc;
u8 log_cq_size;
+ /* nvme-tcp offload mandates 128 byte cqes */
+ param->force_cqe128 |= IS_ENABLED(CONFIG_MLX5_EN_NVMEOTCP) && params->nvmeotcp;
+
switch (params->rq_wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index);
@@ -1166,9 +1170,9 @@ static u8 mlx5e_build_async_icosq_log_wq_sz(struct mlx5_core_dev *mdev)
return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
}
-static void mlx5e_build_icosq_param(struct mlx5_core_dev *mdev,
- u8 log_wq_size,
- struct mlx5e_sq_param *param)
+void mlx5e_build_icosq_param(struct mlx5_core_dev *mdev,
+ u8 log_wq_size,
+ struct mlx5e_sq_param *param)
{
void *sqc = param->sqc;
void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
index c9be6eb88012..d5b3455c4875 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
@@ -17,6 +17,7 @@ struct mlx5e_cq_param {
struct mlx5_wq_param wq;
u16 eq_ix;
u8 cq_period_mode;
+ bool force_cqe128;
};
struct mlx5e_rq_param {
@@ -146,6 +147,8 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_sq_param *param);
+void mlx5e_build_icosq_param(struct mlx5_core_dev *mdev,
+ u8 log_wq_size, struct mlx5e_sq_param *param);
int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
u16 q_counter,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
index 07187028f0d3..e38656229399 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
@@ -40,6 +40,7 @@
#include "en_accel/ktls.h"
#include "en_accel/ktls_txrx.h"
#include <en_accel/macsec.h>
+#include "en_accel/nvmeotcp.h"
#include "en.h"
#include "en/txrx.h"
@@ -202,11 +203,13 @@ static inline void mlx5e_accel_tx_finish(struct mlx5e_txqsq *sq,
static inline int mlx5e_accel_init_rx(struct mlx5e_priv *priv)
{
+ mlx5e_nvmeotcp_init_rx(priv);
return mlx5e_ktls_init_rx(priv);
}
static inline void mlx5e_accel_cleanup_rx(struct mlx5e_priv *priv)
{
+ mlx5e_nvmeotcp_cleanup_rx(priv);
mlx5e_ktls_cleanup_rx(priv);
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h
index a032bff482a6..d907e352ffae 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h
@@ -6,7 +6,7 @@
#include "en/fs.h"
-#ifdef CONFIG_MLX5_EN_TLS
+#if defined(CONFIG_MLX5_EN_TLS) || defined(CONFIG_MLX5_EN_NVMEOTCP)
int mlx5e_accel_fs_tcp_create(struct mlx5e_flow_steering *fs);
void mlx5e_accel_fs_tcp_destroy(struct mlx5e_flow_steering *fs);
struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
new file mode 100644
index 000000000000..5462df75fff6
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
+
+#include <linux/netdevice.h>
+#include <linux/idr.h>
+#include "en_accel/nvmeotcp.h"
+#include "en_accel/fs_tcp.h"
+#include "en/txrx.h"
+
+#define MAX_NUM_NVMEOTCP_QUEUES (4000)
+#define MIN_NUM_NVMEOTCP_QUEUES (1)
+
+static const struct rhashtable_params rhash_queues = {
+ .key_len = sizeof(int),
+ .key_offset = offsetof(struct mlx5e_nvmeotcp_queue, id),
+ .head_offset = offsetof(struct mlx5e_nvmeotcp_queue, hash),
+ .automatic_shrinking = true,
+ .min_size = MIN_NUM_NVMEOTCP_QUEUES,
+ .max_size = MAX_NUM_NVMEOTCP_QUEUES,
+};
+
+static int
+mlx5e_nvmeotcp_offload_limits(struct net_device *netdev,
+ struct ulp_ddp_limits *limits)
+{
+ return 0;
+}
+
+static int
+mlx5e_nvmeotcp_queue_init(struct net_device *netdev,
+ struct sock *sk,
+ struct ulp_ddp_config *tconfig)
+{
+ return 0;
+}
+
+static void
+mlx5e_nvmeotcp_queue_teardown(struct net_device *netdev,
+ struct sock *sk)
+{
+}
+
+static int
+mlx5e_nvmeotcp_ddp_setup(struct net_device *netdev,
+ struct sock *sk,
+ struct ulp_ddp_io *ddp)
+{
+ return 0;
+}
+
+static void
+mlx5e_nvmeotcp_ddp_teardown(struct net_device *netdev,
+ struct sock *sk,
+ struct ulp_ddp_io *ddp,
+ void *ddp_ctx)
+{
+}
+
+static void
+mlx5e_nvmeotcp_ddp_resync(struct net_device *netdev,
+ struct sock *sk, u32 seq)
+{
+}
+
+const struct ulp_ddp_dev_ops mlx5e_nvmeotcp_ops = {
+ .limits = mlx5e_nvmeotcp_offload_limits,
+ .sk_add = mlx5e_nvmeotcp_queue_init,
+ .sk_del = mlx5e_nvmeotcp_queue_teardown,
+ .setup = mlx5e_nvmeotcp_ddp_setup,
+ .teardown = mlx5e_nvmeotcp_ddp_teardown,
+ .resync = mlx5e_nvmeotcp_ddp_resync,
+};
+
+int set_ulp_ddp_nvme_tcp(struct net_device *netdev, bool enable)
+{
+ struct mlx5e_priv *priv = netdev_priv(netdev);
+ struct mlx5e_params new_params;
+ int err = 0;
+
+ /* There may be offloaded queues when an ethtool callback to disable the feature is made.
+ * Hence, we can't destroy the tcp flow-table since it may be referenced by the offload
+ * related flows and we'll keep the 128B CQEs on the channel RQs. Also, since we don't
+ * deref/destroy the fs tcp table when the feature is disabled, we don't ref it again
+ * if the feature is enabled multiple times.
+ */
+ if (!enable || priv->nvmeotcp->enabled)
+ return 0;
+
+ err = mlx5e_accel_fs_tcp_create(priv->fs);
+ if (err)
+ return err;
+
+ new_params = priv->channels.params;
+ new_params.nvmeotcp = enable;
+ err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, true);
+ if (err)
+ goto fs_tcp_destroy;
+
+ priv->nvmeotcp->enabled = true;
+ return 0;
+
+fs_tcp_destroy:
+ mlx5e_accel_fs_tcp_destroy(priv->fs);
+ return err;
+}
+
+void mlx5e_nvmeotcp_build_netdev(struct mlx5e_priv *priv)
+{
+ struct net_device *netdev = priv->netdev;
+ struct mlx5_core_dev *mdev = priv->mdev;
+
+ if (!(MLX5_CAP_GEN(mdev, nvmeotcp) &&
+ MLX5_CAP_DEV_NVMEOTCP(mdev, zerocopy) &&
+ MLX5_CAP_DEV_NVMEOTCP(mdev, crc_rx) && MLX5_CAP_GEN(mdev, cqe_128_always)))
+ return;
+
+ /* report ULP DPP as supported, but don't enable it by default */
+ set_bit(ULP_DDP_C_NVME_TCP_BIT, netdev->ulp_ddp_caps.hw);
+ set_bit(ULP_DDP_C_NVME_TCP_DDGST_RX_BIT, netdev->ulp_ddp_caps.hw);
+}
+
+void mlx5e_nvmeotcp_cleanup_rx(struct mlx5e_priv *priv)
+{
+ if (priv->nvmeotcp && priv->nvmeotcp->enabled)
+ mlx5e_accel_fs_tcp_destroy(priv->fs);
+}
+
+int mlx5e_nvmeotcp_init(struct mlx5e_priv *priv)
+{
+ struct mlx5e_nvmeotcp *nvmeotcp = NULL;
+ int ret = 0;
+
+ if (!MLX5_CAP_GEN(priv->mdev, nvmeotcp))
+ return 0;
+
+ nvmeotcp = kzalloc(sizeof(*nvmeotcp), GFP_KERNEL);
+
+ if (!nvmeotcp)
+ return -ENOMEM;
+
+ ida_init(&nvmeotcp->queue_ids);
+ ret = rhashtable_init(&nvmeotcp->queue_hash, &rhash_queues);
+ if (ret)
+ goto err_ida;
+
+ nvmeotcp->enabled = false;
+
+ priv->nvmeotcp = nvmeotcp;
+ return 0;
+
+err_ida:
+ ida_destroy(&nvmeotcp->queue_ids);
+ kfree(nvmeotcp);
+ return ret;
+}
+
+void mlx5e_nvmeotcp_cleanup(struct mlx5e_priv *priv)
+{
+ struct mlx5e_nvmeotcp *nvmeotcp = priv->nvmeotcp;
+
+ if (!nvmeotcp)
+ return;
+
+ rhashtable_destroy(&nvmeotcp->queue_hash);
+ ida_destroy(&nvmeotcp->queue_ids);
+ kfree(nvmeotcp);
+ priv->nvmeotcp = NULL;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.h
new file mode 100644
index 000000000000..a665b7a72bc2
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. */
+#ifndef __MLX5E_NVMEOTCP_H__
+#define __MLX5E_NVMEOTCP_H__
+
+#ifdef CONFIG_MLX5_EN_NVMEOTCP
+
+#include <net/ulp_ddp.h>
+#include "en.h"
+#include "en/params.h"
+
+struct mlx5e_nvmeotcp_queue_entry {
+ struct mlx5e_nvmeotcp_queue *queue;
+ u32 sgl_length;
+ u32 klm_mkey;
+ struct scatterlist *sgl;
+ u32 ccid_gen;
+ u64 size;
+
+ /* for the ddp invalidate done callback */
+ void *ddp_ctx;
+ struct ulp_ddp_io *ddp;
+};
+
+struct mlx5e_nvmeotcp_queue_handler {
+ struct napi_struct napi;
+ struct mlx5e_cq *cq;
+};
+
+/**
+ * struct mlx5e_nvmeotcp_queue - mlx5 metadata for NVMEoTCP queue
+ * @ulp_ddp_ctx: Generic ulp ddp context
+ * @tir: Destination TIR created for NVMEoTCP offload
+ * @fh: Flow handle representing the 5-tuple steering for this flow
+ * @id: Flow tag ID used to identify this queue
+ * @size: NVMEoTCP queue depth
+ * @ccid_gen: Generation ID for the CCID, used to avoid conflicts in DDP
+ * @max_klms_per_wqe: Number of KLMs per DDP operation
+ * @hash: Hash table of queues mapped by @id
+ * @pda: Padding alignment
+ * @tag_buf_table_id: Tag buffer table for CCIDs
+ * @dgst: Digest supported (header and/or data)
+ * @sq: Send queue used for posting umrs
+ * @ref_count: Reference count for this structure
+ * @after_resync_cqe: Indicate if resync occurred
+ * @ccid_table: Table holding metadata for each CC (Command Capsule)
+ * @ccid: ID of the current CC
+ * @ccsglidx: Index within the scatter-gather list (SGL) of the current CC
+ * @ccoff: Offset within the current CC
+ * @ccoff_inner: Current offset within the @ccsglidx element
+ * @channel_ix: Channel IX for this nvmeotcp_queue
+ * @sk: The socket used by the NVMe-TCP queue
+ * @crc_rx: CRC Rx offload indication for this queue
+ * @priv: mlx5e netdev priv
+ * @static_params_done: Async completion structure for the initial umr mapping
+ * synchronization
+ * @sq_lock: Spin lock for the icosq
+ * @qh: Completion queue handler for processing umr completions
+ */
+struct mlx5e_nvmeotcp_queue {
+ struct ulp_ddp_ctx ulp_ddp_ctx;
+ struct mlx5e_tir tir;
+ struct mlx5_flow_handle *fh;
+ int id;
+ u32 size;
+ /* needed when the upper layer immediately reuses CCID + some packet loss happens */
+ u32 ccid_gen;
+ u32 max_klms_per_wqe;
+ struct rhash_head hash;
+ int pda;
+ u32 tag_buf_table_id;
+ u8 dgst;
+ struct mlx5e_icosq sq;
+
+ /* data-path section cache aligned */
+ refcount_t ref_count;
+ /* for MASK HW resync cqe */
+ bool after_resync_cqe;
+ struct mlx5e_nvmeotcp_queue_entry *ccid_table;
+ /* current ccid fields */
+ int ccid;
+ int ccsglidx;
+ off_t ccoff;
+ int ccoff_inner;
+
+ u32 channel_ix;
+ struct sock *sk;
+ u8 crc_rx:1;
+ /* for ddp invalidate flow */
+ struct mlx5e_priv *priv;
+ /* end of data-path section */
+
+ struct completion static_params_done;
+ /* spin lock for the ico sq, ULP can issue requests from multiple contexts */
+ spinlock_t sq_lock;
+ struct mlx5e_nvmeotcp_queue_handler qh;
+};
+
+struct mlx5e_nvmeotcp {
+ struct ida queue_ids;
+ struct rhashtable queue_hash;
+ bool enabled;
+};
+
+void mlx5e_nvmeotcp_build_netdev(struct mlx5e_priv *priv);
+int mlx5e_nvmeotcp_init(struct mlx5e_priv *priv);
+int set_ulp_ddp_nvme_tcp(struct net_device *netdev, bool enable);
+void mlx5e_nvmeotcp_cleanup(struct mlx5e_priv *priv);
+static inline void mlx5e_nvmeotcp_init_rx(struct mlx5e_priv *priv) {}
+void mlx5e_nvmeotcp_cleanup_rx(struct mlx5e_priv *priv);
+extern const struct ulp_ddp_dev_ops mlx5e_nvmeotcp_ops;
+#else
+
+static inline void mlx5e_nvmeotcp_build_netdev(struct mlx5e_priv *priv) {}
+static inline int mlx5e_nvmeotcp_init(struct mlx5e_priv *priv) { return 0; }
+static inline void mlx5e_nvmeotcp_cleanup(struct mlx5e_priv *priv) {}
+static inline int set_ulp_ddp_nvme_tcp(struct net_device *dev, bool en) { return -EOPNOTSUPP; }
+static inline void mlx5e_nvmeotcp_init_rx(struct mlx5e_priv *priv) {}
+static inline void mlx5e_nvmeotcp_cleanup_rx(struct mlx5e_priv *priv) {}
+#endif
+#endif /* __MLX5E_NVMEOTCP_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 7708acc9b2ab..7f763152f989 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -38,6 +38,7 @@
#include "en/ptp.h"
#include "lib/clock.h"
#include "en/fs_ethtool.h"
+#include "en_accel/nvmeotcp.h"
void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
struct ethtool_drvinfo *drvinfo)
@@ -1939,6 +1940,11 @@ int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool new_val
return -EINVAL;
}
+ if (priv->channels.params.nvmeotcp) {
+ netdev_warn(priv->netdev, "Can't set CQE compression after ULP DDP NVMe-TCP offload\n");
+ return -EINVAL;
+ }
+
new_params = priv->channels.params;
MLX5E_SET_PFLAG(&new_params, MLX5E_PFLAG_RX_CQE_COMPRESS, new_val);
if (rx_filter)
@@ -2393,6 +2399,54 @@ static void mlx5e_get_rmon_stats(struct net_device *netdev,
mlx5e_stats_rmon_get(priv, rmon_stats, ranges);
}
+#ifdef CONFIG_MLX5_EN_NVMEOTCP
+static int mlx5e_set_ulp_ddp_capabilities(struct net_device *netdev, unsigned long *new_caps)
+{
+ struct mlx5e_priv *priv = netdev_priv(netdev);
+ DECLARE_BITMAP(old_caps, ULP_DDP_C_COUNT);
+ struct mlx5e_params *params;
+ int ret = 0;
+ int nvme = -1;
+
+ mutex_lock(&priv->state_lock);
+ params = &priv->channels.params;
+ bitmap_copy(old_caps, netdev->ulp_ddp_caps.active, ULP_DDP_C_COUNT);
+
+ /* always handle nvme-tcp-ddp and nvme-tcp-ddgst-rx together (all or nothing) */
+
+ if (ulp_ddp_cap_turned_on(old_caps, new_caps, ULP_DDP_C_NVME_TCP_BIT) &&
+ ulp_ddp_cap_turned_on(old_caps, new_caps, ULP_DDP_C_NVME_TCP_DDGST_RX_BIT)) {
+ if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
+ netdev_warn(netdev,
+ "NVMe-TCP offload not supported when CQE compress is active. Disable rx_cqe_compress ethtool private flag first\n");
+ goto out;
+ }
+
+ if (netdev->features & (NETIF_F_LRO | NETIF_F_GRO_HW)) {
+ netdev_warn(netdev,
+ "NVMe-TCP offload not supported when HW_GRO/LRO is active. Disable rx-gro-hw ethtool feature first\n");
+ goto out;
+ }
+ nvme = 1;
+ } else if (ulp_ddp_cap_turned_off(old_caps, new_caps, ULP_DDP_C_NVME_TCP_BIT) &&
+ ulp_ddp_cap_turned_off(old_caps, new_caps, ULP_DDP_C_NVME_TCP_DDGST_RX_BIT)) {
+ nvme = 0;
+ }
+
+ if (nvme >= 0) {
+ ret = set_ulp_ddp_nvme_tcp(netdev, nvme);
+ if (ret)
+ goto out;
+ change_bit(ULP_DDP_C_NVME_TCP_BIT, netdev->ulp_ddp_caps.active);
+ change_bit(ULP_DDP_C_NVME_TCP_DDGST_RX_BIT, netdev->ulp_ddp_caps.active);
+ }
+
+out:
+ mutex_unlock(&priv->state_lock);
+ return ret;
+}
+#endif
+
const struct ethtool_ops mlx5e_ethtool_ops = {
.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
ETHTOOL_COALESCE_MAX_FRAMES |
@@ -2445,5 +2499,8 @@ const struct ethtool_ops mlx5e_ethtool_ops = {
.get_eth_mac_stats = mlx5e_get_eth_mac_stats,
.get_eth_ctrl_stats = mlx5e_get_eth_ctrl_stats,
.get_rmon_stats = mlx5e_get_rmon_stats,
- .get_link_ext_stats = mlx5e_get_link_ext_stats
+ .get_link_ext_stats = mlx5e_get_link_ext_stats,
+#ifdef CONFIG_MLX5_EN_NVMEOTCP
+ .set_ulp_ddp_capabilities = mlx5e_set_ulp_ddp_capabilities,
+#endif
};
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index 7298fe782e9e..63df22d9ad99 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -62,7 +62,7 @@ struct mlx5e_flow_steering {
#ifdef CONFIG_MLX5_EN_ARFS
struct mlx5e_arfs_tables *arfs;
#endif
-#ifdef CONFIG_MLX5_EN_TLS
+#if defined(CONFIG_MLX5_EN_TLS) || defined(CONFIG_MLX5_EN_NVMEOTCP)
struct mlx5e_accel_fs_tcp *accel_tcp;
#endif
struct mlx5e_fs_udp *udp;
@@ -1560,7 +1560,7 @@ void mlx5e_fs_set_any(struct mlx5e_flow_steering *fs, struct mlx5e_fs_any *any)
fs->any = any;
}
-#ifdef CONFIG_MLX5_EN_TLS
+#if defined(CONFIG_MLX5_EN_TLS) || defined(CONFIG_MLX5_EN_NVMEOTCP)
struct mlx5e_accel_fs_tcp *mlx5e_fs_get_accel_tcp(struct mlx5e_flow_steering *fs)
{
return fs->accel_tcp;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 61a6b52d011f..55071a454db4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -49,6 +49,7 @@
#include "en_accel/macsec.h"
#include "en_accel/en_accel.h"
#include "en_accel/ktls.h"
+#include "en_accel/nvmeotcp.h"
#include "lib/vxlan.h"
#include "lib/clock.h"
#include "en/port.h"
@@ -4146,6 +4147,13 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
}
}
+ if (features & (NETIF_F_LRO | NETIF_F_GRO_HW)) {
+ if (params->nvmeotcp) {
+ netdev_warn(netdev, "Disabling HW-GRO/LRO, not supported after ULP DDP NVMe-TCP offload\n");
+ features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
+ }
+ }
+
if (mlx5e_is_uplink_rep(priv))
features = mlx5e_fix_uplink_rep_features(netdev, features);
@@ -4914,6 +4922,9 @@ const struct net_device_ops mlx5e_netdev_ops = {
.ndo_has_offload_stats = mlx5e_has_offload_stats,
.ndo_get_offload_stats = mlx5e_get_offload_stats,
#endif
+#ifdef CONFIG_MLX5_EN_NVMEOTCP
+ .ulp_ddp_ops = &mlx5e_nvmeotcp_ops,
+#endif
};
static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
@@ -5180,6 +5191,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
mlx5e_macsec_build_netdev(priv);
mlx5e_ipsec_build_netdev(priv);
mlx5e_ktls_build_netdev(priv);
+ mlx5e_nvmeotcp_build_netdev(priv);
}
void mlx5e_create_q_counters(struct mlx5e_priv *priv)
@@ -5246,13 +5258,19 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
if (err)
mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
+ err = mlx5e_nvmeotcp_init(priv);
+ if (err)
+ mlx5_core_err(mdev, "NVMEoTCP initialization failed, %d\n", err);
+
mlx5e_health_create_reporters(priv);
+
return 0;
}
static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
{
mlx5e_health_destroy_reporters(priv);
+ mlx5e_nvmeotcp_cleanup(priv);
mlx5e_ktls_cleanup(priv);
mlx5e_fs_cleanup(priv->fs);
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index df134f6d32dc..40c597d9a55d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -1555,6 +1555,7 @@ static const int types[] = {
MLX5_CAP_DEV_SHAMPO,
MLX5_CAP_MACSEC,
MLX5_CAP_ADV_VIRTUALIZATION,
+ MLX5_CAP_DEV_NVMEOTCP,
};
static void mlx5_hca_caps_free(struct mlx5_core_dev *dev)
--
2.31.1
More information about the Linux-nvme
mailing list