[PATCH 1/3] nvme: check IO queues liveness during controller reconnection

[Jiewei Ke]

During the controller reconnection process, nvme_unquiesce_io_queues
temporarily allows IO submissions. If any IO times out,
nvme_[tcp|rdma]_complete_timed_out may close an individual IO queue. Due
to timing gaps, nvme_wait_freeze_timeout might not yet have timed out,
causing the controller to continue operating with some closed IO queues.
This issue can be reproduced by injecting high IO latency and resetting
the controller.

Add a check after nvme_wait_freeze_timeout to verify whether all IO
queues are in the LIVE state. If any IO queue is not LIVE, the
controller initialization will fail, and the controller will be cleaned
up accordingly.

Signed-off-by: Jiewei Ke <jiewei at smartx.com>
---
 drivers/nvme/host/rdma.c | 24 +++++++++++++++++++++++-
 drivers/nvme/host/tcp.c  | 24 +++++++++++++++++++++++-
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index b5a0295b5bf4..4297507725cf 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -861,6 +861,23 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
 	return error;
 }
 
+static bool nvme_rdma_check_io_queues_liveness(struct nvme_rdma_ctrl *ctrl,
+					       int first, int last)
+{
+	int i;
+	struct nvme_rdma_queue *queue;
+
+	for (i = first; i < last; i++) {
+		queue = &ctrl->queues[i];
+		if (!test_bit(NVME_RDMA_Q_LIVE, &queue->flags)) {
+			dev_warn(ctrl->ctrl.device, "I/O queue %d not live", i);
+			return false;
+		}
+	}
+
+	return true;
+}
+
 static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
 {
 	int ret, nr_queues;
@@ -888,11 +905,16 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
 	if (!new) {
 		nvme_start_freeze(&ctrl->ctrl);
 		nvme_unquiesce_io_queues(&ctrl->ctrl);
-		if (!nvme_wait_freeze_timeout(&ctrl->ctrl, NVME_IO_TIMEOUT)) {
+		if (!nvme_wait_freeze_timeout(&ctrl->ctrl, NVME_IO_TIMEOUT) ||
+		    !nvme_rdma_check_io_queues_liveness(ctrl, 1, nr_queues)) {
 			/*
 			 * If we timed out waiting for freeze we are likely to
 			 * be stuck.  Fail the controller initialization just
 			 * to be safe.
+			 * If any IO timed out during waiting for freeze, some IO
+			 * queue may be stopped by nvme_rdma_complete_timed_out.
+			 * Fail the controller initialization to avoid running
+			 * with missing IO queues.
 			 */
 			ret = -ENODEV;
 			nvme_unfreeze(&ctrl->ctrl);
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 26c459f0198d..9109d5476417 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -2152,6 +2152,23 @@ static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
 	return __nvme_tcp_alloc_io_queues(ctrl);
 }
 
+static bool nvme_tcp_check_io_queues_liveness(struct nvme_tcp_ctrl *ctrl,
+					      int first, int last)
+{
+	int i;
+	struct nvme_tcp_queue *queue;
+
+	for (i = first; i < last; i++) {
+		queue = &ctrl->queues[i];
+		if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) {
+			dev_warn(ctrl->ctrl.device, "I/O queue %d not live", i);
+			return false;
+		}
+	}
+
+	return true;
+}
+
 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
 {
 	int ret, nr_queues;
@@ -2182,11 +2199,16 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
 	if (!new) {
 		nvme_start_freeze(ctrl);
 		nvme_unquiesce_io_queues(ctrl);
-		if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
+		if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT) ||
+		    !nvme_tcp_check_io_queues_liveness(to_tcp_ctrl(ctrl), 1, nr_queues)) {
 			/*
 			 * If we timed out waiting for freeze we are likely to
 			 * be stuck.  Fail the controller initialization just
 			 * to be safe.
+			 * If any IO timed out during waiting for freeze, some IO
+			 * queue may be stopped by nvme_tcp_complete_timed_out.
+			 * Fail the controller initialization to avoid running
+			 * with missing IO queues.
 			 */
 			ret = -ENODEV;
 			nvme_unfreeze(ctrl);
-- 
2.36.0