[PATCH 3/3] nvme: improve the quiesce time for blocking transports

[Chao Leng]

When work with nvme multipathing, if one path failed, the requests will
be canceled and retry on another path. Before cancel the requests,
nvme_stop_queues quiesce queues for all namespaces, now quiesce one by
one, if there is a large set of namespaces, it will take long time.
Because every synchronize_srcu may need more than 10 milliseconds,
the total waiting time will be long.
Quiesce all queues in parallel and start syncing for srcus of all
queues, and then wait for all srcus to complete synchronization.
The quiesce time and I/O fail over time can be largely reduced.

Signed-off-by: Chao Leng <lengchao at huawei.com>
---
 drivers/nvme/host/core.c | 38 ++++++++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index fcfa27e1078a..937d5802d41b 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -21,6 +21,7 @@
 #include <linux/nvme_ioctl.h>
 #include <linux/pm_qos.h>
 #include <asm/unaligned.h>
+#include <linux/rcupdate_wait.h>
 
 #include "nvme.h"
 #include "fabrics.h"
@@ -4916,14 +4917,6 @@ static void nvme_start_ns_queue(struct nvme_ns *ns)
 		blk_mq_unquiesce_queue(ns->queue);
 }
 
-static void nvme_stop_ns_queue(struct nvme_ns *ns)
-{
-	if (!test_and_set_bit(NVME_NS_STOPPED, &ns->flags))
-		blk_mq_quiesce_queue(ns->queue);
-	else
-		blk_mq_wait_quiesce_done(ns->queue);
-}
-
 static void nvme_stop_ns_queue_nosync(struct nvme_ns *ns)
 {
 	if (!test_and_set_bit(NVME_NS_STOPPED, &ns->flags))
@@ -4933,9 +4926,34 @@ static void nvme_stop_ns_queue_nosync(struct nvme_ns *ns)
 static void nvme_stop_blocking_queues(struct nvme_ctrl *ctrl)
 {
 	struct nvme_ns *ns;
+	int i, count;
+	struct rcu_synchronize *rcu;
 
-	list_for_each_entry(ns, &ctrl->namespaces, list)
-		nvme_stop_ns_queue(ns);
+	count = 0;
+	list_for_each_entry(ns, &ctrl->namespaces, list) {
+		nvme_stop_ns_queue_nosync(ns);
+		count++;
+	}
+
+	rcu = kvmalloc(count * sizeof(*rcu), GFP_KERNEL);
+	if (rcu) {
+		i = 0;
+		list_for_each_entry(ns, &ctrl->namespaces, list) {
+			init_rcu_head(&rcu[i].head);
+			init_completion(&rcu[i].completion);
+			call_srcu(ns->queue->srcu, &rcu[i].head, wakeme_after_rcu);
+			i++;
+		}
+
+		for (i = 0; i < count; i++) {
+			wait_for_completion(&rcu[i].completion);
+			destroy_rcu_head(&rcu[i].head);
+		}
+		kvfree(rcu);
+	} else {
+		list_for_each_entry(ns, &ctrl->namespaces, list)
+			blk_mq_wait_quiesce_done(ns->queue);
+	}
 }
 
 static void nvme_stop_nonblocking_queues(struct nvme_ctrl *ctrl)
-- 
2.16.4