[RFC-PATCH 1/2] nvme-multipath: delete gendisk under subsys lock

[Keith Busch]

From: Keith Busch <kbusch at kernel.org>

To maintain symmetry with the disk add side, perform the disk deletion
under the nvme subsystem lock. This will ensure consistent ordering of
device naming when adding namespaces while concurrently deleting stale
references.

The deferred removal requires special attention to the nvme_subsystem:
the head may be holding the final reference in the deferred path, so
take a reference on it while removing the head.

Signed-off-by: Keith Busch <kbusch at kernel.org>
---
 drivers/nvme/host/core.c      |  3 +--
 drivers/nvme/host/multipath.c | 28 +++++++++++-----------------
 drivers/nvme/host/nvme.h      |  1 +
 3 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index a5c1af443420b..7a558d5103a21 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -147,7 +147,6 @@ static const struct class nvme_ns_chr_class = {
 	.name = "nvme-generic",
 };
 
-static void nvme_put_subsystem(struct nvme_subsystem *subsys);
 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
 					   unsigned nsid);
 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
@@ -3119,7 +3118,7 @@ static void nvme_destroy_subsystem(struct kref *ref)
 	put_device(&subsys->dev);
 }
 
-static void nvme_put_subsystem(struct nvme_subsystem *subsys)
+void nvme_put_subsystem(struct nvme_subsystem *subsys)
 {
 	kref_put(&subsys->ref, nvme_destroy_subsystem);
 }
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index fc6800a9f7f94..02a50181d1dd6 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -682,6 +682,8 @@ static void nvme_requeue_work(struct work_struct *work)
 
 static void nvme_remove_head(struct nvme_ns_head *head)
 {
+	list_del_init(&head->entry);
+
 	if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
 		/*
 		 * requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared
@@ -700,16 +702,14 @@ static void nvme_remove_head_work(struct work_struct *work)
 {
 	struct nvme_ns_head *head = container_of(to_delayed_work(work),
 			struct nvme_ns_head, remove_work);
-	bool remove = false;
+	struct nvme_subsystem *subsys = head->subsys;
 
-	mutex_lock(&head->subsys->lock);
-	if (list_empty(&head->list)) {
-		list_del_init(&head->entry);
-		remove = true;
-	}
-	mutex_unlock(&head->subsys->lock);
-	if (remove)
+	kref_get(&subsys->ref);
+	mutex_lock(&subsys->lock);
+	if (list_empty(&head->list))
 		nvme_remove_head(head);
+	mutex_unlock(&subsys->lock);
+	nvme_put_subsystem(subsys);
 
 	module_put(THIS_MODULE);
 }
@@ -1292,8 +1292,6 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid)
 
 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 {
-	bool remove = false;
-
 	if (!head->disk)
 		return;
 
@@ -1314,17 +1312,13 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 	 * Ensure that no one could remove this module while the head
 	 * remove work is pending.
 	 */
-	if (head->delayed_removal_secs && try_module_get(THIS_MODULE)) {
+	if (head->delayed_removal_secs && try_module_get(THIS_MODULE))
 		mod_delayed_work(nvme_wq, &head->remove_work,
 				head->delayed_removal_secs * HZ);
-	} else {
-		list_del_init(&head->entry);
-		remove = true;
-	}
+	else
+		nvme_remove_head(head);
 out:
 	mutex_unlock(&head->subsys->lock);
-	if (remove)
-		nvme_remove_head(head);
 }
 
 void nvme_mpath_put_disk(struct nvme_ns_head *head)
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 44801801fc289..ed8ce356c363e 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -1283,6 +1283,7 @@ struct nvme_ctrl *nvme_ctrl_from_file(struct file *file);
 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
 bool nvme_get_ns(struct nvme_ns *ns);
 void nvme_put_ns(struct nvme_ns *ns);
+void nvme_put_subsystem(struct nvme_subsystem *subsys);
 
 static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)
 {
-- 
2.47.3