[PATCH v2 04/13] libmultipath: Add bio handling
John Garry
john.g.garry at oracle.com
Tue Apr 28 04:10:56 PDT 2026
Add support to submit a bio per-path. In addition, for failover, add
support to requeue a failed bio.
NVMe has almost like-for-like equivalents here:
- nvme_available_path() -> mpath_available_path()
- nvme_requeue_work() -> mpath_requeue_work()
- nvme_ns_head_submit_bio() -> mpath_bdev_submit_bio()
For failover, a driver may want to re-submit a bio, so add support to
clone a bio prior to submission.
A bio which is submitted to a per-path device has flag REQ_MPATH set,
same as what is done for NVMe with REQ_NVME_MPATH.
Signed-off-by: John Garry <john.g.garry at oracle.com>
---
include/linux/multipath.h | 18 +++++++
lib/multipath.c | 100 +++++++++++++++++++++++++++++++++++++-
2 files changed, 116 insertions(+), 2 deletions(-)
diff --git a/include/linux/multipath.h b/include/linux/multipath.h
index 13d810148a96a..2a5a9236480f7 100644
--- a/include/linux/multipath.h
+++ b/include/linux/multipath.h
@@ -3,6 +3,7 @@
#define _LIBMULTIPATH_H
#include <linux/blkdev.h>
+#include <linux/blk-mq.h>
#include <linux/srcu.h>
extern const struct block_device_operations mpath_ops;
@@ -32,10 +33,12 @@ struct mpath_device {
};
struct mpath_head_template {
+ bool (*available_path)(struct mpath_device *);
bool (*is_disabled)(struct mpath_device *);
bool (*is_optimized)(struct mpath_device *);
int (*get_nr_active)(struct mpath_device *);
enum mpath_iopolicy_e (*get_iopolicy)(struct mpath_head *);
+ struct bio *(*clone_bio)(struct bio *);
const struct attribute_group **device_groups;
};
@@ -48,6 +51,10 @@ struct mpath_head {
struct kref ref;
+ struct bio_list requeue_list; /* list for requeing bio */
+ spinlock_t requeue_lock;
+ struct work_struct requeue_work; /* work struct for requeue */
+
void *drvdata;
unsigned long flags;
struct gendisk *disk;
@@ -58,6 +65,13 @@ struct mpath_head {
struct mpath_device __rcu *current_path[];
};
+#define REQ_MPATH REQ_DRV
+
+static inline bool is_mpath_request(struct request *req)
+{
+ return req->cmd_flags & REQ_MPATH;
+}
+
static inline struct mpath_head *mpath_bd_device_to_head(struct device *dev)
{
return dev_get_drvdata(dev);
@@ -100,4 +114,8 @@ static inline bool mpath_qd_iopolicy(struct mpath_iopolicy *mpath_iopolicy)
return mpath_read_iopolicy(mpath_iopolicy) == MPATH_IOPOLICY_QD;
}
+static inline void mpath_schedule_requeue_work(struct mpath_head *mpath_head)
+{
+ kblockd_schedule_work(&mpath_head->requeue_work);
+}
#endif // _LIBMULTIPATH_H
diff --git a/lib/multipath.c b/lib/multipath.c
index fa211420b72c3..eabf1347d9acc 100644
--- a/lib/multipath.c
+++ b/lib/multipath.c
@@ -5,6 +5,7 @@
*/
#include <linux/module.h>
#include <linux/multipath.h>
+#include <trace/events/block.h>
static struct mpath_device *mpath_find_path(struct mpath_head *mpath_head);
@@ -39,7 +40,6 @@ int mpath_get_iopolicy(char *buf, int iopolicy)
}
EXPORT_SYMBOL_GPL(mpath_get_iopolicy);
-
void mpath_synchronize(struct mpath_head *mpath_head)
{
synchronize_srcu(&mpath_head->srcu);
@@ -226,7 +226,6 @@ static struct mpath_device *mpath_numa_path(struct mpath_head *mpath_head)
return mpath_device;
}
-__maybe_unused
static struct mpath_device *mpath_find_path(struct mpath_head *mpath_head)
{
enum mpath_iopolicy_e iopolicy =
@@ -242,6 +241,73 @@ static struct mpath_device *mpath_find_path(struct mpath_head *mpath_head)
}
}
+static bool mpath_available_path(struct mpath_head *mpath_head)
+{
+ struct mpath_device *mpath_device;
+
+ if (!test_bit(MPATH_HEAD_DISK_LIVE, &mpath_head->flags))
+ return false;
+
+ list_for_each_entry_srcu(mpath_device, &mpath_head->dev_list, siblings,
+ srcu_read_lock_held(&mpath_head->srcu)) {
+ if (mpath_head->mpdt->available_path(mpath_device))
+ return true;
+ }
+
+ return false;
+}
+
+static void mpath_bdev_submit_bio(struct bio *bio)
+{
+ struct mpath_head *mpath_head = bio->bi_bdev->bd_disk->private_data;
+ struct device *dev = mpath_head->parent;
+ struct mpath_device *mpath_device;
+ int srcu_idx;
+
+ /*
+ * The mpath_devuce might be going away and the bio might be moved to a
+ * different queue in failover, so we need to use the bio_split
+ * pool from the original queue to allocate the bvecs from.
+ */
+ bio = bio_split_to_limits(bio);
+ if (!bio)
+ return;
+
+ srcu_idx = srcu_read_lock(&mpath_head->srcu);
+ mpath_device = mpath_find_path(mpath_head);
+
+ if (likely(mpath_device)) {
+ if (mpath_head->mpdt->clone_bio) {
+ struct bio *orig = bio;
+
+ bio = mpath_head->mpdt->clone_bio(bio);
+ if (!bio) {
+ bio_io_error(orig);
+ goto out;
+ }
+ }
+ trace_block_bio_remap(bio, disk_devt(mpath_device->disk),
+ bio->bi_iter.bi_sector);
+ bio_set_dev(bio, mpath_device->disk->part0);
+ bio->bi_opf |= REQ_MPATH;
+
+ submit_bio_noacct(bio);
+ } else if (mpath_available_path(mpath_head)) {
+ dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
+
+ spin_lock_irq(&mpath_head->requeue_lock);
+ bio_list_add(&mpath_head->requeue_list, bio);
+ spin_unlock_irq(&mpath_head->requeue_lock);
+ } else {
+ dev_warn_ratelimited(dev, "no available path - failing I/O\n");
+
+ bio_io_error(bio);
+ }
+
+out:
+ srcu_read_unlock(&mpath_head->srcu, srcu_idx);
+}
+
static void mpath_free_head(struct kref *ref)
{
struct mpath_head *mpath_head =
@@ -283,6 +349,7 @@ const struct block_device_operations mpath_ops = {
.owner = THIS_MODULE,
.open = mpath_bdev_open,
.release = mpath_bdev_release,
+ .submit_bio = mpath_bdev_submit_bio,
};
EXPORT_SYMBOL_GPL(mpath_ops);
@@ -300,11 +367,34 @@ static void multipath_partition_scan_work(struct work_struct *work)
mutex_unlock(&mpath_head->disk->open_mutex);
}
+static void mpath_requeue_work(struct work_struct *work)
+{
+ struct mpath_head *mpath_head =
+ container_of(work, struct mpath_head, requeue_work);
+ struct bio *bio, *next;
+
+ spin_lock_irq(&mpath_head->requeue_lock);
+ next = bio_list_get(&mpath_head->requeue_list);
+ spin_unlock_irq(&mpath_head->requeue_lock);
+
+ while ((bio = next) != NULL) {
+ next = bio->bi_next;
+ bio->bi_next = NULL;
+ submit_bio_noacct(bio);
+ }
+}
+
void mpath_remove_disk(struct mpath_head *mpath_head)
{
if (test_and_clear_bit(MPATH_HEAD_DISK_LIVE, &mpath_head->flags)) {
struct gendisk *disk = mpath_head->disk;
+ /*
+ * requeue I/O after MPATH_HEAD_DISK_LIVE has been cleared
+ * to allow multipath to fail all I/O.
+ */
+ mpath_schedule_requeue_work(mpath_head);
+
mpath_synchronize(mpath_head);
del_gendisk(disk);
}
@@ -317,6 +407,8 @@ void mpath_put_disk(struct mpath_head *mpath_head)
return;
/* make sure all pending bios are cleaned up */
+ kblockd_schedule_work(&mpath_head->requeue_work);
+ flush_work(&mpath_head->requeue_work);
flush_work(&mpath_head->partition_scan_work);
put_disk(mpath_head->disk);
}
@@ -369,6 +461,7 @@ void mpath_device_set_live(struct mpath_device *mpath_device)
mutex_unlock(&mpath_head->lock);
mpath_synchronize(mpath_head);
+ mpath_schedule_requeue_work(mpath_head);
}
EXPORT_SYMBOL_GPL(mpath_device_set_live);
@@ -387,6 +480,9 @@ struct mpath_head *mpath_alloc_head(void)
INIT_WORK(&mpath_head->partition_scan_work,
multipath_partition_scan_work);
+ INIT_WORK(&mpath_head->requeue_work, mpath_requeue_work);
+ spin_lock_init(&mpath_head->requeue_lock);
+ bio_list_init(&mpath_head->requeue_list);
ret = init_srcu_struct(&mpath_head->srcu);
if (ret) {
--
2.43.5
More information about the Linux-nvme
mailing list