[PATCH v3 04/13] libmultipath: Add bio handling
John Garry
john.g.garry at oracle.com
Fri Jul 3 03:29:09 PDT 2026
Add support to submit a bio per-path. In addition, for failover, add
support to requeue a failed bio.
NVMe has almost like-for-like equivalents here:
- nvme_available_path() -> mpath_available_path()
- nvme_requeue_work() -> mpath_requeue_work()
- nvme_ns_head_submit_bio() -> mpath_bdev_submit_bio()
For failover, a driver may want to re-submit a bio, so add support to
clone a bio prior to submission.
A bio which is submitted to a per-path device has flag REQ_MPATH set,
same as what is done for NVMe with REQ_NVME_MPATH.
Signed-off-by: John Garry <john.g.garry at oracle.com>
---
include/linux/multipath.h | 22 ++++++++
lib/multipath.c | 108 +++++++++++++++++++++++++++++++++++++-
2 files changed, 128 insertions(+), 2 deletions(-)
diff --git a/include/linux/multipath.h b/include/linux/multipath.h
index 6c75654c12f8f..57d13eb1450cd 100644
--- a/include/linux/multipath.h
+++ b/include/linux/multipath.h
@@ -3,6 +3,7 @@
#define _LIBMULTIPATH_H
#include <linux/blkdev.h>
+#include <linux/blk-mq.h>
#include <linux/srcu.h>
extern const struct block_device_operations mpath_ops;
@@ -29,8 +30,10 @@ struct mpath_device {
};
struct mpath_head_template {
+ bool (*available_path)(struct mpath_device *);
bool (*is_disabled)(struct mpath_device *);
bool (*is_optimized)(struct mpath_device *);
+ struct bio *(*clone_bio)(struct bio *);
const struct attribute_group **device_groups;
};
@@ -44,6 +47,14 @@ struct mpath_head {
refcount_t refcount;
enum mpath_iopolicy_e *iopolicy;
+
+ struct bio_list requeue_list; /* list for requeing bio */
+ spinlock_t requeue_lock;
+ struct work_struct requeue_work; /* work struct for requeue */
+
+ atomic_long_t requeue_no_usable_path_cnt;
+ atomic_long_t fail_no_avail_path_cnt;
+
unsigned long flags;
struct gendisk *disk;
struct work_struct partition_scan_work;
@@ -53,6 +64,13 @@ struct mpath_head {
struct mpath_device __rcu *current_path[MAX_NUMNODES];
};
+#define REQ_MPATH REQ_DRV
+
+static inline bool is_mpath_request(struct request *req)
+{
+ return req->cmd_flags & REQ_MPATH;
+}
+
static inline struct mpath_head *mpath_bd_device_to_head(struct device *dev)
{
return dev_get_drvdata(dev);
@@ -96,4 +114,8 @@ static inline bool mpath_qd_iopolicy(enum mpath_iopolicy_e *iopolicy)
return READ_ONCE(*iopolicy) == MPATH_IOPOLICY_QD;
}
+static inline void mpath_schedule_requeue_work(struct mpath_head *mpath_head)
+{
+ kblockd_schedule_work(&mpath_head->requeue_work);
+}
#endif // _LIBMULTIPATH_H
diff --git a/lib/multipath.c b/lib/multipath.c
index 21f7ffdb22d60..81e737c1ce469 100644
--- a/lib/multipath.c
+++ b/lib/multipath.c
@@ -5,6 +5,7 @@
*/
#include <linux/module.h>
#include <linux/multipath.h>
+#include <trace/events/block.h>
static struct mpath_device *mpath_find_path(struct mpath_head *mpath_head);
@@ -43,7 +44,6 @@ int mpath_get_iopolicy(char *buf, int iopolicy)
}
EXPORT_SYMBOL_GPL(mpath_get_iopolicy);
-
void mpath_synchronize(struct mpath_head *mpath_head)
{
synchronize_srcu(&mpath_head->srcu);
@@ -228,7 +228,6 @@ static struct mpath_device *mpath_numa_path(struct mpath_head *mpath_head)
return mpath_device;
}
-__maybe_unused
static struct mpath_device *mpath_find_path(struct mpath_head *mpath_head)
{
enum mpath_iopolicy_e iopolicy = mpath_read_iopolicy(mpath_head);
@@ -243,6 +242,81 @@ static struct mpath_device *mpath_find_path(struct mpath_head *mpath_head)
}
}
+static bool mpath_available_path(struct mpath_head *mpath_head)
+{
+ struct mpath_device *mpath_device;
+
+ if (!test_bit(MPATH_HEAD_DISK_LIVE, &mpath_head->flags))
+ return false;
+
+ list_for_each_entry_srcu(mpath_device, &mpath_head->dev_list, siblings,
+ srcu_read_lock_held(&mpath_head->srcu)) {
+ if (mpath_head->mpdt->available_path(mpath_device))
+ return true;
+ }
+
+ return false;
+}
+
+static void mpath_bdev_submit_bio(struct bio *bio)
+{
+ struct mpath_head *mpath_head = bio->bi_bdev->bd_disk->private_data;
+ struct device *dev = mpath_head->parent;
+ struct mpath_device *mpath_device;
+ int srcu_idx;
+
+ /*
+ * The mpath_device might be going away and the bio might be moved to a
+ * different queue in failover, so we need to use the bio_split
+ * pool from the original queue to allocate the bvecs from.
+ */
+ bio = bio_split_to_limits(bio);
+ if (!bio)
+ return;
+
+ srcu_idx = srcu_read_lock(&mpath_head->srcu);
+ mpath_device = mpath_find_path(mpath_head);
+
+ if (likely(mpath_device)) {
+ if (mpath_head->mpdt->clone_bio) {
+ struct bio *orig = bio;
+
+ bio = mpath_head->mpdt->clone_bio(bio);
+ if (!bio) {
+ bio_io_error(orig);
+ goto out;
+ }
+ }
+
+ bio_set_dev(bio, mpath_device->disk->part0);
+ /*
+ * Use BIO_REMAPPED to skip bio_check_eod() when this bio
+ * enters submit_bio_noacct() for the per-path device. The EOD
+ * check already passed on the multipath head.
+ */
+ bio_set_flag(bio, BIO_REMAPPED);
+ bio->bi_opf |= REQ_MPATH;
+ trace_block_bio_remap(bio, disk_devt(mpath_device->disk),
+ bio->bi_iter.bi_sector);
+ submit_bio_noacct(bio);
+ } else if (mpath_available_path(mpath_head)) {
+ dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
+
+ spin_lock_irq(&mpath_head->requeue_lock);
+ bio_list_add(&mpath_head->requeue_list, bio);
+ spin_unlock_irq(&mpath_head->requeue_lock);
+ atomic_long_inc(&mpath_head->requeue_no_usable_path_cnt);
+ } else {
+ dev_warn_ratelimited(dev, "no available path - failing I/O\n");
+
+ bio_io_error(bio);
+ atomic_long_inc(&mpath_head->fail_no_avail_path_cnt);
+ }
+
+out:
+ srcu_read_unlock(&mpath_head->srcu, srcu_idx);
+}
+
int mpath_get_head(struct mpath_head *mpath_head)
{
if (!refcount_inc_not_zero(&mpath_head->refcount))
@@ -297,6 +371,7 @@ const struct block_device_operations mpath_ops = {
.owner = THIS_MODULE,
.open = mpath_bdev_open,
.release = mpath_bdev_release,
+ .submit_bio = mpath_bdev_submit_bio,
};
EXPORT_SYMBOL_GPL(mpath_ops);
@@ -314,11 +389,34 @@ static void multipath_partition_scan_work(struct work_struct *work)
mutex_unlock(&mpath_head->disk->open_mutex);
}
+static void mpath_requeue_work(struct work_struct *work)
+{
+ struct mpath_head *mpath_head =
+ container_of(work, struct mpath_head, requeue_work);
+ struct bio *bio, *next;
+
+ spin_lock_irq(&mpath_head->requeue_lock);
+ next = bio_list_get(&mpath_head->requeue_list);
+ spin_unlock_irq(&mpath_head->requeue_lock);
+
+ while ((bio = next) != NULL) {
+ next = bio->bi_next;
+ bio->bi_next = NULL;
+ submit_bio_noacct(bio);
+ }
+}
+
void mpath_remove_disk(struct mpath_head *mpath_head)
{
if (test_and_clear_bit(MPATH_HEAD_DISK_LIVE, &mpath_head->flags)) {
struct gendisk *disk = mpath_head->disk;
+ /*
+ * requeue I/O after MPATH_HEAD_DISK_LIVE has been cleared
+ * to allow multipath to fail all I/O.
+ */
+ mpath_schedule_requeue_work(mpath_head);
+
mpath_synchronize(mpath_head);
del_gendisk(disk);
}
@@ -331,6 +429,8 @@ void mpath_put_disk(struct mpath_head *mpath_head)
return;
/* make sure all pending bios are cleaned up */
+ kblockd_schedule_work(&mpath_head->requeue_work);
+ flush_work(&mpath_head->requeue_work);
flush_work(&mpath_head->partition_scan_work);
put_disk(mpath_head->disk);
}
@@ -387,6 +487,7 @@ void mpath_device_set_live(struct mpath_device *mpath_device)
mutex_unlock(&mpath_head->lock);
mpath_synchronize(mpath_head);
+ mpath_schedule_requeue_work(mpath_head);
}
EXPORT_SYMBOL_GPL(mpath_device_set_live);
@@ -398,6 +499,9 @@ int mpath_head_init(struct mpath_head *mpath_head)
INIT_WORK(&mpath_head->partition_scan_work,
multipath_partition_scan_work);
+ INIT_WORK(&mpath_head->requeue_work, mpath_requeue_work);
+ spin_lock_init(&mpath_head->requeue_lock);
+ bio_list_init(&mpath_head->requeue_list);
return init_srcu_struct(&mpath_head->srcu);
}
--
2.43.7
More information about the Linux-nvme
mailing list