[PATCH v5 2/2] nvme-multipath: add fail_on_ctrl_loss sysfs attribute
Krishna Iyer
kiyer at crusoe.ai
Thu Oct 8 07:14:07 PDT 2026
When no usable path exists, I/O on a multipath namespace is queued
until a path returns. With ctrl_loss_tmo=-1 that can be forever:
during a long fabric outage any process waiting on the I/O is stuck in
D state. We hit this on virtualization hosts, where a SIGKILLed VM
process cannot exit while draining I/O to an unreachable NVMe/TCP
target.
Nothing can fail this I/O without tearing something down: controller
deletion takes every namespace on the controller with it.
Add a fail_on_ctrl_loss attribute on the ns-head disk: a persistent
per-namespace policy to fail parked and newly arriving I/O instead of
queueing it when no usable path exists. It is the namespace-scoped
counterpart to the controller-scoped fast_io_fail_tmo: the trigger is
an event userspace observes (a consumer known to be gone, for us a
SIGKILLed VM the host must reap), not a duration picked up front, and
sibling namespaces behind the same controllers keep queueing and ride
out the outage.
A CONNECTING controller stops counting as an available path, and with
no controllers left the policy overrides the delayed_removal_secs
queue-if-no-path window; the two are opposites, so fail_on_ctrl_loss
takes precedence when both are set. ANA change and controller resetting
still queue, since both are transient. Controller state is untouched and
reconnects continue. Like dm-multipath, the policy is transport
agnostic.
Assisted-by: Claude:claude-fable-5
Signed-off-by: Krishna Iyer <kiyer at crusoe.ai>
---
Documentation/ABI/stable/sysfs-nvme | 16 +++++++++
drivers/nvme/host/multipath.c | 50 +++++++++++++++++++++++++++--
drivers/nvme/host/nvme.h | 2 ++
drivers/nvme/host/sysfs.c | 4 ++-
4 files changed, 69 insertions(+), 3 deletions(-)
diff --git a/Documentation/ABI/stable/sysfs-nvme b/Documentation/ABI/stable/sysfs-nvme
index a0bb88ca1694..20f398b1bcb1 100644
--- a/Documentation/ABI/stable/sysfs-nvme
+++ b/Documentation/ABI/stable/sysfs-nvme
@@ -358,6 +358,22 @@ Description:
is deferred. Only visible on multipath head devices.
Requires CONFIG_NVME_MULTIPATH.
+What: /sys/block/nvmeXnY/fail_on_ctrl_loss
+Date: October 2026
+KernelVersion: 7.4
+Contact: Krishna Iyer <kiyer at crusoe.ai>
+Description:
+ Shows or sets the fail-on-ctrl-loss policy of the multipath
+ head device ("on" or "off", default "off"). When on, I/O
+ queued or arriving while no usable path exists is failed
+ immediately instead of being queued. This is the opposite of
+ the queue-if-no-path behavior enabled by a nonzero
+ delayed_removal_secs: when both are set fail_on_ctrl_loss takes
+ precedence and the parked I/O is failed rather than held for
+ the delayed_removal_secs window. Reconnect attempts are not
+ affected. Only visible on multipath head devices.
+ Requires CONFIG_NVME_MULTIPATH.
+
What: /sys/block/nvmeXnY/csi
What: /sys/block/nvmeXnY/metadata_bytes
What: /sys/block/nvmeXnY/nuse
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 0d6c05f0808b..7e5b50f158cf 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -517,6 +517,8 @@ static inline bool nvme_state_is_live(enum nvme_ana_state state)
static bool nvme_available_path(struct nvme_ns_head *head)
__must_hold_shared(&head->srcu)
{
+ bool fail_on_ctrl_loss = test_bit(NVME_NSHEAD_FAIL_ON_CTRL_LOSS,
+ &head->flags);
bool failfast = false;
struct nvme_ns *ns;
@@ -536,14 +538,17 @@ static bool nvme_available_path(struct nvme_ns_head *head)
continue;
return true;
case NVME_CTRL_RESETTING:
- case NVME_CTRL_CONNECTING:
return true;
+ case NVME_CTRL_CONNECTING:
+ if (!fail_on_ctrl_loss)
+ return true;
+ continue;
default:
break;
}
}
- if (failfast)
+ if (failfast || fail_on_ctrl_loss)
return false;
/*
@@ -1242,6 +1247,47 @@ static ssize_t delayed_removal_secs_store(struct device *dev,
DEVICE_ATTR_RW(delayed_removal_secs);
+static ssize_t fail_on_ctrl_loss_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct gendisk *disk = dev_to_disk(dev);
+ struct nvme_ns_head *head = disk->private_data;
+
+ return sysfs_emit(buf, test_bit(NVME_NSHEAD_FAIL_ON_CTRL_LOSS,
+ &head->flags) ? "on\n" : "off\n");
+}
+
+static ssize_t fail_on_ctrl_loss_store(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct gendisk *disk = dev_to_disk(dev);
+ struct nvme_ns_head *head = disk->private_data;
+ bool enable;
+ int ret;
+
+ ret = kstrtobool(buf, &enable);
+ if (ret < 0)
+ return ret;
+
+ if (enable) {
+ if (test_and_set_bit(NVME_NSHEAD_FAIL_ON_CTRL_LOSS, &head->flags))
+ return count;
+ } else {
+ if (!test_and_clear_bit(NVME_NSHEAD_FAIL_ON_CTRL_LOSS,
+ &head->flags))
+ return count;
+ }
+
+ synchronize_srcu(&head->srcu);
+
+ if (enable)
+ kblockd_schedule_work(&head->requeue_work);
+
+ return count;
+}
+
+DEVICE_ATTR_RW(fail_on_ctrl_loss);
+
static ssize_t multipath_failover_count_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index bac25b287d25..a8ef62b8e34b 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -589,6 +589,7 @@ struct nvme_ns_head {
#define NVME_NSHEAD_DISK_LIVE 0
#define NVME_NSHEAD_QUEUE_IF_NO_PATH 1
#define NVME_NSHEAD_CDEV_LIVE 2
+#define NVME_NSHEAD_FAIL_ON_CTRL_LOSS 3
struct nvme_ns __rcu_guarded *current_path[];
#endif
};
@@ -1097,6 +1098,7 @@ extern struct device_attribute dev_attr_queue_depth;
extern struct device_attribute dev_attr_numa_nodes;
extern struct device_attribute dev_attr_path_state;
extern struct device_attribute dev_attr_delayed_removal_secs;
+extern struct device_attribute dev_attr_fail_on_ctrl_loss;
extern struct device_attribute dev_attr_multipath_failover_count;
extern struct device_attribute dev_attr_io_requeue_no_usable_path_count;
extern struct device_attribute dev_attr_io_fail_no_available_path_count;
diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index 4ac3ea3a86fd..b3abb852a3ec 100644
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -265,6 +265,7 @@ static struct attribute *nvme_ns_attrs[] = {
&dev_attr_numa_nodes.attr,
&dev_attr_path_state.attr,
&dev_attr_delayed_removal_secs.attr,
+ &dev_attr_fail_on_ctrl_loss.attr,
#endif
&dev_attr_io_passthru_err_log_enabled.attr,
NULL,
@@ -302,7 +303,8 @@ static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
if (nvme_disk_is_ns_head(dev_to_disk(dev)))
return 0;
}
- if (a == &dev_attr_delayed_removal_secs.attr) {
+ if (a == &dev_attr_delayed_removal_secs.attr ||
+ a == &dev_attr_fail_on_ctrl_loss.attr) {
struct gendisk *disk = dev_to_disk(dev);
if (!nvme_disk_is_ns_head(disk))
--
2.54.0
More information about the Linux-nvme
mailing list