[RFC PATCH v1 2/9] iommu/arm-smmu-v3: Implement KHO preservation for STEs
Pranjal Shrivastava
praan at google.com
Tue Sep 29 00:19:43 PDT 2026
Introduce the foundation for SMMUv3 Live Update (LU) using the Kexec
Handover (KHO) framework. Implement the preserve and preserve_device
hooks to serialize SMMU & preserved masters' state for KHO, along with
the respective unpreserve hooks.
Since .preserve only runs for the first preserved device, preserve the
L2 Stream Tables in .preserve_device, recording their tokens by L1 index.
Signed-off-by: Pranjal Shrivastava <praan at google.com>
---
drivers/iommu/arm/arm-smmu-v3/Makefile | 1 +
.../arm/arm-smmu-v3/arm-smmu-v3-liveupdate.c | 277 ++++++++++++++++++
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 7 +
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 14 +
4 files changed, 299 insertions(+)
create mode 100644 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-liveupdate.c
diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile
index 2bc52473d960..57db7621972c 100644
--- a/drivers/iommu/arm/arm-smmu-v3/Makefile
+++ b/drivers/iommu/arm/arm-smmu-v3/Makefile
@@ -3,6 +3,7 @@ obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o
arm_smmu_v3-y := arm-smmu-v3.o
arm_smmu_v3-$(CONFIG_ARM_SMMU_V3_IOMMUFD) += arm-smmu-v3-iommufd.o
arm_smmu_v3-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o
+arm_smmu_v3-$(CONFIG_IOMMU_LIVEUPDATE) += arm-smmu-v3-liveupdate.o
arm_smmu_v3-$(CONFIG_ARM_SMMU_V3_KEXEC) += arm-smmu-v3-kexec.o
arm_smmu_v3-$(CONFIG_CRASH_DUMP) += arm-smmu-v3-kdump.o
arm_smmu_v3-$(CONFIG_TEGRA241_CMDQV) += tegra241-cmdqv.o
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-liveupdate.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-liveupdate.c
new file mode 100644
index 000000000000..515a266ac22e
--- /dev/null
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-liveupdate.c
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 Google LLC
+ * Author: Pranjal Shrivastava <praan at google.com>
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/iommu.h>
+#include <linux/iommu-liveupdate.h>
+#include <linux/kexec_handover.h>
+#include "arm-smmu-v3.h"
+
+#ifdef CONFIG_IOMMU_LIVEUPDATE
+static u64 *arm_smmu_l2_strtab_states(struct arm_smmu_device *smmu)
+{
+ struct iommu_hw_ser *iommu_ser = iommu_preserved_state(&smmu->iommu);
+
+ return phys_to_virt(iommu_ser->smmuv3.l2_strtab_lu_states_phys);
+}
+
+static bool arm_smmu_l2_strtab_in_use(struct arm_smmu_device *smmu, u32 idx)
+{
+ struct rb_node *node;
+
+ lockdep_assert_held(&smmu->streams_mutex);
+
+ for (node = rb_first(&smmu->streams); node; node = rb_next(node)) {
+ struct arm_smmu_stream *stream =
+ rb_entry(node, struct arm_smmu_stream, node);
+
+ if (stream->master->preserved &&
+ arm_smmu_strtab_l1_idx(stream->id) == idx)
+ return true;
+ }
+ return false;
+}
+
+/* Unpreserve the L2 tables of the first @num_streams, unless still in use */
+static void arm_smmu_unpreserve_l2_strtabs(struct arm_smmu_master *master,
+ unsigned int num_streams)
+{
+ struct arm_smmu_device *smmu = master->smmu;
+ struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
+ u64 *l2_states;
+ unsigned int i;
+
+ if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB))
+ return;
+
+ l2_states = arm_smmu_l2_strtab_states(smmu);
+
+ mutex_lock(&smmu->streams_mutex);
+ for (i = 0; i < num_streams; i++) {
+ u32 idx = arm_smmu_strtab_l1_idx(master->streams[i].id);
+ dma_addr_t l2_dma;
+
+ if (!l2_states[idx] || arm_smmu_l2_strtab_in_use(smmu, idx))
+ continue;
+
+ l2_dma = le64_to_cpu(cfg->l2.l1tab[idx].l2ptr) & STRTAB_L1_DESC_L2PTR_MASK;
+ dmam_unpreserve_coherent_allocation(smmu->dev, cfg->l2.l2ptrs[idx],
+ sizeof(struct arm_smmu_strtab_l2),
+ l2_dma, l2_states[idx]);
+ l2_states[idx] = 0;
+ }
+ mutex_unlock(&smmu->streams_mutex);
+}
+
+/* Preserve the L2 tables holding the STEs of @master */
+static int arm_smmu_preserve_l2_strtabs(struct arm_smmu_master *master)
+{
+ struct arm_smmu_device *smmu = master->smmu;
+ struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
+ u64 *l2_states;
+ unsigned int i;
+ int ret;
+
+ if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB))
+ return 0;
+
+ l2_states = arm_smmu_l2_strtab_states(smmu);
+
+ for (i = 0; i < master->num_streams; i++) {
+ u32 idx = arm_smmu_strtab_l1_idx(master->streams[i].id);
+ dma_addr_t l2_dma;
+
+ if (l2_states[idx])
+ continue;
+
+ l2_dma = le64_to_cpu(cfg->l2.l1tab[idx].l2ptr) & STRTAB_L1_DESC_L2PTR_MASK;
+ ret = dmam_preserve_coherent_allocation(smmu->dev,
+ cfg->l2.l2ptrs[idx],
+ sizeof(struct arm_smmu_strtab_l2),
+ l2_dma, &l2_states[idx]);
+ if (ret) {
+ arm_smmu_unpreserve_l2_strtabs(master, i);
+ return ret;
+ }
+ }
+ return 0;
+}
+
+int arm_smmu_preserve_device(struct device *dev,
+ struct iommu_device_ser *device_ser)
+{
+ struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+ struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+ struct iommu_domain_ser *domain_ser;
+ int ret;
+
+ /*
+ * We'd anyway configure abort STEs for non-preserved masters.
+ * TODO: Re-visit for identity once IOMMUFD noIOMMU is merged
+ */
+ if (domain->type == IOMMU_DOMAIN_IDENTITY ||
+ domain->type == IOMMU_DOMAIN_BLOCKED)
+ return 0;
+
+ if (domain->preserved_state) {
+ domain_ser = domain->preserved_state;
+ } else {
+ /* Fallback for kernel-managed domains */
+ ret = iommu_preserve_domain(domain, &domain_ser);
+ if (ret)
+ return ret;
+ }
+
+ /* Link this master to the preserved IOMMU domain in the ABI */
+ device_ser->domain_iommu_ser.domain_phys = virt_to_phys(domain_ser);
+
+ ret = arm_smmu_preserve_l2_strtabs(master);
+ if (ret)
+ return ret;
+
+ /* Mark the master as preserved to track state across disable */
+ master->preserved = true;
+
+ /*
+ * TODO: Preserve CD tables for Stage-1 domains here using
+ * dmam_preserve_allocation_attrs() on master->cd_table.
+ */
+
+ return 0;
+}
+
+void arm_smmu_unpreserve_device(struct device *dev,
+ struct iommu_device_ser *device_ser)
+{
+ struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+
+ if (!master->preserved)
+ return;
+
+ master->preserved = false;
+ arm_smmu_unpreserve_l2_strtabs(master, master->num_streams);
+}
+
+static int arm_smmu_preserve_strtab_2lvl(struct arm_smmu_device *smmu,
+ struct iommu_hw_ser *iommu_ser)
+{
+ struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
+ u32 l1size = cfg->l2.num_l1_ents * sizeof(struct arm_smmu_strtab_l1);
+ u64 *l2_states;
+ u64 state;
+ int ret;
+
+ /* Preserve the L1 Stream table */
+ ret = dmam_preserve_coherent_allocation(smmu->dev, cfg->l2.l1tab,
+ l1size, cfg->l2.l1_dma, &state);
+ if (ret) {
+ dev_err(smmu->dev, "L1 table preservation failed\n");
+ return ret;
+ }
+ iommu_ser->smmuv3.l1_strtab_lu_state = state;
+
+ /* The L2 tables are preserved along with the masters using them */
+ l2_states = kho_alloc_preserve(sizeof(*l2_states) * cfg->l2.num_l1_ents);
+ if (IS_ERR(l2_states)) {
+ dmam_unpreserve_coherent_allocation(smmu->dev, cfg->l2.l1tab,
+ l1size, cfg->l2.l1_dma, state);
+ return PTR_ERR(l2_states);
+ }
+
+ iommu_ser->smmuv3.l2_strtab_lu_states_phys = virt_to_phys(l2_states);
+ return 0;
+}
+
+static void arm_smmu_unpreserve_strtab_2lvl(struct arm_smmu_device *smmu,
+ struct iommu_hw_ser *iommu_ser)
+{
+ struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
+ u32 l1size = cfg->l2.num_l1_ents * sizeof(struct arm_smmu_strtab_l1);
+ u64 *l2_states = phys_to_virt(iommu_ser->smmuv3.l2_strtab_lu_states_phys);
+ u32 i;
+
+ for (i = 0; i < cfg->l2.num_l1_ents; i++) {
+ dma_addr_t l2_dma;
+
+ if (!l2_states[i])
+ continue;
+
+ l2_dma = le64_to_cpu(cfg->l2.l1tab[i].l2ptr) & STRTAB_L1_DESC_L2PTR_MASK;
+ dmam_unpreserve_coherent_allocation(smmu->dev, cfg->l2.l2ptrs[i],
+ sizeof(struct arm_smmu_strtab_l2),
+ l2_dma, l2_states[i]);
+ }
+ kho_unpreserve_free(l2_states);
+
+ dmam_unpreserve_coherent_allocation(smmu->dev, cfg->l2.l1tab, l1size,
+ cfg->l2.l1_dma,
+ iommu_ser->smmuv3.l1_strtab_lu_state);
+}
+
+static int arm_smmu_preserve_strtab_linear(struct arm_smmu_device *smmu,
+ struct iommu_hw_ser *iommu_ser)
+{
+ struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
+ u32 size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste);
+ u64 state;
+ int ret;
+
+ /* STRTAB BASE can't be changed hitlessly, preserve the whole table */
+ ret = dmam_preserve_coherent_allocation(smmu->dev, cfg->linear.table,
+ size, cfg->linear.ste_dma,
+ &state);
+ if (ret)
+ return ret;
+
+ iommu_ser->smmuv3.l1_strtab_lu_state = state;
+ iommu_ser->smmuv3.l2_strtab_lu_states_phys = 0;
+ return 0;
+}
+
+static void arm_smmu_unpreserve_strtab_linear(struct arm_smmu_device *smmu,
+ struct iommu_hw_ser *iommu_ser)
+{
+ struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
+ u32 size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste);
+
+ dmam_unpreserve_coherent_allocation(smmu->dev, cfg->linear.table, size,
+ cfg->linear.ste_dma,
+ iommu_ser->smmuv3.l1_strtab_lu_state);
+}
+
+int arm_smmu_preserve(struct iommu_device *iommu,
+ struct iommu_hw_ser *iommu_ser)
+{
+ struct arm_smmu_device *smmu =
+ container_of(iommu, struct arm_smmu_device, iommu);
+
+ /* Basic info */
+ iommu_ser->smmuv3.phys_addr = smmu->base_phys;
+ iommu_ser->token = smmu->base_phys;
+ iommu_ser->type = IOMMU_ARM_SMMUV3;
+ iommu_ser->smmuv3.strtab_base_cfg =
+ readl_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
+
+ /* We always implements 2-level when supported by HW */
+ if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
+ return arm_smmu_preserve_strtab_2lvl(smmu, iommu_ser);
+ else
+ return arm_smmu_preserve_strtab_linear(smmu, iommu_ser);
+}
+
+void arm_smmu_unpreserve(struct iommu_device *iommu,
+ struct iommu_hw_ser *iommu_ser)
+{
+ struct arm_smmu_device *smmu =
+ container_of(iommu, struct arm_smmu_device, iommu);
+
+ if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
+ arm_smmu_unpreserve_strtab_2lvl(smmu, iommu_ser);
+ else
+ arm_smmu_unpreserve_strtab_linear(smmu, iommu_ser);
+}
+
+#endif
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 670487613459..ae4a1c98228c 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -4548,6 +4548,12 @@ static const struct iommu_ops arm_smmu_ops = {
.def_domain_type = arm_smmu_def_domain_type,
.get_viommu_size = arm_smmu_get_viommu_size,
.viommu_init = arm_vsmmu_init,
+#ifdef CONFIG_IOMMU_LIVEUPDATE
+ .preserve = arm_smmu_preserve,
+ .unpreserve = arm_smmu_unpreserve,
+ .preserve_device = arm_smmu_preserve_device,
+ .unpreserve_device = arm_smmu_unpreserve_device,
+#endif
.user_pasid_table = 1,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
@@ -5807,6 +5813,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
return -EINVAL;
}
ioaddr = res->start;
+ smmu->base_phys = ioaddr;
/*
* Don't map the IMPLEMENTATION DEFINED regions, since they may contain
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 69455eced889..7bce5bbe75e5 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -952,6 +952,7 @@ struct arm_smmu_device {
void __iomem *base;
void __iomem *page1;
+ phys_addr_t base_phys;
#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
@@ -1078,6 +1079,8 @@ struct arm_smmu_master {
bool ste_ats_enabled : 1;
bool stall_enabled;
bool ats_always_on;
+ /* Preserved for a KHO Live Update */
+ bool preserved;
unsigned int ssid_bits;
unsigned int iopf_refcount;
};
@@ -1196,6 +1199,17 @@ extern struct mutex arm_smmu_asid_lock;
struct arm_smmu_domain *arm_smmu_domain_alloc(void);
+#ifdef CONFIG_IOMMU_LIVEUPDATE
+int arm_smmu_preserve_device(struct device *dev,
+ struct iommu_device_ser *device_ser);
+int arm_smmu_preserve(struct iommu_device *iommu,
+ struct iommu_hw_ser *iommu_ser);
+void arm_smmu_unpreserve_device(struct device *dev,
+ struct iommu_device_ser *device_ser);
+void arm_smmu_unpreserve(struct iommu_device *iommu,
+ struct iommu_hw_ser *iommu_ser);
+#endif
+
static inline void arm_smmu_domain_free(struct arm_smmu_domain *smmu_domain)
{
/* No concurrency with invalidation is possible at this point */
--
2.56.0.rc1.315.gc6ed9934b7-goog
More information about the kexec
mailing list