[PATCH 6/8] iommu/arm-smmu-v3: Populate the tlbi at the top of the call chain
Jason Gunthorpe
jgg at nvidia.com
Mon May 18 12:43:43 PDT 2026
Each of these has their own unique situation, populate the tlbi right
at the top and pass it into arm_smmu_domain_inv_range(). They will
diverge further when the iommupt invalidation scheme is introduced.
Signed-off-by: Jason Gunthorpe <jgg at nvidia.com>
---
.../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 21 ++++----
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 51 +++++++++----------
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 13 +++--
3 files changed, 45 insertions(+), 40 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index 4649717df151ba..c708fefb053771 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -129,16 +129,19 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
{
struct arm_smmu_domain *smmu_domain =
container_of(mn, struct arm_smmu_domain, mmu_notifier);
- size_t size;
+ struct arm_smmu_tlbi tlbi = {
+ .smmu_domain = smmu_domain,
+ .iova = start,
+ /*
+ * The mm_types defines vm_end as the first byte after the end
+ * address, different from IOMMU subsystem using the last
+ * address of an address range.
+ */
+ .size = end - start,
+ .iopte_granule = PAGE_SIZE,
+ };
- /*
- * The mm_types defines vm_end as the first byte after the end address,
- * different from IOMMU subsystem using the last address of an address
- * range. So do a simple translation here by calculating size correctly.
- */
- size = end - start;
-
- arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false);
+ arm_smmu_domain_tlbi(&tlbi);
}
static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 9aa08f782e8986..94f742de90330c 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2642,25 +2642,12 @@ static void arm_smmu_domain_tlbi_inv(struct arm_smmu_tlbi *tlbi,
}
}
-void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
- unsigned long iova, size_t size,
- unsigned int granule, bool leaf)
+void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi)
{
- struct arm_smmu_tlbi tlbi = {
- .smmu_domain = smmu_domain,
- .iova = iova,
- .size = size,
- .iopte_granule = granule,
- .leaf_only = leaf,
- };
struct arm_smmu_invs *invs;
- if (!size || size == SIZE_MAX) {
- tlbi.single.use_full_inv = true;
- tlbi.range.use_full_inv = true;
- } else {
- arm_smmu_tlbi_calc_single(&tlbi);
- }
+ if (!tlbi->single.use_full_inv)
+ arm_smmu_tlbi_calc_single(tlbi);
/*
* An invalidation request must follow some IOPTE change and then load
@@ -2690,14 +2677,14 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
smp_mb();
rcu_read_lock();
- invs = rcu_dereference(smmu_domain->invs);
+ invs = rcu_dereference(tlbi->smmu_domain->invs);
/* Only precaculate RIL if it will be used. */
if (invs->has_range_inv) {
- if (!tlbi.range.use_full_inv)
- arm_smmu_tlbi_calc_range(&tlbi);
+ if (!tlbi->range.use_full_inv)
+ arm_smmu_tlbi_calc_range(tlbi);
} else {
- tlbi.range.use_full_inv = true;
+ tlbi->range.use_full_inv = true;
}
/*
@@ -2708,10 +2695,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned long flags;
read_lock_irqsave(&invs->rwlock, flags);
- arm_smmu_domain_tlbi_inv(&tlbi, invs);
+ arm_smmu_domain_tlbi_inv(tlbi, invs);
read_unlock_irqrestore(&invs->rwlock, flags);
} else {
- arm_smmu_domain_tlbi_inv(&tlbi, invs);
+ arm_smmu_domain_tlbi_inv(tlbi, invs);
}
rcu_read_unlock();
@@ -2731,8 +2718,14 @@ static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
+ struct arm_smmu_tlbi tlbi = {
+ .smmu_domain = smmu_domain,
+ .iova = iova,
+ .size = size,
+ .iopte_granule = granule,
+ };
- arm_smmu_domain_inv_range(smmu_domain, iova, size, granule, false);
+ arm_smmu_domain_tlbi(&tlbi);
}
static const struct iommu_flush_ops arm_smmu_flush_ops = {
@@ -3994,14 +3987,18 @@ static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct arm_smmu_tlbi tlbi = {
+ .smmu_domain = to_smmu_domain(domain),
+ .iova = gather->start,
+ .size = gather->end - gather->start + 1,
+ .iopte_granule = gather->pgsize,
+ .leaf_only = true,
+ };
if (!gather->pgsize)
return;
- arm_smmu_domain_inv_range(smmu_domain, gather->start,
- gather->end - gather->start + 1,
- gather->pgsize, true);
+ arm_smmu_domain_tlbi(&tlbi);
}
static phys_addr_t
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index b5e214b428d644..95da62d64df171 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -1168,13 +1168,18 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
struct arm_smmu_cd *cd, struct iommu_domain *old);
-void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
- unsigned long iova, size_t size,
- unsigned int granule, bool leaf);
+void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi);
static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain)
{
- arm_smmu_domain_inv_range(smmu_domain, 0, 0, 0, false);
+ /* Prefilled for invalidate all */
+ struct arm_smmu_tlbi tlbi = {
+ .smmu_domain = smmu_domain,
+ .single.use_full_inv = true,
+ .range.use_full_inv = true,
+ };
+
+ arm_smmu_domain_tlbi(&tlbi);
}
void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
--
2.43.0
More information about the linux-arm-kernel
mailing list