[PATCH v4 6/8] iommu/arm-smmu-v3: Populate the tlbi at the top of the call chain
Jason Gunthorpe
jgg at nvidia.com
Wed Aug 5 12:15:24 PDT 2026
Each of these has their own unique situation, populate the tlbi right
at the top and pass it into arm_smmu_domain_inv_range(). They will
diverge further when the iommupt invalidation scheme is introduced.
Reviewed-by: Nicolin Chen <nicolinc at nvidia.com>
Tested-by: Nicolin Chen <nicolinc at nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg at nvidia.com>
---
.../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 21 ++++----
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 54 ++++++++++---------
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 15 ++++--
3 files changed, 51 insertions(+), 39 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index 002e50d222ca47..20f3ca7f5472c6 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -140,16 +140,19 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
{
struct arm_smmu_domain *smmu_domain =
container_of(mn, struct arm_smmu_domain, mmu_notifier);
- size_t size;
+ struct arm_smmu_tlbi tlbi = {
+ .tgsz_lg2 = smmu_domain->tgsz_lg2,
+ .iova = start,
+ /*
+ * The mm_types defines vm_end as the first byte after the end
+ * address, different from IOMMU subsystem using the last
+ * address of an address range.
+ */
+ .size = end - start,
+ .iopte_size = PAGE_SIZE,
+ };
- /*
- * The mm_types defines vm_end as the first byte after the end address,
- * different from IOMMU subsystem using the last address of an address
- * range. So do a simple translation here by calculating size correctly.
- */
- size = end - start;
-
- arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false);
+ arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 55a47a5bdc184f..3c812df9d01473 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2729,25 +2729,13 @@ static void arm_smmu_domain_tlbi_inv(struct arm_smmu_tlbi *tlbi,
}
}
-void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
- unsigned long iova, size_t size,
- unsigned int granule, bool leaf)
+void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi,
+ struct arm_smmu_domain *smmu_domain)
{
- struct arm_smmu_tlbi tlbi = {
- .tgsz_lg2 = smmu_domain->tgsz_lg2,
- .iova = iova,
- .size = size,
- .iopte_size = granule,
- .leaf_only = leaf,
- };
struct arm_smmu_invs *invs;
- if (!size || size == SIZE_MAX) {
- tlbi.single.use_full_inv = true;
- tlbi.range.use_full_inv = true;
- } else {
- arm_smmu_tlbi_calc_single(&tlbi);
- }
+ if (!tlbi->single.use_full_inv)
+ arm_smmu_tlbi_calc_single(tlbi);
/*
* An invalidation request must follow some IOPTE change and then load
@@ -2766,7 +2754,7 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
*
* [CPU0] | [CPU1]
* change IOPTE on new domain: |
- * arm_smmu_domain_inv_range() { | arm_smmu_install_new_domain_invs()
+ * arm_smmu_domain_tlbi() { | arm_smmu_install_new_domain_invs()
* smp_mb(); // ensures IOPTE | arm_smmu_install_ste_for_dev {
* // seen by SMMU | dma_wmb(); // ensures invs update
* // load the updated invs | // before updating STE
@@ -2784,8 +2772,8 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
* this matches the instances used for invalidation.
*/
if (invs->has_range_inv) {
- if (!tlbi.range.use_full_inv)
- arm_smmu_tlbi_calc_range(&tlbi);
+ if (!tlbi->range.use_full_inv)
+ arm_smmu_tlbi_calc_range(tlbi);
}
/*
@@ -2796,10 +2784,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned long flags;
read_lock_irqsave(&invs->rwlock, flags);
- arm_smmu_domain_tlbi_inv(&tlbi, invs);
+ arm_smmu_domain_tlbi_inv(tlbi, invs);
read_unlock_irqrestore(&invs->rwlock, flags);
} else {
- arm_smmu_domain_tlbi_inv(&tlbi, invs);
+ arm_smmu_domain_tlbi_inv(tlbi, invs);
}
rcu_read_unlock();
@@ -2815,12 +2803,23 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
iommu_iotlb_gather_add_page(domain, gather, iova, granule);
}
+/*
+ * Called by io-pgtable-arm.c for each single table level it wants to remove.
+ * size is the size of the table level and granule is the tg in bytes. This must
+ * clear the walk cache and any leaves within the range.
+ */
static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
+ struct arm_smmu_tlbi tlbi = {
+ .tgsz_lg2 = smmu_domain->tgsz_lg2,
+ .iova = iova,
+ .size = size,
+ .iopte_size = 1 << smmu_domain->tgsz_lg2,
+ };
- arm_smmu_domain_inv_range(smmu_domain, iova, size, granule, false);
+ arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
static const struct iommu_flush_ops arm_smmu_flush_ops = {
@@ -4088,13 +4087,18 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct arm_smmu_tlbi tlbi = {
+ .tgsz_lg2 = smmu_domain->tgsz_lg2,
+ .iova = gather->start,
+ .size = gather->end - gather->start + 1,
+ .iopte_size = gather->pgsize,
+ .leaf_only = true,
+ };
if (!gather->pgsize)
return;
- arm_smmu_domain_inv_range(smmu_domain, gather->start,
- gather->end - gather->start + 1,
- gather->pgsize, true);
+ arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
static phys_addr_t
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 4f57cd58197b0a..187669b8e7e34a 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -1177,14 +1177,19 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
struct arm_smmu_cd *cd, struct iommu_domain *old);
-void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
- unsigned long iova, size_t size,
- unsigned int granule, bool leaf);
+void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi,
+ struct arm_smmu_domain *smmu_domain);
static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain)
{
- arm_smmu_domain_inv_range(smmu_domain, 0, 0, smmu_domain->tgsz_lg2,
- false);
+ /* Prefilled for invalidate all */
+ struct arm_smmu_tlbi tlbi = {
+ .tgsz_lg2 = smmu_domain->tgsz_lg2,
+ .single.use_full_inv = true,
+ .range.use_full_inv = true,
+ };
+
+ arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
--
2.43.0
More information about the linux-arm-kernel
mailing list