[PATCH v5 8/9] iommu/arm-smmu-v3: Change how the tlbi describes the invalidation
Jason Gunthorpe
jgg at nvidia.com
Wed Sep 9 08:57:21 PDT 2026
On Mon, Sep 07, 2026 at 12:34:29PM -0300, Jason Gunthorpe wrote:
> > > + tlbi.leaf_levels_bitmap = BIT((ilog2(gather->pgsize) - tg) / (tg - 3));
> >
> > Having some page table macros would be helpful (and in other places in
> > this patch)
>
> At least this one gets deleted in the next series, so I left it like
> this deliberately. Was there something else you saw that had
> duplication?
I ended up with this:
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index 07ba4e7910ab84..fc9622cdbac808 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -155,7 +155,7 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
*/
.table_levels_bitmap = 0xfe,
};
- u8 pmd_lg2sz = (tgsz_lg2 - 3) * 1 + tgsz_lg2;
+ u8 pmd_lg2sz = arm_smmu_pt_level_to_lg2sz(tgsz_lg2, 1);
/*
* If the size is small then we can infer the invalidation is PTE only
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 2a159be751891e..8dd5d30c569771 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2485,7 +2485,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
static bool arm_smmu_ttl_addr_aligned(u64 address, unsigned int tg,
unsigned int ttl)
{
- unsigned int pgsz_lg2 = (tg - 3) * (3 - ttl) + tg;
+ unsigned int pgsz_lg2 = arm_smmu_pt_level_to_lg2sz(tg, 3 - ttl);
return !(address & GENMASK_U64(pgsz_lg2 - 1, 0));
}
@@ -2497,6 +2497,16 @@ struct arm_smmu_ril_range {
unsigned int scale;
};
+static unsigned int arm_smmu_ril_calc_scale(u64 num_tg)
+{
+ return fls64((num_tg - 1) / (CMDQ_TLBI_RANGE_NUM_MAX + 1));
+}
+
+static u64 arm_smmu_ril_calc_num(u64 num_tg, unsigned int scale)
+{
+ return DIV_ROUND_UP_ULL(num_tg, 1ULL << scale);
+}
+
/*
* Initialize the smallest RIL covering num_tg and ending at last_tg.
*/
@@ -2507,8 +2517,8 @@ static struct arm_smmu_ril_range arm_smmu_ril_init_end(u64 last_tg, u64 num_tg)
if (!num_tg)
return ril;
- ril.scale = fls64((num_tg - 1) / 32);
- ril.num = DIV_ROUND_UP_ULL(num_tg, 1ULL << ril.scale);
+ ril.scale = arm_smmu_ril_calc_scale(num_tg);
+ ril.num = arm_smmu_ril_calc_num(num_tg, ril.scale);
ril.start_tg = last_tg - ((ril.num << ril.scale) - 1);
return ril;
}
@@ -2654,7 +2664,7 @@ static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi,
* Unlike other IOMMUs the spec has no alignment requirement on the
* address beyond alignment to tg (so long as TTL=0).
*/
- first.scale = fls64((num_tg - 1) / 32);
+ first.scale = arm_smmu_ril_calc_scale(num_tg);
if (first.scale > scale_max) {
/* Range too large for a single command do full invalidation */
tlbi->range.use_full_inv = true;
@@ -2666,7 +2676,7 @@ static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi,
* Produce a single invalidation by rounding up and disabling
* the trailer.
*/
- first.num = DIV_ROUND_UP_ULL(num_tg, 1ULL << first.scale);
+ first.num = arm_smmu_ril_calc_num(num_tg, first.scale);
trail.num = 0;
} else {
/*
@@ -2694,11 +2704,10 @@ static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi,
static u8 arm_smmu_tlbi_calc_stride(struct arm_smmu_tlbi *tlbi)
{
u8 combined = tlbi->table_levels_bitmap | tlbi->leaf_levels_bitmap;
- u8 tg_szlg2 = tlbi->tgsz_lg2;
if (WARN_ON(!combined))
return U8_MAX;
- return (tg_szlg2 - 3) * __ffs(combined) + tg_szlg2;
+ return arm_smmu_pt_level_to_lg2sz(tlbi->tgsz_lg2, __ffs(combined));
}
/*
@@ -2972,7 +2981,8 @@ static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
.start = iova,
.last = iova + size - 1,
};
- u8 table_levels = BIT((ilog2(size) - tgsz_lg2) / (tgsz_lg2 - 3));
+ u8 table_levels =
+ BIT(arm_smmu_pt_lg2sz_to_level(tgsz_lg2, ilog2(size)));
tlbi.table_levels_bitmap = table_levels;
tlbi.leaf_levels_bitmap = table_levels - 1;
@@ -4247,10 +4257,12 @@ static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
}
/*
- * Called by io-pgtable-arm.c for each run of same pgsize leaf only
- * invalidation. If it has to change to a different leaf level then it flushes
- * the gather and starts a fresh one. Thus this always targets only a single
- * leaf level.
+ * io-pgtable-arm.c calls this function either under
+ * arm_smmu_tlb_inv_page_nosync() or via the normal iommu code to flush the
+ * gather. Due to how iommu_iotlb_gather_add_page() works the gather will end up
+ * with a single uniform pgsize leaf. If it has to change to a different leaf
+ * level then it flushes the gather and starts a fresh one. Thus this always
+ * targets only a single leaf level.
*/
static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
@@ -4266,7 +4278,8 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
if (WARN_ON(gather->pgsize < BIT(tg)))
return;
- tlbi.leaf_levels_bitmap = BIT((ilog2(gather->pgsize) - tg) / (tg - 3));
+ tlbi.leaf_levels_bitmap =
+ BIT(arm_smmu_pt_lg2sz_to_level(tg, ilog2(gather->pgsize)));
arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 8b251a188810c9..70d732a96be27f 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -810,12 +810,24 @@ static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs)
return new_invs;
}
+/* Generic page-table level 0 is the leaf-only level. */
+static inline unsigned int arm_smmu_pt_level_to_lg2sz(unsigned int tgsz_lg2,
+ unsigned int level)
+{
+ return tgsz_lg2 + (tgsz_lg2 - ilog2(sizeof(u64))) * level;
+}
+
+static inline unsigned int arm_smmu_pt_lg2sz_to_level(unsigned int tgsz_lg2,
+ unsigned int lg2sz)
+{
+ return (lg2sz - tgsz_lg2) / (tgsz_lg2 - ilog2(sizeof(u64)));
+}
+
struct arm_smmu_tlbi {
unsigned long start;
unsigned long last;
/* Base Translation Granule of the page table */
u8 tgsz_lg2;
- bool leaf_only;
/*
* Level bitmaps use iommupt numbering: bit 0 is the leaf-only level
* (ARM level 3), bit 1 is the next level up (ARM level 2), etc. These
More information about the linux-arm-kernel
mailing list