[PATCH 7/8] iommu/arm-smmu-v3: Change how the tlbi describes the invalidation
Jason Gunthorpe
jgg at nvidia.com
Mon May 18 12:43:44 PDT 2026
The RIL logic has long had a FIXME that there is not enough
information to properly compute the RIL. There is also subtly not
enough information to properly compute the single stride either.
Change tlbi to use the information format that iommupt is going to
use for ARM. This prepares the invalidation code to support iommupt
and fixes two small limitations with the current code.
iommupt is designed to accumulate all invalidation into a single
gather, then the iommu driver should issue a small number of commands
to execute the gather to control invalidation latency. This is in
contrast to io-pgtable-arm.c which generates many gather flushes and
direct walk cache flushes as it progresses.
To accommodate this the gather will accumulate "damage" in bitmaps,
one for leaf changes and one for table changes. This is enough
information for SMMUv3 to compute the proper stride for single
invalidation and to generate ideal hints for range invalidation.
Change the inner workings of the tlbi process to directly use this
new-style gather description with the idea that the iommupt
conversion will just direct assign the gather fields to the tlbi.
Rework the three places creating the tlbi to express their needs in
terms of the new bitmaps.
1) Simple iotlb invalidation always gets a single range of leaf
levels, so it can set a single leaf bit
2) Walk invalidation always gets a single table level so it can set a
single table bit.
This corrects a weakness in the existing design where single
invalidation would walk the entire table level issuing 4k
invalidations, now it will just push a single invalidation.
3) SVA invalidation has no idea what the MM did, so it will set all
the bits in the bitmaps.
This corrects another weakness where the RIL invalidation logic
was generating hints assuming the #2 rules which isn't correct
for SVA.
Signed-off-by: Jason Gunthorpe <jgg at nvidia.com>
---
.../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 28 ++-
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 163 +++++++++++++-----
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 20 ++-
3 files changed, 157 insertions(+), 54 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index c708fefb053771..d7f88866469846 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -129,17 +129,33 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
{
struct arm_smmu_domain *smmu_domain =
container_of(mn, struct arm_smmu_domain, mmu_notifier);
+ unsigned int tg_lg2 = smmu_domain->tgsz_lg2;
struct arm_smmu_tlbi tlbi = {
.smmu_domain = smmu_domain,
- .iova = start,
+ .start = start,
+ .last = end - 1,
/*
- * The mm_types defines vm_end as the first byte after the end
- * address, different from IOMMU subsystem using the last
- * address of an address range.
+ * No information comes from the mm, assume the worst case that
+ * it changed every table level. The way this is hooked into the
+ * mm is tricky, the range won't be expanded to include an
+ * entire table level if one was removed like the iommu gather
+ * does. Thus even if this is a 4k invalidation it may be
+ * including any table level too.
*/
- .size = end - start,
- .iopte_granule = PAGE_SIZE,
+ .table_levels_bitmap = 0xfe,
};
+ unsigned int pmd_lg2sz = (tg_lg2 - 3) * 1 + tg_lg2;
+
+ /*
+ * If the size is small then we can infer the invalidation is PTE only
+ * and set the PTE level only. Otherwise it could be some other
+ * combination so just set them all. This allows RIL to use TTL=3 in
+ * cases of PTE only changes.
+ */
+ if (end - start < BIT_U64(pmd_lg2sz))
+ tlbi.leaf_levels_bitmap = 1;
+ else
+ tlbi.leaf_levels_bitmap = 0xff;
arm_smmu_domain_tlbi(&tlbi);
}
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 94f742de90330c..0841ab053f903e 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2312,8 +2312,8 @@ static struct arm_smmu_cmd arm_smmu_atc_inv_to_cmd(u32 sid, int ssid,
* This has the unpleasant side-effect of invalidating all PASID-tagged
* ATC entries within the address range.
*/
- page_start = tlbi->iova >> inval_grain_shift;
- page_end = (tlbi->iova + tlbi->size - 1) >> inval_grain_shift;
+ page_start = tlbi->start >> inval_grain_shift;
+ page_end = tlbi->last >> inval_grain_shift;
/*
* In an ATS Invalidate Request, the address must be aligned on the
@@ -2399,7 +2399,49 @@ static bool arm_smmu_ttl_addr_aligned(u64 address, unsigned int tg,
}
/*
- * Generate a single range TLBI command covering [iova, iova+size). Sets
+ * Compute the TTL hint from leaf/table level bitmaps. 0 ttlt means no hint
+ * invalidate all levels.
+ */
+static unsigned int arm_smmu_compute_ttl(u8 leaf_bitmap, u8 table_bitmap,
+ unsigned int tg)
+{
+ int ttl;
+
+ if (leaf_bitmap) {
+ if (is_power_of_2(leaf_bitmap))
+ ttl = 3 - (int)__ffs(leaf_bitmap);
+ else
+ ttl = 0;
+
+ if (table_bitmap) {
+ int table_ttl = 3 - (int)__ffs(table_bitmap) + 1;
+
+ /*
+ * A RIL invalidation with !leaf_only clears out all
+ * table levels above the leaf level ttl only.
+ */
+ if (table_ttl > ttl)
+ ttl = 0;
+ }
+ } else if (table_bitmap) {
+ ttl = 3 - (int)__ffs(table_bitmap) + 1;
+ } else {
+ /* Both bitmaps zero is not allowed */
+ return 0;
+ }
+
+ /* 16K granule, ARM TTL=1 is reserved (SMMUv3 F.b Section 4.4.1) */
+ if (tg == 14 && ttl == 1)
+ return 0;
+
+ /* ARM levels -1 and 0 cannot be hinted */
+ if (ttl <= 0 || ttl > 3)
+ return 0;
+ return ttl;
+}
+
+/*
+ * Generate a single range TLBI command covering [start, last]. Sets
* use_full_inv if the range is too large for a single command.
*
* The algorithm finds the smallest SCALE where the range (in tg-sized pages)
@@ -2409,20 +2451,13 @@ static bool arm_smmu_ttl_addr_aligned(u64 address, unsigned int tg,
static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi)
{
unsigned int tg_lg2 = tlbi->smmu_domain->tgsz_lg2;
- u64 cur_tg = tlbi->iova >> tg_lg2;
- u64 last_tg = (tlbi->iova + tlbi->size - 1) >> tg_lg2;
+ unsigned int ttl = arm_smmu_compute_ttl(
+ tlbi->leaf_levels_bitmap, tlbi->table_levels_bitmap, tg_lg2);
+ u64 cur_tg = tlbi->start >> tg_lg2;
+ u64 last_tg = tlbi->last >> tg_lg2;
u64 num_tg = last_tg - cur_tg + 1;
u8 tg_enc = (tg_lg2 - 10) / 2;
unsigned int scale;
- u8 ttl = 0;
-
- /*
- * Determine what level the granule is at. For non-leaf, both
- * io-pgtable and SVA pass a nominal last-level granule because they
- * don't know what level(s) actually apply, so leave TTL=0.
- */
- if (tlbi->leaf_only)
- ttl = 4 - ((ilog2(tlbi->iopte_granule) - 3) / (tg_lg2 - 3));
/*
* SMMUv3 F.b Section 4.4.1: TG!=0, NUM==0, SCALE==0, TTL==0 is
@@ -2430,14 +2465,18 @@ static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi)
* a TTL hint to target only the exact leaf entry.
*/
if (num_tg == 1) {
- if (!ttl)
+ /*
+ * The two io-pgtable ops filling the tlbi won't generate ttl=0.
+ * sva sets constants for single page that give ttl=3
+ */
+ if (WARN_ON(!ttl))
ttl = 3;
tlbi->range.data0 = 0;
- tlbi->range.data1 =
- FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) |
- FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
- FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) |
- (cur_tg << tg_lg2);
+ tlbi->range.data1 = FIELD_PREP(CMDQ_TLBI_1_LEAF,
+ !tlbi->table_levels_bitmap) |
+ FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
+ FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) |
+ (cur_tg << tg_lg2);
return;
}
@@ -2461,10 +2500,6 @@ static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi)
return;
}
- /* 16K granule TTL=1 is reserved (Section 4.4.1) */
- if (tg_lg2 == 14 && ttl == 1)
- ttl = 0;
-
/* Verify address alignment for the TTL hint */
if (ttl && !arm_smmu_ttl_addr_aligned(cur_tg << tg_lg2, tg_lg2, ttl))
ttl = 0;
@@ -2473,27 +2508,52 @@ static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi)
FIELD_PREP(CMDQ_TLBI_0_NUM,
DIV_ROUND_UP_ULL(num_tg, 1ULL << scale) - 1) |
FIELD_PREP(CMDQ_TLBI_0_SCALE, scale);
- tlbi->range.data1 = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) |
- FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
- FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) |
- (cur_tg << tg_lg2);
+ tlbi->range.data1 =
+ FIELD_PREP(CMDQ_TLBI_1_LEAF, !tlbi->table_levels_bitmap) |
+ FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
+ FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) |
+ (cur_tg << tg_lg2);
}
/*
- * One TLBI command per IOTLB entry, assuming the entries are all at least
- * iopte_granule sized. Sets use_full_inv if too many commands would be needed
- * which indicates too high a latency. The threshold is similar to MAX_DVM_OPS
- * in arch/arm64/include/asm/tlbflush.h for the 4k PAGE_SIZE.
+ * Compute the stride for non-RIL single-page invalidation. Returns the log2
+ * stride of the lowest affected level. Single invalidation removes all IOPTEs
+ * that contain the IOVA invalidated, and we can reliably assume that the
+ * architected page size and table sizes (not contiguous!) are reflected in the
+ * IOTLB. Thus if there is a 2M leaf entry we only need to issue a single IOTLB
+ * invalidation within that 2M IOVA.
+ */
+static u8 arm_smmu_tlbi_calc_stride(struct arm_smmu_tlbi *tlbi)
+{
+ unsigned int tg_lg2 = tlbi->smmu_domain->tgsz_lg2;
+ u8 combined = tlbi->table_levels_bitmap | tlbi->leaf_levels_bitmap;
+
+ if (!combined)
+ return U8_MAX;
+ return (tg_lg2 - 3) * __ffs(combined) + tg_lg2;
+}
+
+/*
+ * One TLBI command per stride-sized entry. Sets use_full_inv if too many
+ * commands would be needed. The threshold is similar to MAX_DVM_OPS in
+ * arch/arm64/include/asm/tlbflush.h.
*/
static void arm_smmu_tlbi_calc_single(struct arm_smmu_tlbi *tlbi)
{
- unsigned long num_ops = tlbi->size / tlbi->iopte_granule;
+ u8 stride_lg2 = arm_smmu_tlbi_calc_stride(tlbi);
+ unsigned long num_ops;
+ if (stride_lg2 == U8_MAX) {
+ tlbi->single.use_full_inv = true;
+ return;
+ }
+ num_ops = (tlbi->last - tlbi->start + 1) >> stride_lg2;
if (!num_ops || num_ops > 512) {
tlbi->single.use_full_inv = true;
return;
}
tlbi->single.num = num_ops;
+ tlbi->single.stride_lg2 = stride_lg2;
}
static void arm_smmu_inv_all_cmd(struct arm_smmu_inv *inv,
@@ -2513,7 +2573,7 @@ static bool arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
struct arm_smmu_cmd *cmd,
struct arm_smmu_tlbi *tlbi)
{
- u64 iova = tlbi->iova;
+ u64 iova = tlbi->start;
unsigned int i;
if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
@@ -2533,9 +2593,10 @@ static bool arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
}
for (i = 0; i < tlbi->single.num; i++) {
- cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) |
+ cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF,
+ !tlbi->table_levels_bitmap) |
(iova & ~GENMASK_U64(11, 0));
- iova += tlbi->iopte_granule;
+ iova += BIT_U64(tlbi->single.stride_lg2);
arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, cmd);
}
return false;
@@ -2714,15 +2775,21 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
iommu_iotlb_gather_add_page(domain, gather, iova, granule);
}
+/*
+ * Called by io-pgtable-arm.c for each single table level it wants to remove.
+ * size is the size of the table level and granule is the tg in bytes.
+ */
static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
+ unsigned int tg_lg2 = smmu_domain->tgsz_lg2;
struct arm_smmu_tlbi tlbi = {
.smmu_domain = smmu_domain,
- .iova = iova,
- .size = size,
- .iopte_granule = granule,
+ .start = iova,
+ .last = iova + size - 1,
+ .table_levels_bitmap =
+ BIT((ilog2(size) - tg_lg2) / (tg_lg2 - 3)),
};
arm_smmu_domain_tlbi(&tlbi);
@@ -3984,15 +4051,23 @@ static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
arm_smmu_tlb_inv_context(smmu_domain);
}
+/*
+ * Called by io-pgtable-arm.c for each run of same pgsize leaf only
+ * invalidation. If it has to change to a different leaf level then it flushes
+ * the gather and starts a fresh one. Thus this always targets only a single
+ * leaf level.
+ */
static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ unsigned int tg = smmu_domain->tgsz_lg2;
struct arm_smmu_tlbi tlbi = {
- .smmu_domain = to_smmu_domain(domain),
- .iova = gather->start,
- .size = gather->end - gather->start + 1,
- .iopte_granule = gather->pgsize,
- .leaf_only = true,
+ .smmu_domain = smmu_domain,
+ .start = gather->start,
+ .last = gather->end,
+ .leaf_levels_bitmap =
+ BIT((ilog2(gather->pgsize) - tg) / (tg - 3)),
};
if (!gather->pgsize)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 95da62d64df171..d6c548ade41f01 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -805,14 +805,26 @@ static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs)
struct arm_smmu_tlbi {
struct arm_smmu_domain *smmu_domain;
- unsigned long iova;
- size_t size;
- unsigned int iopte_granule;
- bool leaf_only;
+ unsigned long start;
+ unsigned long last;
+ /*
+ * Level bitmaps use iommupt numbering: bit 0 is the leaf-only level
+ * (ARM level 3), bit 1 is the next level up (ARM level 2), etc. These
+ * match the iommu_iotlb_gather.pt fields. Each set bit indicates a
+ * change at that level. The contiguous hint has no effect on
+ * invalidation processing because HW can ignore the hint.
+ *
+ * If leaf_levels_bitmap is 0 then this is a walk cache only
+ * invalidation. If table_levels_bitmap is 0 then this is a leaf only
+ * invalidation.
+ */
+ u8 leaf_levels_bitmap;
+ u8 table_levels_bitmap;
struct {
bool use_full_inv;
u16 num;
+ u8 stride_lg2;
} single;
struct {
--
2.43.0
More information about the linux-arm-kernel
mailing list