[PATCH 3/8] iommu/arm-smmu-v3: Optimize range invalidation for latency
Jason Gunthorpe
jgg at nvidia.com
Mon May 18 12:43:40 PDT 2026
The server IOMMU drivers focus on invalidation latency by default,
over-invalidating if necessary, to round the invalidation range up to a
single command. I think this represents a trade off for DMA non-FQ and SVA
where stalling the operation is overall worse than re-loading the IOTLB.
For instance AMD and VT-d both round the range up to the largest aligned
power of two and invalidate that. This causes over-invalidation but that
is preferred on real HW over trying to issue a number of smaller
range invalidations.
Only if a para-virtualizating hypervisor is detected do they switch to
using more accurate invalidation. This also triggers using
iommu_iotlb_gather_is_disjoint() (ie PT_FEAT_FLUSH_RANGE_NO_GAPS) to
remove over invalidation from the gather. A pvIOMMU has a hypervisor that
will walk the IOPTEs and resync them. Over invalidation, especially
significant over invalidation, can incur a big latency cost reloading alot
of page table. x86 IOMMUs have aligned range restrictions so there are
some pretty nasty corner cases that can trigger huge over invalidation.
Currently SMMUv3 doesn't support detecting a hypervisor, and it
unconditionally runs in a NO_GAPS mode. This makes some sense for the
single invalidation flow where there is little reason to push single
commands across a gap.
When we get to RIL hardware, this doesn't look so good. On real HW the
best option is the same as x86: issue a single RIL per gather and optimize
for latency. SMMUv3 has a significant advantage as its RIL does not have
alignment limitations so it's single-command over-invalidation is capped
at < 1/32 of the gather's size, making it much more suitable for a
pvIOMMU.
However even with RIL SMMUv3 still uses NO_GAPS and it breaks down the
gather into several exactly sized RILs to avoid any over-invalidation,
costing latency on real HW.
When the HW has RIL support follow the x86 approach in SMMUv3 and
calculate a single RIL per gather that will cover the required
invalidation.
Calculate the smallest SCALE such that NUM can cover the range to minimize
over-invalidation. Always use a RIL command if RIL is possible working
around the spec limitations to form a valid one. If RIL is not possible
then do full invalidation.
At least one invalidation errata is avoided by 'always use RIL'.
Since the normal path is now the only one with a loop, split them into two
functions and fold a simplified version of arm_smmu_inv_size_too_big()
directly into the normal flow in a way that directly limits the number of
single invalidation commands generated, again focusing on controlling
latency.
The end result is any gather is converted into either:
- One invalidate all
- One range invalidate op
- At most 512 single invalidation ops
Signed-off-by: Jason Gunthorpe <jgg at nvidia.com>
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 257 ++++++++++++--------
1 file changed, 153 insertions(+), 104 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index cd0ab518712cd6..59634a5a5c0640 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2376,124 +2376,166 @@ static void arm_smmu_tlb_inv_context(void *cookie)
arm_smmu_domain_inv(smmu_domain);
}
-static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
+/*
+ * Check address alignment for TTL hint per SMMUv3 F.b Section 4.4.1.
+ * Address bits below the alignment must be zero, otherwise UNPREDICTABLE.
+ */
+static bool arm_smmu_ttl_addr_aligned(u64 address, unsigned int tg,
+ unsigned int ttl)
+{
+ unsigned int pgsz_lg2 = (tg - 3) * (3 - ttl) + tg;
+
+ return !(address & GENMASK_U64(pgsz_lg2 - 1, 0));
+}
+
+static void arm_smmu_cmdq_batch_add_ril(struct arm_smmu_device *smmu,
+ struct arm_smmu_cmdq_batch *cmds,
+ struct arm_smmu_cmd *cmd, bool leaf,
+ u64 address, unsigned int num,
+ unsigned int scale, u8 ttl, u8 tg_enc)
+{
+ cmd->data[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, num) |
+ FIELD_PREP(CMDQ_TLBI_0_SCALE, scale);
+ cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf) |
+ FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
+ FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | address;
+ arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd);
+}
+
+/*
+ * Issue a single range TLBI command covering [iova, iova+size). Returns true if
+ * successful, false if the range is too large for a single command.
+ *
+ * The algorithm finds the smallest SCALE where the range (in tg-sized pages)
+ * fits in the 5-bit NUM field (max 32 units of 2^SCALE pages). This may widen
+ * the invalidation range.
+ */
+static bool arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd,
struct arm_smmu_tlbi *tlbi)
{
- size_t inv_range = tlbi->iopte_granule;
- unsigned long iova = tlbi->iova;
- unsigned long end = iova + tlbi->size;
- unsigned long num_pages = 0;
- unsigned int tg = tlbi->smmu_domain->tgsz_lg2;
- u64 orig_data0 = cmd->data[0];
- u8 ttl = 0, tg_enc = 0;
+ unsigned int tg_lg2 = tlbi->smmu_domain->tgsz_lg2;
+ u64 cur_tg = tlbi->iova >> tg_lg2;
+ u64 last_tg = (tlbi->iova + tlbi->size - 1) >> tg_lg2;
+ u64 num_tg = last_tg - cur_tg + 1;
+ u8 tg_enc = (tg_lg2 - 10) / 2;
+ unsigned int scale;
+ u8 ttl = 0;
- if (WARN_ON_ONCE(!tlbi->size))
- return;
-
- if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
- num_pages = tlbi->size >> tg;
-
- /* Convert page size of 12,14,16 (log2) to 1,2,3 */
- tg_enc = (tg - 10) / 2;
-
- /*
- * Determine what level the granule is at. For non-leaf, both
- * io-pgtable and SVA pass a nominal last-level granule because
- * they don't know what level(s) actually apply, so ignore that
- * and leave TTL=0. However for various errata reasons we still
- * want to use a range command, so avoid the SVA corner case
- * where both scale and num could be 0 as well.
- */
- if (tlbi->leaf_only)
- ttl = 4 - ((ilog2(tlbi->iopte_granule) - 3) / (tg - 3));
- else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
- num_pages++;
- }
-
- while (iova < end) {
- if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
- /*
- * On each iteration of the loop, the range is 5 bits
- * worth of the aligned size remaining.
- * The range in pages is:
- *
- * range = (num_pages & (0x1f << __ffs(num_pages)))
- */
- unsigned long scale, num;
-
- /* Determine the power of 2 multiple number of pages */
- scale = __ffs(num_pages);
-
- /* Determine how many chunks of 2^scale size we have */
- num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
-
- cmd->data[0] = orig_data0 |
- FIELD_PREP(CMDQ_TLBI_0_NUM, num - 1) |
- FIELD_PREP(CMDQ_TLBI_0_SCALE, scale);
-
- /* range is num * 2^scale * pgsize */
- inv_range = num << (scale + tg);
-
- /* Clear out the lower order bits for the next iteration */
- num_pages -= num << scale;
- }
-
- /*
- * IPA has fewer bits than VA, but they are reserved in the
- * command and something would be very broken if iova had them
- * set.
- */
- cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) |
- FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
- FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) |
- (iova & ~GENMASK_U64(11, 0));
-
- arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd);
- iova += inv_range;
- }
-}
-
-static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu,
- struct arm_smmu_tlbi *tlbi)
-{
- size_t max_tlbi_ops;
-
- /* 0 size means invalidate all */
- if (!tlbi->size || tlbi->size == SIZE_MAX)
- return true;
-
- if (smmu->features & ARM_SMMU_FEAT_RANGE_INV)
+ if (!tlbi->size)
return false;
/*
- * Borrowed from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h,
- * this is used as a threshold to replace "size_opcode" commands with a
- * single "nsize_opcode" command, when SMMU doesn't implement the range
- * invalidation feature, where there can be too many per-granule TLBIs,
- * resulting in a soft lockup.
+ * Determine what level the granule is at. For non-leaf, both
+ * io-pgtable and SVA pass a nominal last-level granule because they
+ * don't know what level(s) actually apply, so leave TTL=0.
*/
- max_tlbi_ops = 1 << (ilog2(tlbi->iopte_granule) - 3);
- return tlbi->size >= max_tlbi_ops * tlbi->iopte_granule;
+ if (tlbi->leaf_only)
+ ttl = 4 - ((ilog2(tlbi->iopte_granule) - 3) / (tg_lg2 - 3));
+
+ /*
+ * SMMUv3 F.b Section 4.4.1: TG!=0, NUM==0, SCALE==0, TTL==0 is
+ * Reserved and causes CERROR_ILL. Single page uses NUM=0, SCALE=0 with
+ * a TTL hint to target only the exact leaf entry.
+ */
+ if (num_tg == 1) {
+ if (!ttl)
+ ttl = 3;
+ arm_smmu_cmdq_batch_add_ril(smmu, cmds, cmd, tlbi->leaf_only,
+ cur_tg << tg_lg2, 0, 0, ttl,
+ tg_enc);
+ return true;
+ }
+
+ /*
+ * There are at most 5 possible values for NUM based on SCALE. The
+ * highest NUM is at the lowest SCALE where:
+ * ceil(num_tg / 2^SCALE) <= 32
+ * scale >= ceil(log2(num_tg / 32))
+ * The lowest value is 1 where 2^SCALE covers the whole range. Pick the
+ * highest since it trivially also gives the tightest range.
+ *
+ * Unlike other IOMMUs the spec doesn't have any alignment requirements
+ * on the address beyond it must be aligned to tg (so long as TTL=0)
+ */
+ scale = fls64((num_tg - 1) / 32);
+ if (scale > 31) {
+ /*
+ * Range too large for a single command, use full invalidation.
+ */
+ return false;
+ }
+
+ /* 16K granule TTL=1 is reserved (Section 4.4.1) */
+ if (tg_lg2 == 14 && ttl == 1)
+ ttl = 0;
+
+ /* Verify address alignment for the TTL hint */
+ if (ttl && !arm_smmu_ttl_addr_aligned(cur_tg << tg_lg2, tg_lg2, ttl))
+ ttl = 0;
+
+ arm_smmu_cmdq_batch_add_ril(smmu, cmds, cmd, tlbi->leaf_only,
+ cur_tg << tg_lg2,
+ DIV_ROUND_UP_ULL(num_tg, 1ULL << scale) - 1,
+ scale, ttl, tg_enc);
+ return true;
}
-/* Used by non INV_TYPE_ATS* invalidations */
-static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
+/*
+ * One TLBI command per IOTLB entry, assuming the entries are all at least
+ * iopte_granule sized. Returns false if too many commands would be needed which
+ * indicates too high a latency. The threshold is similar to MAX_DVM_OPS in
+ * arch/arm64/include/asm/tlbflush.h for the 4k PAGE_SIZE.
+ */
+static bool arm_smmu_cmdq_batch_add_single(struct arm_smmu_device *smmu,
+ struct arm_smmu_cmdq_batch *cmds,
+ struct arm_smmu_cmd *cmd,
+ struct arm_smmu_tlbi *tlbi)
+{
+ unsigned long num_ops = tlbi->size / tlbi->iopte_granule;
+ unsigned long iova = tlbi->iova;
+ unsigned long i;
+
+ if (!num_ops || num_ops > 512)
+ return false;
+
+ for (i = 0; i < num_ops; i++) {
+ cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) |
+ (iova & ~GENMASK_U64(11, 0));
+ arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd);
+ iova += tlbi->iopte_granule;
+ }
+ return true;
+}
+
+static void arm_smmu_inv_all_cmd(struct arm_smmu_inv *inv,
+ struct arm_smmu_cmdq_batch *cmds,
+ struct arm_smmu_cmd *cmd)
+{
+ u64p_replace_bits(&cmd->data[0], inv->nsize_opcode, CMDQ_0_OP);
+ arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, cmd);
+}
+
+/*
+ * Used by non INV_TYPE_ATS* invalidations. Returns true if it fell back to
+ * full invalidation using nsize_opcode.
+ */
+static bool arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd,
struct arm_smmu_tlbi *tlbi)
{
- if (arm_smmu_inv_size_too_big(inv->smmu, tlbi)) {
- struct arm_smmu_cmd nsize_cmd = *cmd;
-
- u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode,
- CMDQ_0_OP);
- arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd);
- return;
+ if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
+ if (arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi))
+ return false;
+ } else {
+ if (arm_smmu_cmdq_batch_add_single(inv->smmu, cmds, cmd, tlbi))
+ return false;
}
- arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi);
+ arm_smmu_inv_all_cmd(inv, cmds, cmd);
+ return true;
}
static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
@@ -2516,6 +2558,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_tlbi *tlbi,
struct arm_smmu_invs *invs)
{
struct arm_smmu_cmdq_batch cmds = {};
+ bool used_s12_vmall = false;
struct arm_smmu_inv *cur;
struct arm_smmu_inv *end;
@@ -2548,11 +2591,17 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_tlbi *tlbi,
case INV_TYPE_S2_VMID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
0, cur->id);
- arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi);
+ used_s12_vmall = arm_smmu_inv_to_cmdq_batch(cur, &cmds,
+ &cmd, tlbi);
break;
case INV_TYPE_S2_VMID_S1_CLEAR:
- /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */
- if (arm_smmu_inv_size_too_big(cur->smmu, tlbi))
+ /*
+ * S2_VMID used CMDQ_OP_TLBI_S12_VMALL which already
+ * flushed S1 entries. These two types always come in
+ * pairs and arm_smmu_inv_cmp() ensures that they are
+ * consecutive in the list.
+ */
+ if (used_s12_vmall)
break;
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
--
2.43.0
More information about the linux-arm-kernel
mailing list