[PATCH v7 2/3] iommu/arm-smmu-v3: Introduce CFGI/TLBI-repeat workaround infrastructure
Ashish Mhetre
amhetre at nvidia.com
Tue Jul 14 03:42:01 PDT 2026
Tegra264 SMMU instances need every CFGI/TLBI command sequence issued
twice, with the second issue executing only after the first issue's
CMD_SYNC has completed:
TLBI/CFGI ... CMD_SYNC TLBI/CFGI ... CMD_SYNC
ATC_INV is not affected and must never be doubled.
Add arm_smmu_erratum_repeat_tlbi_cfgi_key and an
arm_smmu_erratum_cmd_needs_repeating() helper that gates on the static
key first and then range-checks the opcode (CFGI_STE .. ATC_INV), so
subsequent changes wiring the workaround into the CMDQ submission and
iommufd batching paths can share a single predicate.
Rename the existing arm_smmu_cmdq_issue_cmdlist() to
__arm_smmu_cmdq_issue_cmdlist() and add a thin wrapper that re-issues
the same cmdlist a second time when the predicate fires. Register the
new condition with arm_smmu_cmdq_batch_force_sync() and add
arm_vsmmu_can_batch_cmd() so iommufd batches split at every "needs
repeating" transition.
No callers enable the static key yet, so there is no functional change.
A subsequent change will enable the key on affected instances.
Suggested-by: Nicolin Chen <nicolinc at nvidia.com>
Reviewed-by: Nicolin Chen <nicolinc at nvidia.com>
Signed-off-by: Ashish Mhetre <amhetre at nvidia.com>
---
.../arm/arm-smmu-v3/arm-smmu-v3-iommufd.c | 14 ++++-
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 57 +++++++++++++++++--
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 +
3 files changed, 67 insertions(+), 5 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
index 1e9f7d2de344..143f0d015aeb 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
@@ -350,6 +350,17 @@ static int arm_vsmmu_convert_user_cmd(struct arm_vsmmu *vsmmu,
return 0;
}
+static bool arm_vsmmu_can_batch_cmd(struct arm_vsmmu_invalidation_cmd *last,
+ struct arm_vsmmu_invalidation_cmd *next)
+{
+ struct arm_smmu_cmd next_cmd = {
+ .data[0] = le64_to_cpu(next->ucmd.cmd[0]),
+ };
+
+ return arm_smmu_erratum_cmd_needs_repeating(&last->cmd) ==
+ arm_smmu_erratum_cmd_needs_repeating(&next_cmd);
+}
+
int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu,
struct iommu_user_data_array *array)
{
@@ -382,7 +393,8 @@ int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu,
/* FIXME work in blocks of CMDQ_BATCH_ENTRIES and copy each block? */
cur++;
- if (cur != end && (cur - last) != CMDQ_BATCH_ENTRIES - 1)
+ if (cur != end && (cur - last) != CMDQ_BATCH_ENTRIES - 1 &&
+ arm_vsmmu_can_batch_cmd(last, cur))
continue;
/* FIXME always uses the main cmdq rather than trying to group by type */
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index dd7475c50afc..9c49e6412053 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -18,6 +18,7 @@
#include <linux/interrupt.h>
#include <linux/io-pgtable.h>
#include <linux/iopoll.h>
+#include <linux/jump_label.h>
#include <linux/module.h>
#include <linux/msi.h>
#include <linux/of.h>
@@ -42,6 +43,14 @@ MODULE_PARM_DESC(disable_msipolling,
static const struct iommu_ops arm_smmu_ops;
static struct iommu_dirty_ops arm_smmu_dirty_ops;
+/*
+ * Repeat every {CFGI,TLBI};CMD_SYNC command sequence so that the second
+ * issue executes only after the first issue's CMD_SYNC has completed.
+ * Does not apply to ATC_INV. The key is global and is enabled from DT
+ * probe on affected hardware (currently Tegra264 only).
+ */
+static DEFINE_STATIC_KEY_FALSE(arm_smmu_erratum_repeat_tlbi_cfgi_key);
+
enum arm_smmu_msi_index {
EVTQ_MSI_INDEX,
GERROR_MSI_INDEX,
@@ -698,10 +707,10 @@ static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq,
* insert their own list of commands then all of the commands from one
* CPU will appear before any of the commands from the other CPU.
*/
-int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
- struct arm_smmu_cmdq *cmdq,
- struct arm_smmu_cmd *cmds, int n,
- bool sync)
+static int __arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
+ struct arm_smmu_cmdq *cmdq,
+ struct arm_smmu_cmd *cmds, int n,
+ bool sync)
{
struct arm_smmu_cmd cmd_sync;
u32 prod;
@@ -820,6 +829,38 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
return ret;
}
+bool arm_smmu_erratum_cmd_needs_repeating(struct arm_smmu_cmd *cmd)
+{
+ u8 opcode;
+
+ if (!static_branch_unlikely(&arm_smmu_erratum_repeat_tlbi_cfgi_key))
+ return false;
+
+ opcode = FIELD_GET(CMDQ_0_OP, cmd->data[0]);
+ return opcode >= CMDQ_OP_CFGI_STE && opcode < CMDQ_OP_ATC_INV;
+}
+
+int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
+ struct arm_smmu_cmdq *cmdq,
+ struct arm_smmu_cmd *cmds, int n,
+ bool sync)
+{
+ int ret = __arm_smmu_cmdq_issue_cmdlist(smmu, cmdq, cmds, n, sync);
+
+ /*
+ * A bare CMD_SYNC can be issued with n == 0 (e.g. an empty
+ * batch_submit()), in which case there is no cmds[0] to inspect
+ * and nothing to repeat.
+ */
+ if (!n || ret || !sync)
+ return ret;
+
+ if (arm_smmu_erratum_cmd_needs_repeating(&cmds[0]))
+ ret = __arm_smmu_cmdq_issue_cmdlist(smmu, cmdq, cmds, n, sync);
+
+ return ret;
+}
+
static int arm_smmu_cmdq_issue_cmd_p(struct arm_smmu_device *smmu,
struct arm_smmu_cmd *cmd, bool sync)
{
@@ -860,6 +901,14 @@ static bool arm_smmu_cmdq_batch_force_sync(struct arm_smmu_device *smmu,
(smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC))
return true;
+ /*
+ * See the description at arm_smmu_erratum_repeat_tlbi_cfgi_key. Batches
+ * never mix CFGI/TLBI with others, so checking cmds[0] alone is enough.
+ */
+ if (cmds->num == CMDQ_BATCH_ENTRIES &&
+ arm_smmu_erratum_cmd_needs_repeating(&cmds->cmds[0]))
+ return true;
+
return false;
}
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index c909c9a88538..e175dedf7c77 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -1211,6 +1211,7 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds, int n,
bool sync);
+bool arm_smmu_erratum_cmd_needs_repeating(struct arm_smmu_cmd *cmd);
#ifdef CONFIG_ARM_SMMU_V3_SVA
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
--
2.50.1
More information about the linux-arm-kernel
mailing list