[PATCH v6 15/17] iommu/arm-smmu-v3: Add INV_TYPE_ATS_BROKEN for quarantined masters
Nicolin Chen
nicolinc at nvidia.com
Wed Sep 23 13:11:34 PDT 2026
Clearing STE.EATS makes the SMMU reject ATS requests from any quarantined
master, but does not stop the driver from issuing ATC invalidations for it.
Those commands would continue to time out.
Introduce INV_TYPE_ATS_BROKEN, which a subsequent change will assign to the
master's ATS invalidation entries. Treat it as an ATS type during iteration
and ordering, but skip it when issuing commands.
The subsequent quarantine path changes inv->type in place via WRITE_ONCE()
while readers hold the invalidation array's read lock. Since a u8 load is
already atomic, arm_smmu_inv_type() uses READ_ONCE() to annotate the race
for KCSAN. Whole-struct copies in merge and purge cannot use READ_ONCE(),
so annotate those copies with data_race().
A stale read at worst issues one more ATC_INV, which times out and causes
the master to be quarantined again.
Treat ATS, ATS_FULL, and ATS_BROKEN as one comparator class. This preserves
the sort position and matching identity when an entry is changed in place.
No functional change yet; the new type is never set anywhere.
Suggested-by: Jason Gunthorpe <jgg at nvidia.com>
Assisted-by: LLM
Signed-off-by: Nicolin Chen <nicolinc at nvidia.com>
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 12 +++++++-
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 33 +++++++++++++++------
2 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 6fa4f390ad133..1394a14241fd3 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -739,6 +739,7 @@ enum arm_smmu_inv_type {
INV_TYPE_S2_VMID_S1_CLEAR,
INV_TYPE_ATS,
INV_TYPE_ATS_FULL,
+ INV_TYPE_ATS_BROKEN,
};
struct arm_smmu_inv {
@@ -755,9 +756,18 @@ struct arm_smmu_inv {
int users; /* users=0 to mark as a trash to be purged */
};
+/* cur->type may flip to INV_TYPE_ATS_BROKEN concurrently with readers */
+static inline u8 arm_smmu_inv_type(const struct arm_smmu_inv *inv)
+{
+ return READ_ONCE(inv->type);
+}
+
static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv)
{
- return inv->type == INV_TYPE_ATS || inv->type == INV_TYPE_ATS_FULL;
+ u8 type = arm_smmu_inv_type(inv);
+
+ return type == INV_TYPE_ATS || type == INV_TYPE_ATS_FULL ||
+ type == INV_TYPE_ATS_BROKEN;
}
/**
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 607246a965a91..359cbee20ce5a 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1094,13 +1094,21 @@ arm_smmu_invs_iter_next(struct arm_smmu_invs *invs, size_t next, size_t *idx)
static int arm_smmu_inv_cmp(const struct arm_smmu_inv *inv_l,
const struct arm_smmu_inv *inv_r)
{
+ /*
+ * Treat all ATS types as one class, so an in-place flip to ATS_BROKEN
+ * preserves the sort order and still matches the original ATS entry.
+ */
+ bool are_ats = arm_smmu_inv_is_ats(inv_l) & arm_smmu_inv_is_ats(inv_r);
+ u8 type_l = arm_smmu_inv_type(inv_l);
+ u8 type_r = arm_smmu_inv_type(inv_r);
+
if (inv_l->smmu != inv_r->smmu)
return cmp_int((uintptr_t)inv_l->smmu, (uintptr_t)inv_r->smmu);
- if (inv_l->type != inv_r->type)
- return cmp_int(inv_l->type, inv_r->type);
+ if (!are_ats && type_l != type_r)
+ return cmp_int(type_l, type_r);
if (inv_l->id != inv_r->id)
return cmp_int(inv_l->id, inv_r->id);
- if (arm_smmu_inv_is_ats(inv_l))
+ if (are_ats)
return cmp_int(inv_l->ssid, inv_r->ssid);
return 0;
}
@@ -1186,11 +1194,12 @@ struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
return ERR_PTR(-ENOMEM);
new = new_invs->inv;
+ /* data_race(): a racing quarantine may flip ->type; the u8 is safe */
arm_smmu_invs_for_each_cmp(invs, i, to_merge, j, cmp) {
if (cmp < 0) {
- *new = invs->inv[i];
+ *new = data_race(invs->inv[i]);
} else if (cmp == 0) {
- *new = invs->inv[i];
+ *new = data_race(invs->inv[i]);
WRITE_ONCE(new->users, READ_ONCE(new->users) + 1);
} else {
*new = to_merge->inv[j];
@@ -1312,8 +1321,9 @@ struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs)
if (!new_invs)
return NULL;
+ /* data_race(): a racing quarantine may flip ->type; the u8 is safe */
arm_smmu_invs_for_each_entry(invs, i, inv) {
- new_invs->inv[num_invs] = *inv;
+ new_invs->inv[num_invs] = data_race(*inv);
if (arm_smmu_inv_is_ats(inv))
new_invs->has_ats = true;
num_invs++;
@@ -2700,8 +2710,8 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
if (cur->smmu != next->smmu)
return true;
/* The batch for S2 TLBI must be done before nested S1 ASIDs */
- if (cur->type != INV_TYPE_S2_VMID_S1_CLEAR &&
- next->type == INV_TYPE_S2_VMID_S1_CLEAR)
+ if (arm_smmu_inv_type(cur) != INV_TYPE_S2_VMID_S1_CLEAR &&
+ arm_smmu_inv_type(next) == INV_TYPE_S2_VMID_S1_CLEAR)
return true;
/* ATS must be after a sync of the S1/S2 invalidations */
if (!arm_smmu_inv_is_ats(cur) && arm_smmu_inv_is_ats(next))
@@ -2737,7 +2747,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
if (!cmds.num)
arm_smmu_cmdq_batch_init_cmd(smmu, &cmds, &cmd);
- switch (cur->type) {
+ switch (arm_smmu_inv_type(cur)) {
case INV_TYPE_S1_ASID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
cur->id, 0);
@@ -2771,6 +2781,9 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
arm_smmu_make_cmd_atc_inv_all(cur->id,
IOMMU_NO_PASID));
break;
+ case INV_TYPE_ATS_BROKEN:
+ /* Master is quarantined; skip its ATC_INV */
+ break;
default:
WARN_ON_ONCE(1);
break;
@@ -3326,6 +3339,8 @@ arm_smmu_master_build_inv(struct arm_smmu_master *master,
cur->size_opcode = cur->nsize_opcode = CMDQ_OP_ATC_INV;
cur->ssid = ssid;
break;
+ case INV_TYPE_ATS_BROKEN:
+ break;
}
return cur;
--
2.43.0
More information about the linux-arm-kernel
mailing list