[PATCH v5 15/18] iommu/arm-smmu-v3: Add INV_TYPE_ATS_BROKEN to skip quarantined ATS masters
Nicolin Chen
nicolinc at nvidia.com
Thu Jul 2 21:06:40 PDT 2026
A subsequent change quarantines a master whose ATC invalidation timed out,
marking its INV_TYPE_ATS / INV_TYPE_ATS_FULL entries in that domain's invs
as broken. Clearing STE.EATS makes the SMMU reject the device's ATS but
does not stop the driver from issuing ATC_INV, so without a marker those
commands would keep timing out on that master.
Add the INV_TYPE_ATS_BROKEN type. __arm_smmu_domain_inv_range() skips it in
its switch without issuing commands. arm_smmu_inv_is_ats() recognizes it so
the iter's batch-boundary logic places it next to other ATS group entries.
The setter writes cur->type via WRITE_ONCE while the inv_range iter holds
read_lock on invs->rwlock, so its loads of cur->type and next->type race
with it. A new arm_smmu_inv_type() helper wraps the load in READ_ONCE, and
the iter and arm_smmu_inv_cmp() read through it. cur->type is u8 so the
access is already atomic; READ_ONCE annotates it for KCSAN and matches the
cur->users pattern.
arm_smmu_invs_merge() and arm_smmu_invs_purge() also read cur->type, but
through a whole-struct copy of the live array that cannot be a READ_ONCE.
Wrap those copies in data_race(): the u8 load returns the old or flipped
type, and a stale read at worst yields one more ATC_INV timeout, which
re-quarantines. The hot-path WRITE_ONCE and READ_ONCE stay; data_race()
only covers the cold copies that cannot be marked.
An in-place flip of cur->type to INV_TYPE_ATS_BROKEN must not change its
sort position, or arm_smmu_invs_merge() and unref() walks would no longer
match it against an incoming ATS / ATS_FULL identity. Treat all three ATS
variants as one sort class in arm_smmu_inv_cmp() so a flip stays in place
and the flipped entry still matches its pre-flip ssid on attach and detach.
No functional change yet; the new type is introduced but never set
anywhere.
Suggested-by: Jason Gunthorpe <jgg at nvidia.com>
Assisted-by: Claude:claude-opus-4-7
Signed-off-by: Nicolin Chen <nicolinc at nvidia.com>
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 12 +++++++-
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 33 +++++++++++++++------
2 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 56e9a94826a12..8eb5684696316 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -736,6 +736,7 @@ enum arm_smmu_inv_type {
INV_TYPE_S2_VMID_S1_CLEAR,
INV_TYPE_ATS,
INV_TYPE_ATS_FULL,
+ INV_TYPE_ATS_BROKEN,
};
struct arm_smmu_inv {
@@ -752,9 +753,18 @@ struct arm_smmu_inv {
int users; /* users=0 to mark as a trash to be purged */
};
+/* cur->type may flip to INV_TYPE_ATS_BROKEN concurrently with readers */
+static inline u8 arm_smmu_inv_type(const struct arm_smmu_inv *inv)
+{
+ return READ_ONCE(inv->type);
+}
+
static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv)
{
- return inv->type == INV_TYPE_ATS || inv->type == INV_TYPE_ATS_FULL;
+ u8 type = arm_smmu_inv_type(inv);
+
+ return type == INV_TYPE_ATS || type == INV_TYPE_ATS_FULL ||
+ type == INV_TYPE_ATS_BROKEN;
}
/**
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 78e2559bdc491..a18a56ceeb7fb 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1029,13 +1029,21 @@ arm_smmu_invs_iter_next(struct arm_smmu_invs *invs, size_t next, size_t *idx)
static int arm_smmu_inv_cmp(const struct arm_smmu_inv *inv_l,
const struct arm_smmu_inv *inv_r)
{
+ /*
+ * Treat all ATS types as one class, so an in-place flip to ATS_BROKEN
+ * preserves the sort order and still matches the original ATS entry.
+ */
+ bool are_ats = arm_smmu_inv_is_ats(inv_l) & arm_smmu_inv_is_ats(inv_r);
+ u8 type_l = arm_smmu_inv_type(inv_l);
+ u8 type_r = arm_smmu_inv_type(inv_r);
+
if (inv_l->smmu != inv_r->smmu)
return cmp_int((uintptr_t)inv_l->smmu, (uintptr_t)inv_r->smmu);
- if (inv_l->type != inv_r->type)
- return cmp_int(inv_l->type, inv_r->type);
+ if (!are_ats && type_l != type_r)
+ return cmp_int(type_l, type_r);
if (inv_l->id != inv_r->id)
return cmp_int(inv_l->id, inv_r->id);
- if (arm_smmu_inv_is_ats(inv_l))
+ if (are_ats)
return cmp_int(inv_l->ssid, inv_r->ssid);
return 0;
}
@@ -1121,11 +1129,12 @@ struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
return ERR_PTR(-ENOMEM);
new = new_invs->inv;
+ /* data_race(): a racing quarantine may flip ->type; the u8 is safe */
arm_smmu_invs_for_each_cmp(invs, i, to_merge, j, cmp) {
if (cmp < 0) {
- *new = invs->inv[i];
+ *new = data_race(invs->inv[i]);
} else if (cmp == 0) {
- *new = invs->inv[i];
+ *new = data_race(invs->inv[i]);
WRITE_ONCE(new->users, READ_ONCE(new->users) + 1);
} else {
*new = to_merge->inv[j];
@@ -1247,8 +1256,9 @@ struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs)
if (!new_invs)
return NULL;
+ /* data_race(): a racing quarantine may flip ->type; the u8 is safe */
arm_smmu_invs_for_each_entry(invs, i, inv) {
- new_invs->inv[num_invs] = *inv;
+ new_invs->inv[num_invs] = data_race(*inv);
if (arm_smmu_inv_is_ats(inv))
new_invs->has_ats = true;
num_invs++;
@@ -2635,8 +2645,8 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
if (cur->smmu != next->smmu)
return true;
/* The batch for S2 TLBI must be done before nested S1 ASIDs */
- if (cur->type != INV_TYPE_S2_VMID_S1_CLEAR &&
- next->type == INV_TYPE_S2_VMID_S1_CLEAR)
+ if (arm_smmu_inv_type(cur) != INV_TYPE_S2_VMID_S1_CLEAR &&
+ arm_smmu_inv_type(next) == INV_TYPE_S2_VMID_S1_CLEAR)
return true;
/* ATS must be after a sync of the S1/S2 invalidations */
if (!arm_smmu_inv_is_ats(cur) && arm_smmu_inv_is_ats(next))
@@ -2672,7 +2682,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
if (!cmds.num)
arm_smmu_cmdq_batch_init_cmd(smmu, &cmds, &cmd);
- switch (cur->type) {
+ switch (arm_smmu_inv_type(cur)) {
case INV_TYPE_S1_ASID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
cur->id, 0);
@@ -2706,6 +2716,9 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
arm_smmu_make_cmd_atc_inv_all(cur->id,
IOMMU_NO_PASID));
break;
+ case INV_TYPE_ATS_BROKEN:
+ /* Master is quarantined; skip its ATC_INV */
+ break;
default:
WARN_ON_ONCE(1);
break;
@@ -3256,6 +3269,8 @@ arm_smmu_master_build_inv(struct arm_smmu_master *master,
cur->size_opcode = cur->nsize_opcode = CMDQ_OP_ATC_INV;
cur->ssid = ssid;
break;
+ case INV_TYPE_ATS_BROKEN:
+ break;
}
return cur;
--
2.43.0
More information about the linux-arm-kernel
mailing list