[PATCH v4 23/24] iommu/arm-smmu-v3: Serialize STE.EATS and ats_broken updates

[Nicolin Chen]

A subsequent change adding an ATS-broken path will set master->ats_broken
flag and overwrite STE.EATS to 0 for a broken master. This will introduce
race conditions:
 - A concurrent attachment that read ats_broken=false prior to ats_broken
   being set to true could re-enable STE.EATS.
 - A concurrent reset_device_done callback could clear master->ats_broken
   during the ATS-broken path, leading to iommu_report_device_broken() on
   a pre-reset fault.
 - When the ATS-broken path reads the old_data[1] and writes (old_data[1]
   & ~EATS), a concurrent attachment could write a new_data[1] in-between.

Due to an ATS-broken path can run in an atomic context (invalidation), it
cannot use mutex.

Introduce a per-master spinlock_t ats_broken_lock to fence these cases so
as to guarantee that in concurrent cases:
 a) A master->ats_broken update is observed by every concurrent attach
 b) STE.EATS is never re-enabled while master->ats_broken is true
 c) data[1] writes are not lost to a concurrent ATS-broken path

Note IRQ has to be disabled while holding the lock, because an ATS-broken
path can be entered via a hardirq that might interrupt another caller and
lead to deadlock.

Assisted-by: Claude:claude-opus-4-7
Signed-off-by: Nicolin Chen <nicolinc at nvidia.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |  5 +++++
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 22 ++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 186efcbed1ea9..e3eb4c4a62d3a 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -1048,6 +1048,11 @@ struct arm_smmu_master {
 	/* Scratch memory for arm_smmu_atc_inv_master() to build an ATS array */
 	struct arm_smmu_invs		*ats_invs;
 	struct arm_smmu_vmaster		*vmaster; /* use smmu->streams_mutex */
+	/*
+	 * Serializes arm_smmu_write_ste(), reset_device_done, and an ATS-broken
+	 * path, preventing races on ats_broken flag and STE updates.
+	 */
+	spinlock_t			ats_broken_lock;
 	/* Locked by the iommu core using the group mutex */
 	struct arm_smmu_ctx_desc_cfg	cd_table;
 	unsigned int			num_streams;
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 9591e4ab2b14a..ee864046f0baa 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1791,7 +1791,23 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
 		.sid = sid,
 	};
 
-	arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
+	/*
+	 * Fence against the ATS-broken path concurrently overwriting STE.EATS.
+	 * It's fine if the ATS-broken path writes after arm_smmu_write_entry.
+	 * Otherwise, we must clear STE.EATS before sending a CFGI_STE command.
+	 *
+	 * Must disable IRQs; otherwise a hardirq-context invalidation path on
+	 * this CPU could deadlock at ats_broken_lock on an ATC_INV timeout.
+	 */
+	scoped_guard(spinlock_irqsave, &master->ats_broken_lock) {
+		struct arm_smmu_ste local_target = *target;
+
+		if (master->ats_broken)
+			local_target.data[1] &=
+				~cpu_to_le64(STRTAB_STE_1_EATS);
+		arm_smmu_write_entry(&ste_writer.writer, ste->data,
+				     local_target.data);
+	}
 
 	/* It's likely that we'll want to use the new STE soon */
 	if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
@@ -3000,6 +3016,9 @@ static void arm_smmu_reset_device_done(struct device *dev)
 
 	if (WARN_ON(!master))
 		return;
+
+	/* Ensure the device recovery is seen, to flush any pre-reset fault */
+	guard(spinlock_irqsave)(&master->ats_broken_lock);
 	/* Pair with lockless readers */
 	WRITE_ONCE(master->ats_broken, false);
 }
@@ -4236,6 +4255,7 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 
 	master->dev = dev;
 	master->smmu = smmu;
+	spin_lock_init(&master->ats_broken_lock);
 	dev_iommu_priv_set(dev, master);
 
 	ret = arm_smmu_insert_master(smmu, master);
-- 
2.43.0