[PATCH v4 17/24] iommu/arm-smmu-v3: Add master in arm_smmu_inv for ATS entries

[Nicolin Chen]

Storing the master pointer allows backtracking it from an ATS invalidation
entry, which will be useful when handling ATC invalidation timeouts.

Don't simply swap the "smmu" pointer for the "master": a non-ATS entry may
be shared across multiple devices (masters). An ATS entry is okay since it
is tied to a unique SID.

Master must outlive any concurrent RCU reader iterating the domain->invs,
because inv->master is dereferenced inside the read-side critical section.

Add a synchronize_rcu() in arm_smmu_release_device() before freeing master.

Signed-off-by: Nicolin Chen <nicolinc at nvidia.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |  1 +
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 14 +++++++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 604f7edf54158..df6e539f75274 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -738,6 +738,7 @@ enum arm_smmu_inv_type {
 
 struct arm_smmu_inv {
 	struct arm_smmu_device *smmu;
+	struct arm_smmu_master *master; /* INV_TYPE_ATS* */
 	u8 type;
 	u8 size_opcode;
 	u8 nsize_opcode;
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 0e4f34ed036c6..cde2ff2dcc49b 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3211,6 +3211,7 @@ arm_smmu_master_build_inv(struct arm_smmu_master *master,
 	case INV_TYPE_ATS_FULL:
 		cur->size_opcode = cur->nsize_opcode = CMDQ_OP_ATC_INV;
 		cur->ssid = ssid;
+		cur->master = master;
 		break;
 	}
 
@@ -4168,9 +4169,6 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master)
 	for (i = 0; i < fwspec->num_ids; i++)
 		rb_erase(&master->streams[i].node, &smmu->streams);
 	mutex_unlock(&smmu->streams_mutex);
-
-	kfree(master->streams);
-	kfree(master->build_invs);
 }
 
 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
@@ -4244,6 +4242,16 @@ static void arm_smmu_release_device(struct device *dev)
 	arm_smmu_remove_master(master);
 	if (arm_smmu_cdtab_allocated(&master->cd_table))
 		arm_smmu_free_cd_tables(master);
+
+	/*
+	 * The iommu core detaches @dev from every iommu domain before invoking
+	 * release_device. So the updated domain->invs no longer references the
+	 * @master; IOW, new RCU readers cannot reach it. Wait one grace period
+	 * for in-flight readers to drop their references.
+	 */
+	synchronize_rcu();
+	kfree(master->streams);
+	kfree(master->build_invs);
 	kfree(master);
 }
 
-- 
2.43.0