[PATCH 2/2] iommu/io-pgtable-arm: invalidate iotlb on dirty pte read_and_clear

[Seongjun Hong]

Use io_pgtable_tlb_add_page() instead of iommu_iotlb_gather_add_range()
to prevent the undesirable case where iotlb invalidation cannot be executed
because gather->pgsize is not updated before arm_smmu_iotlb_sync().

io_pgtable_tlb_add_page() leads to iommu_iotlb_gather_add_page(), which
updates gather->pgsize. It prevents missing iotlb invalidation because
arm_smmu_iotlb_sync() invalidates iotlb only when gather->pgsize is not
zero.

Update gather iova range only when IOMMU_DIRTY_NO_CLEAR flag is not set,
as record_dirty() does.

Since iommu_dirty_bitmap_record() becomes stale, discard it.

Fixes: 4fe88fd8b4ae ("iommu/io-pgtable-arm: Add read_and_clear_dirty() support")
Cc: stable at vger.kernel.org
Signed-off-by: Seongjun Hong <hsj0512 at snu.ac.kr>
---
 drivers/iommu/io-pgtable-arm.c | 11 +++++++++--
 include/linux/iommu.h          | 17 -----------------
 2 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 01c3f9cd5d9c..6513dac26e79 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -870,9 +870,16 @@ static int visit_dirty(struct io_pgtable_walk_data *walk_data, int lvl,
 	if (iopte_writeable_dirty(*ptep)) {
 		u64 iova = ALIGN_DOWN(walk_data->addr, size);
 
-		iommu_dirty_bitmap_record(dirty, iova, size);
-		if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR))
+		if (dirty->bitmap)
+			iova_bitmap_set(dirty->bitmap, iova, size);
+
+		if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR)) {
 			iopte_set_writeable_clean(ptep);
+			if (dirty->gather)
+				io_pgtable_tlb_add_page(walk_data->iop,
+							dirty->gather, iova,
+							size);
+		}
 	}
 
 	return 0;
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index ef5043752a36..df57ead2cc3c 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -1106,17 +1106,6 @@ static inline void iommu_dirty_bitmap_init(struct iommu_dirty_bitmap *dirty,
 	dirty->gather = gather;
 }
 
-static inline void iommu_dirty_bitmap_record(struct iommu_dirty_bitmap *dirty,
-					     unsigned long iova,
-					     unsigned long length)
-{
-	if (dirty->bitmap)
-		iova_bitmap_set(dirty->bitmap, iova, length);
-
-	if (dirty->gather)
-		iommu_iotlb_gather_add_range(dirty->gather, iova, length);
-}
-
 /* PCI device grouping function */
 extern struct iommu_group *pci_device_group(struct device *dev);
 /* Generic device grouping function */
@@ -1449,12 +1438,6 @@ static inline void iommu_dirty_bitmap_init(struct iommu_dirty_bitmap *dirty,
 {
 }
 
-static inline void iommu_dirty_bitmap_record(struct iommu_dirty_bitmap *dirty,
-					     unsigned long iova,
-					     unsigned long length)
-{
-}
-
 static inline void iommu_device_unregister(struct iommu_device *iommu)
 {
 }

-- 
2.43.0