[PATCH] iommu/rockchip: implement .flush_iotlb_all

[Jiaxing Hu]

rk_iommu invalidates the IOTLB synchronously in .unmap (via
rk_iommu_zap_iova), so it stays consistent without a deferred .iotlb_sync
and does not advertise IOMMU_CAP_DEFERRED_FLUSH.

It never implemented the optional .flush_iotlb_all op, though, and
iommu_flush_iotlb_all() is a no-op when that op is absent -- so on Rockchip
a driver that owns an rk_iommu domain and calls it to flush the whole
domain's TLB (e.g. before reusing a mapping, without unmapping it) gets
nothing, even though the hardware can do it: rk_iommu already issues
ZAP_CACHE for range shootdowns.

Implement it with that same primitive: ZAP_CACHE every bank of every IOMMU
on the domain, under the runtime-PM + clk guard already used by
rk_iommu_zap_iova().

Found while bringing up an out-of-tree RK3576 NPU (accel/rocket) driver
that flushes before each submit; on Rockchip that flush did nothing.

Signed-off-by: Jiaxing Hu <huhuvmb88 at gmail.com>
---
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -1202,6 +1202,30 @@
 	return 0;
 }
 
+/* shootdown the entire tlb on all iommus using this domain */
+static void rk_iommu_flush_iotlb_all(struct iommu_domain *domain)
+{
+	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
+	struct list_head *pos;
+	unsigned long flags;
+
+	spin_lock_irqsave(&rk_domain->iommus_lock, flags);
+	list_for_each(pos, &rk_domain->iommus) {
+		struct rk_iommu *iommu = list_entry(pos, struct rk_iommu, node);
+		int ret = pm_runtime_get_if_in_use(iommu->dev);
+
+		if (WARN_ON_ONCE(ret < 0))
+			continue;
+		if (ret) {
+			WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks));
+			rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE);
+			clk_bulk_disable(iommu->num_clocks, iommu->clocks);
+			pm_runtime_put(iommu->dev);
+		}
+	}
+	spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);
+}
+
 static const struct iommu_ops rk_iommu_ops = {
 	.identity_domain = &rk_identity_domain,
 	.domain_alloc_paging = rk_iommu_domain_alloc_paging,
@@ -1214,6 +1245,7 @@
 		.map_pages	= rk_iommu_map,
 		.unmap_pages	= rk_iommu_unmap,
 		.iova_to_phys	= rk_iommu_iova_to_phys,
+		.flush_iotlb_all = rk_iommu_flush_iotlb_all,
 		.free		= rk_iommu_domain_free,
 	}
 };