[PATCH v3 24/32] iommufd: use iova_to_phys_length for efficient unmap

[Guanghui Feng]

Use iommu_iova_to_phys_length() to get PTE page size in
batch_from_domain and raw_pages_from_domain, allowing traversal
by actual mapping granularity instead of PAGE_SIZE steps.

Signed-off-by: Guanghui Feng <guanghuifeng at linux.alibaba.com>
Acked-by: Shiqiang Zhang <shiyu.zsq at linux.alibaba.com>
Acked-by: Simon Guo <wei.guo.simon at linux.alibaba.com>
---
 drivers/iommu/iommufd/pages.c | 74 +++++++++++++++++++++++++++++------
 1 file changed, 62 insertions(+), 12 deletions(-)

diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c
index 9bdb2945afe1..40a2fe9adf9c 100644
--- a/drivers/iommu/iommufd/pages.c
+++ b/drivers/iommu/iommufd/pages.c
@@ -417,17 +417,42 @@ static void batch_from_domain(struct pfn_batch *batch,
 	if (start_index == iopt_area_index(area))
 		page_offset = area->page_offset;
 	while (start_index <= last_index) {
+		size_t pgsize;
+		unsigned long npages;
+		unsigned long i;
+
 		/*
-		 * This is pretty slow, it would be nice to get the page size
-		 * back from the driver, or have the driver directly fill the
-		 * batch.
+		 * Use iova_to_phys_length to get both the physical address
+		 * and the contiguous mapped length in a single page table
+		 * walk, allowing us to skip ahead by the contiguous region
+		 * size instead of walking page tables for every PAGE_SIZE.
+		 * Query at page-aligned iova so pgsize covers from page start.
 		 */
-		phys = iommu_iova_to_phys(domain, iova) - page_offset;
-		if (!batch_add_pfn(batch, PHYS_PFN(phys)))
-			return;
-		iova += PAGE_SIZE - page_offset;
+		phys = iommu_iova_to_phys_length(domain, iova - page_offset,
+						  &pgsize);
+		if (WARN_ON(phys == PHYS_ADDR_MAX))
+			break;
+		if (WARN_ON(!pgsize || pgsize < PAGE_SIZE))
+			pgsize = PAGE_SIZE;
+
+		/*
+		 * pgsize is the contiguous length from the page-aligned
+		 * iova, so npages is simply pgsize / PAGE_SIZE.
+		 */
+		npages = pgsize / PAGE_SIZE;
+		npages = min_t(unsigned long, npages,
+			       last_index - start_index + 1);
+		if (!npages)
+			npages = 1;
+
+		for (i = 0; i < npages; i++) {
+			if (!batch_add_pfn(batch, PHYS_PFN(phys) + i))
+				return;
+		}
+
+		iova += npages * PAGE_SIZE - page_offset;
 		page_offset = 0;
-		start_index++;
+		start_index += npages;
 	}
 }
 
@@ -445,11 +470,36 @@ static struct page **raw_pages_from_domain(struct iommu_domain *domain,
 	if (start_index == iopt_area_index(area))
 		page_offset = area->page_offset;
 	while (start_index <= last_index) {
-		phys = iommu_iova_to_phys(domain, iova) - page_offset;
-		*(out_pages++) = pfn_to_page(PHYS_PFN(phys));
-		iova += PAGE_SIZE - page_offset;
+		size_t pgsize;
+		unsigned long npages;
+		unsigned long i;
+
+		/*
+		 * Resolve the contiguous mapped length together with the
+		 * physical address so we can fill multiple struct page
+		 * pointers per page table walk when the IOMMU uses large
+		 * pages. Query at page-aligned iova so pgsize covers from
+		 * page start.
+		 */
+		phys = iommu_iova_to_phys_length(domain, iova - page_offset,
+						  &pgsize);
+		if (WARN_ON(phys == PHYS_ADDR_MAX))
+			break;
+		if (WARN_ON(!pgsize || pgsize < PAGE_SIZE))
+			pgsize = PAGE_SIZE;
+
+		npages = pgsize / PAGE_SIZE;
+		npages = min_t(unsigned long, npages,
+			       last_index - start_index + 1);
+		if (!npages)
+			npages = 1;
+
+		for (i = 0; i < npages; i++)
+			*(out_pages++) = pfn_to_page(PHYS_PFN(phys) + i);
+
+		iova += npages * PAGE_SIZE - page_offset;
 		page_offset = 0;
-		start_index++;
+		start_index += npages;
 	}
 	return out_pages;
 }
-- 
2.43.7