[RFC 4/8] arm64/mm: enable __create_pgd_mapping() to run across different pgtable

[Pingfan Liu]

PUP/PMD/PTE fixmap can not be shared across different pgtable
concurrently.

Also change the return type from phys_addr_t to unsigned long, since
allocator may return virtual address directly.

Signed-off-by: Pingfan Liu <kernelfans at gmail.com>
Cc: Catalin Marinas <catalin.marinas at arm.com>
Cc: Will Deacon <will at kernel.org>
Cc: Marc Zyngier <maz at kernel.org>
Cc: Kristina Martsenko <kristina.martsenko at arm.com>
Cc: James Morse <james.morse at arm.com>
Cc: Steven Price <steven.price at arm.com>
Cc: Jonathan Cameron <Jonathan.Cameron at huawei.com>
Cc: Pavel Tatashin <pasha.tatashin at soleen.com>
Cc: Anshuman Khandual <anshuman.khandual at arm.com>
Cc: Atish Patra <atish.patra at wdc.com>
Cc: Mike Rapoport <rppt at kernel.org>
Cc: Logan Gunthorpe <logang at deltatee.com>
Cc: Mark Brown <broonie at kernel.org>
To: linux-arm-kernel at lists.infradead.org
---
 arch/arm64/include/asm/pgalloc.h |  6 +++++-
 arch/arm64/mm/mmu.c              |  6 +++---
 arch/arm64/mm/mmu_include.c      | 31 ++++++++++++++++++++-----------
 3 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 42f602528b90..6e9f1e218300 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -83,7 +83,7 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
 }
 #define pmd_pgtable(pmd) pmd_page(pmd)
 
-typedef phys_addr_t (*pgtable_alloc)(unsigned long shift, void *data);
+typedef unsigned long (*pgtable_alloc)(unsigned long shift, void *data);
 
 extern void __create_pgd_mapping_extend(pgd_t *pgdir,
 		unsigned int entries_cnt, phys_addr_t phys,
@@ -93,4 +93,8 @@ extern void __create_pgd_mapping_extend(pgd_t *pgdir,
 		void *info,
 		int flags);
 
+#define NO_BLOCK_MAPPINGS	BIT(0)
+#define NO_CONT_MAPPINGS	BIT(1)
+#define NO_FIXMAP	BIT(2)
+
 #endif
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 0f183aaf98c9..628752c3cfd0 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -86,7 +86,7 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 }
 EXPORT_SYMBOL(phys_mem_access_prot);
 
-static phys_addr_t __init early_pgtable_alloc(unsigned long unused_a, void *unused_b)
+static unsigned long __init early_pgtable_alloc(unsigned long unused_a, void *unused_b)
 {
 	phys_addr_t phys;
 	void *ptr;
@@ -113,7 +113,7 @@ static phys_addr_t __init early_pgtable_alloc(unsigned long unused_a, void *unus
 	return phys;
 }
 
-static phys_addr_t __pgd_pgtable_alloc(unsigned long unused_a, void *unused_b)
+static unsigned long __pgd_pgtable_alloc(unsigned long unused_a, void *unused_b)
 {
 	void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL);
 	BUG_ON(!ptr);
@@ -123,7 +123,7 @@ static phys_addr_t __pgd_pgtable_alloc(unsigned long unused_a, void *unused_b)
 	return __pa(ptr);
 }
 
-static phys_addr_t pgd_pgtable_alloc(unsigned long shift, void *unused)
+static unsigned long pgd_pgtable_alloc(unsigned long shift, void *unused)
 {
 	phys_addr_t pa = __pgd_pgtable_alloc(shift, unused);
 
diff --git a/arch/arm64/mm/mmu_include.c b/arch/arm64/mm/mmu_include.c
index 371afc7d4502..adad0f93cd53 100644
--- a/arch/arm64/mm/mmu_include.c
+++ b/arch/arm64/mm/mmu_include.c
@@ -1,8 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
-#define NO_BLOCK_MAPPINGS	BIT(0)
-#define NO_CONT_MAPPINGS	BIT(1)
-
 static bool pgattr_change_is_safe(u64 old, u64 new)
 {
 	/*
@@ -38,11 +35,14 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
 }
 
 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
-		     phys_addr_t phys, pgprot_t prot)
+		     phys_addr_t phys, pgprot_t prot, int flags)
 {
 	pte_t *ptep;
 
-	ptep = pte_set_fixmap_offset(pmdp, addr);
+	if (likely(!(flags & NO_FIXMAP)))
+		ptep = pte_set_fixmap_offset(pmdp, addr);
+	else
+		ptep = pte_offset_kernel(pmdp, addr);
 	do {
 		pte_t old_pte = READ_ONCE(*ptep);
 
@@ -58,7 +58,8 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
 		phys += PAGE_SIZE;
 	} while (ptep++, addr += PAGE_SIZE, addr != end);
 
-	pte_clear_fixmap();
+	if (likely(!(flags & NO_FIXMAP)))
+		pte_clear_fixmap();
 }
 
 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
@@ -91,7 +92,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
 		    (flags & NO_CONT_MAPPINGS) == 0)
 			__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
 
-		init_pte(pmdp, addr, next, phys, __prot);
+		init_pte(pmdp, addr, next, phys, __prot, flags);
 
 		phys += next - addr;
 	} while (addr = next, addr != end);
@@ -106,7 +107,10 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
 	unsigned long next;
 	pmd_t *pmdp;
 
-	pmdp = pmd_set_fixmap_offset(pudp, addr);
+	if (likely(!(flags & NO_FIXMAP)))
+		pmdp = pmd_set_fixmap_offset(pudp, addr);
+	else
+		pmdp = pmd_offset(pudp, addr);
 	do {
 		pmd_t old_pmd = READ_ONCE(*pmdp);
 
@@ -133,7 +137,8 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
 		phys += next - addr;
 	} while (pmdp++, addr = next, addr != end);
 
-	pmd_clear_fixmap();
+	if (likely(!(flags & NO_FIXMAP)))
+		pmd_clear_fixmap();
 }
 
 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
@@ -207,7 +212,10 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
 	}
 	BUG_ON(p4d_bad(p4d));
 
-	pudp = pud_set_fixmap_offset(p4dp, addr);
+	if (likely(!(flags & NO_FIXMAP)))
+		pudp = pud_set_fixmap_offset(p4dp, addr);
+	else
+		pudp = pud_offset(p4dp, addr);
 	do {
 		pud_t old_pud = READ_ONCE(*pudp);
 
@@ -236,7 +244,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
 		phys += next - addr;
 	} while (pudp++, addr = next, addr != end);
 
-	pud_clear_fixmap();
+	if (likely(!(flags & NO_FIXMAP)))
+		pud_clear_fixmap();
 }
 
 static void __create_pgd_mapping(pgd_t *pgdir, unsigned int entries_cnt, phys_addr_t phys,
-- 
2.29.2