[PATCH v2 16/20 DO-NOT-MERGE] hugetlb_vmemmap: Use try_populate_vmemmap_pmd for replacing in-use PMDs
James Houghton
jthoughton at google.com
Fri Oct 2 17:21:19 PDT 2026
This routine is to be used for updating in-use, leaf-level PMDs in the
vmemmap without introducing a window where vmemmap accesses might fault.
Because try_populate_vmemmap_pmd() can fail, the split_page() call needs
to be moved to after the PMD update. Because both split_page() and the
PMD update are done under the page table lock, this rearrangement is
safe.
Only leaf-level PMD to non-leaf PMD (mapping the same physical pages) is
the only transition that needs to be supported.
Collapsing a non-leaf PMD to a leaf-level PMD is not used by HVO, so
try_populate_vmemmap_pmd() need not support it.
This patch allows arm64's implementation to replaced with one that does
not require BBML3.
Signed-off-by: James Houghton <jthoughton at google.com>
---
arch/arm64/include/asm/pgalloc.h | 8 ++++++++
arch/loongarch/include/asm/pgalloc.h | 2 ++
arch/riscv/include/asm/pgalloc.h | 2 ++
arch/x86/include/asm/pgalloc.h | 2 ++
include/linux/pgalloc.h | 21 +++++++++++++++++++++
mm/hugetlb_vmemmap.c | 25 ++++++++++++++-----------
6 files changed, 49 insertions(+), 11 deletions(-)
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 1b4509d3382c..0263329fb7b3 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -121,4 +121,12 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN);
}
+static inline int try_populate_vmemmap_pmd(unsigned long addr, pmd_t *pmdp,
+ pte_t *pgtable)
+{
+ /* BBLM3 is required. Its presence has been checked. */
+ pmd_populate_kernel(&init_mm, pmdp, pgtable);
+ return 0;
+}
+
#endif
diff --git a/arch/loongarch/include/asm/pgalloc.h b/arch/loongarch/include/asm/pgalloc.h
index 248f62d0b590..0553addb0afa 100644
--- a/arch/loongarch/include/asm/pgalloc.h
+++ b/arch/loongarch/include/asm/pgalloc.h
@@ -24,6 +24,8 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte)
set_pmd(pmd, __pmd((unsigned long)page_address(pte)));
}
+#define ARCH_WANTS_GENERIC_POPULATE_VMEMMAP_PMD
+
#ifndef __PAGETABLE_PMD_FOLDED
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h
index 770ce18a7328..b9b4b25fc6f0 100644
--- a/arch/riscv/include/asm/pgalloc.h
+++ b/arch/riscv/include/asm/pgalloc.h
@@ -31,6 +31,8 @@ static inline void pmd_populate(struct mm_struct *mm,
set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
}
+#define ARCH_WANTS_GENERIC_POPULATE_VMEMMAP_PMD
+
#ifndef __PAGETABLE_PMD_FOLDED
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index c88691b15f3c..f8d24b97cbf5 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -82,6 +82,8 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE));
}
+#define ARCH_WANTS_GENERIC_POPULATE_VMEMMAP_PMD
+
#if CONFIG_PGTABLE_LEVELS > 2
extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
diff --git a/include/linux/pgalloc.h b/include/linux/pgalloc.h
index 9174fa59bbc5..1dbecb748028 100644
--- a/include/linux/pgalloc.h
+++ b/include/linux/pgalloc.h
@@ -26,4 +26,25 @@
arch_sync_kernel_mappings(addr, addr); \
} while (0)
+#ifdef ARCH_WANTS_GENERIC_POPULATE_VMEMMAP_PMD
+/*
+ * try_populate_vmemmap_pmd - Populate a PMD that is in use by the vmemmap.
+ * @addr: Base address of the remapped PMD.
+ * @pmdp: Page table pointer to be overwritten.
+ * @pgtable: Pointer to the page table that the new PMD will point to.
+ *
+ * This function is only to be used to update PMDs that map the vmemmap to
+ * point to a page of already-populated PTEs that map the same pages.
+ *
+ * Implementations of this function must ensure that, while the update is taking
+ * place, CPUs will not fault on the remapped virtual address range.
+ */
+static inline int try_populate_vmemmap_pmd(unsigned long addr, pmd_t *pmdp,
+ pte_t *pgtable)
+{
+ pmd_populate_kernel(&init_mm, pmdp, pgtable);
+ return 0;
+}
+#endif
+
#endif /* _LINUX_PGALLOC_H */
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 1eca03a3def3..9e0f52474bdb 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -88,6 +88,7 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
struct vmemmap_remap_walk *walk)
{
pmd_t __pmd;
+ int ret;
int i;
unsigned long addr = start;
pte_t *pgtable;
@@ -107,8 +108,15 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
set_pte_at(&init_mm, addr, pte, entry);
}
+ ret = 0;
spin_lock(&init_mm.page_table_lock);
if (likely(pmd_leaf(*pmd))) {
+ /* Make pte visible before pmd. See comment in pmd_install(). */
+ smp_wmb();
+ ret = try_populate_vmemmap_pmd(start, pmd, pgtable);
+ if (ret)
+ goto free;
+
/*
* Higher order allocations from buddy allocator must be able to
* be treated as independent small pages (as they can be freed
@@ -117,21 +125,16 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
if (!PageReserved(head))
split_page(head, get_order(PMD_SIZE));
- /* Make pte visible before pmd. See comment in pmd_install(). */
- smp_wmb();
- /*
- * On arm64, this requires BBML3. Its support has already been
- * checked.
- */
- pmd_populate_kernel(&init_mm, pmd, pgtable);
if (!(walk->flags & VMEMMAP_SPLIT_NO_TLB_FLUSH))
flush_tlb_kernel_range(start, start + PMD_SIZE);
- } else {
- pte_free_kernel(&init_mm, pgtable);
+ goto out;
}
- spin_unlock(&init_mm.page_table_lock);
- return 0;
+free:
+ pte_free_kernel(&init_mm, pgtable);
+out:
+ spin_unlock(&init_mm.page_table_lock);
+ return ret;
}
static int vmemmap_pmd_entry(pmd_t *pmd, unsigned long addr,
--
2.56.0.rc1.315.gc6ed9934b7-goog
More information about the linux-arm-kernel
mailing list