[PATCH 11/18] hugetlb_vmemmap: Use try_populate_vmemmap_pmd for replacing in-use PMDs
James Houghton
jthoughton at google.com
Tue Jul 7 20:11:21 PDT 2026
This routine is to be used for updating in-use, leaf-level PMDs in the
vmemmap without introducing a window where vmemmap accesses might fault.
Because try_populate_vmemmap_pmd() can fail, the split_page() call needs
to be moved to after the PMD update. Because both split_page() and the
PMD update are done under the page table lock, this rearrangement is
safe.
Only leaf-level PMD to non-leaf PMD (mapping the same physical pages) is
the only transition that needs to be supported.
Collapsing a non-leaf PMD to a leaf-level PMD is not used by HVO, so
try_populate_vmemmap_pmd() need not support it.
This patch allows arm64's implementation to replaced with one that does
not require BBML2_NOABORT.
Signed-off-by: James Houghton <jthoughton at google.com>
---
arch/arm64/include/asm/pgalloc.h | 9 +++++++++
arch/loongarch/include/asm/pgalloc.h | 8 ++++++++
arch/riscv/include/asm/pgalloc.h | 8 ++++++++
arch/x86/include/asm/pgalloc.h | 8 ++++++++
include/linux/pgalloc.h | 20 ++++++++++++++++++++
mm/hugetlb_vmemmap.c | 28 ++++++++++++++++------------
6 files changed, 69 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 1b4509d3382c..c8946250d431 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -121,4 +121,13 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN);
}
+#define __HAVE_ARCH_TRY_POPULATE_VMEMMAP_PMD
+static inline int try_populate_vmemmap_pmd(pmd_t *pmdp, pte_t *pgtable,
+ unsigned long addr)
+{
+ /* BBML2_NOABORT is required. Its presence has been checked. */
+ pmd_populate_kernel(&init_mm, pmdp, pgtable);
+ return 0;
+}
+
#endif
diff --git a/arch/loongarch/include/asm/pgalloc.h b/arch/loongarch/include/asm/pgalloc.h
index 248f62d0b590..9322d962c9cc 100644
--- a/arch/loongarch/include/asm/pgalloc.h
+++ b/arch/loongarch/include/asm/pgalloc.h
@@ -24,6 +24,14 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte)
set_pmd(pmd, __pmd((unsigned long)page_address(pte)));
}
+#define __HAVE_ARCH_TRY_POPULATE_VMEMMAP_PMD
+static inline int try_populate_vmemmap_pmd(pmd_t *pmdp, pte_t *pgtable,
+ unsigned long addr)
+{
+ pmd_populate_kernel(&init_mm, pmdp, pgtable);
+ return 0;
+}
+
#ifndef __PAGETABLE_PMD_FOLDED
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h
index 770ce18a7328..8dbbe62b2574 100644
--- a/arch/riscv/include/asm/pgalloc.h
+++ b/arch/riscv/include/asm/pgalloc.h
@@ -31,6 +31,14 @@ static inline void pmd_populate(struct mm_struct *mm,
set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
}
+#define __HAVE_ARCH_TRY_POPULATE_VMEMMAP_PMD
+static inline int try_populate_vmemmap_pmd(pmd_t *pmdp, pte_t *pgtable,
+ unsigned long addr)
+{
+ pmd_populate_kernel(&init_mm, pmdp, pgtable);
+ return 0;
+}
+
#ifndef __PAGETABLE_PMD_FOLDED
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index c88691b15f3c..47b56bbd6236 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -82,6 +82,14 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE));
}
+#define __HAVE_ARCH_TRY_POPULATE_VMEMMAP_PMD
+static inline int try_populate_vmemmap_pmd(pmd_t *pmdp, pte_t *pgtable,
+ unsigned long addr)
+{
+ pmd_populate_kernel(&init_mm, pmdp, pgtable);
+ return 0;
+}
+
#if CONFIG_PGTABLE_LEVELS > 2
extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
diff --git a/include/linux/pgalloc.h b/include/linux/pgalloc.h
index 9174fa59bbc5..ed446d95ca37 100644
--- a/include/linux/pgalloc.h
+++ b/include/linux/pgalloc.h
@@ -26,4 +26,24 @@
arch_sync_kernel_mappings(addr, addr); \
} while (0)
+#ifndef __HAVE_ARCH_TRY_POPULATE_VMEMMAP_PMD
+/*
+ * try_populate_vmemmap_pmd - Populate a PMD that is in use by the vmemmap.
+ * @addr: Base address of the remapped PMD.
+ * @pmdp: Page table pointer to be overwritten.
+ * @pgtable: Pointer to the page table that the new PMD will point to.
+ *
+ * This function is only to be used to update PMDs that map the vmemmap to
+ * point to a page of already-populated PTEs that map the same pages.
+ *
+ * Implementations of this function must ensure that, while the update is taking
+ * place, CPUs will not fault on the remapped virtual address range.
+ */
+static inline int try_populate_vmemmap_pmd(pmd_t *pmdp, pte_t *pgtable,
+ unsigned long addr)
+{
+ return -EOPNOTSUPP;
+}
+#endif
+
#endif /* _LINUX_PGALLOC_H */
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 977249e22ed0..b445febac0d2 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -55,6 +55,7 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
struct vmemmap_remap_walk *walk)
{
pmd_t __pmd;
+ int ret;
int i;
unsigned long addr = start;
pte_t *pgtable;
@@ -74,8 +75,15 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
set_pte_at(&init_mm, addr, pte, entry);
}
+ ret = 0;
spin_lock(&init_mm.page_table_lock);
if (likely(pmd_leaf(*pmd))) {
+ /* Make pte visible before pmd. See comment in pmd_install(). */
+ smp_wmb();
+ ret = try_populate_vmemmap_pmd(pmd, pgtable, start);
+ if (ret)
+ goto free;
+
/*
* Higher order allocations from buddy allocator must be able to
* be treated as independent small pages (as they can be freed
@@ -84,21 +92,17 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
if (!PageReserved(head))
split_page(head, get_order(PMD_SIZE));
- /* Make pte visible before pmd. See comment in pmd_install(). */
- smp_wmb();
- /*
- * On arm64, this requires BBML2_NOABORT. Its support has
- * already been checked.
- */
- pmd_populate_kernel(&init_mm, pmd, pgtable);
if (!(walk->flags & VMEMMAP_SPLIT_NO_TLB_FLUSH))
flush_tlb_kernel_range(start, start + PMD_SIZE);
- } else {
- pte_free_kernel(&init_mm, pgtable);
- }
- spin_unlock(&init_mm.page_table_lock);
+ } else
+ goto free;
- return 0;
+out:
+ spin_unlock(&init_mm.page_table_lock);
+ return ret;
+free:
+ pte_free_kernel(&init_mm, pgtable);
+ goto out;
}
static int vmemmap_pmd_entry(pmd_t *pmd, unsigned long addr,
--
2.55.0.795.g602f6c329a-goog
More information about the linux-arm-kernel
mailing list