[PATCH 02/10] arm64: mm: Introduce mmu_head routines without instrumentation
Pingfan Liu
piliu at redhat.com
Wed Mar 13 05:57:00 PDT 2024
During the early boot stage, the instrumentation can not be handled.
Use a macro INSTRUMENT_OPTION to switch on or off 'noinstr' on these
routines.
Signed-off-by: Pingfan Liu <piliu at redhat.com>
Cc: Ard Biesheuvel <ardb at kernel.org>
Cc: Catalin Marinas <catalin.marinas at arm.com>
Cc: Will Deacon <will at kernel.org>
Cc: Mark Rutland <mark.rutland at arm.com>
To: linux-arm-kernel at lists.infradead.org
---
arch/arm64/mm/Makefile | 2 +-
arch/arm64/mm/mmu.c | 50 +++++++++--------------------
arch/arm64/mm/mmu_head.c | 19 +++++++++++
arch/arm64/mm/mmu_inc.c | 68 +++++++++++++++++++++++++++++++---------
4 files changed, 87 insertions(+), 52 deletions(-)
create mode 100644 arch/arm64/mm/mmu_head.c
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index dbd1bc95967d..0d92fb24a398 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -2,7 +2,7 @@
obj-y := dma-mapping.o extable.o fault.o init.o \
cache.o copypage.o flush.o \
ioremap.o mmap.o pgd.o mmu.o \
- context.o proc.o pageattr.o fixmap.o
+ context.o proc.o pageattr.o fixmap.o mmu_head.o
obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 870be374f458..80e49faaf066 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -131,46 +131,14 @@ static phys_addr_t __init early_pgtable_alloc(int shift)
return phys;
}
+#define INSTRUMENT_OPTION
+#include "mmu_inc.c"
+
bool pgattr_change_is_safe(u64 old, u64 new)
{
- /*
- * The following mapping attributes may be updated in live
- * kernel mappings without the need for break-before-make.
- */
- pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG;
-
- /* creating or taking down mappings is always safe */
- if (!pte_valid(__pte(old)) || !pte_valid(__pte(new)))
- return true;
-
- /* A live entry's pfn should not change */
- if (pte_pfn(__pte(old)) != pte_pfn(__pte(new)))
- return false;
-
- /* live contiguous mappings may not be manipulated at all */
- if ((old | new) & PTE_CONT)
- return false;
-
- /* Transitioning from Non-Global to Global is unsafe */
- if (old & ~new & PTE_NG)
- return false;
-
- /*
- * Changing the memory type between Normal and Normal-Tagged is safe
- * since Tagged is considered a permission attribute from the
- * mismatched attribute aliases perspective.
- */
- if (((old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) ||
- (old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)) &&
- ((new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) ||
- (new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)))
- mask |= PTE_ATTRINDX_MASK;
-
- return ((old ^ new) & ~mask) == 0;
+ return __pgattr_change_is_safe(old, new);
}
-#include "mmu_inc.c"
-
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot,
@@ -945,6 +913,16 @@ void vmemmap_free(unsigned long start, unsigned long end,
}
#endif /* CONFIG_MEMORY_HOTPLUG */
+int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
+{
+ return __pud_set_huge(pudp, phys, prot);
+}
+
+int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
+{
+ return __pmd_set_huge(pmdp, phys, prot);
+}
+
int pud_clear_huge(pud_t *pudp)
{
if (!pud_sect(READ_ONCE(*pudp)))
diff --git a/arch/arm64/mm/mmu_head.c b/arch/arm64/mm/mmu_head.c
new file mode 100644
index 000000000000..4d65b7368db3
--- /dev/null
+++ b/arch/arm64/mm/mmu_head.c
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <asm/barrier.h>
+#include <asm/kernel-pgtable.h>
+#include <asm/pgalloc.h>
+
+#define INSTRUMENT_OPTION __noinstr_section(".init.text.noinstr")
+#include "mmu_inc.c"
+
+void INSTRUMENT_OPTION mmu_head_create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
+ unsigned long virt, phys_addr_t size,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
+{
+ __create_pgd_mapping_locked(pgdir, phys, virt, size, prot, pgtable_alloc, flags);
+}
diff --git a/arch/arm64/mm/mmu_inc.c b/arch/arm64/mm/mmu_inc.c
index dcd97eea0726..2535927d30ec 100644
--- a/arch/arm64/mm/mmu_inc.c
+++ b/arch/arm64/mm/mmu_inc.c
@@ -1,11 +1,49 @@
// SPDX-License-Identifier: GPL-2.0-only
-int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
+static bool INSTRUMENT_OPTION __pgattr_change_is_safe(u64 old, u64 new)
+{
+ /*
+ * The following mapping attributes may be updated in live
+ * kernel mappings without the need for break-before-make.
+ */
+ pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG;
+
+ /* creating or taking down mappings is always safe */
+ if (!pte_valid(__pte(old)) || !pte_valid(__pte(new)))
+ return true;
+
+ /* A live entry's pfn should not change */
+ if (pte_pfn(__pte(old)) != pte_pfn(__pte(new)))
+ return false;
+
+ /* live contiguous mappings may not be manipulated at all */
+ if ((old | new) & PTE_CONT)
+ return false;
+
+ /* Transitioning from Non-Global to Global is unsafe */
+ if (old & ~new & PTE_NG)
+ return false;
+
+ /*
+ * Changing the memory type between Normal and Normal-Tagged is safe
+ * since Tagged is considered a permission attribute from the
+ * mismatched attribute aliases perspective.
+ */
+ if (((old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) ||
+ (old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)) &&
+ ((new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) ||
+ (new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)))
+ mask |= PTE_ATTRINDX_MASK;
+
+ return ((old ^ new) & ~mask) == 0;
+}
+
+static int INSTRUMENT_OPTION __pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
{
pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot));
/* Only allow permission changes for now */
- if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)),
+ if (!__pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)),
pud_val(new_pud)))
return 0;
@@ -14,12 +52,12 @@ int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
return 1;
}
-int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
+static int INSTRUMENT_OPTION __pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
{
pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot));
/* Only allow permission changes for now */
- if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)),
+ if (!__pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)),
pmd_val(new_pmd)))
return 0;
@@ -28,7 +66,7 @@ int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
return 1;
}
-static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
+static void INSTRUMENT_OPTION init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot)
{
pte_t *ptep;
@@ -43,7 +81,7 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
* After the PTE entry has been populated once, we
* only allow updates to the permission attributes.
*/
- BUG_ON(!pgattr_change_is_safe(pte_val(old_pte),
+ BUG_ON(!__pgattr_change_is_safe(pte_val(old_pte),
READ_ONCE(pte_val(*ptep))));
phys += PAGE_SIZE;
@@ -52,7 +90,7 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
pte_clear_fixmap();
}
-static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
+static void INSTRUMENT_OPTION alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
unsigned long end, phys_addr_t phys,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(int),
@@ -91,7 +129,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
} while (addr = next, addr != end);
}
-static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
+static void INSTRUMENT_OPTION init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
phys_addr_t (*pgtable_alloc)(int), int flags)
{
@@ -107,13 +145,13 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
/* try section mapping first */
if (((addr | next | phys) & ~PMD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0) {
- pmd_set_huge(pmdp, phys, prot);
+ __pmd_set_huge(pmdp, phys, prot);
/*
* After the PMD entry has been populated once, we
* only allow updates to the permission attributes.
*/
- BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
+ BUG_ON(!__pgattr_change_is_safe(pmd_val(old_pmd),
READ_ONCE(pmd_val(*pmdp))));
} else {
alloc_init_cont_pte(pmdp, addr, next, phys, prot,
@@ -128,7 +166,7 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
pmd_clear_fixmap();
}
-static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
+static void INSTRUMENT_OPTION alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
unsigned long end, phys_addr_t phys,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(int), int flags)
@@ -169,7 +207,7 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
} while (addr = next, addr != end);
}
-static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+static void INSTRUMENT_OPTION alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
phys_addr_t (*pgtable_alloc)(int),
int flags)
@@ -204,13 +242,13 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
if (pud_sect_supported() &&
((addr | next | phys) & ~PUD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0) {
- pud_set_huge(pudp, phys, prot);
+ __pud_set_huge(pudp, phys, prot);
/*
* After the PUD entry has been populated once, we
* only allow updates to the permission attributes.
*/
- BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
+ BUG_ON(!__pgattr_change_is_safe(pud_val(old_pud),
READ_ONCE(pud_val(*pudp))));
} else {
alloc_init_cont_pmd(pudp, addr, next, phys, prot,
@@ -225,7 +263,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
pud_clear_fixmap();
}
-static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
+static void INSTRUMENT_OPTION __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(int),
--
2.41.0
More information about the linux-arm-kernel
mailing list