[PATCH v6 11/11] arm64: mm: set the contiguous bit for kernel mappings where appropriate
Mark Rutland
mark.rutland at arm.com
Fri Mar 10 04:30:06 PST 2017
On Thu, Mar 09, 2017 at 09:52:09PM +0100, Ard Biesheuvel wrote:
> This is the third attempt at enabling the use of contiguous hints for
> kernel mappings. The most recent attempt 0bfc445dec9d was reverted after
> it turned out that updating permission attributes on live contiguous ranges
> may result in TLB conflicts. So this time, the contiguous hint is not set
> for .rodata or for the linear alias of .text/.rodata, both of which are
> mapped read-write initially, and remapped read-only at a later stage.
> (Note that the latter region could also be unmapped and remapped again
> with updated permission attributes, given that the region, while live, is
> only mapped for the convenience of the hibernation code, but that also
> means the TLB footprint is negligible anyway, so why bother)
>
> This enables the following contiguous range sizes for the virtual mapping
> of the kernel image, and for the linear mapping:
>
> granule size | cont PTE | cont PMD |
> -------------+------------+------------+
> 4 KB | 64 KB | 32 MB |
> 16 KB | 2 MB | 1 GB* |
> 64 KB | 2 MB | 16 GB* |
>
> * Only when built for 3 or more levels of translation. This is due to the
> fact that a 2 level configuration only consists of PGDs and PTEs, and the
> added complexity of dealing with folded PMDs is not justified considering
> that 16 GB contiguous ranges are likely to be ignored by the hardware (and
> 16k/2 levels is a niche configuration)
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel at linaro.org>
This looks great; thanks for slogging through this!
To the best of my understanding this should avoid the issue seen by
Jean-Philippe with the last attempt.
As with the rest of the series:
Reviewed-by: Mark Rutland <mark.rutland at arm.com>
I've given this a spin with 4K and 64K pages (2 and 3 level for the
latter) on Juno, with debug_pagealloc dynamically disabled/enabled. The
page tables look as expected in all cases, and memtest can happily poke
all memory without issue regardless.
FWIW, for the series:
Tested-by: Mark Rutland <mark.rutland at arm.com>
Hopefully Catalin/Will are happy to pick this up soon!
Thanks,
Mark.
> ---
> arch/arm64/include/asm/pgtable.h | 10 ++
> arch/arm64/mm/mmu.c | 140 ++++++++++++++------
> 2 files changed, 107 insertions(+), 43 deletions(-)
>
> diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
> index 0eef6064bf3b..c213fdbd056c 100644
> --- a/arch/arm64/include/asm/pgtable.h
> +++ b/arch/arm64/include/asm/pgtable.h
> @@ -74,6 +74,16 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
> #define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN))
> #define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))
>
> +#define pte_cont_addr_end(addr, end) \
> +({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \
> + (__boundary - 1 < (end) - 1) ? __boundary : (end); \
> +})
> +
> +#define pmd_cont_addr_end(addr, end) \
> +({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \
> + (__boundary - 1 < (end) - 1) ? __boundary : (end); \
> +})
> +
> #ifdef CONFIG_ARM64_HW_AFDBM
> #define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
> #else
> diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
> index 85ab82f5a0bc..91502e36e6d9 100644
> --- a/arch/arm64/mm/mmu.c
> +++ b/arch/arm64/mm/mmu.c
> @@ -44,6 +44,7 @@
> #include <asm/ptdump.h>
>
> #define NO_BLOCK_MAPPINGS BIT(0)
> +#define NO_CONT_MAPPINGS BIT(1)
>
> u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
>
> @@ -116,22 +117,11 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
> return ((old ^ new) & ~mask) == 0;
> }
>
> -static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
> - unsigned long end, phys_addr_t phys,
> - pgprot_t prot,
> - phys_addr_t (*pgtable_alloc)(void))
> +static void init_pte(pmd_t *pmd, unsigned long addr, unsigned long end,
> + phys_addr_t phys, pgprot_t prot)
> {
> pte_t *pte;
>
> - BUG_ON(pmd_sect(*pmd));
> - if (pmd_none(*pmd)) {
> - phys_addr_t pte_phys;
> - BUG_ON(!pgtable_alloc);
> - pte_phys = pgtable_alloc();
> - __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
> - }
> - BUG_ON(pmd_bad(*pmd));
> -
> pte = pte_set_fixmap_offset(pmd, addr);
> do {
> pte_t old_pte = *pte;
> @@ -150,25 +140,45 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
> pte_clear_fixmap();
> }
>
> -static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
> - phys_addr_t phys, pgprot_t prot,
> - phys_addr_t (*pgtable_alloc)(void),
> - int flags)
> +static void alloc_init_cont_pte(pmd_t *pmd, unsigned long addr,
> + unsigned long end, phys_addr_t phys,
> + pgprot_t prot,
> + phys_addr_t (*pgtable_alloc)(void),
> + int flags)
> {
> - pmd_t *pmd;
> unsigned long next;
>
> - /*
> - * Check for initial section mappings in the pgd/pud and remove them.
> - */
> - BUG_ON(pud_sect(*pud));
> - if (pud_none(*pud)) {
> - phys_addr_t pmd_phys;
> + BUG_ON(pmd_sect(*pmd));
> + if (pmd_none(*pmd)) {
> + phys_addr_t pte_phys;
> BUG_ON(!pgtable_alloc);
> - pmd_phys = pgtable_alloc();
> - __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
> + pte_phys = pgtable_alloc();
> + __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
> }
> - BUG_ON(pud_bad(*pud));
> + BUG_ON(pmd_bad(*pmd));
> +
> + do {
> + pgprot_t __prot = prot;
> +
> + next = pte_cont_addr_end(addr, end);
> +
> + /* use a contiguous mapping if the range is suitably aligned */
> + if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
> + (flags & NO_CONT_MAPPINGS) == 0)
> + __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
> +
> + init_pte(pmd, addr, next, phys, __prot);
> +
> + phys += next - addr;
> + } while (addr = next, addr != end);
> +}
> +
> +static void init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
> + phys_addr_t phys, pgprot_t prot,
> + phys_addr_t (*pgtable_alloc)(void), int flags)
> +{
> + unsigned long next;
> + pmd_t *pmd;
>
> pmd = pmd_set_fixmap_offset(pud, addr);
> do {
> @@ -188,8 +198,8 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
> BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
> pmd_val(*pmd)));
> } else {
> - alloc_init_pte(pmd, addr, next, phys,
> - prot, pgtable_alloc);
> + alloc_init_cont_pte(pmd, addr, next, phys, prot,
> + pgtable_alloc, flags);
>
> BUG_ON(pmd_val(old_pmd) != 0 &&
> pmd_val(old_pmd) != pmd_val(*pmd));
> @@ -200,6 +210,41 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
> pmd_clear_fixmap();
> }
>
> +static void alloc_init_cont_pmd(pud_t *pud, unsigned long addr,
> + unsigned long end, phys_addr_t phys,
> + pgprot_t prot,
> + phys_addr_t (*pgtable_alloc)(void), int flags)
> +{
> + unsigned long next;
> +
> + /*
> + * Check for initial section mappings in the pgd/pud.
> + */
> + BUG_ON(pud_sect(*pud));
> + if (pud_none(*pud)) {
> + phys_addr_t pmd_phys;
> + BUG_ON(!pgtable_alloc);
> + pmd_phys = pgtable_alloc();
> + __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
> + }
> + BUG_ON(pud_bad(*pud));
> +
> + do {
> + pgprot_t __prot = prot;
> +
> + next = pmd_cont_addr_end(addr, end);
> +
> + /* use a contiguous mapping if the range is suitably aligned */
> + if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
> + (flags & NO_CONT_MAPPINGS) == 0)
> + __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
> +
> + init_pmd(pud, addr, next, phys, __prot, pgtable_alloc, flags);
> +
> + phys += next - addr;
> + } while (addr = next, addr != end);
> +}
> +
> static inline bool use_1G_block(unsigned long addr, unsigned long next,
> unsigned long phys)
> {
> @@ -248,8 +293,8 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
> BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
> pud_val(*pud)));
> } else {
> - alloc_init_pmd(pud, addr, next, phys, prot,
> - pgtable_alloc, flags);
> + alloc_init_cont_pmd(pud, addr, next, phys, prot,
> + pgtable_alloc, flags);
>
> BUG_ON(pud_val(old_pud) != 0 &&
> pud_val(old_pud) != pud_val(*pud));
> @@ -313,7 +358,8 @@ static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
> &phys, virt);
> return;
> }
> - __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0);
> + __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
> + NO_CONT_MAPPINGS);
> }
>
> void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
> @@ -325,7 +371,7 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
> BUG_ON(mm == &init_mm);
>
> if (page_mappings_only)
> - flags = NO_BLOCK_MAPPINGS;
> + flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
>
> __create_pgd_mapping(mm->pgd, phys, virt, size, prot,
> pgd_pgtable_alloc, flags);
> @@ -340,7 +386,8 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
> return;
> }
>
> - __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0);
> + __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
> + NO_CONT_MAPPINGS);
>
> /* flush the TLBs after updating live kernel mappings */
> flush_tlb_kernel_range(virt, virt + size);
> @@ -353,7 +400,7 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end
> int flags = 0;
>
> if (debug_pagealloc_enabled())
> - flags = NO_BLOCK_MAPPINGS;
> + flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
>
> /*
> * Take care not to create a writable alias for the
> @@ -390,10 +437,12 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end
> * alternative patching has completed). This makes the contents
> * of the region accessible to subsystems such as hibernate,
> * but protects it from inadvertent modification or execution.
> + * Note that contiguous mappings cannot be remapped in this way,
> + * so we should avoid them here.
> */
> __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start),
> kernel_end - kernel_start, PAGE_KERNEL,
> - early_pgtable_alloc, 0);
> + early_pgtable_alloc, NO_CONT_MAPPINGS);
> }
>
> void __init mark_linear_text_alias_ro(void)
> @@ -440,7 +489,8 @@ void mark_rodata_ro(void)
> }
>
> static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
> - pgprot_t prot, struct vm_struct *vma)
> + pgprot_t prot, struct vm_struct *vma,
> + int flags)
> {
> phys_addr_t pa_start = __pa_symbol(va_start);
> unsigned long size = va_end - va_start;
> @@ -449,7 +499,7 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
> BUG_ON(!PAGE_ALIGNED(size));
>
> __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
> - early_pgtable_alloc, 0);
> + early_pgtable_alloc, flags);
>
> vma->addr = va_start;
> vma->phys_addr = pa_start;
> @@ -481,14 +531,18 @@ static void __init map_kernel(pgd_t *pgd)
> */
> pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
>
> - map_kernel_segment(pgd, _text, _etext, text_prot, &vmlinux_text);
> + /*
> + * Only rodata will be remapped with different permissions later on,
> + * all other segments are allowed to use contiguous mappings.
> + */
> + map_kernel_segment(pgd, _text, _etext, text_prot, &vmlinux_text, 0);
> map_kernel_segment(pgd, __start_rodata, __inittext_begin, PAGE_KERNEL,
> - &vmlinux_rodata);
> + &vmlinux_rodata, NO_CONT_MAPPINGS);
> map_kernel_segment(pgd, __inittext_begin, __inittext_end, text_prot,
> - &vmlinux_inittext);
> + &vmlinux_inittext, 0);
> map_kernel_segment(pgd, __initdata_begin, __initdata_end, PAGE_KERNEL,
> - &vmlinux_initdata);
> - map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
> + &vmlinux_initdata, 0);
> + map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data, 0);
>
> if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
> /*
> --
> 2.7.4
>
More information about the linux-arm-kernel
mailing list