[PATCH v5 5/7] arm64: head: Clean the ID map and the HYP text to the PoC if needed

[Ard Biesheuvel]

On Tue, 8 Nov 2022 at 19:22, Ard Biesheuvel <ardb at kernel.org> wrote:
>
> If we enter with the MMU and caches enabled, the bootloader may not have
> performed any cache maintenance to the PoC. So clean the ID mapped page
> to the PoC, to ensure that instruction and data accesses with the MMU
> off see the correct data. For similar reasons, clean all the HYP text to
> the PoC as well when entering at EL2 with the MMU and caches enabled.
>
> Note that this means primary_entry() itself needs to be moved into the
> ID map as well, as we will return from init_kernel_el() with the MMU and
> caches off.
>
> Signed-off-by: Ard Biesheuvel <ardb at kernel.org>
> ---
>  arch/arm64/kernel/head.S  | 31 +++++++++++++++++---
>  arch/arm64/kernel/sleep.S |  1 +
>  2 files changed, 28 insertions(+), 4 deletions(-)
>
> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> index a7c84cde67c5c652..825f1d0549661030 100644
> --- a/arch/arm64/kernel/head.S
> +++ b/arch/arm64/kernel/head.S
> @@ -70,7 +70,7 @@
>
>         __EFI_PE_HEADER
>
> -       __INIT
> +       .section ".idmap.text","awx"
>
>         /*
>          * The following callee saved general purpose registers are used on the
> @@ -90,6 +90,17 @@ SYM_CODE_START(primary_entry)
>         bl      record_mmu_state
>         bl      preserve_boot_args
>         bl      create_idmap
> +
> +       /*
> +        * If we entered with the MMU and caches on, clean the ID mapped part
> +        * of the primary boot code to the PoC so we can safely execute it with
> +        * the MMU off.
> +        */
> +       cbz     x19, 0f
> +       adrp    x0, __idmap_text_start
> +       adr_l   x1, __idmap_text_end
> +       bl      dcache_clean_poc
> +0:     mov     x19, x0

This is wrong, it should be

mov x0, x19


>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         mov     x20, x0
>
> @@ -111,6 +122,7 @@ SYM_CODE_START(primary_entry)
>         b       __primary_switch
>  SYM_CODE_END(primary_entry)
>
> +       __INIT
>  SYM_CODE_START_LOCAL(record_mmu_state)
>         mrs     x19, CurrentEL
>         cmp     x19, #CurrentEL_EL2
> @@ -505,10 +517,12 @@ SYM_FUNC_END(__primary_switched)
>   * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x0 if
>   * booted in EL1 or EL2 respectively, with the top 32 bits containing
>   * potential context flags. These flags are *not* stored in __boot_cpu_mode.
> + *
> + * x0: whether we are being called from the primary boot path with the MMU on
>   */
>  SYM_FUNC_START(init_kernel_el)
> -       mrs     x0, CurrentEL
> -       cmp     x0, #CurrentEL_EL2
> +       mrs     x1, CurrentEL
> +       cmp     x1, #CurrentEL_EL2
>         b.eq    init_el2
>
>  SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
> @@ -523,6 +537,14 @@ SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
>         eret
>
>  SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
> +       msr     elr_el2, lr
> +
> +       // clean all HYP code to the PoC if we booted at EL2 with the MMU on
> +       cbz     x0, 0f
> +       adrp    x0, __hyp_idmap_text_start
> +       adr_l   x1, __hyp_text_end
> +       bl      dcache_clean_poc
> +0:
>         mov_q   x0, HCR_HOST_NVHE_FLAGS
>         msr     hcr_el2, x0
>         isb
> @@ -556,7 +578,6 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
>         msr     sctlr_el1, x1
>         mov     x2, xzr
>  2:
> -       msr     elr_el2, lr
>         mov     w0, #BOOT_CPU_MODE_EL2
>         orr     x0, x0, x2
>         eret
> @@ -567,6 +588,7 @@ SYM_FUNC_END(init_kernel_el)
>          * cores are held until we're ready for them to initialise.
>          */
>  SYM_FUNC_START(secondary_holding_pen)
> +       mov     x0, xzr
>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         mrs     x2, mpidr_el1
>         mov_q   x1, MPIDR_HWID_BITMASK
> @@ -584,6 +606,7 @@ SYM_FUNC_END(secondary_holding_pen)
>          * be used where CPUs are brought online dynamically by the kernel.
>          */
>  SYM_FUNC_START(secondary_entry)
> +       mov     x0, xzr
>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         b       secondary_startup
>  SYM_FUNC_END(secondary_entry)
> diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
> index 7b7c56e048346e97..2ae7cff1953aaf87 100644
> --- a/arch/arm64/kernel/sleep.S
> +++ b/arch/arm64/kernel/sleep.S
> @@ -99,6 +99,7 @@ SYM_FUNC_END(__cpu_suspend_enter)
>
>         .pushsection ".idmap.text", "awx"
>  SYM_CODE_START(cpu_resume)
> +       mov     x0, xzr
>         bl      init_kernel_el
>         mov     x19, x0                 // preserve boot mode
>  #if VA_BITS > 48
> --
> 2.35.1
>