[PATCH v1 13/13] arm64: Rename arm64-internal cache maintenance functions

Fuad Tabba tabba at google.com
Wed May 12 02:56:09 PDT 2021


Hi Ard,

> I am a big fan of this change: code is so much easier to read if the
> names of subroutines match their intent. I would suggest, though, that
> we get rid of all the leading underscores while at it: we often use
> them when refactoring existing routines into separate pieces (which is
> where at least some of these came from), but here, they seem to have
> little value.

Thank you. I'll remove the underscores in v2.

Thanks,
/fuad

>
> > ---
> >  arch/arm64/include/asm/arch_gicv3.h |  2 +-
> >  arch/arm64/include/asm/cacheflush.h | 36 +++++++++----------
> >  arch/arm64/include/asm/efi.h        |  2 +-
> >  arch/arm64/include/asm/kvm_mmu.h    |  6 ++--
> >  arch/arm64/kernel/alternative.c     |  2 +-
> >  arch/arm64/kernel/efi-entry.S       |  4 +--
> >  arch/arm64/kernel/head.S            |  8 ++---
> >  arch/arm64/kernel/hibernate.c       | 12 +++----
> >  arch/arm64/kernel/idreg-override.c  |  2 +-
> >  arch/arm64/kernel/image-vars.h      |  2 +-
> >  arch/arm64/kernel/insn.c            |  2 +-
> >  arch/arm64/kernel/kaslr.c           |  6 ++--
> >  arch/arm64/kernel/machine_kexec.c   | 10 +++---
> >  arch/arm64/kernel/smp.c             |  4 +--
> >  arch/arm64/kernel/smp_spin_table.c  |  4 +--
> >  arch/arm64/kernel/sys_compat.c      |  2 +-
> >  arch/arm64/kvm/arm.c                |  2 +-
> >  arch/arm64/kvm/hyp/nvhe/cache.S     |  4 +--
> >  arch/arm64/kvm/hyp/nvhe/setup.c     |  2 +-
> >  arch/arm64/kvm/hyp/nvhe/tlb.c       |  2 +-
> >  arch/arm64/kvm/hyp/pgtable.c        |  4 +--
> >  arch/arm64/lib/uaccess_flushcache.c |  4 +--
> >  arch/arm64/mm/cache.S               | 56 ++++++++++++++---------------
> >  arch/arm64/mm/flush.c               | 12 +++----
> >  24 files changed, 95 insertions(+), 95 deletions(-)
> >
> > diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h
> > index ed1cc9d8e6df..4b7ac9098e8f 100644
> > --- a/arch/arm64/include/asm/arch_gicv3.h
> > +++ b/arch/arm64/include/asm/arch_gicv3.h
> > @@ -125,7 +125,7 @@ static inline u32 gic_read_rpr(void)
> >  #define gic_write_lpir(v, c)           writeq_relaxed(v, c)
> >
> >  #define gic_flush_dcache_to_poc(a,l)   \
> > -       __flush_dcache_area((unsigned long)(a), (unsigned long)(a)+(l))
> > +       __clean_inval_dcache_poc((unsigned long)(a), (unsigned long)(a)+(l))
> >
> >  #define gits_read_baser(c)             readq_relaxed(c)
> >  #define gits_write_baser(v, c)         writeq_relaxed(v, c)
> > diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
> > index 4b91d3530013..526eee4522eb 100644
> > --- a/arch/arm64/include/asm/cacheflush.h
> > +++ b/arch/arm64/include/asm/cacheflush.h
> > @@ -34,54 +34,54 @@
> >   *             - start  - virtual start address
> >   *             - end    - virtual end address
> >   *
> > - *     __flush_icache_range(start, end)
> > + *     __clean_inval_cache_pou(start, end)
> >   *
> >   *             Ensure coherency between the I-cache and the D-cache region to
> >   *             the Point of Unification.
> >   *
> > - *     __flush_cache_user_range(start, end)
> > + *     __clean_inval_cache_user_pou(start, end)
> >   *
> >   *             Ensure coherency between the I-cache and the D-cache region to
> >   *             the Point of Unification.
> >   *             Use only if the region might access user memory.
> >   *
> > - *     invalidate_icache_range(start, end)
> > + *     __inval_icache_pou(start, end)
> >   *
> >   *             Invalidate I-cache region to the Point of Unification.
> >   *
> > - *     __flush_dcache_area(start, end)
> > + *     __clean_inval_dcache_poc(start, end)
> >   *
> >   *             Clean and invalidate D-cache region to the Point of Coherence.
> >   *
> > - *     __inval_dcache_area(start, end)
> > + *     __inval_dcache_poc(start, end)
> >   *
> >   *             Invalidate D-cache region to the Point of Coherence.
> >   *
> > - *     __clean_dcache_area_poc(start, end)
> > + *     __clean_dcache_poc(start, end)
> >   *
> >   *             Clean D-cache region to the Point of Coherence.
> >   *
> > - *     __clean_dcache_area_pop(start, end)
> > + *     __clean_dcache_pop(start, end)
> >   *
> >   *             Clean D-cache region to the Point of Persistence.
> >   *
> > - *     __clean_dcache_area_pou(start, end)
> > + *     __clean_dcache_pou(start, end)
> >   *
> >   *             Clean D-cache region to the Point of Unification.
> >   */
> > -extern void __flush_icache_range(unsigned long start, unsigned long end);
> > -extern void invalidate_icache_range(unsigned long start, unsigned long end);
> > -extern void __flush_dcache_area(unsigned long start, unsigned long end);
> > -extern void __inval_dcache_area(unsigned long start, unsigned long end);
> > -extern void __clean_dcache_area_poc(unsigned long start, unsigned long end);
> > -extern void __clean_dcache_area_pop(unsigned long start, unsigned long end);
> > -extern void __clean_dcache_area_pou(unsigned long start, unsigned long end);
> > -extern long __flush_cache_user_range(unsigned long start, unsigned long end);
> > +extern void __clean_inval_cache_pou(unsigned long start, unsigned long end);
> > +extern void __inval_icache_pou(unsigned long start, unsigned long end);
> > +extern void __clean_inval_dcache_poc(unsigned long start, unsigned long end);
> > +extern void __inval_dcache_poc(unsigned long start, unsigned long end);
> > +extern void __clean_dcache_poc(unsigned long start, unsigned long end);
> > +extern void __clean_dcache_pop(unsigned long start, unsigned long end);
> > +extern void __clean_dcache_pou(unsigned long start, unsigned long end);
> > +extern long __clean_inval_cache_user_pou(unsigned long start, unsigned long end);
> >  extern void sync_icache_aliases(unsigned long start, unsigned long end);
> >
> >  static inline void flush_icache_range(unsigned long start, unsigned long end)
> >  {
> > -       __flush_icache_range(start, end);
> > +       __clean_inval_cache_pou(start, end);
> >
> >         /*
> >          * IPI all online CPUs so that they undergo a context synchronization
> > @@ -135,7 +135,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *,
> >  #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
> >  extern void flush_dcache_page(struct page *);
> >
> > -static __always_inline void __flush_icache_all(void)
> > +static __always_inline void __clean_inval_all_icache_pou(void)
> >  {
> >         if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
> >                 return;
> > diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
> > index 0ae2397076fd..d1e2a4bf8def 100644
> > --- a/arch/arm64/include/asm/efi.h
> > +++ b/arch/arm64/include/asm/efi.h
> > @@ -137,7 +137,7 @@ void efi_virtmap_unload(void);
> >
> >  static inline void efi_capsule_flush_cache_range(void *addr, int size)
> >  {
> > -       __flush_dcache_area((unsigned long)addr, (unsigned long)addr + size);
> > +       __clean_inval_dcache_poc((unsigned long)addr, (unsigned long)addr + size);
> >  }
> >
> >  #endif /* _ASM_EFI_H */
> > diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
> > index 33293d5855af..29d2aa6f3940 100644
> > --- a/arch/arm64/include/asm/kvm_mmu.h
> > +++ b/arch/arm64/include/asm/kvm_mmu.h
> > @@ -181,7 +181,7 @@ static inline void *__kvm_vector_slot2addr(void *base,
> >  struct kvm;
> >
> >  #define kvm_flush_dcache_to_poc(a,l)   \
> > -       __flush_dcache_area((unsigned long)(a), (unsigned long)(a)+(l))
> > +       __clean_inval_dcache_poc((unsigned long)(a), (unsigned long)(a)+(l))
> >
> >  static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
> >  {
> > @@ -209,12 +209,12 @@ static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn,
> >  {
> >         if (icache_is_aliasing()) {
> >                 /* any kind of VIPT cache */
> > -               __flush_icache_all();
> > +               __clean_inval_all_icache_pou();
> >         } else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
> >                 /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
> >                 void *va = page_address(pfn_to_page(pfn));
> >
> > -               invalidate_icache_range((unsigned long)va,
> > +               __inval_icache_pou((unsigned long)va,
> >                                         (unsigned long)va + size);
> >         }
> >  }
> > diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
> > index c906d20c7b52..ea2d52fa9a0c 100644
> > --- a/arch/arm64/kernel/alternative.c
> > +++ b/arch/arm64/kernel/alternative.c
> > @@ -181,7 +181,7 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu
> >          */
> >         if (!is_module) {
> >                 dsb(ish);
> > -               __flush_icache_all();
> > +               __clean_inval_all_icache_pou();
> >                 isb();
> >
> >                 /* Ignore ARM64_CB bit from feature mask */
> > diff --git a/arch/arm64/kernel/efi-entry.S b/arch/arm64/kernel/efi-entry.S
> > index 72e6a580290a..230506f460ec 100644
> > --- a/arch/arm64/kernel/efi-entry.S
> > +++ b/arch/arm64/kernel/efi-entry.S
> > @@ -29,7 +29,7 @@ SYM_CODE_START(efi_enter_kernel)
> >          */
> >         ldr     w1, =kernel_size
> >         add     x1, x0, x1
> > -       bl      __clean_dcache_area_poc
> > +       bl      __clean_dcache_poc
> >         ic      ialluis
> >
> >         /*
> > @@ -38,7 +38,7 @@ SYM_CODE_START(efi_enter_kernel)
> >          */
> >         adr     x0, 0f
> >         adr     x1, 3f
> > -       bl      __clean_dcache_area_poc
> > +       bl      __clean_dcache_poc
> >  0:
> >         /* Turn off Dcache and MMU */
> >         mrs     x0, CurrentEL
> > diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> > index 8df0ac8d9123..ea0447c5010a 100644
> > --- a/arch/arm64/kernel/head.S
> > +++ b/arch/arm64/kernel/head.S
> > @@ -118,7 +118,7 @@ SYM_CODE_START_LOCAL(preserve_boot_args)
> >                                                 // MMU off
> >
> >         add     x1, x0, #0x20                   // 4 x 8 bytes
> > -       b       __inval_dcache_area             // tail call
> > +       b       __inval_dcache_poc              // tail call
> >  SYM_CODE_END(preserve_boot_args)
> >
> >  /*
> > @@ -268,7 +268,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables)
> >          */
> >         adrp    x0, init_pg_dir
> >         adrp    x1, init_pg_end
> > -       bl      __inval_dcache_area
> > +       bl      __inval_dcache_poc
> >
> >         /*
> >          * Clear the init page tables.
> > @@ -381,11 +381,11 @@ SYM_FUNC_START_LOCAL(__create_page_tables)
> >
> >         adrp    x0, idmap_pg_dir
> >         adrp    x1, idmap_pg_end
> > -       bl      __inval_dcache_area
> > +       bl      __inval_dcache_poc
> >
> >         adrp    x0, init_pg_dir
> >         adrp    x1, init_pg_end
> > -       bl      __inval_dcache_area
> > +       bl      __inval_dcache_poc
> >
> >         ret     x28
> >  SYM_FUNC_END(__create_page_tables)
> > diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
> > index b40ddce71507..ec871b24fd5b 100644
> > --- a/arch/arm64/kernel/hibernate.c
> > +++ b/arch/arm64/kernel/hibernate.c
> > @@ -210,7 +210,7 @@ static int create_safe_exec_page(void *src_start, size_t length,
> >                 return -ENOMEM;
> >
> >         memcpy(page, src_start, length);
> > -       __flush_icache_range((unsigned long)page, (unsigned long)page + length);
> > +       __clean_inval_cache_pou((unsigned long)page, (unsigned long)page + length);
> >         rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page);
> >         if (rc)
> >                 return rc;
> > @@ -381,17 +381,17 @@ int swsusp_arch_suspend(void)
> >                 ret = swsusp_save();
> >         } else {
> >                 /* Clean kernel core startup/idle code to PoC*/
> > -               __flush_dcache_area((unsigned long)__mmuoff_data_start,
> > +               __clean_inval_dcache_poc((unsigned long)__mmuoff_data_start,
> >                                     (unsigned long)__mmuoff_data_end);
> > -               __flush_dcache_area((unsigned long)__idmap_text_start,
> > +               __clean_inval_dcache_poc((unsigned long)__idmap_text_start,
> >                                     (unsigned long)__idmap_text_end);
> >
> >                 /* Clean kvm setup code to PoC? */
> >                 if (el2_reset_needed()) {
> > -                       __flush_dcache_area(
> > +                       __clean_inval_dcache_poc(
> >                                 (unsigned long)__hyp_idmap_text_start,
> >                                 (unsigned long)__hyp_idmap_text_end);
> > -                       __flush_dcache_area((unsigned long)__hyp_text_start,
> > +                       __clean_inval_dcache_poc((unsigned long)__hyp_text_start,
> >                                             (unsigned long)__hyp_text_end);
> >                 }
> >
> > @@ -477,7 +477,7 @@ int swsusp_arch_resume(void)
> >          * The hibernate exit text contains a set of el2 vectors, that will
> >          * be executed at el2 with the mmu off in order to reload hyp-stub.
> >          */
> > -       __flush_dcache_area((unsigned long)hibernate_exit,
> > +       __clean_inval_dcache_poc((unsigned long)hibernate_exit,
> >                             (unsigned long)hibernate_exit + exit_size);
> >
> >         /*
> > diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c
> > index 3dd515baf526..6b4b5727f2db 100644
> > --- a/arch/arm64/kernel/idreg-override.c
> > +++ b/arch/arm64/kernel/idreg-override.c
> > @@ -237,7 +237,7 @@ asmlinkage void __init init_feature_override(void)
> >
> >         for (i = 0; i < ARRAY_SIZE(regs); i++) {
> >                 if (regs[i]->override)
> > -                       __flush_dcache_area((unsigned long)regs[i]->override,
> > +                       __clean_inval_dcache_poc((unsigned long)regs[i]->override,
> >                                             (unsigned long)regs[i]->override +
> >                                             sizeof(*regs[i]->override));
> >         }
> > diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
> > index bcf3c2755370..14beda6a573d 100644
> > --- a/arch/arm64/kernel/image-vars.h
> > +++ b/arch/arm64/kernel/image-vars.h
> > @@ -35,7 +35,7 @@ __efistub_strnlen             = __pi_strnlen;
> >  __efistub_strcmp               = __pi_strcmp;
> >  __efistub_strncmp              = __pi_strncmp;
> >  __efistub_strrchr              = __pi_strrchr;
> > -__efistub___clean_dcache_area_poc = __pi___clean_dcache_area_poc;
> > +__efistub___clean_dcache_poc = __pi___clean_dcache_poc;
> >
> >  #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
> >  __efistub___memcpy             = __pi_memcpy;
> > diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c
> > index 6c0de2f60ea9..11c7be09e305 100644
> > --- a/arch/arm64/kernel/insn.c
> > +++ b/arch/arm64/kernel/insn.c
> > @@ -198,7 +198,7 @@ int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
> >
> >         ret = aarch64_insn_write(tp, insn);
> >         if (ret == 0)
> > -               __flush_icache_range((uintptr_t)tp,
> > +               __clean_inval_cache_pou((uintptr_t)tp,
> >                                      (uintptr_t)tp + AARCH64_INSN_SIZE);
> >
> >         return ret;
> > diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c
> > index 49cccd03cb37..038a4cc7de93 100644
> > --- a/arch/arm64/kernel/kaslr.c
> > +++ b/arch/arm64/kernel/kaslr.c
> > @@ -72,7 +72,7 @@ u64 __init kaslr_early_init(void)
> >          * we end up running with module randomization disabled.
> >          */
> >         module_alloc_base = (u64)_etext - MODULES_VSIZE;
> > -       __flush_dcache_area((unsigned long)&module_alloc_base,
> > +       __clean_inval_dcache_poc((unsigned long)&module_alloc_base,
> >                             (unsigned long)&module_alloc_base +
> >                                     sizeof(module_alloc_base));
> >
> > @@ -172,10 +172,10 @@ u64 __init kaslr_early_init(void)
> >         module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21;
> >         module_alloc_base &= PAGE_MASK;
> >
> > -       __flush_dcache_area((unsigned long)&module_alloc_base,
> > +       __clean_inval_dcache_poc((unsigned long)&module_alloc_base,
> >                             (unsigned long)&module_alloc_base +
> >                                     sizeof(module_alloc_base));
> > -       __flush_dcache_area((unsigned long)&memstart_offset_seed,
> > +       __clean_inval_dcache_poc((unsigned long)&memstart_offset_seed,
> >                             (unsigned long)&memstart_offset_seed +
> >                                     sizeof(memstart_offset_seed));
> >
> > diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
> > index 4cada9000acf..0e20a789b03e 100644
> > --- a/arch/arm64/kernel/machine_kexec.c
> > +++ b/arch/arm64/kernel/machine_kexec.c
> > @@ -69,10 +69,10 @@ int machine_kexec_post_load(struct kimage *kimage)
> >         kexec_image_info(kimage);
> >
> >         /* Flush the reloc_code in preparation for its execution. */
> > -       __flush_dcache_area((unsigned long)reloc_code,
> > +       __clean_inval_dcache_poc((unsigned long)reloc_code,
> >                             (unsigned long)reloc_code +
> >                                     arm64_relocate_new_kernel_size);
> > -       invalidate_icache_range((uintptr_t)reloc_code,
> > +       __inval_icache_pou((uintptr_t)reloc_code,
> >                                 (uintptr_t)reloc_code +
> >                                         arm64_relocate_new_kernel_size);
> >
> > @@ -108,7 +108,7 @@ static void kexec_list_flush(struct kimage *kimage)
> >                 unsigned long addr;
> >
> >                 /* flush the list entries. */
> > -               __flush_dcache_area((unsigned long)entry,
> > +               __clean_inval_dcache_poc((unsigned long)entry,
> >                                     (unsigned long)entry +
> >                                             sizeof(kimage_entry_t));
> >
> > @@ -125,7 +125,7 @@ static void kexec_list_flush(struct kimage *kimage)
> >                         break;
> >                 case IND_SOURCE:
> >                         /* flush the source pages. */
> > -                       __flush_dcache_area(addr, addr + PAGE_SIZE);
> > +                       __clean_inval_dcache_poc(addr, addr + PAGE_SIZE);
> >                         break;
> >                 case IND_DESTINATION:
> >                         break;
> > @@ -152,7 +152,7 @@ static void kexec_segment_flush(const struct kimage *kimage)
> >                         kimage->segment[i].memsz,
> >                         kimage->segment[i].memsz /  PAGE_SIZE);
> >
> > -               __flush_dcache_area(
> > +               __clean_inval_dcache_poc(
> >                         (unsigned long)phys_to_virt(kimage->segment[i].mem),
> >                         (unsigned long)phys_to_virt(kimage->segment[i].mem) +
> >                                 kimage->segment[i].memsz);
> > diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
> > index 5fcdee331087..2044210ed15a 100644
> > --- a/arch/arm64/kernel/smp.c
> > +++ b/arch/arm64/kernel/smp.c
> > @@ -122,7 +122,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
> >         secondary_data.task = idle;
> >         secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
> >         update_cpu_boot_status(CPU_MMU_OFF);
> > -       __flush_dcache_area((unsigned long)&secondary_data,
> > +       __clean_inval_dcache_poc((unsigned long)&secondary_data,
> >                             (unsigned long)&secondary_data +
> >                                     sizeof(secondary_data));
> >
> > @@ -145,7 +145,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
> >         pr_crit("CPU%u: failed to come online\n", cpu);
> >         secondary_data.task = NULL;
> >         secondary_data.stack = NULL;
> > -       __flush_dcache_area((unsigned long)&secondary_data,
> > +       __clean_inval_dcache_poc((unsigned long)&secondary_data,
> >                             (unsigned long)&secondary_data +
> >                                     sizeof(secondary_data));
> >         status = READ_ONCE(secondary_data.status);
> > diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c
> > index 58d804582a35..a946ccb9791e 100644
> > --- a/arch/arm64/kernel/smp_spin_table.c
> > +++ b/arch/arm64/kernel/smp_spin_table.c
> > @@ -36,7 +36,7 @@ static void write_pen_release(u64 val)
> >         unsigned long size = sizeof(secondary_holding_pen_release);
> >
> >         secondary_holding_pen_release = val;
> > -       __flush_dcache_area((unsigned long)start, (unsigned long)start + size);
> > +       __clean_inval_dcache_poc((unsigned long)start, (unsigned long)start + size);
> >  }
> >
> >
> > @@ -90,7 +90,7 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu)
> >          * the boot protocol.
> >          */
> >         writeq_relaxed(pa_holding_pen, release_addr);
> > -       __flush_dcache_area((__force unsigned long)release_addr,
> > +       __clean_inval_dcache_poc((__force unsigned long)release_addr,
> >                             (__force unsigned long)release_addr +
> >                                     sizeof(*release_addr));
> >
> > diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c
> > index 265fe3eb1069..fdd415f8d841 100644
> > --- a/arch/arm64/kernel/sys_compat.c
> > +++ b/arch/arm64/kernel/sys_compat.c
> > @@ -41,7 +41,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end)
> >                         dsb(ish);
> >                 }
> >
> > -               ret = __flush_cache_user_range(start, start + chunk);
> > +               ret = __clean_inval_cache_user_pou(start, start + chunk);
> >                 if (ret)
> >                         return ret;
> >
> > diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
> > index 1cb39c0803a4..edeca89405ff 100644
> > --- a/arch/arm64/kvm/arm.c
> > +++ b/arch/arm64/kvm/arm.c
> > @@ -1064,7 +1064,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
> >                 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
> >                         stage2_unmap_vm(vcpu->kvm);
> >                 else
> > -                       __flush_icache_all();
> > +                       __clean_inval_all_icache_pou();
> >         }
> >
> >         vcpu_reset_hcr(vcpu);
> > diff --git a/arch/arm64/kvm/hyp/nvhe/cache.S b/arch/arm64/kvm/hyp/nvhe/cache.S
> > index 36cef6915428..a906dd596e66 100644
> > --- a/arch/arm64/kvm/hyp/nvhe/cache.S
> > +++ b/arch/arm64/kvm/hyp/nvhe/cache.S
> > @@ -7,7 +7,7 @@
> >  #include <asm/assembler.h>
> >  #include <asm/alternative.h>
> >
> > -SYM_FUNC_START_PI(__flush_dcache_area)
> > +SYM_FUNC_START_PI(__clean_inval_dcache_poc)
> >         dcache_by_line_op civac, sy, x0, x1, x2, x3
> >         ret
> > -SYM_FUNC_END_PI(__flush_dcache_area)
> > +SYM_FUNC_END_PI(__clean_inval_dcache_poc)
> > diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
> > index 5dffe928f256..a16719f5068d 100644
> > --- a/arch/arm64/kvm/hyp/nvhe/setup.c
> > +++ b/arch/arm64/kvm/hyp/nvhe/setup.c
> > @@ -134,7 +134,7 @@ static void update_nvhe_init_params(void)
> >         for (i = 0; i < hyp_nr_cpus; i++) {
> >                 params = per_cpu_ptr(&kvm_init_params, i);
> >                 params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd);
> > -               __flush_dcache_area((unsigned long)params,
> > +               __clean_inval_dcache_poc((unsigned long)params,
> >                                     (unsigned long)params + sizeof(*params));
> >         }
> >  }
> > diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c
> > index 83dc3b271bc5..184c9c7c13bd 100644
> > --- a/arch/arm64/kvm/hyp/nvhe/tlb.c
> > +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c
> > @@ -104,7 +104,7 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
> >          * you should be running with VHE enabled.
> >          */
> >         if (icache_is_vpipt())
> > -               __flush_icache_all();
> > +               __clean_inval_all_icache_pou();
> >
> >         __tlb_switch_to_host(&cxt);
> >  }
> > diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
> > index 10d2f04013d4..fb2613f458de 100644
> > --- a/arch/arm64/kvm/hyp/pgtable.c
> > +++ b/arch/arm64/kvm/hyp/pgtable.c
> > @@ -841,7 +841,7 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
> >         if (need_flush) {
> >                 kvm_pte_t *pte_follow = kvm_pte_follow(pte, mm_ops);
> >
> > -               __flush_dcache_area((unsigned long)pte_follow,
> > +               __clean_inval_dcache_poc((unsigned long)pte_follow,
> >                                     (unsigned long)pte_follow +
> >                                             kvm_granule_size(level));
> >         }
> > @@ -997,7 +997,7 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
> >                 return 0;
> >
> >         pte_follow = kvm_pte_follow(pte, mm_ops);
> > -       __flush_dcache_area((unsigned long)pte_follow,
> > +       __clean_inval_dcache_poc((unsigned long)pte_follow,
> >                             (unsigned long)pte_follow +
> >                                     kvm_granule_size(level));
> >         return 0;
> > diff --git a/arch/arm64/lib/uaccess_flushcache.c b/arch/arm64/lib/uaccess_flushcache.c
> > index 62ea989effe8..b1a6d9823864 100644
> > --- a/arch/arm64/lib/uaccess_flushcache.c
> > +++ b/arch/arm64/lib/uaccess_flushcache.c
> > @@ -15,7 +15,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt)
> >          * barrier to order the cache maintenance against the memcpy.
> >          */
> >         memcpy(dst, src, cnt);
> > -       __clean_dcache_area_pop((unsigned long)dst, (unsigned long)dst + cnt);
> > +       __clean_dcache_pop((unsigned long)dst, (unsigned long)dst + cnt);
> >  }
> >  EXPORT_SYMBOL_GPL(memcpy_flushcache);
> >
> > @@ -33,6 +33,6 @@ unsigned long __copy_user_flushcache(void *to, const void __user *from,
> >         rc = raw_copy_from_user(to, from, n);
> >
> >         /* See above */
> > -       __clean_dcache_area_pop((unsigned long)to, (unsigned long)to + n - rc);
> > +       __clean_dcache_pop((unsigned long)to, (unsigned long)to + n - rc);
> >         return rc;
> >  }
> > diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
> > index d8434e57fab3..2df7212de799 100644
> > --- a/arch/arm64/mm/cache.S
> > +++ b/arch/arm64/mm/cache.S
> > @@ -15,7 +15,7 @@
> >  #include <asm/asm-uaccess.h>
> >
> >  /*
> > - *     __flush_cache_range(start,end) [needs_uaccess]
> > + *     __clean_inval_cache_pou_macro(start,end) [needs_uaccess]
> >   *
> >   *     Ensure that the I and D caches are coherent within specified region.
> >   *     This is typically used when code has been written to a memory region,
> > @@ -25,7 +25,7 @@
> >   *     - end           - virtual end address of region
> >   *     - needs_uaccess - (macro parameter) might access user space memory
> >   */
> > -.macro __flush_cache_range, needs_uaccess
> > +.macro __clean_inval_cache_pou_macro, needs_uaccess
> >         .if     \needs_uaccess
> >         uaccess_ttbr0_enable x2, x3, x4
> >         .endif
> > @@ -77,12 +77,12 @@ alternative_else_nop_endif
> >   *     - start   - virtual start address of region
> >   *     - end     - virtual end address of region
> >   */
> > -SYM_FUNC_START(__flush_icache_range)
> > -       __flush_cache_range needs_uaccess=0
> > -SYM_FUNC_END(__flush_icache_range)
> > +SYM_FUNC_START(__clean_inval_cache_pou)
> > +       __clean_inval_cache_pou_macro needs_uaccess=0
> > +SYM_FUNC_END(__clean_inval_cache_pou)
> >
> >  /*
> > - *     __flush_cache_user_range(start,end)
> > + *     __clean_inval_cache_user_pou(start,end)
> >   *
> >   *     Ensure that the I and D caches are coherent within specified region.
> >   *     This is typically used when code has been written to a memory region,
> > @@ -91,19 +91,19 @@ SYM_FUNC_END(__flush_icache_range)
> >   *     - start   - virtual start address of region
> >   *     - end     - virtual end address of region
> >   */
> > -SYM_FUNC_START(__flush_cache_user_range)
> > -       __flush_cache_range needs_uaccess=1
> > -SYM_FUNC_END(__flush_cache_user_range)
> > +SYM_FUNC_START(__clean_inval_cache_user_pou)
> > +       __clean_inval_cache_pou_macro needs_uaccess=1
> > +SYM_FUNC_END(__clean_inval_cache_user_pou)
> >
> >  /*
> > - *     invalidate_icache_range(start,end)
> > + *     __inval_icache_pou(start,end)
> >   *
> >   *     Ensure that the I cache is invalid within specified region.
> >   *
> >   *     - start   - virtual start address of region
> >   *     - end     - virtual end address of region
> >   */
> > -SYM_FUNC_START(invalidate_icache_range)
> > +SYM_FUNC_START(__inval_icache_pou)
> >  alternative_if ARM64_HAS_CACHE_DIC
> >         isb
> >         ret
> > @@ -111,10 +111,10 @@ alternative_else_nop_endif
> >
> >         invalidate_icache_by_line x0, x1, x2, x3, 0, 0f
> >         ret
> > -SYM_FUNC_END(invalidate_icache_range)
> > +SYM_FUNC_END(__inval_icache_pou)
> >
> >  /*
> > - *     __flush_dcache_area(start, end)
> > + *     __clean_inval_dcache_poc(start, end)
> >   *
> >   *     Ensure that any D-cache lines for the interval [start, end)
> >   *     are cleaned and invalidated to the PoC.
> > @@ -122,13 +122,13 @@ SYM_FUNC_END(invalidate_icache_range)
> >   *     - start   - virtual start address of region
> >   *     - end     - virtual end address of region
> >   */
> > -SYM_FUNC_START_PI(__flush_dcache_area)
> > +SYM_FUNC_START_PI(__clean_inval_dcache_poc)
> >         dcache_by_line_op civac, sy, x0, x1, x2, x3
> >         ret
> > -SYM_FUNC_END_PI(__flush_dcache_area)
> > +SYM_FUNC_END_PI(__clean_inval_dcache_poc)
> >
> >  /*
> > - *     __clean_dcache_area_pou(start, end)
> > + *     __clean_dcache_pou(start, end)
> >   *
> >   *     Ensure that any D-cache lines for the interval [start, end)
> >   *     are cleaned to the PoU.
> > @@ -136,17 +136,17 @@ SYM_FUNC_END_PI(__flush_dcache_area)
> >   *     - start   - virtual start address of region
> >   *     - end     - virtual end address of region
> >   */
> > -SYM_FUNC_START(__clean_dcache_area_pou)
> > +SYM_FUNC_START(__clean_dcache_pou)
> >  alternative_if ARM64_HAS_CACHE_IDC
> >         dsb     ishst
> >         ret
> >  alternative_else_nop_endif
> >         dcache_by_line_op cvau, ish, x0, x1, x2, x3
> >         ret
> > -SYM_FUNC_END(__clean_dcache_area_pou)
> > +SYM_FUNC_END(__clean_dcache_pou)
> >
> >  /*
> > - *     __inval_dcache_area(start, end)
> > + *     __inval_dcache_poc(start, end)
> >   *
> >   *     Ensure that any D-cache lines for the interval [start, end)
> >   *     are invalidated. Any partial lines at the ends of the interval are
> > @@ -156,7 +156,7 @@ SYM_FUNC_END(__clean_dcache_area_pou)
> >   *     - end     - kernel end address of region
> >   */
> >  SYM_FUNC_START_LOCAL(__dma_inv_area)
> > -SYM_FUNC_START_PI(__inval_dcache_area)
> > +SYM_FUNC_START_PI(__inval_dcache_poc)
> >         /* FALLTHROUGH */
> >
> >  /*
> > @@ -181,11 +181,11 @@ SYM_FUNC_START_PI(__inval_dcache_area)
> >         b.lo    2b
> >         dsb     sy
> >         ret
> > -SYM_FUNC_END_PI(__inval_dcache_area)
> > +SYM_FUNC_END_PI(__inval_dcache_poc)
> >  SYM_FUNC_END(__dma_inv_area)
> >
> >  /*
> > - *     __clean_dcache_area_poc(start, end)
> > + *     __clean_dcache_poc(start, end)
> >   *
> >   *     Ensure that any D-cache lines for the interval [start, end)
> >   *     are cleaned to the PoC.
> > @@ -194,7 +194,7 @@ SYM_FUNC_END(__dma_inv_area)
> >   *     - end     - virtual end address of region
> >   */
> >  SYM_FUNC_START_LOCAL(__dma_clean_area)
> > -SYM_FUNC_START_PI(__clean_dcache_area_poc)
> > +SYM_FUNC_START_PI(__clean_dcache_poc)
> >         /* FALLTHROUGH */
> >
> >  /*
> > @@ -204,11 +204,11 @@ SYM_FUNC_START_PI(__clean_dcache_area_poc)
> >   */
> >         dcache_by_line_op cvac, sy, x0, x1, x2, x3
> >         ret
> > -SYM_FUNC_END_PI(__clean_dcache_area_poc)
> > +SYM_FUNC_END_PI(__clean_dcache_poc)
> >  SYM_FUNC_END(__dma_clean_area)
> >
> >  /*
> > - *     __clean_dcache_area_pop(start, end)
> > + *     __clean_dcache_pop(start, end)
> >   *
> >   *     Ensure that any D-cache lines for the interval [start, end)
> >   *     are cleaned to the PoP.
> > @@ -216,13 +216,13 @@ SYM_FUNC_END(__dma_clean_area)
> >   *     - start   - virtual start address of region
> >   *     - end     - virtual end address of region
> >   */
> > -SYM_FUNC_START_PI(__clean_dcache_area_pop)
> > +SYM_FUNC_START_PI(__clean_dcache_pop)
> >         alternative_if_not ARM64_HAS_DCPOP
> > -       b       __clean_dcache_area_poc
> > +       b       __clean_dcache_poc
> >         alternative_else_nop_endif
> >         dcache_by_line_op cvap, sy, x0, x1, x2, x3
> >         ret
> > -SYM_FUNC_END_PI(__clean_dcache_area_pop)
> > +SYM_FUNC_END_PI(__clean_dcache_pop)
> >
> >  /*
> >   *     __dma_flush_area(start, size)
> > diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
> > index 143f625e7727..005b92148252 100644
> > --- a/arch/arm64/mm/flush.c
> > +++ b/arch/arm64/mm/flush.c
> > @@ -17,14 +17,14 @@
> >  void sync_icache_aliases(unsigned long start, unsigned long end)
> >  {
> >         if (icache_is_aliasing()) {
> > -               __clean_dcache_area_pou(start, end);
> > -               __flush_icache_all();
> > +               __clean_dcache_pou(start, end);
> > +               __clean_inval_all_icache_pou();
> >         } else {
> >                 /*
> >                  * Don't issue kick_all_cpus_sync() after I-cache invalidation
> >                  * for user mappings.
> >                  */
> > -               __flush_icache_range(start, end);
> > +               __clean_inval_cache_pou(start, end);
> >         }
> >  }
> >
> > @@ -76,20 +76,20 @@ EXPORT_SYMBOL(flush_dcache_page);
> >  /*
> >   * Additional functions defined in assembly.
> >   */
> > -EXPORT_SYMBOL(__flush_icache_range);
> > +EXPORT_SYMBOL(__clean_inval_cache_pou);
> >
> >  #ifdef CONFIG_ARCH_HAS_PMEM_API
> >  void arch_wb_cache_pmem(void *addr, size_t size)
> >  {
> >         /* Ensure order against any prior non-cacheable writes */
> >         dmb(osh);
> > -       __clean_dcache_area_pop((unsigned long)addr, (unsigned long)addr + size);
> > +       __clean_dcache_pop((unsigned long)addr, (unsigned long)addr + size);
> >  }
> >  EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
> >
> >  void arch_invalidate_pmem(void *addr, size_t size)
> >  {
> > -       __inval_dcache_area((unsigned long)addr, (unsigned long)addr + size);
> > +       __inval_dcache_poc((unsigned long)addr, (unsigned long)addr + size);
> >  }
> >  EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
> >  #endif
> > --
> > 2.31.1.607.g51e8a6a459-goog
> >



More information about the linux-arm-kernel mailing list