[PATCH v1 13/13] arm64: Rename arm64-internal cache maintenance functions
Fuad Tabba
tabba at google.com
Wed May 12 03:00:00 PDT 2021
Hi Mark,
> > > "s/\b__flush_icache_range\b/__clean_inval_cache_pou/g;"\
>
> For icaches, a "flush" is just an invalidate, so this doesn't need
> "clean".
This is one of the reasons for this patch. Although you are correct
when it comes to what the name __flush_icache_range implies, it wasn't
doing only that. It's flushing both I and D caches. Therefore, the new
naming scheme with the cache type as a prefix that you suggest below
should make that clearer.
> > > "s/\b__flush_icache_all\b/__clean_inval_all_icache_pou/g;"
>
> Likewise here.
I'll fix that in v2.
> > >
> > > Note that __clean_dcache_area_poc is deliberately missing a word
> > > boundary check to match the efistub symbols in image-vars.h.
> > >
> > > No functional change intended.
> > >
> > > Signed-off-by: Fuad Tabba <tabba at google.com>
> >
> > I am a big fan of this change: code is so much easier to read if the
> > names of subroutines match their intent.
>
> Likewise!
Thanks!
> > I would suggest, though, that we get rid of all the leading
> > underscores while at it: we often use them when refactoring existing
> > routines into separate pieces (which is where at least some of these
> > came from), but here, they seem to have little value.
>
> That all makes sense to me; I'd also suggest we make the cache type the
> prefix, e.g.
>
> * icache_clean_pou
> * dcache_clean_inval_poc
> * caches_clean_inval_user_pou // D+I caches
>
> ... since then it's easier to read consistently, rather than having to
> search for the cache type midway through the name.
I'll fix that as well in v2.
Thanks,
/fuad
>
> Thanks,
> Mark.
>
> >
> >
> > > ---
> > > arch/arm64/include/asm/arch_gicv3.h | 2 +-
> > > arch/arm64/include/asm/cacheflush.h | 36 +++++++++----------
> > > arch/arm64/include/asm/efi.h | 2 +-
> > > arch/arm64/include/asm/kvm_mmu.h | 6 ++--
> > > arch/arm64/kernel/alternative.c | 2 +-
> > > arch/arm64/kernel/efi-entry.S | 4 +--
> > > arch/arm64/kernel/head.S | 8 ++---
> > > arch/arm64/kernel/hibernate.c | 12 +++----
> > > arch/arm64/kernel/idreg-override.c | 2 +-
> > > arch/arm64/kernel/image-vars.h | 2 +-
> > > arch/arm64/kernel/insn.c | 2 +-
> > > arch/arm64/kernel/kaslr.c | 6 ++--
> > > arch/arm64/kernel/machine_kexec.c | 10 +++---
> > > arch/arm64/kernel/smp.c | 4 +--
> > > arch/arm64/kernel/smp_spin_table.c | 4 +--
> > > arch/arm64/kernel/sys_compat.c | 2 +-
> > > arch/arm64/kvm/arm.c | 2 +-
> > > arch/arm64/kvm/hyp/nvhe/cache.S | 4 +--
> > > arch/arm64/kvm/hyp/nvhe/setup.c | 2 +-
> > > arch/arm64/kvm/hyp/nvhe/tlb.c | 2 +-
> > > arch/arm64/kvm/hyp/pgtable.c | 4 +--
> > > arch/arm64/lib/uaccess_flushcache.c | 4 +--
> > > arch/arm64/mm/cache.S | 56 ++++++++++++++---------------
> > > arch/arm64/mm/flush.c | 12 +++----
> > > 24 files changed, 95 insertions(+), 95 deletions(-)
> > >
> > > diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h
> > > index ed1cc9d8e6df..4b7ac9098e8f 100644
> > > --- a/arch/arm64/include/asm/arch_gicv3.h
> > > +++ b/arch/arm64/include/asm/arch_gicv3.h
> > > @@ -125,7 +125,7 @@ static inline u32 gic_read_rpr(void)
> > > #define gic_write_lpir(v, c) writeq_relaxed(v, c)
> > >
> > > #define gic_flush_dcache_to_poc(a,l) \
> > > - __flush_dcache_area((unsigned long)(a), (unsigned long)(a)+(l))
> > > + __clean_inval_dcache_poc((unsigned long)(a), (unsigned long)(a)+(l))
> > >
> > > #define gits_read_baser(c) readq_relaxed(c)
> > > #define gits_write_baser(v, c) writeq_relaxed(v, c)
> > > diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
> > > index 4b91d3530013..526eee4522eb 100644
> > > --- a/arch/arm64/include/asm/cacheflush.h
> > > +++ b/arch/arm64/include/asm/cacheflush.h
> > > @@ -34,54 +34,54 @@
> > > * - start - virtual start address
> > > * - end - virtual end address
> > > *
> > > - * __flush_icache_range(start, end)
> > > + * __clean_inval_cache_pou(start, end)
> > > *
> > > * Ensure coherency between the I-cache and the D-cache region to
> > > * the Point of Unification.
> > > *
> > > - * __flush_cache_user_range(start, end)
> > > + * __clean_inval_cache_user_pou(start, end)
> > > *
> > > * Ensure coherency between the I-cache and the D-cache region to
> > > * the Point of Unification.
> > > * Use only if the region might access user memory.
> > > *
> > > - * invalidate_icache_range(start, end)
> > > + * __inval_icache_pou(start, end)
> > > *
> > > * Invalidate I-cache region to the Point of Unification.
> > > *
> > > - * __flush_dcache_area(start, end)
> > > + * __clean_inval_dcache_poc(start, end)
> > > *
> > > * Clean and invalidate D-cache region to the Point of Coherence.
> > > *
> > > - * __inval_dcache_area(start, end)
> > > + * __inval_dcache_poc(start, end)
> > > *
> > > * Invalidate D-cache region to the Point of Coherence.
> > > *
> > > - * __clean_dcache_area_poc(start, end)
> > > + * __clean_dcache_poc(start, end)
> > > *
> > > * Clean D-cache region to the Point of Coherence.
> > > *
> > > - * __clean_dcache_area_pop(start, end)
> > > + * __clean_dcache_pop(start, end)
> > > *
> > > * Clean D-cache region to the Point of Persistence.
> > > *
> > > - * __clean_dcache_area_pou(start, end)
> > > + * __clean_dcache_pou(start, end)
> > > *
> > > * Clean D-cache region to the Point of Unification.
> > > */
> > > -extern void __flush_icache_range(unsigned long start, unsigned long end);
> > > -extern void invalidate_icache_range(unsigned long start, unsigned long end);
> > > -extern void __flush_dcache_area(unsigned long start, unsigned long end);
> > > -extern void __inval_dcache_area(unsigned long start, unsigned long end);
> > > -extern void __clean_dcache_area_poc(unsigned long start, unsigned long end);
> > > -extern void __clean_dcache_area_pop(unsigned long start, unsigned long end);
> > > -extern void __clean_dcache_area_pou(unsigned long start, unsigned long end);
> > > -extern long __flush_cache_user_range(unsigned long start, unsigned long end);
> > > +extern void __clean_inval_cache_pou(unsigned long start, unsigned long end);
> > > +extern void __inval_icache_pou(unsigned long start, unsigned long end);
> > > +extern void __clean_inval_dcache_poc(unsigned long start, unsigned long end);
> > > +extern void __inval_dcache_poc(unsigned long start, unsigned long end);
> > > +extern void __clean_dcache_poc(unsigned long start, unsigned long end);
> > > +extern void __clean_dcache_pop(unsigned long start, unsigned long end);
> > > +extern void __clean_dcache_pou(unsigned long start, unsigned long end);
> > > +extern long __clean_inval_cache_user_pou(unsigned long start, unsigned long end);
> > > extern void sync_icache_aliases(unsigned long start, unsigned long end);
> > >
> > > static inline void flush_icache_range(unsigned long start, unsigned long end)
> > > {
> > > - __flush_icache_range(start, end);
> > > + __clean_inval_cache_pou(start, end);
> > >
> > > /*
> > > * IPI all online CPUs so that they undergo a context synchronization
> > > @@ -135,7 +135,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *,
> > > #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
> > > extern void flush_dcache_page(struct page *);
> > >
> > > -static __always_inline void __flush_icache_all(void)
> > > +static __always_inline void __clean_inval_all_icache_pou(void)
> > > {
> > > if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
> > > return;
> > > diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
> > > index 0ae2397076fd..d1e2a4bf8def 100644
> > > --- a/arch/arm64/include/asm/efi.h
> > > +++ b/arch/arm64/include/asm/efi.h
> > > @@ -137,7 +137,7 @@ void efi_virtmap_unload(void);
> > >
> > > static inline void efi_capsule_flush_cache_range(void *addr, int size)
> > > {
> > > - __flush_dcache_area((unsigned long)addr, (unsigned long)addr + size);
> > > + __clean_inval_dcache_poc((unsigned long)addr, (unsigned long)addr + size);
> > > }
> > >
> > > #endif /* _ASM_EFI_H */
> > > diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
> > > index 33293d5855af..29d2aa6f3940 100644
> > > --- a/arch/arm64/include/asm/kvm_mmu.h
> > > +++ b/arch/arm64/include/asm/kvm_mmu.h
> > > @@ -181,7 +181,7 @@ static inline void *__kvm_vector_slot2addr(void *base,
> > > struct kvm;
> > >
> > > #define kvm_flush_dcache_to_poc(a,l) \
> > > - __flush_dcache_area((unsigned long)(a), (unsigned long)(a)+(l))
> > > + __clean_inval_dcache_poc((unsigned long)(a), (unsigned long)(a)+(l))
> > >
> > > static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
> > > {
> > > @@ -209,12 +209,12 @@ static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn,
> > > {
> > > if (icache_is_aliasing()) {
> > > /* any kind of VIPT cache */
> > > - __flush_icache_all();
> > > + __clean_inval_all_icache_pou();
> > > } else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
> > > /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
> > > void *va = page_address(pfn_to_page(pfn));
> > >
> > > - invalidate_icache_range((unsigned long)va,
> > > + __inval_icache_pou((unsigned long)va,
> > > (unsigned long)va + size);
> > > }
> > > }
> > > diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
> > > index c906d20c7b52..ea2d52fa9a0c 100644
> > > --- a/arch/arm64/kernel/alternative.c
> > > +++ b/arch/arm64/kernel/alternative.c
> > > @@ -181,7 +181,7 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu
> > > */
> > > if (!is_module) {
> > > dsb(ish);
> > > - __flush_icache_all();
> > > + __clean_inval_all_icache_pou();
> > > isb();
> > >
> > > /* Ignore ARM64_CB bit from feature mask */
> > > diff --git a/arch/arm64/kernel/efi-entry.S b/arch/arm64/kernel/efi-entry.S
> > > index 72e6a580290a..230506f460ec 100644
> > > --- a/arch/arm64/kernel/efi-entry.S
> > > +++ b/arch/arm64/kernel/efi-entry.S
> > > @@ -29,7 +29,7 @@ SYM_CODE_START(efi_enter_kernel)
> > > */
> > > ldr w1, =kernel_size
> > > add x1, x0, x1
> > > - bl __clean_dcache_area_poc
> > > + bl __clean_dcache_poc
> > > ic ialluis
> > >
> > > /*
> > > @@ -38,7 +38,7 @@ SYM_CODE_START(efi_enter_kernel)
> > > */
> > > adr x0, 0f
> > > adr x1, 3f
> > > - bl __clean_dcache_area_poc
> > > + bl __clean_dcache_poc
> > > 0:
> > > /* Turn off Dcache and MMU */
> > > mrs x0, CurrentEL
> > > diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> > > index 8df0ac8d9123..ea0447c5010a 100644
> > > --- a/arch/arm64/kernel/head.S
> > > +++ b/arch/arm64/kernel/head.S
> > > @@ -118,7 +118,7 @@ SYM_CODE_START_LOCAL(preserve_boot_args)
> > > // MMU off
> > >
> > > add x1, x0, #0x20 // 4 x 8 bytes
> > > - b __inval_dcache_area // tail call
> > > + b __inval_dcache_poc // tail call
> > > SYM_CODE_END(preserve_boot_args)
> > >
> > > /*
> > > @@ -268,7 +268,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables)
> > > */
> > > adrp x0, init_pg_dir
> > > adrp x1, init_pg_end
> > > - bl __inval_dcache_area
> > > + bl __inval_dcache_poc
> > >
> > > /*
> > > * Clear the init page tables.
> > > @@ -381,11 +381,11 @@ SYM_FUNC_START_LOCAL(__create_page_tables)
> > >
> > > adrp x0, idmap_pg_dir
> > > adrp x1, idmap_pg_end
> > > - bl __inval_dcache_area
> > > + bl __inval_dcache_poc
> > >
> > > adrp x0, init_pg_dir
> > > adrp x1, init_pg_end
> > > - bl __inval_dcache_area
> > > + bl __inval_dcache_poc
> > >
> > > ret x28
> > > SYM_FUNC_END(__create_page_tables)
> > > diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
> > > index b40ddce71507..ec871b24fd5b 100644
> > > --- a/arch/arm64/kernel/hibernate.c
> > > +++ b/arch/arm64/kernel/hibernate.c
> > > @@ -210,7 +210,7 @@ static int create_safe_exec_page(void *src_start, size_t length,
> > > return -ENOMEM;
> > >
> > > memcpy(page, src_start, length);
> > > - __flush_icache_range((unsigned long)page, (unsigned long)page + length);
> > > + __clean_inval_cache_pou((unsigned long)page, (unsigned long)page + length);
> > > rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page);
> > > if (rc)
> > > return rc;
> > > @@ -381,17 +381,17 @@ int swsusp_arch_suspend(void)
> > > ret = swsusp_save();
> > > } else {
> > > /* Clean kernel core startup/idle code to PoC*/
> > > - __flush_dcache_area((unsigned long)__mmuoff_data_start,
> > > + __clean_inval_dcache_poc((unsigned long)__mmuoff_data_start,
> > > (unsigned long)__mmuoff_data_end);
> > > - __flush_dcache_area((unsigned long)__idmap_text_start,
> > > + __clean_inval_dcache_poc((unsigned long)__idmap_text_start,
> > > (unsigned long)__idmap_text_end);
> > >
> > > /* Clean kvm setup code to PoC? */
> > > if (el2_reset_needed()) {
> > > - __flush_dcache_area(
> > > + __clean_inval_dcache_poc(
> > > (unsigned long)__hyp_idmap_text_start,
> > > (unsigned long)__hyp_idmap_text_end);
> > > - __flush_dcache_area((unsigned long)__hyp_text_start,
> > > + __clean_inval_dcache_poc((unsigned long)__hyp_text_start,
> > > (unsigned long)__hyp_text_end);
> > > }
> > >
> > > @@ -477,7 +477,7 @@ int swsusp_arch_resume(void)
> > > * The hibernate exit text contains a set of el2 vectors, that will
> > > * be executed at el2 with the mmu off in order to reload hyp-stub.
> > > */
> > > - __flush_dcache_area((unsigned long)hibernate_exit,
> > > + __clean_inval_dcache_poc((unsigned long)hibernate_exit,
> > > (unsigned long)hibernate_exit + exit_size);
> > >
> > > /*
> > > diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c
> > > index 3dd515baf526..6b4b5727f2db 100644
> > > --- a/arch/arm64/kernel/idreg-override.c
> > > +++ b/arch/arm64/kernel/idreg-override.c
> > > @@ -237,7 +237,7 @@ asmlinkage void __init init_feature_override(void)
> > >
> > > for (i = 0; i < ARRAY_SIZE(regs); i++) {
> > > if (regs[i]->override)
> > > - __flush_dcache_area((unsigned long)regs[i]->override,
> > > + __clean_inval_dcache_poc((unsigned long)regs[i]->override,
> > > (unsigned long)regs[i]->override +
> > > sizeof(*regs[i]->override));
> > > }
> > > diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
> > > index bcf3c2755370..14beda6a573d 100644
> > > --- a/arch/arm64/kernel/image-vars.h
> > > +++ b/arch/arm64/kernel/image-vars.h
> > > @@ -35,7 +35,7 @@ __efistub_strnlen = __pi_strnlen;
> > > __efistub_strcmp = __pi_strcmp;
> > > __efistub_strncmp = __pi_strncmp;
> > > __efistub_strrchr = __pi_strrchr;
> > > -__efistub___clean_dcache_area_poc = __pi___clean_dcache_area_poc;
> > > +__efistub___clean_dcache_poc = __pi___clean_dcache_poc;
> > >
> > > #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
> > > __efistub___memcpy = __pi_memcpy;
> > > diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c
> > > index 6c0de2f60ea9..11c7be09e305 100644
> > > --- a/arch/arm64/kernel/insn.c
> > > +++ b/arch/arm64/kernel/insn.c
> > > @@ -198,7 +198,7 @@ int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
> > >
> > > ret = aarch64_insn_write(tp, insn);
> > > if (ret == 0)
> > > - __flush_icache_range((uintptr_t)tp,
> > > + __clean_inval_cache_pou((uintptr_t)tp,
> > > (uintptr_t)tp + AARCH64_INSN_SIZE);
> > >
> > > return ret;
> > > diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c
> > > index 49cccd03cb37..038a4cc7de93 100644
> > > --- a/arch/arm64/kernel/kaslr.c
> > > +++ b/arch/arm64/kernel/kaslr.c
> > > @@ -72,7 +72,7 @@ u64 __init kaslr_early_init(void)
> > > * we end up running with module randomization disabled.
> > > */
> > > module_alloc_base = (u64)_etext - MODULES_VSIZE;
> > > - __flush_dcache_area((unsigned long)&module_alloc_base,
> > > + __clean_inval_dcache_poc((unsigned long)&module_alloc_base,
> > > (unsigned long)&module_alloc_base +
> > > sizeof(module_alloc_base));
> > >
> > > @@ -172,10 +172,10 @@ u64 __init kaslr_early_init(void)
> > > module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21;
> > > module_alloc_base &= PAGE_MASK;
> > >
> > > - __flush_dcache_area((unsigned long)&module_alloc_base,
> > > + __clean_inval_dcache_poc((unsigned long)&module_alloc_base,
> > > (unsigned long)&module_alloc_base +
> > > sizeof(module_alloc_base));
> > > - __flush_dcache_area((unsigned long)&memstart_offset_seed,
> > > + __clean_inval_dcache_poc((unsigned long)&memstart_offset_seed,
> > > (unsigned long)&memstart_offset_seed +
> > > sizeof(memstart_offset_seed));
> > >
> > > diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
> > > index 4cada9000acf..0e20a789b03e 100644
> > > --- a/arch/arm64/kernel/machine_kexec.c
> > > +++ b/arch/arm64/kernel/machine_kexec.c
> > > @@ -69,10 +69,10 @@ int machine_kexec_post_load(struct kimage *kimage)
> > > kexec_image_info(kimage);
> > >
> > > /* Flush the reloc_code in preparation for its execution. */
> > > - __flush_dcache_area((unsigned long)reloc_code,
> > > + __clean_inval_dcache_poc((unsigned long)reloc_code,
> > > (unsigned long)reloc_code +
> > > arm64_relocate_new_kernel_size);
> > > - invalidate_icache_range((uintptr_t)reloc_code,
> > > + __inval_icache_pou((uintptr_t)reloc_code,
> > > (uintptr_t)reloc_code +
> > > arm64_relocate_new_kernel_size);
> > >
> > > @@ -108,7 +108,7 @@ static void kexec_list_flush(struct kimage *kimage)
> > > unsigned long addr;
> > >
> > > /* flush the list entries. */
> > > - __flush_dcache_area((unsigned long)entry,
> > > + __clean_inval_dcache_poc((unsigned long)entry,
> > > (unsigned long)entry +
> > > sizeof(kimage_entry_t));
> > >
> > > @@ -125,7 +125,7 @@ static void kexec_list_flush(struct kimage *kimage)
> > > break;
> > > case IND_SOURCE:
> > > /* flush the source pages. */
> > > - __flush_dcache_area(addr, addr + PAGE_SIZE);
> > > + __clean_inval_dcache_poc(addr, addr + PAGE_SIZE);
> > > break;
> > > case IND_DESTINATION:
> > > break;
> > > @@ -152,7 +152,7 @@ static void kexec_segment_flush(const struct kimage *kimage)
> > > kimage->segment[i].memsz,
> > > kimage->segment[i].memsz / PAGE_SIZE);
> > >
> > > - __flush_dcache_area(
> > > + __clean_inval_dcache_poc(
> > > (unsigned long)phys_to_virt(kimage->segment[i].mem),
> > > (unsigned long)phys_to_virt(kimage->segment[i].mem) +
> > > kimage->segment[i].memsz);
> > > diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
> > > index 5fcdee331087..2044210ed15a 100644
> > > --- a/arch/arm64/kernel/smp.c
> > > +++ b/arch/arm64/kernel/smp.c
> > > @@ -122,7 +122,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
> > > secondary_data.task = idle;
> > > secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
> > > update_cpu_boot_status(CPU_MMU_OFF);
> > > - __flush_dcache_area((unsigned long)&secondary_data,
> > > + __clean_inval_dcache_poc((unsigned long)&secondary_data,
> > > (unsigned long)&secondary_data +
> > > sizeof(secondary_data));
> > >
> > > @@ -145,7 +145,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
> > > pr_crit("CPU%u: failed to come online\n", cpu);
> > > secondary_data.task = NULL;
> > > secondary_data.stack = NULL;
> > > - __flush_dcache_area((unsigned long)&secondary_data,
> > > + __clean_inval_dcache_poc((unsigned long)&secondary_data,
> > > (unsigned long)&secondary_data +
> > > sizeof(secondary_data));
> > > status = READ_ONCE(secondary_data.status);
> > > diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c
> > > index 58d804582a35..a946ccb9791e 100644
> > > --- a/arch/arm64/kernel/smp_spin_table.c
> > > +++ b/arch/arm64/kernel/smp_spin_table.c
> > > @@ -36,7 +36,7 @@ static void write_pen_release(u64 val)
> > > unsigned long size = sizeof(secondary_holding_pen_release);
> > >
> > > secondary_holding_pen_release = val;
> > > - __flush_dcache_area((unsigned long)start, (unsigned long)start + size);
> > > + __clean_inval_dcache_poc((unsigned long)start, (unsigned long)start + size);
> > > }
> > >
> > >
> > > @@ -90,7 +90,7 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu)
> > > * the boot protocol.
> > > */
> > > writeq_relaxed(pa_holding_pen, release_addr);
> > > - __flush_dcache_area((__force unsigned long)release_addr,
> > > + __clean_inval_dcache_poc((__force unsigned long)release_addr,
> > > (__force unsigned long)release_addr +
> > > sizeof(*release_addr));
> > >
> > > diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c
> > > index 265fe3eb1069..fdd415f8d841 100644
> > > --- a/arch/arm64/kernel/sys_compat.c
> > > +++ b/arch/arm64/kernel/sys_compat.c
> > > @@ -41,7 +41,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end)
> > > dsb(ish);
> > > }
> > >
> > > - ret = __flush_cache_user_range(start, start + chunk);
> > > + ret = __clean_inval_cache_user_pou(start, start + chunk);
> > > if (ret)
> > > return ret;
> > >
> > > diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
> > > index 1cb39c0803a4..edeca89405ff 100644
> > > --- a/arch/arm64/kvm/arm.c
> > > +++ b/arch/arm64/kvm/arm.c
> > > @@ -1064,7 +1064,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
> > > if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
> > > stage2_unmap_vm(vcpu->kvm);
> > > else
> > > - __flush_icache_all();
> > > + __clean_inval_all_icache_pou();
> > > }
> > >
> > > vcpu_reset_hcr(vcpu);
> > > diff --git a/arch/arm64/kvm/hyp/nvhe/cache.S b/arch/arm64/kvm/hyp/nvhe/cache.S
> > > index 36cef6915428..a906dd596e66 100644
> > > --- a/arch/arm64/kvm/hyp/nvhe/cache.S
> > > +++ b/arch/arm64/kvm/hyp/nvhe/cache.S
> > > @@ -7,7 +7,7 @@
> > > #include <asm/assembler.h>
> > > #include <asm/alternative.h>
> > >
> > > -SYM_FUNC_START_PI(__flush_dcache_area)
> > > +SYM_FUNC_START_PI(__clean_inval_dcache_poc)
> > > dcache_by_line_op civac, sy, x0, x1, x2, x3
> > > ret
> > > -SYM_FUNC_END_PI(__flush_dcache_area)
> > > +SYM_FUNC_END_PI(__clean_inval_dcache_poc)
> > > diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
> > > index 5dffe928f256..a16719f5068d 100644
> > > --- a/arch/arm64/kvm/hyp/nvhe/setup.c
> > > +++ b/arch/arm64/kvm/hyp/nvhe/setup.c
> > > @@ -134,7 +134,7 @@ static void update_nvhe_init_params(void)
> > > for (i = 0; i < hyp_nr_cpus; i++) {
> > > params = per_cpu_ptr(&kvm_init_params, i);
> > > params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd);
> > > - __flush_dcache_area((unsigned long)params,
> > > + __clean_inval_dcache_poc((unsigned long)params,
> > > (unsigned long)params + sizeof(*params));
> > > }
> > > }
> > > diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c
> > > index 83dc3b271bc5..184c9c7c13bd 100644
> > > --- a/arch/arm64/kvm/hyp/nvhe/tlb.c
> > > +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c
> > > @@ -104,7 +104,7 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
> > > * you should be running with VHE enabled.
> > > */
> > > if (icache_is_vpipt())
> > > - __flush_icache_all();
> > > + __clean_inval_all_icache_pou();
> > >
> > > __tlb_switch_to_host(&cxt);
> > > }
> > > diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
> > > index 10d2f04013d4..fb2613f458de 100644
> > > --- a/arch/arm64/kvm/hyp/pgtable.c
> > > +++ b/arch/arm64/kvm/hyp/pgtable.c
> > > @@ -841,7 +841,7 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
> > > if (need_flush) {
> > > kvm_pte_t *pte_follow = kvm_pte_follow(pte, mm_ops);
> > >
> > > - __flush_dcache_area((unsigned long)pte_follow,
> > > + __clean_inval_dcache_poc((unsigned long)pte_follow,
> > > (unsigned long)pte_follow +
> > > kvm_granule_size(level));
> > > }
> > > @@ -997,7 +997,7 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
> > > return 0;
> > >
> > > pte_follow = kvm_pte_follow(pte, mm_ops);
> > > - __flush_dcache_area((unsigned long)pte_follow,
> > > + __clean_inval_dcache_poc((unsigned long)pte_follow,
> > > (unsigned long)pte_follow +
> > > kvm_granule_size(level));
> > > return 0;
> > > diff --git a/arch/arm64/lib/uaccess_flushcache.c b/arch/arm64/lib/uaccess_flushcache.c
> > > index 62ea989effe8..b1a6d9823864 100644
> > > --- a/arch/arm64/lib/uaccess_flushcache.c
> > > +++ b/arch/arm64/lib/uaccess_flushcache.c
> > > @@ -15,7 +15,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt)
> > > * barrier to order the cache maintenance against the memcpy.
> > > */
> > > memcpy(dst, src, cnt);
> > > - __clean_dcache_area_pop((unsigned long)dst, (unsigned long)dst + cnt);
> > > + __clean_dcache_pop((unsigned long)dst, (unsigned long)dst + cnt);
> > > }
> > > EXPORT_SYMBOL_GPL(memcpy_flushcache);
> > >
> > > @@ -33,6 +33,6 @@ unsigned long __copy_user_flushcache(void *to, const void __user *from,
> > > rc = raw_copy_from_user(to, from, n);
> > >
> > > /* See above */
> > > - __clean_dcache_area_pop((unsigned long)to, (unsigned long)to + n - rc);
> > > + __clean_dcache_pop((unsigned long)to, (unsigned long)to + n - rc);
> > > return rc;
> > > }
> > > diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
> > > index d8434e57fab3..2df7212de799 100644
> > > --- a/arch/arm64/mm/cache.S
> > > +++ b/arch/arm64/mm/cache.S
> > > @@ -15,7 +15,7 @@
> > > #include <asm/asm-uaccess.h>
> > >
> > > /*
> > > - * __flush_cache_range(start,end) [needs_uaccess]
> > > + * __clean_inval_cache_pou_macro(start,end) [needs_uaccess]
> > > *
> > > * Ensure that the I and D caches are coherent within specified region.
> > > * This is typically used when code has been written to a memory region,
> > > @@ -25,7 +25,7 @@
> > > * - end - virtual end address of region
> > > * - needs_uaccess - (macro parameter) might access user space memory
> > > */
> > > -.macro __flush_cache_range, needs_uaccess
> > > +.macro __clean_inval_cache_pou_macro, needs_uaccess
> > > .if \needs_uaccess
> > > uaccess_ttbr0_enable x2, x3, x4
> > > .endif
> > > @@ -77,12 +77,12 @@ alternative_else_nop_endif
> > > * - start - virtual start address of region
> > > * - end - virtual end address of region
> > > */
> > > -SYM_FUNC_START(__flush_icache_range)
> > > - __flush_cache_range needs_uaccess=0
> > > -SYM_FUNC_END(__flush_icache_range)
> > > +SYM_FUNC_START(__clean_inval_cache_pou)
> > > + __clean_inval_cache_pou_macro needs_uaccess=0
> > > +SYM_FUNC_END(__clean_inval_cache_pou)
> > >
> > > /*
> > > - * __flush_cache_user_range(start,end)
> > > + * __clean_inval_cache_user_pou(start,end)
> > > *
> > > * Ensure that the I and D caches are coherent within specified region.
> > > * This is typically used when code has been written to a memory region,
> > > @@ -91,19 +91,19 @@ SYM_FUNC_END(__flush_icache_range)
> > > * - start - virtual start address of region
> > > * - end - virtual end address of region
> > > */
> > > -SYM_FUNC_START(__flush_cache_user_range)
> > > - __flush_cache_range needs_uaccess=1
> > > -SYM_FUNC_END(__flush_cache_user_range)
> > > +SYM_FUNC_START(__clean_inval_cache_user_pou)
> > > + __clean_inval_cache_pou_macro needs_uaccess=1
> > > +SYM_FUNC_END(__clean_inval_cache_user_pou)
> > >
> > > /*
> > > - * invalidate_icache_range(start,end)
> > > + * __inval_icache_pou(start,end)
> > > *
> > > * Ensure that the I cache is invalid within specified region.
> > > *
> > > * - start - virtual start address of region
> > > * - end - virtual end address of region
> > > */
> > > -SYM_FUNC_START(invalidate_icache_range)
> > > +SYM_FUNC_START(__inval_icache_pou)
> > > alternative_if ARM64_HAS_CACHE_DIC
> > > isb
> > > ret
> > > @@ -111,10 +111,10 @@ alternative_else_nop_endif
> > >
> > > invalidate_icache_by_line x0, x1, x2, x3, 0, 0f
> > > ret
> > > -SYM_FUNC_END(invalidate_icache_range)
> > > +SYM_FUNC_END(__inval_icache_pou)
> > >
> > > /*
> > > - * __flush_dcache_area(start, end)
> > > + * __clean_inval_dcache_poc(start, end)
> > > *
> > > * Ensure that any D-cache lines for the interval [start, end)
> > > * are cleaned and invalidated to the PoC.
> > > @@ -122,13 +122,13 @@ SYM_FUNC_END(invalidate_icache_range)
> > > * - start - virtual start address of region
> > > * - end - virtual end address of region
> > > */
> > > -SYM_FUNC_START_PI(__flush_dcache_area)
> > > +SYM_FUNC_START_PI(__clean_inval_dcache_poc)
> > > dcache_by_line_op civac, sy, x0, x1, x2, x3
> > > ret
> > > -SYM_FUNC_END_PI(__flush_dcache_area)
> > > +SYM_FUNC_END_PI(__clean_inval_dcache_poc)
> > >
> > > /*
> > > - * __clean_dcache_area_pou(start, end)
> > > + * __clean_dcache_pou(start, end)
> > > *
> > > * Ensure that any D-cache lines for the interval [start, end)
> > > * are cleaned to the PoU.
> > > @@ -136,17 +136,17 @@ SYM_FUNC_END_PI(__flush_dcache_area)
> > > * - start - virtual start address of region
> > > * - end - virtual end address of region
> > > */
> > > -SYM_FUNC_START(__clean_dcache_area_pou)
> > > +SYM_FUNC_START(__clean_dcache_pou)
> > > alternative_if ARM64_HAS_CACHE_IDC
> > > dsb ishst
> > > ret
> > > alternative_else_nop_endif
> > > dcache_by_line_op cvau, ish, x0, x1, x2, x3
> > > ret
> > > -SYM_FUNC_END(__clean_dcache_area_pou)
> > > +SYM_FUNC_END(__clean_dcache_pou)
> > >
> > > /*
> > > - * __inval_dcache_area(start, end)
> > > + * __inval_dcache_poc(start, end)
> > > *
> > > * Ensure that any D-cache lines for the interval [start, end)
> > > * are invalidated. Any partial lines at the ends of the interval are
> > > @@ -156,7 +156,7 @@ SYM_FUNC_END(__clean_dcache_area_pou)
> > > * - end - kernel end address of region
> > > */
> > > SYM_FUNC_START_LOCAL(__dma_inv_area)
> > > -SYM_FUNC_START_PI(__inval_dcache_area)
> > > +SYM_FUNC_START_PI(__inval_dcache_poc)
> > > /* FALLTHROUGH */
> > >
> > > /*
> > > @@ -181,11 +181,11 @@ SYM_FUNC_START_PI(__inval_dcache_area)
> > > b.lo 2b
> > > dsb sy
> > > ret
> > > -SYM_FUNC_END_PI(__inval_dcache_area)
> > > +SYM_FUNC_END_PI(__inval_dcache_poc)
> > > SYM_FUNC_END(__dma_inv_area)
> > >
> > > /*
> > > - * __clean_dcache_area_poc(start, end)
> > > + * __clean_dcache_poc(start, end)
> > > *
> > > * Ensure that any D-cache lines for the interval [start, end)
> > > * are cleaned to the PoC.
> > > @@ -194,7 +194,7 @@ SYM_FUNC_END(__dma_inv_area)
> > > * - end - virtual end address of region
> > > */
> > > SYM_FUNC_START_LOCAL(__dma_clean_area)
> > > -SYM_FUNC_START_PI(__clean_dcache_area_poc)
> > > +SYM_FUNC_START_PI(__clean_dcache_poc)
> > > /* FALLTHROUGH */
> > >
> > > /*
> > > @@ -204,11 +204,11 @@ SYM_FUNC_START_PI(__clean_dcache_area_poc)
> > > */
> > > dcache_by_line_op cvac, sy, x0, x1, x2, x3
> > > ret
> > > -SYM_FUNC_END_PI(__clean_dcache_area_poc)
> > > +SYM_FUNC_END_PI(__clean_dcache_poc)
> > > SYM_FUNC_END(__dma_clean_area)
> > >
> > > /*
> > > - * __clean_dcache_area_pop(start, end)
> > > + * __clean_dcache_pop(start, end)
> > > *
> > > * Ensure that any D-cache lines for the interval [start, end)
> > > * are cleaned to the PoP.
> > > @@ -216,13 +216,13 @@ SYM_FUNC_END(__dma_clean_area)
> > > * - start - virtual start address of region
> > > * - end - virtual end address of region
> > > */
> > > -SYM_FUNC_START_PI(__clean_dcache_area_pop)
> > > +SYM_FUNC_START_PI(__clean_dcache_pop)
> > > alternative_if_not ARM64_HAS_DCPOP
> > > - b __clean_dcache_area_poc
> > > + b __clean_dcache_poc
> > > alternative_else_nop_endif
> > > dcache_by_line_op cvap, sy, x0, x1, x2, x3
> > > ret
> > > -SYM_FUNC_END_PI(__clean_dcache_area_pop)
> > > +SYM_FUNC_END_PI(__clean_dcache_pop)
> > >
> > > /*
> > > * __dma_flush_area(start, size)
> > > diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
> > > index 143f625e7727..005b92148252 100644
> > > --- a/arch/arm64/mm/flush.c
> > > +++ b/arch/arm64/mm/flush.c
> > > @@ -17,14 +17,14 @@
> > > void sync_icache_aliases(unsigned long start, unsigned long end)
> > > {
> > > if (icache_is_aliasing()) {
> > > - __clean_dcache_area_pou(start, end);
> > > - __flush_icache_all();
> > > + __clean_dcache_pou(start, end);
> > > + __clean_inval_all_icache_pou();
> > > } else {
> > > /*
> > > * Don't issue kick_all_cpus_sync() after I-cache invalidation
> > > * for user mappings.
> > > */
> > > - __flush_icache_range(start, end);
> > > + __clean_inval_cache_pou(start, end);
> > > }
> > > }
> > >
> > > @@ -76,20 +76,20 @@ EXPORT_SYMBOL(flush_dcache_page);
> > > /*
> > > * Additional functions defined in assembly.
> > > */
> > > -EXPORT_SYMBOL(__flush_icache_range);
> > > +EXPORT_SYMBOL(__clean_inval_cache_pou);
> > >
> > > #ifdef CONFIG_ARCH_HAS_PMEM_API
> > > void arch_wb_cache_pmem(void *addr, size_t size)
> > > {
> > > /* Ensure order against any prior non-cacheable writes */
> > > dmb(osh);
> > > - __clean_dcache_area_pop((unsigned long)addr, (unsigned long)addr + size);
> > > + __clean_dcache_pop((unsigned long)addr, (unsigned long)addr + size);
> > > }
> > > EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
> > >
> > > void arch_invalidate_pmem(void *addr, size_t size)
> > > {
> > > - __inval_dcache_area((unsigned long)addr, (unsigned long)addr + size);
> > > + __inval_dcache_poc((unsigned long)addr, (unsigned long)addr + size);
> > > }
> > > EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
> > > #endif
> > > --
> > > 2.31.1.607.g51e8a6a459-goog
> > >
More information about the linux-arm-kernel
mailing list