[PATCH V11 03/17] riscv: Use Zicbop in arch_xchg when available

Guo Ren guoren at kernel.org
Tue Sep 19 00:53:22 PDT 2023


On Tue, Sep 19, 2023 at 1:13 PM Leonardo Bras <leobras at redhat.com> wrote:
>
> On Sun, Sep 17, 2023 at 10:34:36PM +0800, Guo Ren wrote:
> > On Sat, Sep 16, 2023 at 9:25 AM Leonardo Bras <leobras at redhat.com> wrote:
> > >
> > > On Fri, Sep 15, 2023 at 08:36:31PM +0800, Guo Ren wrote:
> > > > On Wed, Sep 13, 2023 at 4:50 PM Leonardo Bras <leobras at redhat.com> wrote:
> > > > >
> > > > > On Sun, Sep 10, 2023 at 04:28:57AM -0400, guoren at kernel.org wrote:
> > > > > > From: Guo Ren <guoren at linux.alibaba.com>
> > > > > >
> > > > > > Cache-block prefetch instructions are HINTs to the hardware to
> > > > > > indicate that software intends to perform a particular type of
> > > > > > memory access in the near future. Enable ARCH_HAS_PREFETCHW and
> > > > > > improve the arch_xchg for qspinlock xchg_tail.
> > > > > >
> > > > > > Signed-off-by: Guo Ren <guoren at linux.alibaba.com>
> > > > > > Signed-off-by: Guo Ren <guoren at kernel.org>
> > > > > > ---
> > > > > >  arch/riscv/Kconfig                 | 15 +++++++++++++++
> > > > > >  arch/riscv/include/asm/cmpxchg.h   |  4 +++-
> > > > > >  arch/riscv/include/asm/hwcap.h     |  1 +
> > > > > >  arch/riscv/include/asm/insn-def.h  |  5 +++++
> > > > > >  arch/riscv/include/asm/processor.h | 13 +++++++++++++
> > > > > >  arch/riscv/kernel/cpufeature.c     |  1 +
> > > > > >  6 files changed, 38 insertions(+), 1 deletion(-)
> > > > > >
> > > > > > diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> > > > > > index e9ae6fa232c3..2c346fe169c1 100644
> > > > > > --- a/arch/riscv/Kconfig
> > > > > > +++ b/arch/riscv/Kconfig
> > > > > > @@ -617,6 +617,21 @@ config RISCV_ISA_ZICBOZ
> > > > > >
> > > > > >          If you don't know what to do here, say Y.
> > > > > >
> > > > > > +config RISCV_ISA_ZICBOP
> > > > > > +     bool "Zicbop extension support for cache block prefetch"
> > > > > > +     depends on MMU
> > > > > > +     depends on RISCV_ALTERNATIVE
> > > > > > +     default y
> > > > > > +     help
> > > > > > +        Adds support to dynamically detect the presence of the ZICBOP
> > > > > > +        extension (Cache Block Prefetch Operations) and enable its
> > > > > > +        usage.
> > > > > > +
> > > > > > +        The Zicbop extension can be used to prefetch cache block for
> > > > > > +        read/write/instruction fetch.
> > > > > > +
> > > > > > +        If you don't know what to do here, say Y.
> > > > > > +
> > > > > >  config TOOLCHAIN_HAS_ZIHINTPAUSE
> > > > > >       bool
> > > > > >       default y
> > > > > > diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
> > > > > > index 702725727671..56eff7a9d2d2 100644
> > > > > > --- a/arch/riscv/include/asm/cmpxchg.h
> > > > > > +++ b/arch/riscv/include/asm/cmpxchg.h
> > > > > > @@ -11,6 +11,7 @@
> > > > > >
> > > > > >  #include <asm/barrier.h>
> > > > > >  #include <asm/fence.h>
> > > > > > +#include <asm/processor.h>
> > > > > >
> > > > > >  #define __arch_xchg_masked(prepend, append, r, p, n)                 \
> > > > > >  ({                                                                   \
> > > > > > @@ -25,6 +26,7 @@
> > > > > >                                                                       \
> > > > > >       __asm__ __volatile__ (                                          \
> > > > > >              prepend                                                  \
> > > > > > +            PREFETCHW_ASM(%5)                                        \
> > > > > >              "0:      lr.w %0, %2\n"                                  \
> > > > > >              "        and  %1, %0, %z4\n"                             \
> > > > > >              "        or   %1, %1, %z3\n"                             \
> > > > > > @@ -32,7 +34,7 @@
> > > > > >              "        bnez %1, 0b\n"                                  \
> > > > > >              append                                                   \
> > > > > >              : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b))       \
> > > > > > -            : "rJ" (__newx), "rJ" (~__mask)                          \
> > > > > > +            : "rJ" (__newx), "rJ" (~__mask), "rJ" (__ptr32b)         \
> > > > > >              : "memory");                                             \
> > > > > >                                                                       \
> > > > > >       r = (__typeof__(*(p)))((__retx & __mask) >> __s);               \
> > > > > > diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
> > > > > > index b7b58258f6c7..78b7b8b53778 100644
> > > > > > --- a/arch/riscv/include/asm/hwcap.h
> > > > > > +++ b/arch/riscv/include/asm/hwcap.h
> > > > > > @@ -58,6 +58,7 @@
> > > > > >  #define RISCV_ISA_EXT_ZICSR          40
> > > > > >  #define RISCV_ISA_EXT_ZIFENCEI               41
> > > > > >  #define RISCV_ISA_EXT_ZIHPM          42
> > > > > > +#define RISCV_ISA_EXT_ZICBOP         43
> > > > > >
> > > > > >  #define RISCV_ISA_EXT_MAX            64
> > > > > >
> > > > > > diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h
> > > > > > index 6960beb75f32..dc590d331894 100644
> > > > > > --- a/arch/riscv/include/asm/insn-def.h
> > > > > > +++ b/arch/riscv/include/asm/insn-def.h
> > > > > > @@ -134,6 +134,7 @@
> > > > > >
> > > > > >  #define RV_OPCODE_MISC_MEM   RV_OPCODE(15)
> > > > > >  #define RV_OPCODE_SYSTEM     RV_OPCODE(115)
> > > > > > +#define RV_OPCODE_PREFETCH   RV_OPCODE(19)
> > > > > >
> > > > > >  #define HFENCE_VVMA(vaddr, asid)                             \
> > > > > >       INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(17),              \
> > > > > > @@ -196,4 +197,8 @@
> > > > > >       INSN_I(OPCODE_MISC_MEM, FUNC3(2), __RD(0),              \
> > > > > >              RS1(base), SIMM12(4))
> > > > > >
> > > > > > +#define CBO_prefetchw(base)                                  \
> > > > > > +     INSN_R(OPCODE_PREFETCH, FUNC3(6), FUNC7(0),             \
> > > > > > +            RD(x0), RS1(base), RS2(x0))
> > > > > > +
> > > > >
> > > > > I understand that here you create the instruction via bitfield, following
> > > > > the ISA, and this enables using instructions not available on the
> > > > > toolchain.
> > > > >
> > > > > It took me some time to find the document with this instruction, so please
> > > > > add this to the commit msg:
> > > > >
> > > > > https://github.com/riscv/riscv-CMOs/blob/master/specifications/cmobase-v1.0.pdf
> > > > > Page 23.
> > > > >
> > > > > IIUC, the instruction is "prefetch.w".
> > > > >
> > > > > Maybe I am missing something, but in the document the rs2 field
> > > > > (PREFETCH.W) contains a 0x3, while the above looks to have a 0 instead.
> > > > >
> > > > > rs2 field = 0x0 would be a prefetch.i (instruction prefetch) instead.
> > > > >
> > > > > Is the above correct, or am I missing something?
> > > > Oh, you are right. My fault, thx for pointing out. It should be:
> > > > +       INSN_R(OPCODE_PREFETCH, FUNC3(6), FUNC7(0),             \
> > > > +              RD(x0), RS1(base), RS2(x3))
> > >
> > > Now I am curious to check if / how will this impact performance. :)
> > > (Please let me know)
> > Ref:
> > commit 0ea366f5e1b6 ("arm64: atomics: prefetch the destination word
> > for write prior to stxr")
> > commit 86d231459d6d ("bpf: cpumap memory prefetchw optimizations for
> > struct page")
>
> Oh, I understand that prefetch.w is very useful for performance :)
>
> What I meant is that previously this patch was issuing a prefetch.i,
> and now it's issuing a prefetch.w (as intended).
>
> What got me curious is how much would it impact the performance to change
> the prefetch.i to prefetch.w. :)
The current SOPHO sg2042 hardware platform didn't support prefetch.w
instruction. So there is no performance result I could share with you.

Our next generation of processors would support ZICBOP.

>
> Thanks!
> Leo
>
>
> >
> > >
> > >
> > > >
> > > > >
> > > > >
> > > > > Thanks!
> > > > > Leo
> > > > >
> > > > > >  #endif /* __ASM_INSN_DEF_H */
> > > > > > diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
> > > > > > index de9da852f78d..7ad3a24212e8 100644
> > > > > > --- a/arch/riscv/include/asm/processor.h
> > > > > > +++ b/arch/riscv/include/asm/processor.h
> > > > > > @@ -12,6 +12,8 @@
> > > > > >  #include <vdso/processor.h>
> > > > > >
> > > > > >  #include <asm/ptrace.h>
> > > > > > +#include <asm/insn-def.h>
> > > > > > +#include <asm/hwcap.h>
> > > > > >
> > > > > >  #ifdef CONFIG_64BIT
> > > > > >  #define DEFAULT_MAP_WINDOW   (UL(1) << (MMAP_VA_BITS - 1))
> > > > > > @@ -103,6 +105,17 @@ static inline void arch_thread_struct_whitelist(unsigned long *offset,
> > > > > >  #define KSTK_EIP(tsk)                (ulong)(task_pt_regs(tsk)->epc)
> > > > > >  #define KSTK_ESP(tsk)                (ulong)(task_pt_regs(tsk)->sp)
> > > > > >
> > > > > > +#define ARCH_HAS_PREFETCHW
> > > > > > +#define PREFETCHW_ASM(base)  ALTERNATIVE(__nops(1), \
> > > > > > +                                         CBO_prefetchw(base), \
> > > > > > +                                         0, \
> > > > > > +                                         RISCV_ISA_EXT_ZICBOP, \
> > > > > > +                                         CONFIG_RISCV_ISA_ZICBOP)
> > > > > > +static inline void prefetchw(const void *ptr)
> > > > > > +{
> > > > > > +     asm volatile(PREFETCHW_ASM(%0)
> > > > > > +             : : "r" (ptr) : "memory");
> > > > > > +}
> > > > > >
> > > > > >  /* Do necessary setup to start up a newly executed thread. */
> > > > > >  extern void start_thread(struct pt_regs *regs,
> > > > > > diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
> > > > > > index ef7b4fd9e876..e0b897db0b97 100644
> > > > > > --- a/arch/riscv/kernel/cpufeature.c
> > > > > > +++ b/arch/riscv/kernel/cpufeature.c
> > > > > > @@ -159,6 +159,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
> > > > > >       __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h),
> > > > > >       __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM),
> > > > > >       __RISCV_ISA_EXT_DATA(zicboz, RISCV_ISA_EXT_ZICBOZ),
> > > > > > +     __RISCV_ISA_EXT_DATA(zicbop, RISCV_ISA_EXT_ZICBOP),
> > > > > >       __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
> > > > > >       __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR),
> > > > > >       __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI),
> > > > > > --
> > > > > > 2.36.1
> > > > > >
> > > > >
> > > >
> > > >
> > > > --
> > > > Best Regards
> > > >  Guo Ren
> > > >
> > >
> >
> >
> > --
> > Best Regards
> >  Guo Ren
> >
>


-- 
Best Regards
 Guo Ren



More information about the linux-riscv mailing list