[PATCH v2 5/6] RISC-V: Report vector unaligned access speed hwprobe

Evan Green evan at rivosinc.com
Thu Jun 20 11:51:35 PDT 2024


On Thu, Jun 13, 2024 at 12:17 PM Jesse Taube <jesse at rivosinc.com> wrote:
>
> Detect if vector misaligned accesses are faster or slower than
> equivalent vector byte accesses. This is useful for usermode to know
> whether vector byte accesses or vector misaligned accesses have a better
> bandwidth for operations like memcpy.
>
> Signed-off-by: Jesse Taube <jesse at rivosinc.com>
> ---
> V1 -> V2:
>  - Add Kconfig options
>  - Add WORD_EEW to vec-copy-unaligned.S
> ---
>  arch/riscv/Kconfig                         |  18 +++
>  arch/riscv/kernel/Makefile                 |   3 +-
>  arch/riscv/kernel/copy-unaligned.h         |   5 +
>  arch/riscv/kernel/sys_hwprobe.c            |   6 +
>  arch/riscv/kernel/unaligned_access_speed.c | 130 ++++++++++++++++++++-
>  arch/riscv/kernel/vec-copy-unaligned.S     |  58 +++++++++
>  6 files changed, 217 insertions(+), 3 deletions(-)
>  create mode 100644 arch/riscv/kernel/vec-copy-unaligned.S
>
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index f12df0ca6c18..b181b35f37ca 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -813,6 +813,24 @@ config CONFIG_RISCV_UNALIGNED_ACCESS_UNSUPPORTED
>           The kernel and userspace programs may run them successfully on systems
>           that do support vector unaligned memory accesses.
>
> +config RISCV_SLOW_VEC_UNALIGNED_ACCESS
> +       bool "Assume the system supports slow vector unaligned memory accesses"
> +       depends on NONPORTABLE
> +       help
> +         Assume that the system supports slow vector unaligned memory accesses. The
> +         kernel and userspace programs may not be able to run at all on systems
> +         that do not support unaligned memory accesses.
> +
> +config RISCV_EFFICIENT_VEC_UNALIGNED_ACCESS
> +       bool "Assume the system supports fast vector unaligned memory accesses"
> +       depends on NONPORTABLE
> +       help
> +         Assume that the system supports fast vector unaligned memory accesses. When
> +         enabled, this option improves the performance of the kernel on such
> +         systems. However, the kernel and userspace programs will run much more
> +         slowly, or will not be able to run at all, on systems that do not
> +         support efficient unaligned memory accesses.
> +
>  endchoice
>
>  endmenu # "Platform type"
> diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
> index 62ac19c029f1..c4d6f54705a4 100644
> --- a/arch/riscv/kernel/Makefile
> +++ b/arch/riscv/kernel/Makefile
> @@ -64,7 +64,8 @@ obj-$(CONFIG_MMU) += vdso.o vdso/
>
>  obj-y  += traps_misaligned.o
>  obj-y  += unaligned_access_speed.o
> -obj-$(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)     += copy-unaligned.o
> +obj-$(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)             += copy-unaligned.o
> +obj-$(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)      += vec-copy-unaligned.o
>
>  obj-$(CONFIG_FPU)              += fpu.o
>  obj-$(CONFIG_FPU)              += kernel_mode_fpu.o
> diff --git a/arch/riscv/kernel/copy-unaligned.h b/arch/riscv/kernel/copy-unaligned.h
> index e3d70d35b708..85d4d11450cb 100644
> --- a/arch/riscv/kernel/copy-unaligned.h
> +++ b/arch/riscv/kernel/copy-unaligned.h
> @@ -10,4 +10,9 @@
>  void __riscv_copy_words_unaligned(void *dst, const void *src, size_t size);
>  void __riscv_copy_bytes_unaligned(void *dst, const void *src, size_t size);
>
> +#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
> +void __riscv_copy_vec_words_unaligned(void *dst, const void *src, size_t size);
> +void __riscv_copy_vec_bytes_unaligned(void *dst, const void *src, size_t size);
> +#endif
> +
>  #endif /* __RISCV_KERNEL_COPY_UNALIGNED_H */
> diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c
> index c40df314058b..1679db4fe360 100644
> --- a/arch/riscv/kernel/sys_hwprobe.c
> +++ b/arch/riscv/kernel/sys_hwprobe.c
> @@ -203,6 +203,12 @@ static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
>         if (IS_ENABLED(CONFIG_RISCV_UNALIGNED_ACCESS_UNSUPPORTED))
>                 return RISCV_HWPROBE_VEC_MISALIGNED_UNSUPPORTED;
>
> +       if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VEC_UNALIGNED_ACCESS))
> +               return RISCV_HWPROBE_VEC_MISALIGNED_FAST;
> +
> +       if (IS_ENABLED(CONFIG_RISCV_SLOW_VEC_UNALIGNED_ACCESS))
> +               return RISCV_HWPROBE_VEC_MISALIGNED_SLOW;
> +
>         /* Return if supported or not even if speed wasn't probed */
>         for_each_cpu(cpu, cpus) {
>                 int this_perf = per_cpu(vector_misaligned_access, cpu);
> diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c
> index c6106bd4a25a..ddc49cce54e3 100644
> --- a/arch/riscv/kernel/unaligned_access_speed.c
> +++ b/arch/riscv/kernel/unaligned_access_speed.c
> @@ -8,9 +8,11 @@
>  #include <linux/jump_label.h>
>  #include <linux/mm.h>
>  #include <linux/smp.h>
> +#include <linux/kthread.h>
>  #include <linux/types.h>
>  #include <asm/cpufeature.h>
>  #include <asm/hwprobe.h>
> +#include <asm/vector.h>
>
>  #include "copy-unaligned.h"
>
> @@ -262,9 +264,126 @@ static int check_unaligned_access_speed_all_cpus(void)
>  }
>  #endif /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
>
> +#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
> +static void check_vector_unaligned_access(struct work_struct *unused)
> +{
> +       int cpu = smp_processor_id();
> +       u64 start_cycles, end_cycles;
> +       u64 word_cycles;
> +       u64 byte_cycles;
> +       int ratio;
> +       unsigned long start_jiffies, now;
> +       struct page *page;
> +       void *dst;
> +       void *src;
> +       long speed = RISCV_HWPROBE_VEC_MISALIGNED_SLOW;
> +
> +       if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_VEC_MISALIGNED_SLOW)
> +               return;
> +
> +       page = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
> +       if (!page) {
> +               pr_warn("Allocation failure, not measuring vector misaligned performance\n");
> +               return;
> +       }
> +
> +       /* Make an unaligned destination buffer. */
> +       dst = (void *)((unsigned long)page_address(page) | 0x1);
> +       /* Unalign src as well, but differently (off by 1 + 2 = 3). */
> +       src = dst + (MISALIGNED_BUFFER_SIZE / 2);
> +       src += 2;
> +       word_cycles = -1ULL;
> +
> +       /* Do a warmup. */
> +       local_irq_enable();

Another one here, discussed in the previous patch.

> +       kernel_vector_begin();
> +       __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
> +
> +       start_jiffies = jiffies;
> +       while ((now = jiffies) == start_jiffies)
> +               cpu_relax();
> +
> +       /*
> +        * For a fixed amount of time, repeatedly try the function, and take
> +        * the best time in cycles as the measurement.
> +        */
> +       while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
> +               start_cycles = get_cycles64();
> +               /* Ensure the CSR read can't reorder WRT to the copy. */
> +               mb();
> +               __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
> +               /* Ensure the copy ends before the end time is snapped. */
> +               mb();
> +               end_cycles = get_cycles64();
> +               if ((end_cycles - start_cycles) < word_cycles)
> +                       word_cycles = end_cycles - start_cycles;
> +       }
> +
> +       byte_cycles = -1ULL;
> +       __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
> +       start_jiffies = jiffies;
> +       while ((now = jiffies) == start_jiffies)
> +               cpu_relax();
> +
> +       while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
> +               start_cycles = get_cycles64();
> +               mb();
> +               __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
> +               mb();
> +               end_cycles = get_cycles64();
> +               if ((end_cycles - start_cycles) < byte_cycles)
> +                       byte_cycles = end_cycles - start_cycles;
> +       }
> +
> +       kernel_vector_end();
> +
> +       /* Don't divide by zero. */
> +       if (!word_cycles || !byte_cycles) {
> +               pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n",
> +                       cpu);
> +
> +               return;
> +       }
> +
> +       if (word_cycles < byte_cycles)
> +               speed = RISCV_HWPROBE_VEC_MISALIGNED_FAST;
> +
> +       ratio = div_u64((byte_cycles * 100), word_cycles);
> +       pr_info("cpu%d: Ratio of vector byte access time to vector unaligned word access is %d.%02d, unaligned accesses are %s\n",
> +               cpu,
> +               ratio / 100,
> +               ratio % 100,
> +               (speed ==  RISCV_HWPROBE_VEC_MISALIGNED_FAST) ? "fast" : "slow");
> +
> +       per_cpu(vector_misaligned_access, cpu) = speed;
> +}
> +
> +static int riscv_online_cpu_vec(unsigned int cpu)
> +{
> +       check_vector_unaligned_access(NULL);
> +       return 0;
> +}
> +
> +/* Measure unaligned access speed on all CPUs present at boot in parallel. */
> +static int vec_check_unaligned_access_speed_all_cpus(void *unused)
> +{
> +       /* Schecudles work que*/

Tpyos!


> +       schedule_on_each_cpu(check_vector_unaligned_access);
> +
> +       /*
> +        * Setup hotplug callbacks for any new CPUs that come online or go
> +        * offline.
> +        */
> +       cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
> +                                 riscv_online_cpu_vec, NULL);
> +
> +       return 0;
> +}
> +#endif /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */
> +
>  static int check_unaligned_access_all_cpus(void)
>  {
> -       bool all_cpus_emulated;
> +       bool all_cpus_emulated, all_cpus_vec_supported;
>         int cpu;
>
>         if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICCLSM)) {
> @@ -280,7 +399,14 @@ static int check_unaligned_access_all_cpus(void)
>         }
>
>         all_cpus_emulated = check_unaligned_access_emulated_all_cpus();
> -       check_vector_unaligned_access_emulated_all_cpus();
> +       all_cpus_vec_supported = check_vector_unaligned_access_emulated_all_cpus();
> +
> +#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
> +       if (all_cpus_vec_supported) {
> +               kthread_run(vec_check_unaligned_access_speed_all_cpus,
> +                           NULL, "vec_check_unaligned_access_speed_all_cpus");
> +       }
> +#endif
>
>  #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
>         if (!all_cpus_emulated)
> diff --git a/arch/riscv/kernel/vec-copy-unaligned.S b/arch/riscv/kernel/vec-copy-unaligned.S
> new file mode 100644
> index 000000000000..e5bc94917e60
> --- /dev/null
> +++ b/arch/riscv/kernel/vec-copy-unaligned.S
> @@ -0,0 +1,58 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright (C) 2024 Rivos Inc. */
> +
> +#include <linux/linkage.h>
> +#include <asm/asm.h>
> +#include <linux/args.h>
> +
> +       .text
> +
> +#define WORD_EEW 32
> +
> +#define WORD_SEW CONCATENATE(e, WORD_EEW)
> +#define VEC_L CONCATENATE(vle, WORD_EEW).v
> +#define VEC_S CONCATENATE(vle, WORD_EEW).v
> +
> +/* void __riscv_copy_vec_words_unaligned(void *, const void *, size_t) */
> +/* Performs a memcpy without aligning buffers, using word loads and stores. */
> +/* Note: The size is truncated to a multiple of WORD_EEW */
> +SYM_FUNC_START(__riscv_copy_vec_words_unaligned)
> +       andi  a4, a2, ~(WORD_EEW-1)
> +       beqz  a4, 2f
> +       add   a3, a1, a4
> +       .option push
> +       .option arch, +zve32x
> +1:
> +       vsetivli t0, 8, WORD_SEW, m8, ta, ma
> +       VEC_L v0, (a1)
> +       VEC_S v0, (a0)
> +       addi  a0, a0, WORD_EEW
> +       addi  a1, a1, WORD_EEW
> +       bltu  a1, a3, 1b
> +
> +2:
> +       .option pop
> +       ret
> +SYM_FUNC_END(__riscv_copy_vec_words_unaligned)
> +
> +/* void __riscv_copy_vec_bytes_unaligned(void *, const void *, size_t) */
> +/* Performs a memcpy without aligning buffers, using only byte accesses. */
> +/* Note: The size is truncated to a multiple of 8 */
> +SYM_FUNC_START(__riscv_copy_vec_bytes_unaligned)
> +       andi a4, a2, ~(8-1)
> +       beqz a4, 2f
> +       add  a3, a1, a4
> +       .option push
> +       .option arch, +zve32x
> +1:
> +       vsetivli t0, 8, e8, m8, ta, ma
> +       vle8.v v0, (a1)
> +       vse8.v v0, (a0)
> +       addi a0, a0, 8
> +       addi a1, a1, 8
> +       bltu a1, a3, 1b
> +
> +2:
> +       .option pop
> +       ret
> +SYM_FUNC_END(__riscv_copy_vec_bytes_unaligned)
> --
> 2.43.0
>



More information about the linux-riscv mailing list