[RFC patch 3/4] riscv: Introduce alternative mechanism to apply errata solution
Palmer Dabbelt
palmer at dabbelt.com
Wed Mar 10 04:39:15 GMT 2021
On Sun, 07 Mar 2021 19:58:16 PST (-0800), vincent.chen at sifive.com wrote:
> Introduce the "alternative" mechanism from ARM64 and x86 to apply the CPU
> vendors' errata solution at runtime. The main purpose of this patch is
> to provide a framework. Therefore, the implementation is quite basic for
> now so that some scenarios could not use this schemei, such as patching
> code to a module, relocating the patching code and heterogeneous CPU
> topology.
>
> Users could use the macro ALTERNATIVE to apply an errata to the existing
> code flow. In the macro ALTERNATIVE, users need to specify the manufacturer
> information(vendorid, archid, and impid) for this errata. Therefore, kernel
> will know this errata is suitable for which CPU core. During the booting
> procedure, kernel will select the errata required by the CPU core and then
> patch it. It means that the kernel only applies the errata to the specified
> CPU core. In this case the vendor's errata does not affect each other at
> runtime. The above patching procedure only occurs during the booting phase,
> so we only take the overhead of the "alternative" mechanism once.
>
> This "alternative" mechanism is enabled by default to ensure that all
> required errata will be applied. However, users can disable this feature by
> the Kconfig "CONFIG_RISCV_ERRATA_ALTERNATIVE".
>
> Signed-off-by: Vincent Chen <vincent.chen at sifive.com>
> ---
> arch/riscv/Kconfig | 1 +
> arch/riscv/Kconfig.erratas | 12 +++
> arch/riscv/Makefile | 1 +
> arch/riscv/errata/Makefile | 1 +
> arch/riscv/errata/alternative.c | 69 +++++++++++++++++
> arch/riscv/include/asm/alternative-macros.h | 110 ++++++++++++++++++++++++++++
> arch/riscv/include/asm/alternative.h | 44 +++++++++++
> arch/riscv/include/asm/asm.h | 1 +
> arch/riscv/include/asm/errata_list.h | 8 ++
> arch/riscv/include/asm/sections.h | 2 +
> arch/riscv/include/asm/vendorid_list.h | 6 ++
> arch/riscv/kernel/smpboot.c | 4 +
> arch/riscv/kernel/vmlinux.lds.S | 14 ++++
> 13 files changed, 273 insertions(+)
> create mode 100644 arch/riscv/Kconfig.erratas
> create mode 100644 arch/riscv/errata/Makefile
> create mode 100644 arch/riscv/errata/alternative.c
> create mode 100644 arch/riscv/include/asm/alternative-macros.h
> create mode 100644 arch/riscv/include/asm/alternative.h
> create mode 100644 arch/riscv/include/asm/errata_list.h
> create mode 100644 arch/riscv/include/asm/vendorid_list.h
>
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index a998babc1237..2e26251fe1f2 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -204,6 +204,7 @@ config LOCKDEP_SUPPORT
> def_bool y
>
> source "arch/riscv/Kconfig.socs"
> +source "arch/riscv/Kconfig.erratas"
>
> menu "Platform type"
>
> diff --git a/arch/riscv/Kconfig.erratas b/arch/riscv/Kconfig.erratas
> new file mode 100644
> index 000000000000..4d0bafc536df
> --- /dev/null
> +++ b/arch/riscv/Kconfig.erratas
> @@ -0,0 +1,12 @@
> +menu "CPU errata selection"
> +
> +config RISCV_ERRATA_ALTERNATIVE
> + bool "RISC-V alternative scheme"
> + default y
> + help
> + This Kconfig allows the kernel to automatically patch the
> + errata required by the execution platform at run time. The
> + code patching is performed once in the boot stages. It means
> + that the overhead from this mechanism is just taken once.
> +
> +endmenu
> diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
> index 1368d943f1f3..1f5c03082976 100644
> --- a/arch/riscv/Makefile
> +++ b/arch/riscv/Makefile
> @@ -87,6 +87,7 @@ KBUILD_IMAGE := $(boot)/Image.gz
> head-y := arch/riscv/kernel/head.o
>
> core-y += arch/riscv/
> +core-$(CONFIG_RISCV_ERRATA_ALTERNATIVE) += arch/riscv/errata/
>
> libs-y += arch/riscv/lib/
> libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
> diff --git a/arch/riscv/errata/Makefile b/arch/riscv/errata/Makefile
> new file mode 100644
> index 000000000000..43e6d5424367
> --- /dev/null
> +++ b/arch/riscv/errata/Makefile
> @@ -0,0 +1 @@
> +obj-y += alternative.o
> diff --git a/arch/riscv/errata/alternative.c b/arch/riscv/errata/alternative.c
> new file mode 100644
> index 000000000000..052affdce6eb
> --- /dev/null
> +++ b/arch/riscv/errata/alternative.c
> @@ -0,0 +1,69 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * alternative runtime patching
> + * inspired by the ARM64 and x86 version
> + *
> + * Copyright (C) 2021 Sifive.
> + */
> +
> +#include <linux/init.h>
> +#include <linux/cpu.h>
> +#include <linux/uaccess.h>
> +#include <asm/patch.h>
> +#include <asm/alternative.h>
> +#include <asm/sections.h>
> +
> +struct alt_region {
> + struct alt_entry *begin;
> + struct alt_entry *end;
> +};
> +
> +static bool (*errata_checkfunc)(struct alt_entry *alt);
> +typedef int (*patch_func_t)(void *addr, const void *insn, size_t size);
> +
> +static void __apply_alternatives(void *alt_region, void *alt_patch_func)
> +{
> + struct alt_entry *alt;
> + struct alt_region *region = alt_region;
> +
> + for (alt = region->begin; alt < region->end; alt++) {
> + if (!errata_checkfunc(alt))
> + continue;
> + ((patch_func_t)alt_patch_func)(alt->old_ptr, alt->alt_ptr, alt->alt_len);
> + }
> +}
> +
> +static void __init init_alternative(void)
> +{
> + struct errata_checkfunc_id *ptr;
> +
> + for (ptr = (struct errata_checkfunc_id *)__alt_checkfunc_table;
> + ptr < (struct errata_checkfunc_id *)__alt_checkfunc_table_end;
> + ptr++) {
> + if (cpu_mfr_info.vendor_id == ptr->vendor_id)
Of those three new SBI calls this is the only one I see used. You could get
rid of the whole global caching thing and just call it once here, which would
be a bit simpler.
I'm also not convinced that it's worth all this table stuff here. It's not
like we have new vendors every day, we could just have a switch statement on
sbi_get_mvendorid() and stick every vendor's errata probe in there.
At that point this would be simple enough that I'd be inclined to
unconditionally (at least on S-mode kernels, the M-mode kernels aren't portable
so who cares) include the errata checking. I can understand not wanting to
take all the errata fixing, as some of those could get complex, but if we have
the check in there we can provide a warning to users who don't have the errata
fix.
If that gets out of hand or we have non-portable kernels we can always hoist
the ifdefs to avoid the check functions as well, but I'd prefer that portable
kernels are at least capable of detecting they've been built in a fashion that
will subtly break things.
> + errata_checkfunc = ptr->func;
> + }
> +}
> +
> +/*
> + * This is called very early in the boot process (directly after we run
> + * a feature detect on the boot CPU). No need to worry about other CPUs
> + * here.
> + */
> +void __init apply_boot_alternatives(void)
> +{
> + struct alt_region region;
> +
> + /* If called on non-boot cpu things could go wrong */
> + WARN_ON(smp_processor_id() != 0);
> +
> + init_alternative();
> +
> + if (!errata_checkfunc)
> + return;
> +
> + region.begin = (struct alt_entry *)__alt_start;
> + region.end = (struct alt_entry *)__alt_end;
> + __apply_alternatives(®ion, patch_text_nosync);
> +}
> +
> diff --git a/arch/riscv/include/asm/alternative-macros.h b/arch/riscv/include/asm/alternative-macros.h
> new file mode 100644
> index 000000000000..7b6f0c94b1fb
> --- /dev/null
> +++ b/arch/riscv/include/asm/alternative-macros.h
> @@ -0,0 +1,110 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef __ASM_ALTERNATIVE_MACROS_H
> +#define __ASM_ALTERNATIVE_MACROS_H
> +
> +#ifdef CONFIG_RISCV_ERRATA_ALTERNATIVE
> +
> +#ifndef __ASSEMBLY__
> +
> +#include <asm/asm.h>
> +#include <linux/stringify.h>
> +
> +#define ALT_ENTRY(oldptr, altptr, vendor_id, errata_id, altlen) \
> + RISCV_PTR " " oldptr "\n" \
> + RISCV_PTR " " altptr "\n" \
> + REG_ASM " " vendor_id "\n" \
> + REG_ASM " " altlen "\n" \
> + ".word " errata_id "\n"
> +
> +#define __ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, enable) \
> + "886 :\n" \
> + oldinsn "\n" \
> + ".if " __stringify(enable) " == 1\n" \
> + "887 :\n" \
> + ".pushsection .alternative, \"a\"\n" \
> + ALT_ENTRY("886b", "888f", __stringify(vendor_id), __stringify(errata_id), "889f - 888f") \
> + ".popsection\n" \
> + ".subsection 1\n" \
> + "888 :\n" \
> + altinsn "\n" \
> + "889 :\n" \
> + ".previous\n" \
> + ".org . - (887b - 886b) + (889b - 888b)\n" \
> + ".org . - (889b - 888b) + (887b - 886b)\n" \
> + ".endif\n"
> +
> +#define _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
> + __ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, IS_ENABLED(CONFIG_k))
> +
> +#else /* __ASSEMBLY__ */
> +
> +.macro ALT_ENTRY oldptr altptr vendor_id errata_id alt_len
> + RISCV_PTR \oldptr
> + RISCV_PTR \altptr
> + REG_ASM \vendor_id
> + REG_ASM \alt_len
> + .word \errata_id
> +.endm
> +
> +.macro __ALTERNATIVE_CFG insn1 insn2 vendor_id errata_id enable = 1
> +886 :
> + \insn1
> + .if \enable
> +887 :
> + .pushsection .alternative, "a"
> + ALT_ENTRY 886b, 888f, \vendor_id, \errata_id, 889f - 888f
> + .popsection
> + .subsection 1
> +888 :
> + \insn2
> +889 :
> + .previous
> + .org . - (889b - 888b) + (887b - 886b)
> + .org . - (887b - 886b) + (889b - 888b)
> + .endif
> +.endm
> +
> +#define _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
> + __ALTERNATIVE_CFG oldinsn, altinsn, vendor_id, errata_id, IS_ENABLED(CONFIG_k)
> +
> +#endif /* !__ASSEMBLY__ */
> +
> +#else /* !CONFIG_RISCV_ERRATA_ALTERNATIVE*/
> +#ifndef __ASSEMBLY__
> +
> +#define __ALTERNATIVE_CFG(oldinsn) \
> + oldinsn "\n"
> +
> +#define _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
> + __ALTERNATIVE_CFG(oldinsn)
> +
> +#else /* __ASSEMBLY__ */
> +
> +.macro __ALTERNATIVE_CFG insn1
> + \insn1
> +.endm
> +
> +#define _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
> + __ALTERNATIVE_CFG oldinsn
> +
> +#endif /* !__ASSEMBLY__ */
> +#endif /* CONFIG_RISCV_ERRATA_ALTERNATIVE */
> +
> +/*
> + * Usage:
> + * ALTERNATIVE(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k)
> + * in the assembly code. Otherwise,
> + * asm(ALTERNATIVE(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k));
> + *
> + * oldinsn: The old instruction which will be replaced.
> + * altinsn: The replacement instruction.
> + * vendor_id: The CPU vendor ID.
> + * errata_id: The errata ID.
> + * CONFIG_k: The Kconfig of this errata. The instructions replacement can
> + * be disabled by this Kconfig. When Kconfig is disabled, the
> + * oldinsn will always be executed.
> + */
> +#define ALTERNATIVE(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
> + _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k)
> +
> +#endif
> diff --git a/arch/riscv/include/asm/alternative.h b/arch/riscv/include/asm/alternative.h
> new file mode 100644
> index 000000000000..98a0ea331a27
> --- /dev/null
> +++ b/arch/riscv/include/asm/alternative.h
> @@ -0,0 +1,44 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * Copyright (C) 2021 Sifive.
> + */
> +
> +#ifndef __ASM_ALTERNATIVE_H
> +#define __ASM_ALTERNATIVE_H
> +
> +#define ERRATA_STRING_LENGTH_MAX 32
> +
> +#include <asm/alternative-macros.h>
> +
> +#ifndef __ASSEMBLY__
> +
> +#include <linux/init.h>
> +#include <linux/types.h>
> +#include <linux/stddef.h>
> +#include <asm/hwcap.h>
> +
> +void __init apply_boot_alternatives(void);
> +
> +struct alt_entry {
> + void *old_ptr; /* address of original instruciton or data */
> + void *alt_ptr; /* address of replacement instruction or data */
> + unsigned long vendor_id; /* cpu vendor id */
> + unsigned long alt_len; /* The replacement size */
> + unsigned int errata_id; /* The errata id */
> +} __packed;
> +
> +struct errata_checkfunc_id {
> + unsigned long vendor_id;
> + bool (*func)(struct alt_entry *alt);
> +};
> +
> +extern struct cpu_manufacturer_info_t cpu_mfr_info;
> +
> +#define REGISTER_ERRATA_CHECKFUNC(checkfunc, vendorid) \
> + static const struct errata_checkfunc_id _errata_check_##vendorid \
> + __used __section(".alt_checkfunc_table") \
> + __aligned(__alignof__(struct errata_checkfunc_id)) = \
> + { .vendor_id = vendorid, \
> + .func = checkfunc }
> +#endif
> +#endif
> diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h
> index 9c992a88d858..e20646aa97e5 100644
> --- a/arch/riscv/include/asm/asm.h
> +++ b/arch/riscv/include/asm/asm.h
> @@ -23,6 +23,7 @@
> #define REG_L __REG_SEL(ld, lw)
> #define REG_S __REG_SEL(sd, sw)
> #define REG_SC __REG_SEL(sc.d, sc.w)
> +#define REG_ASM __REG_SEL(.dword, .word)
> #define SZREG __REG_SEL(8, 4)
> #define LGREG __REG_SEL(3, 2)
>
> diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h
> new file mode 100644
> index 000000000000..5e5a1fcd90ba
> --- /dev/null
> +++ b/arch/riscv/include/asm/errata_list.h
> @@ -0,0 +1,8 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * Copyright (C) 2021 Sifive.
> + */
> +
> +#ifdef CONFIG_SOC_SIFIVE
> +#define ERRATA_NUMBER 0
> +#endif
> diff --git a/arch/riscv/include/asm/sections.h b/arch/riscv/include/asm/sections.h
> index 1595c5b60cfd..d13160f46d4e 100644
> --- a/arch/riscv/include/asm/sections.h
> +++ b/arch/riscv/include/asm/sections.h
> @@ -11,5 +11,7 @@ extern char _start[];
> extern char _start_kernel[];
> extern char __init_data_begin[], __init_data_end[];
> extern char __init_text_begin[], __init_text_end[];
> +extern char __alt_checkfunc_table[], __alt_checkfunc_table_end[];
> +extern char __alt_start[], __alt_end[];
>
> #endif /* __ASM_SECTIONS_H */
> diff --git a/arch/riscv/include/asm/vendorid_list.h b/arch/riscv/include/asm/vendorid_list.h
> new file mode 100644
> index 000000000000..1f3be47decb6
> --- /dev/null
> +++ b/arch/riscv/include/asm/vendorid_list.h
> @@ -0,0 +1,6 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * Copyright (C) 2021 SiFive
> + */
> +
> +#define SIFIVE_VENDOR_ID 0x489
> diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
> index 5e276c25646f..9a408e2942ac 100644
> --- a/arch/riscv/kernel/smpboot.c
> +++ b/arch/riscv/kernel/smpboot.c
> @@ -32,6 +32,7 @@
> #include <asm/sections.h>
> #include <asm/sbi.h>
> #include <asm/smp.h>
> +#include <asm/alternative.h>
>
> #include "head.h"
>
> @@ -40,6 +41,9 @@ static DECLARE_COMPLETION(cpu_running);
> void __init smp_prepare_boot_cpu(void)
> {
> init_cpu_topology();
> +#ifdef CONFIG_RISCV_ERRATA_ALTERNATIVE
> + apply_boot_alternatives();
> +#endif
> }
>
> void __init smp_prepare_cpus(unsigned int max_cpus)
> diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S
> index de03cb22d0e9..6503dfca65b0 100644
> --- a/arch/riscv/kernel/vmlinux.lds.S
> +++ b/arch/riscv/kernel/vmlinux.lds.S
> @@ -90,6 +90,20 @@ SECTIONS
> }
>
> __init_data_end = .;
> +
> + . = ALIGN(8);
> + .alt_checkfunc_table : {
> + __alt_checkfunc_table = .;
> + *(.alt_checkfunc_table)
> + __alt_checkfunc_table_end = .;
> + }
> +
> + . = ALIGN(8);
> + .alternative : {
> + __alt_start = .;
> + *(.alternative)
> + __alt_end = .;
> + }
> __init_end = .;
>
> /* Start of data section */
More information about the linux-riscv
mailing list