[RFC 10/14] lib: sbi: Add PMU support

Anup Patel Anup.Patel at wdc.com
Mon Apr 19 12:44:44 BST 2021



> -----Original Message-----
> From: Atish Patra <atish.patra at wdc.com>
> Sent: 20 March 2021 03:43
> To: opensbi at lists.infradead.org
> Cc: Atish Patra <Atish.Patra at wdc.com>; Anup Patel <Anup.Patel at wdc.com>
> Subject: [RFC 10/14] lib: sbi: Add PMU support
> 
> RISC-V SBI v0.3 specification defined a PMU extension to
> configure/start/stop the hardware/firmware pmu events.
> 
> Implement PMU support in OpenSBI library. The implementation is agnostic
> of event to counter mapping & mhpmevent value configuration. That means,
> it expects platform hooks will be used to set up the mapping and provide the
> mhpmevent value at runtime.
> 
> Signed-off-by: Atish Patra <atish.patra at wdc.com>

Please check the revised v5 PMU specification patch. I had changed
event_data usage for RAW events.

> ---
>  include/sbi/sbi_ecall_interface.h | 129 +++++++-
>  include/sbi/sbi_error.h           |   2 +
>  include/sbi/sbi_pmu.h             |  71 ++++
>  lib/sbi/objects.mk                |   1 +
>  lib/sbi/sbi_hart.c                |   1 +
>  lib/sbi/sbi_init.c                |   9 +
>  lib/sbi/sbi_pmu.c                 | 534 ++++++++++++++++++++++++++++++
>  7 files changed, 746 insertions(+), 1 deletion(-)  create mode 100644
> include/sbi/sbi_pmu.h  create mode 100644 lib/sbi/sbi_pmu.c
> 
> diff --git a/include/sbi/sbi_ecall_interface.h
> b/include/sbi/sbi_ecall_interface.h
> index 559a33e7ced0..63b1ca7c75b4 100644
> --- a/include/sbi/sbi_ecall_interface.h
> +++ b/include/sbi/sbi_ecall_interface.h
> @@ -28,6 +28,7 @@
>  #define SBI_EXT_RFENCE				0x52464E43
>  #define SBI_EXT_HSM				0x48534D
>  #define SBI_EXT_SRST				0x53525354
> +#define SBI_EXT_PMU				0x504D55
> 
>  /* SBI function IDs for BASE extension*/
>  #define SBI_EXT_BASE_GET_SPEC_VERSION		0x0
> @@ -91,6 +92,130 @@
>  #define SBI_SRST_RESET_REASON_NONE	0x0
>  #define SBI_SRST_RESET_REASON_SYSFAIL	0x1
> 
> +/* SBI function IDs for PMU extension */
> +#define SBI_EXT_PMU_NUM_COUNTERS	0x0
> +#define SBI_EXT_PMU_COUNTER_GET_INFO	0x1
> +#define SBI_EXT_PMU_COUNTER_CFG_MATCH	0x2
> +#define SBI_EXT_PMU_COUNTER_FW_READ	0x3
> +#define SBI_EXT_PMU_COUNTER_START	0x4
> +#define SBI_EXT_PMU_COUNTER_STOP	0x5
> +
> +/** General pmu event codes specified in SBI PMU extension */ enum
> +sbi_pmu_hw_generic_events_t {
> +	SBI_PMU_HW_NO_EVENT = 0,
> +	SBI_PMU_HW_CPU_CYCLES,
> +	SBI_PMU_HW_INSTRUCTIONS,
> +	SBI_PMU_HW_CACHE_REFERENCES,
> +	SBI_PMU_HW_CACHE_MISSES,
> +	SBI_PMU_HW_BRANCH_INSTRUCTIONS,
> +	SBI_PMU_HW_BRANCH_MISSES,
> +	SBI_PMU_HW_BUS_CYCLES,
> +	SBI_PMU_HW_STALLED_CYCLES_FRONTEND,
> +	SBI_PMU_HW_STALLED_CYCLES_BACKEND,
> +	SBI_PMU_HW_REF_CPU_CYCLES,
> +
> +	SBI_PMU_HW_GENERIC_MAX,
> +};
> +
> +/**
> + * Generalized hardware cache events:
> + *
> + *       { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x
> + *       { read, write, prefetch } x
> + *       { accesses, misses }
> + */
> +enum sbi_pmu_hw_cache_id {
> +	SBI_PMU_HW_CACHE_L1D		= 0,
> +	SBI_PMU_HW_CACHE_L1I		= 1,
> +	SBI_PMU_HW_CACHE_LL		= 2,
> +	SBI_PMU_HW_CACHE_DTLB		= 3,
> +	SBI_PMU_HW_CACHE_ITLB		= 4,
> +	SBI_PMU_HW_CACHE_BPU		= 5,
> +	SBI_PMU_HW_CACHE_NODE		= 6,
> +
> +	SBI_PMU_HW_CACHE_MAX,
> +};
> +
> +enum sbi_pmu_hw_cache_op_id {
> +	SBI_PMU_HW_CACHE_OP_READ	= 0,
> +	SBI_PMU_HW_CACHE_OP_WRITE	= 1,
> +	SBI_PMU_HW_CACHE_OP_PREFETCH	= 2,
> +
> +	SBI_PMU_HW_CACHE_OP_MAX,
> +};
> +
> +enum sbi_pmu_hw_cache_op_result_id {
> +	SBI_PMU_HW_CACHE_RESULT_ACCESS	= 0,
> +	SBI_PMU_HW_CACHE_RESULT_MISS	= 1,
> +
> +	SBI_PMU_HW_CACHE_RESULT_MAX,
> +};
> +
> +/**
> + * Special "firmware" events provided by the OpenSBI, even if the
> +hardware
> + * does not support performance events. These events are encoded as a
> +raw
> + * event type in Linux kernel perf framework.
> + */
> +enum sbi_pmu_fw_event_code_id {
> +	SBI_PMU_FW_MISALIGNED_LOAD	= 0,
> +	SBI_PMU_FW_MISALIGNED_STORE	= 1,
> +	SBI_PMU_FW_ACCESS_LOAD		= 2,
> +	SBI_PMU_FW_ACCESS_STORE		= 3,
> +	SBI_PMU_FW_ILLEGAL_INSN		= 4,
> +	SBI_PMU_FW_SET_TIMER		= 5,
> +	SBI_PMU_FW_IPI_SENT		= 6,
> +	SBI_PMU_FW_IPI_RECVD		= 7,
> +	SBI_PMU_FW_FENCE_I_SENT		= 8,
> +	SBI_PMU_FW_FENCE_I_RECVD	= 9,
> +	SBI_PMU_FW_SFENCE_VMA_SENT	= 10,
> +	SBI_PMU_FW_SFENCE_VMA_RCVD	= 11,
> +	SBI_PMU_FW_SFENCE_VMA_ASID_SENT	= 12,
> +	SBI_PMU_FW_SFENCE_VMA_ASID_RCVD	= 13,
> +
> +	SBI_PMU_FW_HFENCE_GVMA_SENT	= 14,
> +	SBI_PMU_FW_HFENCE_GVMA_RCVD	= 15,
> +	SBI_PMU_FW_HFENCE_GVMA_VMID_SENT = 16,
> +	SBI_PMU_FW_HFENCE_GVMA_VMID_RCVD = 17,
> +
> +	SBI_PMU_FW_HFENCE_VVMA_SENT	= 18,
> +	SBI_PMU_FW_HFENCE_VVMA_RCVD	= 19,
> +	SBI_PMU_FW_HFENCE_VVMA_ASID_SENT = 20,
> +	SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD = 21,
> +	SBI_PMU_FW_MAX,
> +};
> +
> +/** SBI PMU event idx type */
> +enum sbi_pmu_event_type_id {
> +	SBI_PMU_EVENT_TYPE_HW				= 0x0,
> +	SBI_PMU_EVENT_TYPE_HW_CACHE			= 0x1,
> +	SBI_PMU_EVENT_TYPE_HW_RAW			= 0x2,
> +	SBI_PMU_EVENT_TYPE_FW				= 0xf,
> +	SBI_PMU_EVENT_TYPE_MAX,
> +};
> +
> +/** SBI PMU counter type */
> +enum sbi_pmu_ctr_type {
> +	SBI_PMU_CTR_TYPE_HW = 0,
> +	SBI_PMU_CTR_TYPE_FW,
> +};
> +
> +/* Helper macros to decode event idx */ #define
> +SBI_PMU_EVENT_IDX_OFFSET 20 #define SBI_PMU_EVENT_IDX_MASK
> 0xFFFFF
> +#define SBI_PMU_EVENT_IDX_CODE_MASK 0xFFFF #define
> +SBI_PMU_EVENT_IDX_TYPE_MASK 0xF0000 #define
> SBI_PMU_EVENT_RAW_IDX
> +0x20000
> +
> +#define SBI_PMU_EVENT_IDX_INVALID 0xFFFFFFFF
> +
> +/* Flags defined for config matching function */ #define
> +SBI_PMU_CFG_FLAG_AUTOSTART 0x1 #define
> SBI_PMU_CFG_FLAG_SKIPMATCH 0x2
> +
> +/* Flags defined for counter stop function */ #define
> +SBI_PMU_STOP_FLAG_RESET 0x1
> +
> +/* SBI base specification related macros */
>  #define SBI_SPEC_VERSION_MAJOR_OFFSET		24
>  #define SBI_SPEC_VERSION_MAJOR_MASK		0x7f
>  #define SBI_SPEC_VERSION_MINOR_MASK		0xffffff
> @@ -107,8 +232,10 @@
>  #define SBI_ERR_DENIED				-4
>  #define SBI_ERR_INVALID_ADDRESS			-5
>  #define SBI_ERR_ALREADY_AVAILABLE		-6
> +#define SBI_ERR_ALREADY_STARTED			-7
> +#define SBI_ERR_ALREADY_STOPPED			-8
> 
> -#define SBI_LAST_ERR
> 	SBI_ERR_ALREADY_AVAILABLE
> +#define SBI_LAST_ERR
> 	SBI_ERR_ALREADY_STOPPED
> 
>  /* clang-format on */
> 
> diff --git a/include/sbi/sbi_error.h b/include/sbi/sbi_error.h index
> 3655d122006b..dd65e14b6fcd 100644
> --- a/include/sbi/sbi_error.h
> +++ b/include/sbi/sbi_error.h
> @@ -21,6 +21,8 @@
>  #define SBI_EDENIED		SBI_ERR_DENIED
>  #define SBI_EINVALID_ADDR	SBI_ERR_INVALID_ADDRESS
>  #define SBI_EALREADY		SBI_ERR_ALREADY_AVAILABLE
> +#define SBI_EALREADY_STARTED	SBI_ERR_ALREADY_STARTED
> +#define SBI_EALREADY_STOPPED	SBI_ERR_ALREADY_STOPPED
> 
>  #define SBI_ENODEV		-1000
>  #define SBI_ENOSYS		-1001
> diff --git a/include/sbi/sbi_pmu.h b/include/sbi/sbi_pmu.h new file mode
> 100644 index 000000000000..b6f25de44c6c
> --- /dev/null
> +++ b/include/sbi/sbi_pmu.h
> @@ -0,0 +1,71 @@
> +/*
> + * SPDX-License-Identifier: BSD-2-Clause
> + *
> + * Copyright (c) 2021 Western Digital Corporation or its affiliates.
> + *
> + * Authors:
> + *   Atish Patra <atish.patra at wdc.com>
> + */
> +
> +#ifndef __SBI_PMU_H__
> +#define __SBI_PMU_H__
> +
> +#include <sbi/sbi_types.h>
> +#include <sbi/sbi_hartmask.h>
> +#include <sbi/sbi_scratch.h>
> +#include <sbi/sbi_ecall_interface.h>
> +
> +/* Event related macros */
> +/* Maximum number of hardware events that can mapped by OpenSBI */
> +#define SBI_PMU_HW_EVENT_MAX 64
> +
> +/* Maximum number of firmware events that can mapped by OpenSBI */
> +#define SBI_PMU_FW_EVENT_MAX 32
> +
> +/* Counter related macros */
> +#define SBI_PMU_FW_CTR_MAX 16
> +#define SBI_PMU_HW_CTR_MAX 32
> +#define SBI_PMU_CTR_MAX	   (SBI_PMU_HW_CTR_MAX +
> SBI_PMU_FW_CTR_MAX)
> +
> +/** Initialize PMU */
> +int sbi_pmu_init(struct sbi_scratch *scratch, bool cold_boot);
> +
> +/** Reset PMU during hart exit */
> +void sbi_pmu_exit(struct sbi_scratch *scratch);
> +
> +/**
> + * Add the hardware event to counter mapping information. This should
> +be called
> + * from the platform code to update the mapping table.
> + * @param eidx_start Start of the event idx range for supported counters
> + * @param eidx_end   End of the event idx range for supported counters
> + * @param cmap       A bitmap representing counters supporting the event
> range
> + * @return 0 on success, error otherwise.
> + */
> +int sbi_pmu_add_hw_event_counter_map(u32 eidx_start, u32 eidx_end,
> u32
> +cmap);
> +
> +/**
> + * Add the raw hardware event selector and supported counter
> +information. This
> + * should be called from the platform code to update the mapping table.
> + * @param info  a pointer to the hardware event info
> + * @return 0 on success, error otherwise.
> + */
> +
> +int sbi_pmu_add_raw_event_counter_map(uint64_t select, u32 cmap);
> +
> +int sbi_pmu_read_ctr(uint32_t cidx, unsigned long *cval);
> +
> +int sbi_pmu_stop_ctr(uint32_t cidx, bool bReset);
> +
> +int sbi_pmu_start_ctr(uint32_t cidx, uint64_t ival);
> +
> +int sbi_pmu_get_ctr_info(uint32_t cidx, unsigned long *ctr_info);
> +
> +unsigned long sbi_pmu_num_ctr(void);
> +
> +int sbi_pmu_get_ctr_match(unsigned long cidx_base, unsigned long
> cidx_mask,
> +			  unsigned long event_idx, uint64_t event_info,
> +			  unsigned long flags);
> +
> +int sbi_pmu_incr_fw_ctr(enum sbi_pmu_fw_event_code_id fw_id);
> +
> +#endif
> diff --git a/lib/sbi/objects.mk b/lib/sbi/objects.mk index
> 6f2c06f5b501..d9068b707854 100644
> --- a/lib/sbi/objects.mk
> +++ b/lib/sbi/objects.mk
> @@ -33,6 +33,7 @@ libsbi-objs-y += sbi_init.o  libsbi-objs-y += sbi_ipi.o
> libsbi-objs-y += sbi_misaligned_ldst.o  libsbi-objs-y += sbi_platform.o
> +libsbi-objs-y += sbi_pmu.o
>  libsbi-objs-y += sbi_scratch.o
>  libsbi-objs-y += sbi_string.o
>  libsbi-objs-y += sbi_system.o
> diff --git a/lib/sbi/sbi_hart.c b/lib/sbi/sbi_hart.c index
> b87d1dbedc0b..2d155c6b6bf4 100644
> --- a/lib/sbi/sbi_hart.c
> +++ b/lib/sbi/sbi_hart.c
> @@ -17,6 +17,7 @@
>  #include <sbi/sbi_csr_detect.h>
>  #include <sbi/sbi_error.h>
>  #include <sbi/sbi_hart.h>
> +#include <sbi/sbi_pmu.h>
>  #include <sbi/sbi_math.h>
>  #include <sbi/sbi_platform.h>
>  #include <sbi/sbi_string.h>

Drop this change from lib/sbi/sbi_hart.c

> diff --git a/lib/sbi/sbi_init.c b/lib/sbi/sbi_init.c index
> acc2a81a1b53..7362430716e6 100644
> --- a/lib/sbi/sbi_init.c
> +++ b/lib/sbi/sbi_init.c
> @@ -19,6 +19,7 @@
>  #include <sbi/sbi_hsm.h>
>  #include <sbi/sbi_ipi.h>
>  #include <sbi/sbi_platform.h>
> +#include <sbi/sbi_pmu.h>
>  #include <sbi/sbi_system.h>
>  #include <sbi/sbi_string.h>
>  #include <sbi/sbi_timer.h>
> @@ -234,6 +235,8 @@ static void __noreturn init_coldboot(struct
> sbi_scratch *scratch, u32 hartid)
>  	if (rc)
>  		sbi_hart_hang();
> 
> +	sbi_pmu_init(scratch, TRUE);
> +
>  	sbi_boot_print_banner(scratch);
> 
>  	rc = sbi_platform_irqchip_init(plat, TRUE); @@ -335,6 +338,8 @@
> static void init_warm_startup(struct sbi_scratch *scratch, u32 hartid)
>  	if (rc)
>  		sbi_hart_hang();
> 
> +	sbi_pmu_init(scratch, FALSE);
> +
>  	rc = sbi_platform_irqchip_init(plat, FALSE);
>  	if (rc)
>  		sbi_hart_hang();
> @@ -375,6 +380,8 @@ static void init_warm_resume(struct sbi_scratch
> *scratch)
>  	if (rc)
>  		sbi_hart_hang();
> 
> +	sbi_pmu_init(scratch, FALSE);
> +
>  	rc = sbi_hart_pmp_configure(scratch);
>  	if (rc)
>  		sbi_hart_hang();
> @@ -498,6 +505,8 @@ void __noreturn sbi_exit(struct sbi_scratch *scratch)
> 
>  	sbi_platform_early_exit(plat);
> 
> +	sbi_pmu_exit(scratch);
> +
>  	sbi_timer_exit(scratch);
> 
>  	sbi_ipi_exit(scratch);
> diff --git a/lib/sbi/sbi_pmu.c b/lib/sbi/sbi_pmu.c new file mode 100644 index
> 000000000000..904b583fc7d6
> --- /dev/null
> +++ b/lib/sbi/sbi_pmu.c
> @@ -0,0 +1,534 @@
> +/*
> + * SPDX-License-Identifier: BSD-2-Clause
> + *
> + * Copyright (c) 2020 Western Digital Corporation or its affiliates.

Should be 2021 instead of 2020

> + *
> + * Authors:
> + *   Atish Patra <atish.patra at wdc.com>
> + */
> +
> +#include <sbi/riscv_asm.h>
> +#include <sbi/sbi_bitops.h>
> +#include <sbi/sbi_console.h>
> +#include <sbi/sbi_hart.h>
> +#include <sbi/sbi_platform.h>
> +#include <sbi/sbi_pmu.h>
> +#include <sbi/sbi_scratch.h>
> +#include <sbi/sbi_string.h>
> +
> +/** Information about hardware counters */ struct sbi_pmu_hw_event {
> +	unsigned long counters;
> +	unsigned long start_idx;
> +	unsigned long end_idx;
> +	/* Event selector value used only for raw events */
> +	uint64_t select;
> +};
> +
> +/** Representation of a firmware event */ struct sbi_pmu_fw_event {
> +
> +	/* Event associated with the particular counter */
> +	unsigned long event_idx;
> +
> +	/* Current value of the counter */
> +	unsigned long curr_count;
> +
> +	/* A flag indicating pmu event monitoring is started */
> +	bool bStarted;
> +};
> +
> +/* Information about PMU counters as per SBI specification */ union
> +sbi_pmu_ctr_info {
> +	unsigned long value;
> +	struct {
> +		unsigned long csr:12;
> +		unsigned long width:6;
> +#if __riscv_xlen == 32
> +		unsigned long reserved:13;
> +#else
> +		unsigned long reserved:45;
> +#endif
> +		unsigned long type:1;
> +	};
> +};
> +
> +/* Mapping between event range and possible counters  */ static struct
> +sbi_pmu_hw_event hw_event_map[SBI_PMU_HW_EVENT_MAX] = {0};

In future, you can keep this array sorted so that you can do binary search
based on "event_idx".

> +
> +/* counter to enabled event mapping */
> +static uint32_t
> active_events[SBI_HARTMASK_MAX_BITS][SBI_PMU_HW_CTR_MAX
> ++ SBI_PMU_FW_CTR_MAX];
> +
> +/* Contains all the information about firmwares events */ static struct
> +sbi_pmu_fw_event
> +fw_event_map[SBI_HARTMASK_MAX_BITS][SBI_PMU_FW_EVENT_MAX]
> = {0};
> +
> +/* Maximum number of hardware events available */ static uint32_t
> +num_hw_events;
> +/* Maximum number of hardware counters available */ static uint32_t
> +num_hw_ctrs;
> +
> +/* Maximum number of counters available */ static uint32_t total_ctrs;
> +
> +/* Helper macros to retrieve event idx and code type */ #define
> +get_cidx_type(x) ((x & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16) #define
> +get_cidx_code(x) (x & SBI_PMU_EVENT_IDX_CODE_MASK)
> +
> +/**
> + * Perform a sanity check on event & counter mappings with event range
> +overlap check
> + * @param evtA Pointer to the existing hw event structure
> + * @param evtB Pointer to the new hw event structure
> + *
> + * Return FALSE if the range doesn't overlap, TRUE otherwise  */ static
> +bool pmu_event_range_overlap(struct sbi_pmu_hw_event *evtA,
> +				    struct sbi_pmu_hw_event *evtB)
> +{
> +	/* check if the range of events overlap with a previous entry */
> +	if (((evtA->end_idx < evtB->start_idx) && (evtA->end_idx < evtB-
> >end_idx)) ||
> +	   ((evtA->start_idx > evtB->start_idx) && (evtA->start_idx > evtB-
> >end_idx)))
> +		return FALSE;
> +	return TRUE;
> +}
> +
> +static bool pmu_event_select_overlap(struct sbi_pmu_hw_event *evt,
> +				     uint64_t select_val)
> +{
> +
> +	if (evt->select == select_val)
> +		return TRUE;
> +
> +	return FALSE;
> +}
> +
> +static int pmu_validate_ctr(uint32_t cidx, uint32_t *event_idx_code) {
> +	uint32_t event_idx_val;
> +	uint32_t event_idx_type;
> +	u32 hartid = current_hartid();
> +
> +	event_idx_val = active_events[hartid][cidx];
> +
> +	if (cidx >= total_ctrs || (event_idx_val ==
> SBI_PMU_EVENT_IDX_INVALID))
> +		return SBI_EINVAL;
> +
> +	event_idx_type = get_cidx_type(event_idx_val);
> +	if (event_idx_type >= SBI_PMU_EVENT_TYPE_MAX)
> +		return SBI_EINVAL;
> +
> +	*event_idx_code = get_cidx_code(event_idx_val);
> +
> +	return event_idx_type;
> +}
> +
> +static int sbi_pmu_read_fw_ctr(uint32_t cidx, unsigned long *cval,
> +			       uint32_t fw_evt_code)

For consistency, name this function pmu_read_fw_ctr()

This way local (or static) function names will have just "pmu_" prefix
and public function names will have "sbi_pmu_" prefix.

> +{
> +	u32 hartid = current_hartid();
> +	struct sbi_pmu_fw_event fevent;
> +
> +	fevent = fw_event_map[hartid][fw_evt_code];
> +	*cval = fevent.curr_count;
> +
> +	return 0;
> +}
> +
> +/* Add a hardware counter read for completeness for future purpose */
> +static int sbi_pmu_read_hw_ctr(uint32_t cidx, uint64_t *cval) { #if

Same as above.

> +__riscv_xlen == 32
> +	uint32_t temp, temph = 0;
> +
> +	temp = csr_read_num(CSR_MCYCLE + cidx);
> +	temph = csr_read_num(CSR_MCYCLEH + cidx);
> +	*cval = ((uint64_t)temph << 32) | temp; #else
> +	*cval = csr_read_num(CSR_MCYCLE + cidx); #endif
> +
> +	return 0;
> +}
> +
> +int sbi_pmu_read_ctr(uint32_t cidx, unsigned long *cval) {
> +	int event_idx_type;
> +	uint32_t event_code;
> +	uint64_t cval64;
> +
> +	event_idx_type = pmu_validate_ctr(cidx, &event_code);
> +	if (event_idx_type < 0)
> +		return SBI_EINVAL;
> +	else if (event_idx_type == SBI_PMU_EVENT_TYPE_FW)
> +		sbi_pmu_read_fw_ctr(cidx, cval, event_code);
> +	else
> +		sbi_pmu_read_hw_ctr(cidx, &cval64);
> +
> +	return 0;
> +}
> +
> +static int pmu_add_hw_event_map(u32 eidx_start, u32 eidx_end, u32
> cmap,
> +				uint64_t select)
> +{
> +	int i = 0;
> +	bool is_overlap;
> +	struct sbi_pmu_hw_event *event =
> &hw_event_map[num_hw_events];
> +
> +	/* The first two counters are reserved by priv spec */
> +	if ((eidx_start == SBI_PMU_HW_CPU_CYCLES && cmap != 0x1) ||
> +	    (eidx_start == SBI_PMU_HW_INSTRUCTIONS && cmap != 0x4) ||
> +	    (eidx_start > SBI_PMU_HW_INSTRUCTIONS && cmap < 0x08))
> +		return SBI_EDENIED;
> +
> +	if (num_hw_events >= SBI_PMU_HW_EVENT_MAX - 1) {
> +		sbi_printf("Can not handle more than %d perf events\n",
> +			    SBI_PMU_HW_EVENT_MAX);
> +		return SBI_EFAIL;
> +	}
> +
> +	event->start_idx = eidx_start;
> +	event->end_idx = eidx_end;
> +	event->counters = cmap;
> +	event->select = select;
> +
> +	/* Sanity check */
> +	for (i = 0; i < num_hw_events; i++) {
> +		if (eidx_start == SBI_PMU_EVENT_RAW_IDX)
> +		/* All raw events have same event idx. Just do sanity check
> on select */
> +			is_overlap =
> pmu_event_select_overlap(&hw_event_map[i], select);
> +		else
> +			is_overlap =
> pmu_event_range_overlap(&hw_event_map[i], event);
> +		if (is_overlap)
> +			return SBI_EINVALID_ADDR;
> +	}
> +	num_hw_events++;
> +
> +	return 0;
> +}
> +
> +/**
> + * Logical counter ids are assigned to hardware counters are assigned
> consecutively.
> + * E.g. counter0 must count MCYCLE where counter2 must count minstret.
> +Similarly,
> + * counterX will mhpmcounterX.
> + */
> +int sbi_pmu_add_hw_event_counter_map(u32 eidx_start, u32 eidx_end,
> u32
> +cmap) {
> +	if ((eidx_start > eidx_end) || eidx_start ==
> SBI_PMU_EVENT_RAW_IDX ||
> +	     eidx_end == SBI_PMU_EVENT_RAW_IDX)
> +		return SBI_EINVAL;
> +
> +	return pmu_add_hw_event_map(eidx_start, eidx_end, cmap, 0); }
> +
> +int sbi_pmu_add_raw_event_counter_map(uint64_t select, u32 cmap) {
> +	return pmu_add_hw_event_map(SBI_PMU_EVENT_RAW_IDX,
> +				    SBI_PMU_EVENT_RAW_IDX, cmap, select);
> }
> +
> +static int pmu_start_hw_ctr(uint32_t cidx, uint64_t ival) {
> +	unsigned long mctr_en = csr_read(CSR_MCOUNTEREN);
> +	unsigned long mctr_inhbt = csr_read(CSR_MCOUNTINHIBIT);
> +
> +	if (cidx > num_hw_ctrs)
> +		return SBI_EINVAL;
> +
> +	if (__test_bit(cidx, &mctr_en) && !__test_bit(cidx, &mctr_inhbt))
> +		return SBI_EALREADY_STARTED;
> +
> +	__set_bit(cidx, &mctr_en);
> +	__clear_bit(cidx, &mctr_inhbt);
> +
> +	csr_write(CSR_MCOUNTEREN, mctr_en);
> +	csr_write(CSR_MCOUNTINHIBIT, mctr_inhbt);
> +
> +#if __riscv_xlen == 32
> +	csr_write_num(CSR_MCYCLE + cidx, ival & 0xFFFF);
> +	csr_write_num(CSR_MCYCLEH + cidx, ival >> BITS_PER_LONG); #else
> +	csr_write_num(CSR_MCYCLE + cidx, ival); #endif
> +
> +	return 0;
> +}
> +
> +static int pmu_start_fw_ctr(uint32_t cidx, uint64_t ival, uint32_t
> +fw_evt_code) {
> +	u32 hartid = current_hartid();
> +	struct sbi_pmu_fw_event *fevent;
> +
> +	fevent = &fw_event_map[hartid][fw_evt_code];
> +	fevent->curr_count = ival;
> +	fevent->bStarted = TRUE;
> +
> +	return 0;
> +}
> +
> +int sbi_pmu_start_ctr(uint32_t cidx, uint64_t ival) {
> +	int event_idx_type;
> +	uint32_t event_code;
> +
> +	event_idx_type = pmu_validate_ctr(cidx, &event_code);
> +	if (event_idx_type < 0)
> +		return SBI_EINVAL;
> +	else if (event_idx_type == SBI_PMU_EVENT_TYPE_FW)
> +		return pmu_start_fw_ctr(cidx, ival, event_code);
> +	else
> +		return pmu_start_hw_ctr(cidx, ival);
> +}
> +
> +static int pmu_stop_hw_ctr(uint32_t cidx) {
> +	unsigned long mctr_en = csr_read(CSR_MCOUNTEREN);
> +	unsigned long mctr_inhbt = csr_read(CSR_MCOUNTINHIBIT);
> +
> +	if (__test_bit(cidx, &mctr_en) && !__test_bit(cidx, &mctr_inhbt)) {
> +		__set_bit(cidx, &mctr_inhbt);
> +		__clear_bit(cidx, &mctr_en);
> +		csr_write(CSR_MCOUNTEREN, mctr_en);
> +		csr_write(CSR_MCOUNTINHIBIT, mctr_inhbt);
> +		return 0;
> +	} else
> +		return SBI_EALREADY_STOPPED;
> +}
> +
> +static int pmu_stop_fw_ctr(uint32_t cidx, uint32_t fw_evt_code) {
> +	u32 hartid = current_hartid();
> +
> +	fw_event_map[hartid][fw_evt_code].bStarted = FALSE;
> +
> +	return 0;
> +}
> +
> +int sbi_pmu_stop_ctr(uint32_t cidx, bool bReset) {
> +	u32 hartid = current_hartid();
> +	int ret, event_idx_type;
> +	uint32_t event_code;
> +
> +	event_idx_type = pmu_validate_ctr(cidx, &event_code);
> +	if (event_idx_type < 0)
> +		return SBI_EINVAL;
> +
> +	else if (event_idx_type == SBI_PMU_EVENT_TYPE_FW)
> +		ret = pmu_stop_fw_ctr(cidx, event_code);
> +	else
> +		ret = pmu_stop_hw_ctr(cidx);
> +
> +	if (!ret && bReset)
> +		active_events[hartid][cidx] =
> SBI_PMU_EVENT_IDX_INVALID;
> +
> +	return ret;
> +}
> +
> +static int pmu_update_hw_mhpmevent(struct sbi_pmu_hw_event
> *hw_evt, int ctr_idx,
> +				    unsigned long eindex, uint64_t data) {
> +	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
> +	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
> +	uint64_t mhpmevent_val;
> +
> +	/* Get the final mhpmevent value to be written from platform */
> +	mhpmevent_val = sbi_platform_get_mhpmevent_value(plat,
> eindex, data);
> +
> +	if (!mhpmevent_val || ctr_idx < 3 || ctr_idx >=
> SBI_PMU_HW_CTR_MAX)
> +		return SBI_EFAIL;
> +
> +	csr_write_num(CSR_MCOUNTINHIBIT + ctr_idx, mhpmevent_val);
> +
> +	return 0;
> +}
> +
> +static int pmu_find_hw_ctr(unsigned long cbase, unsigned long cmask,
> +			   unsigned long event_idx, uint64_t data) {
> +	unsigned long ctr_mask;
> +	int i, ret = 0, ctr_idx = SBI_ENOTSUPP;
> +	struct sbi_pmu_hw_event *temp;
> +	unsigned long mctr_en = csr_read(CSR_MCOUNTEREN);
> +	unsigned long mctr_inhbt = csr_read(CSR_MCOUNTINHIBIT);
> +	int evt_idx_code = get_cidx_code(event_idx);
> +
> +	if (cbase > num_hw_ctrs)
> +		return SBI_EINVAL;
> +
> +	for (i = 0; i < num_hw_events; i++) {
> +		temp = &hw_event_map[i];
> +		if ((temp->start_idx > event_idx && event_idx < temp-
> >end_idx) ||
> +		    (temp->start_idx < event_idx && event_idx > temp-
> >end_idx))
> +			continue;
> +
> +		/* For raw events, event data is used as the select value */
> +		if ((event_idx == SBI_PMU_EVENT_RAW_IDX) && temp-
> >select != data)
> +			continue;
> +
> +		ctr_mask = temp->counters & (cmask << cbase);
> +		for_each_set_bit_from(cbase, &ctr_mask,
> SBI_PMU_HW_CTR_MAX) {
> +			if (!__test_bit(cbase, &mctr_en) &&
> +			    __test_bit(cbase, &mctr_inhbt)) {
> +				ctr_idx = cbase;
> +				break;
> +			}
> +		}
> +	}
> +
> +	if (ctr_idx == SBI_ENOTSUPP)
> +		return SBI_EFAIL;
> +
> +	/* No need to update the  event selectors for fixed events */
> +	if (evt_idx_code != SBI_PMU_HW_CPU_CYCLES &&
> +	    evt_idx_code != SBI_PMU_HW_INSTRUCTIONS)
> +		ret = pmu_update_hw_mhpmevent(temp, ctr_idx,
> event_idx, data);
> +
> +	if (!ret)
> +		ret = ctr_idx;
> +
> +	return ret;
> +}
> +
> +
> +/**
> + * Any firmware counter can map to any firmware event.
> + * Thus, select the first available fw counter after sanity
> + * check.
> + */
> +static int pmu_find_fw_ctr(unsigned long cbase, unsigned long cmask,
> +u32 hartid) {
> +	int i = 0;
> +	int fw_base;
> +	unsigned long ctr_mask = cmask << cbase;
> +
> +	if (cbase <= num_hw_ctrs)
> +		fw_base = num_hw_ctrs + 1;
> +	else
> +		fw_base = cbase;
> +
> +	for (i = fw_base; i < total_ctrs; i++)
> +		if ((active_events[hartid][i] ==
> SBI_PMU_EVENT_IDX_INVALID) &&
> +		    ((1UL << i) & ctr_mask))
> +			return i;
> +
> +	return SBI_ENOTSUPP;
> +}
> +
> +int sbi_pmu_get_ctr_match(unsigned long cidx_base, unsigned long
> cidx_mask,
> +			  unsigned long event_idx, uint64_t event_data,
> +			  unsigned long flags)
> +{
> +	int ctr_idx = SBI_ENOTSUPP;
> +	u32 hartid = current_hartid();
> +	int event_type = get_cidx_type(event_idx);
> +
> +	if (cidx_base >= total_ctrs || event_type >=
> SBI_PMU_EVENT_TYPE_MAX)
> +		return SBI_EINVAL;
> +
> +	if (event_type == SBI_PMU_EVENT_TYPE_FW) {
> +		/* Any firmware counter can be used track any firmware
> event */
> +		ctr_idx = pmu_find_fw_ctr(cidx_base, cidx_mask, hartid);
> +	} else {
> +		ctr_idx = pmu_find_hw_ctr(cidx_base, cidx_mask,
> event_idx, event_data);
> +	}
> +
> +	if (ctr_idx < 0)
> +		return SBI_ENOTSUPP;
> +
> +	active_events[hartid][ctr_idx] = event_idx;
> +
> +	return ctr_idx;
> +}
> +
> +inline int sbi_pmu_incr_fw_ctr(enum sbi_pmu_fw_event_code_id fw_id) {
> +	u32 hartid = current_hartid();
> +	struct sbi_pmu_fw_event *fevent;
> +
> +	if (unlikely(fw_id >= SBI_PMU_FW_MAX))
> +		return SBI_EINVAL;
> +
> +	fevent = &fw_event_map[hartid][fw_id];
> +
> +	/* PMU counters will be only enabled during performance debugging
> */
> +	if (unlikely(fevent->bStarted))
> +		fevent->curr_count++;
> +
> +	return 0;
> +}
> +
> +unsigned long sbi_pmu_num_ctr(void)
> +{
> +	return (num_hw_ctrs + SBI_PMU_FW_CTR_MAX); }
> +
> +int sbi_pmu_get_ctr_info(uint32_t cidx, unsigned long *ctr_info) {
> +	union sbi_pmu_ctr_info cinfo = {0};
> +	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
> +
> +	/* Sanity check. Counter1 is not mapped at all */
> +	if (cidx > total_ctrs || cidx == 1)
> +		return SBI_EINVAL;
> +
> +	/* We have 31 HW counters with 31 being the last
> index(MHPMCOUNTER31) */
> +	if (cidx <= num_hw_ctrs) {
> +		cinfo.type = SBI_PMU_CTR_TYPE_HW;
> +		cinfo.csr = CSR_CYCLE + cidx;
> +		/* mcycle & minstret are always 64 bit */
> +		if (cidx == 0 || cidx == 2)
> +			cinfo.width = 63;
> +		else
> +			cinfo.width = sbi_hart_pmu_event_bits(scratch);
> +	} else {
> +		/* it's a firmware counter */
> +		cinfo.type = SBI_PMU_CTR_TYPE_FW;
> +		/* Firmware counters are XLEN bits wide */
> +		cinfo.width = BITS_PER_LONG - 1;
> +	}
> +
> +	*ctr_info = cinfo.value;
> +
> +	return 0;
> +}
> +
> +
> +static void pmu_reset_event_map(u32 hartid) {
> +	int j;
> +
> +	/* Initialize the counter to event mapping table */
> +	for (j = 0; j < total_ctrs; j++)
> +		active_events[hartid][j] = SBI_PMU_EVENT_IDX_INVALID;
> +	for (j = 0; j < SBI_PMU_FW_CTR_MAX; j++)
> +		sbi_memset(&fw_event_map[hartid][j], 0,
> +			   sizeof(struct sbi_pmu_fw_event)); }
> +
> +void sbi_pmu_exit(struct sbi_scratch *scratch) {
> +	u32 hartid = current_hartid();
> +
> +	pmu_reset_event_map(hartid);

We should also disable counters in MCOUNTINHIBIT and MCOUNTEN CSRs
for current HART in sbi_pmu_exit().

> +}
> +
> +int sbi_pmu_init(struct sbi_scratch *scratch, bool cold_boot) {
> +	const struct sbi_platform *plat;
> +
> +	if (!sbi_hart_has_feature(scratch,
> SBI_HART_HAS_MCOUNTINHIBIT))
> +		return SBI_ENOTSUPP;
> +
> +	if (cold_boot) {
> +		plat = sbi_platform_ptr(scratch);
> +		/* Initialize hw pmu events */
> +		sbi_platform_pmu_init(plat);
> +
> +		/* mcycle & minstret is available always */
> +		num_hw_ctrs = sbi_hart_mhpm_count(scratch) + 2;
> +		total_ctrs = num_hw_ctrs + SBI_PMU_FW_CTR_MAX;
> +	}
> +
> +	pmu_reset_event_map(current_hartid());

Same as above.

> +
> +	return 0;
> +}
> --
> 2.25.1

Overall, this patch is in good shape apart from minor comments above.

Regards,
Anup



More information about the opensbi mailing list