[PATCH 1/2] arm64: Support inlined clocksource reads for the arch counter

[Mark Rutland]

Hi David,

On Fri, Oct 02, 2026 at 04:19:37PM +0100, David Woodhouse wrote:
> From: David Woodhouse <dwmw at amazon.co.uk>
> 
> Commit 2e27beeb66e43 ("timekeeping: Allow inlining clocksource::read()")
> allows an architecture's clocksource::read() to be inlined, and was only
> wired up for x86_64 thus far in commit b27801189f7fc ("x86: Inline TSC
> reads in timekeeping").
> 
> Add the corresponding ARCH_WANTS_CLOCKSOURCE_READ_INLINE support for the
> arm64 arch counter, allowing the indirect clock->read() call in the fast
> paths to be replaced with a direct sysreg read. A static key selects
> CNTPCT vs CNTVCT as appropriate, and the inline support is only
> advertised for the plain (no erratum workaround) read paths.
> 
> Measured on a Cortex-A53 at 1.35GHz (12.5MHz arch counter): the
> interval from a raw counter read in the caller to the counter read
> inside ktime_get_real_ts64() drops from 30.2ns to 23.4ns, mean of 1M
> iterations. Every ktime_get*() user of tk_clock_read() benefits
> similarly. Except ktime_get_snapshot_id(), which I shall get to shortly.
> 
> Signed-off-by: David Woodhouse <dwmw at amazon.co.uk>
> Assisted-by: LLM

On IRC you mentioned that the saving here was dominated by other costs
(in the microseconds range). Given that, as I mentioned there, I'd
prefer to avoid this.

The arm_arch_timer driver is complicated enough with all the errata
handling, with which we've had more than one painful problem. I don't
want more complication there just to game a microbenchmark.

Mark.

> ---
>  arch/arm64/Kconfig                     |  1 +
>  arch/arm64/include/asm/clock_inlined.h | 26 ++++++++++++++++++++++++++
>  drivers/clocksource/arm_arch_timer.c   | 21 +++++++++++++++++++++
>  3 files changed, 48 insertions(+)
>  create mode 100644 arch/arm64/include/asm/clock_inlined.h
> 
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index b5a51b0ef944..25795b87cd41 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -94,6 +94,7 @@ config ARM64
>  	select ARCH_WANT_FRAME_POINTERS
>  	select ARCH_WANT_HUGE_PMD_SHARE if ARM64_4K_PAGES || (ARM64_16K_PAGES && !ARM64_VA_BITS_36)
>  	select ARCH_WANT_LD_ORPHAN_WARN
> +	select ARCH_WANTS_CLOCKSOURCE_READ_INLINE
>  	select ARCH_WANTS_EXECMEM_LATE
>  	select ARCH_WANTS_NO_INSTR
>  	select ARCH_WANTS_THP_SWAP if ARM64_4K_PAGES
> diff --git a/arch/arm64/include/asm/clock_inlined.h b/arch/arm64/include/asm/clock_inlined.h
> new file mode 100644
> index 000000000000..fe997412bc7d
> --- /dev/null
> +++ b/arch/arm64/include/asm/clock_inlined.h
> @@ -0,0 +1,26 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef _ASM_ARM64_CLOCK_INLINED_H
> +#define _ASM_ARM64_CLOCK_INLINED_H
> +
> +#include <linux/jump_label.h>
> +#include <asm/arch_timer.h>
> +
> +struct clocksource;
> +
> +/*
> + * Set by the arch timer driver when it selects the physical counter
> + * (kernel booted at EL2); otherwise the virtual counter is used. The
> + * driver only advertises CLOCK_SOURCE_CAN_INLINE_READ when it selected
> + * a plain (no erratum workaround) sysreg read matching this function.
> + */
> +DECLARE_STATIC_KEY_FALSE(arch_timer_inline_read_pct);
> +
> +static __always_inline u64 arch_inlined_clocksource_read(struct clocksource *cs)
> +{
> +	if (static_branch_likely(&arch_timer_inline_read_pct))
> +		return __arch_counter_get_cntpct();
> +
> +	return __arch_counter_get_cntvct();
> +}
> +
> +#endif
> diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
> index cc9a8b399004..1c3d503d881a 100644
> --- a/drivers/clocksource/arm_arch_timer.c
> +++ b/drivers/clocksource/arm_arch_timer.c
> @@ -133,6 +133,13 @@ static noinstr u64 arch_counter_get_cntvct(void)
>  u64 (*arch_timer_read_counter)(void) __ro_after_init = arch_counter_get_cntvct;
>  EXPORT_SYMBOL_GPL(arch_timer_read_counter);
>  
> +#ifdef CONFIG_ARM64
> +#include <asm/clock_inlined.h>
> +
> +/* Selects CNTPCT in arch_inlined_clocksource_read() */
> +DEFINE_STATIC_KEY_FALSE(arch_timer_inline_read_pct);
> +#endif
> +
>  static u64 arch_counter_read(struct clocksource *cs)
>  {
>  	return arch_timer_read_counter();
> @@ -917,6 +924,15 @@ static void __init arch_counter_register(void)
>  		} else {
>  			rd = arch_counter_get_cntvct;
>  			scr = arch_counter_get_cntvct;
> +			/*
> +			 * A plain sysreg read with no erratum workaround
> +			 * matches arch_inlined_clocksource_read(); allow the
> +			 * core timekeeping code to bypass the indirect call.
> +			 */
> +#ifdef CONFIG_ARM64
> +			clocksource_counter.flags |=
> +				CLOCK_SOURCE_CAN_INLINE_READ;
> +#endif
>  		}
>  	} else {
>  		if (arch_timer_counter_has_wa()) {
> @@ -925,6 +941,11 @@ static void __init arch_counter_register(void)
>  		} else {
>  			rd = arch_counter_get_cntpct;
>  			scr = arch_counter_get_cntpct;
> +#ifdef CONFIG_ARM64
> +			static_branch_enable(&arch_timer_inline_read_pct);
> +			clocksource_counter.flags |=
> +				CLOCK_SOURCE_CAN_INLINE_READ;
> +#endif
>  		}
>  	}
>  
> -- 
> 2.43.0
>