[PATCH 08/18] arm64: Implement try_update_vmemmap_pte using the AF trick

[Catalin Marinas]

On Wed, Jul 08, 2026 at 03:11:18AM +0000, James Houghton wrote:
> diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
> index 5f21d3a738ee..7b11aa41d0a0 100644
> --- a/arch/arm64/include/asm/pgtable.h
> +++ b/arch/arm64/include/asm/pgtable.h
> @@ -1302,8 +1302,7 @@ static inline void __pte_clear(struct mm_struct *mm,
>  	__set_pte(ptep, __pte(0));
>  }
>  
> -static inline bool __ptep_test_and_clear_young(struct vm_area_struct *vma,
> -		unsigned long address, pte_t *ptep)
> +static inline pte_t __ptep_clear_young(pte_t *ptep)
>  {
>  	pte_t old_pte, pte;
>  
> @@ -1315,7 +1314,13 @@ static inline bool __ptep_test_and_clear_young(struct vm_area_struct *vma,
>  					       pte_val(old_pte), pte_val(pte));
>  	} while (pte_val(pte) != pte_val(old_pte));
>  
> -	return pte_young(pte);
> +	return pte;
> +}

At some point, we should use LSE atomics directly here if supported
rather than a CAS loop (well, this is LSE as well if supported but it
doesn't guarantee forward progress of the loop). I reckon we can replace
it with some test_and_clear_bit_relaxed() call.

> +static inline bool __ptep_test_and_clear_young(struct vm_area_struct *vma,
> +		unsigned long address, pte_t *ptep)
> +{
> +	return pte_young(__ptep_clear_young(ptep));
>  }
>  
>  static inline bool __ptep_clear_flush_young(struct vm_area_struct *vma,
> @@ -1793,6 +1798,48 @@ static inline void pte_clear(struct mm_struct *mm,
>  	__pte_clear(mm, addr, ptep);
>  }
>  
> +#define __HAVE_ARCH_TRY_UPDATE_VMEMMAP_PTE
> +static inline int try_update_vmemmap_pte(unsigned long addr, pte_t *ptep,
> +					 const pte_t pte)
> +{
> +	const int max_attempts = 16;
> +	int attempts = 0;
> +	pte_t old_pte;
> +
> +	if (!system_supports_hvo())
> +		return -EOPNOTSUPP;
> +
> +	/* This routine is only to be used for valid-to-valid transitions. */
> +	if (WARN_ON_ONCE(!pte_valid(pte)))
> +		return -EINVAL;
> +
> +	old_pte = __ptep_get(ptep);
> +
> +	do {
> +		if (WARN_ON_ONCE(!pte_valid(old_pte)))
> +			return -EINVAL;
> +
> +		/* We should never get a contiguous PTE here. */
> +		if (WARN_ON_ONCE(pte_valid_cont(old_pte)))
> +			return -EINVAL;
> +
> +		if (pte_young(old_pte)) {
> +			/* __ptep_clear_young() returns the overwritten PTE */
> +			old_pte = pte_mkold(__ptep_clear_young(ptep));
> +
> +			flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
> +		}

I think this is going to do a lot of TLBIs given that the default kernel
prot has PTE_AF. It somewhat defeats the VMEMMAP_REMAP_NO_TLB_FLUSH
flag but I haven't figured exactly how this optimisation works.

If it becomes a problem, we could do a first pass to clear AF as an
optimisation or later via vmemmap_split_pmd(), only map with AF=0. We
still have some page copying that touches the vmemmap, bringing AF back.
Of course, you'd still need the above flush, just wondering whether we
can reduce/coalesce it.

> +	/*
> +	 * Translations without AF cannot be cached, so we can replace
> +	 * them without BBM.
> +	 */
> +	} while (!try_cmpxchg_relaxed(&pte_val(*ptep), &pte_val(old_pte),
> +				      pte_val(pte)) &&
> +		 ++attempts < max_attempts);
> +
> +	return attempts == max_attempts ? -EAGAIN : 0;
> +}

Here, indeed, we do need this bounded, otherwise some pathological cases
may set the AF continuously.

-- 
Catalin