[PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()

[Paul Walmsley]

On Mon, 1 Sep 2025, Xu Lu wrote:

> Only flush tlb of the specified mm, and apply svinval if available.
> 
> Signed-off-by: Xu Lu <luxu.kernel at bytedance.com>
> ---
>  arch/riscv/include/asm/pgtable.h  | 16 +++++++++++++++-
>  arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++
>  arch/riscv/mm/tlbflush.c          | 23 -----------------------
>  3 files changed, 38 insertions(+), 24 deletions(-)

And here's the second patch, separated out from your original, which adds 
the Svinval path.  Thoughts?


- Paul

From: Xu Lu <luxu.kernel at bytedance.com>
Date: Mon, 1 Sep 2025 19:41:40 +0800
Subject: [PATCH] riscv: mm: Use Svinval if present in update_mmu_cache_range()

If the Svinval extension is present, use it in
update_mmu_cache_range(), rather than using the standard sfence.vma.
Svinval should be faster.

Signed-off-by: Xu Lu <luxu.kernel at bytedance.com>
Link: https://patch.msgid.link/20250901114141.5438-2-luxu.kernel@bytedance.com
[pjw at kernel.org: separate the non-Svinval path into an earlier patch; update to apply and improve the description]
Signed-off-by: Paul Walmsley <pjw at kernel.org>
---
 arch/riscv/include/asm/pgtable.h  | 16 ++++++++++++++++
 arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 18 ------------------
 3 files changed, 34 insertions(+), 18 deletions(-)

diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 755495a542cc..899d38614bb3 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -562,6 +562,17 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 #define pgd_ERROR(e) \
 	pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e))
 
+static inline void __update_mmu_cache_range_svinval(struct vm_area_struct *vma,
+		unsigned long address, unsigned int nr)
+{
+	int i;
+	unsigned long asid = get_mm_asid(vma->vm_mm);
+
+	local_sfence_w_inval();
+	for (i = 0; i < nr; i++)
+		local_sinval_vma(address + nr * PAGE_SIZE, asid);
+	local_sfence_inval_ir();
+}
 
 /* Commit new configuration to MMU hardware */
 static inline void update_mmu_cache_range(struct vm_fault *vmf,
@@ -578,6 +589,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
 		return;
 
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
+		__update_mmu_cache_range_svinval(vma, address, nr);
+		return;
+	}
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index 7c2cd5cc92d3..9636d07fe9ee 100644
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)
 	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
 }
 
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 73c226f719c7..962db300a166 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
-- 
2.53.0