[PATCH RFC 06/18] x86/mm/pat: introduce an inline helper to check if alias needs update

[Mike Rapoport (Microsoft)]

Once core parts of change_page_attr() would be moved to common code,
the processing of alias mappings would be out of line and would require
a function call for each modified PTE even if the primary target of
change_page_attr() does not have alias mappings.

Calling a function per PTE just to find out it has nothing to do is
expensive and resulted in ~30% regression in instrumented cpa-test.

Although cpa-test is a microbenchmark, ~30% regression is still a lot
and can be easily avoided by inlining the checks that gates processing
of the alias mapping.

Introduce cpa_should_update_alias() inline helper that checks if the
alias update is required.

Since on 64-bit an alias could be in the high kernel mapping, pull
__cpa_pfn_in_highmap() along and rename it to pfn_is_kernel().

No functional change intended.

Assisted-by: Copilot:claude-opus-4.8
Signed-off-by: Mike Rapoport (Microsoft) <rppt at kernel.org>
---
 arch/x86/include/asm/set_memory.h | 50 +++++++++++++++++++++++++++++++
 arch/x86/mm/pat/set_memory.c      | 62 +++------------------------------------
 2 files changed, 54 insertions(+), 58 deletions(-)

diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h
index 4362c26aa992..332ec300c2ed 100644
--- a/arch/x86/include/asm/set_memory.h
+++ b/arch/x86/include/asm/set_memory.h
@@ -3,12 +3,62 @@
 #define _ASM_X86_SET_MEMORY_H
 
 #include <asm/page.h>
+#include <asm/sections.h>
 #include <asm-generic/set_memory.h>
 #include <asm/pgtable.h>
 
 #define set_memory_rox set_memory_rox
 int set_memory_rox(unsigned long addr, int numpages);
 
+#ifdef CONFIG_X86_64
+/*
+ * The kernel image is mapped into two places in the virtual address space
+ * (addresses without KASLR, of course):
+ *
+ * 1. The kernel direct map (0xffff880000000000)
+ * 2. The "high kernel map" (0xffffffff81000000)
+ *
+ * We actually execute out of #2. If we get the address of a kernel symbol, it
+ * points to #2, but almost all physical-to-virtual translations point to #1.
+ *
+ * This is so that we can have both a directmap of all physical memory *and*
+ * take full advantage of the limited (s32) immediate addressing range (2G)
+ * of x86_64.
+ *
+ * See Documentation/arch/x86/x86_64/mm.rst for more detail.
+ */
+static inline bool pfn_is_kernel(unsigned long pfn)
+{
+	unsigned long spfn = __pa_symbol(_text) >> PAGE_SHIFT;
+	/* Do not reference a physical address outside the kernel. */
+	unsigned long epfn = __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
+
+	return pfn >= spfn && pfn <= epfn;
+}
+#else
+static inline bool pfn_is_kernel(unsigned long pfn)
+{
+	/* There is no highmap on 32-bit */
+	return false;
+}
+#endif
+
+static inline bool cpa_should_update_alias(unsigned long vaddr,
+					   unsigned long pfn)
+{
+	/* Primary is not in the direct map, its direct map alias needs update */
+	if (vaddr < PAGE_OFFSET ||
+	    vaddr >= PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))
+		return true;
+
+	/* direct map page that is also part of the kernel highmap */
+	if ((vaddr < (unsigned long)_text || vaddr >= _brk_end) &&
+	    pfn_is_kernel(pfn))
+		return true;
+
+	return false;
+}
+
 /*
  * The set_memory_* API can be used to change various attributes of a virtual
  * address range. The attributes include:
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index ee40962d4ee9..2a087dcedf84 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -224,61 +224,6 @@ within(unsigned long addr, unsigned long start, unsigned long end)
 	return addr >= start && addr < end;
 }
 
-#ifdef CONFIG_X86_64
-
-static inline int
-within_inclusive(unsigned long addr, unsigned long start, unsigned long end)
-{
-	return addr >= start && addr <= end;
-}
-
-/*
- * The kernel image is mapped into two places in the virtual address space
- * (addresses without KASLR, of course):
- *
- * 1. The kernel direct map (0xffff880000000000)
- * 2. The "high kernel map" (0xffffffff81000000)
- *
- * We actually execute out of #2. If we get the address of a kernel symbol, it
- * points to #2, but almost all physical-to-virtual translations point to #1.
- *
- * This is so that we can have both a directmap of all physical memory *and*
- * take full advantage of the limited (s32) immediate addressing range (2G)
- * of x86_64.
- *
- * See Documentation/arch/x86/x86_64/mm.rst for more detail.
- */
-
-static inline unsigned long highmap_start_pfn(void)
-{
-	return __pa_symbol(_text) >> PAGE_SHIFT;
-}
-
-static inline unsigned long highmap_end_pfn(void)
-{
-	/* Do not reference physical address outside the kernel. */
-	return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
-}
-
-static bool __cpa_pfn_in_highmap(unsigned long pfn)
-{
-	/*
-	 * Kernel text has an alias mapping at a high address, known
-	 * here as "highmap".
-	 */
-	return within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn());
-}
-
-#else
-
-static bool __cpa_pfn_in_highmap(unsigned long pfn)
-{
-	/* There is no highmap on 32-bit */
-	return false;
-}
-
-#endif
-
 /*
  * See set_mce_nospec().
  *
@@ -1830,7 +1775,7 @@ static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
 		cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
 		return 0;
 
-	} else if (__cpa_pfn_in_highmap(cpa->pfn)) {
+	} else if (pfn_is_kernel(cpa->pfn)) {
 		/* Faults in the highmap are OK, so do not warn: */
 		return -EFAULT;
 	} else {
@@ -1968,7 +1913,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
 	 * to touch the high mapped kernel as well:
 	 */
 	if (!within(vaddr, (unsigned long)_text, _brk_end) &&
-	    __cpa_pfn_in_highmap(cpa->pfn)) {
+	    pfn_is_kernel(cpa->pfn)) {
 		unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
 					       __START_KERNEL_map - phys_base;
 		alias_cpa = *cpa;
@@ -2026,7 +1971,8 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary)
 		if (ret)
 			goto out;
 
-		if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS)) {
+		if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS) &&
+		    cpa_should_update_alias(__cpa_addr(cpa, cpa->curpage), cpa->pfn)) {
 			ret = cpa_process_alias(cpa);
 			if (ret)
 				goto out;

-- 
2.53.0