[PATCH v3 6/9] mm/mmu_notifier: add MMU_NOTIFIER_RANGE_PERM_UPGRADE

[Carlos López]

Add a range flag for invalidations that only upgrade permissions, so
that secondary MMUs can keep their mappings rather than zap them.
change_protection() sets it for MM_CP_PERM_UPGRADE and for the
userfaultfd resolves, which only restore access.

Now that change_protection() keeps writable entries writable on these
changes, no present entry loses access. Writable migration and
device-private entries are still made read-only, which does not affect
secondary MMUs. Migration entries are only installed after secondary
MMUs are invalidated, and device-private pages are migrated back, with
an invalidation, before the CPU can access them.

hugetlb_change_protection() never sets the flag, as hugetlb does not
restore the write bit and may unshare page tables.

Signed-off-by: Carlos López <clopez at suse.de>
---
 include/linux/mmu_notifier.h | 5 +++++
 mm/hugetlb.c                 | 4 ++++
 mm/internal.h                | 7 +++++++
 mm/mprotect.c                | 5 +++--
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 5db78e5761c5..b2f186ebacf9 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -60,6 +60,11 @@ enum mmu_notifier_event {
 };
 
 #define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
+/*
+ * Set on an invalidation that only expands permissions without
+ * revoking any existing access, nor freeing/moving the underlying page.
+ */
+#define MMU_NOTIFIER_RANGE_PERM_UPGRADE (1 << 1)
 
 struct mmu_notifier_ops {
 	/*
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index ce9c8e57b8e5..06fd31b8fbc4 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -6632,6 +6632,10 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 	 * In the case of shared PMDs, the area to flush could be beyond
 	 * start/end.  Set range.start/range.end to cover the maximum possible
 	 * range if PMD sharing is possible.
+	 *
+	 * MMU_NOTIFIER_RANGE_PERM_UPGRADE is never set here. hugetlb does not
+	 * restore the write bit, and huge_pmd_unshare() may drop the mapping,
+	 * so this is not a strict permission upgrade.
 	 */
 	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
 				0, mm, start, end);
diff --git a/mm/internal.h b/mm/internal.h
index e3f92883bf23..b74ea18c6103 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1453,6 +1453,13 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte
 	return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte);
 }
 
+/* Whether a change_protection() only adds permissions. */
+static inline bool mm_cp_is_perm_upgrade(unsigned long cp_flags)
+{
+	return cp_flags & (MM_CP_PERM_UPGRADE | MM_CP_UFFD_WP_RESOLVE |
+			   MM_CP_UFFD_RWP_RESOLVE);
+}
+
 /* shrinker related functions */
 unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
 			  int priority);
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 8c0fcd10f5a3..b7accf60c3d4 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -608,8 +608,9 @@ static inline long change_pud_range(struct mmu_gather *tlb,
 			continue;
 
 		if (!range.start) {
-			mmu_notifier_range_init(&range,
-						MMU_NOTIFY_PROTECTION_VMA, 0,
+			mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
+						mm_cp_is_perm_upgrade(cp_flags) ?
+						MMU_NOTIFIER_RANGE_PERM_UPGRADE : 0,
 						vma->vm_mm, addr, end);
 			mmu_notifier_invalidate_range_start(&range);
 		}
-- 
2.51.0