[PATCH] ARM: tlb: Prevent flushing insane large ranges one by one

[Thomas Gleixner]

vmalloc uses lazy TLB flushes for unmapped ranges to avoid excessive TLB
flushing on every unmap. The lazy flushing coalesces unmapped ranges and
invokes flush_tlb_kernel_range() with the combined range.

The coalescing can result in ranges which spawn the full vmalloc address
range. In the case of flushing an executable mapping in the module address
space this range is extended to also flush the direct map alias.

flush_tlb_kernel_range() then walks insane large ranges, the worst case
observed was ~1.5GB.

The range is flushed page by page, which takes several milliseconds to
complete in the worst case and obviously affects all processes in the
system. In the worst case observed this causes the runtime of a realtime
task on an isolated CPU to be almost doubled over the normal worst
case, which makes it miss the deadline.

Cure this by sanity checking the range against a threshold and fall back to
tlb_flush_all() when the range is too large.

The default threshold is 32 pages, but for CPUs with CP15 this is evaluated
at boot time via read_cpuid(CPUID_TLBTYPE) and set to the half of the TLB
size.

The vmalloc range coalescing could be improved to provide a list or
array of ranges to flush, which allows to avoid overbroad flushing, but
that's a major surgery and does not solve the problem of actual
justified large range flushes which can happen due to the lazy flush
mechanics in vmalloc. The lazy flush results in batching which is biased
towards large range flushes by design.

Fixes: db64fe02258f ("mm: rewrite vmap layer")
Reported-by: John Ogness <john.ogness at linutronix.de>
Debugged-by: John Ogness <john.ogness at linutronix.de>
Signed-off-by: Thomas Gleixner <tglx at linutronix.de>
Tested-by: John Ogness <john.ogness at linutronix.de>
Link: https://lore.kernel.org/all/87a5y5a6kj.ffs@tglx
---
 arch/arm/include/asm/cputype.h  |    5 +++++
 arch/arm/include/asm/tlbflush.h |    2 ++
 arch/arm/kernel/setup.c         |   10 ++++++++++
 arch/arm/kernel/smp_tlb.c       |    4 ++++
 4 files changed, 21 insertions(+)

--- a/arch/arm/include/asm/cputype.h
+++ b/arch/arm/include/asm/cputype.h
@@ -196,6 +196,11 @@ static inline unsigned int __attribute_c
 	return read_cpuid(CPUID_MPUIR);
 }
 
+static inline unsigned int __attribute_const__ read_cpuid_tlbsize(void)
+{
+	return 64 << ((read_cpuid(CPUID_TLBTYPE) >> 1) & 0x03);
+}
+
 #elif defined(CONFIG_CPU_V7M)
 
 static inline unsigned int __attribute_const__ read_cpuid_id(void)
--- a/arch/arm/include/asm/tlbflush.h
+++ b/arch/arm/include/asm/tlbflush.h
@@ -210,6 +210,8 @@ struct cpu_tlb_fns {
 	unsigned long tlb_flags;
 };
 
+extern unsigned int tlb_flush_all_threshold;
+
 /*
  * Select the calling method
  */
--- a/arch/arm/kernel/setup.c
+++ b/arch/arm/kernel/setup.c
@@ -90,6 +90,8 @@ EXPORT_SYMBOL(__machine_arch_type);
 unsigned int cacheid __read_mostly;
 EXPORT_SYMBOL(cacheid);
 
+unsigned int tlb_flush_all_threshold __ro_after_init = 32;
+
 unsigned int __atags_pointer __initdata;
 
 unsigned int system_rev;
@@ -356,6 +358,13 @@ static void __init cacheid_init(void)
 		cache_is_vipt_nonaliasing() ? "VIPT nonaliasing" : "unknown");
 }
 
+static void __init tlbsize_init(void)
+{
+#ifdef CONFIG_CPU_CP15
+	tlb_flush_all_threshold = read_cpuid_tlbsize() / 2;
+#endif
+}
+
 /*
  * These functions re-use the assembly code in head.S, which
  * already provide the required functionality.
@@ -747,6 +756,7 @@ static void __init setup_processor(void)
 	elf_hwcap_fixup();
 
 	cacheid_init();
+	tlbsize_init();
 	cpu_init();
 }
 
--- a/arch/arm/kernel/smp_tlb.c
+++ b/arch/arm/kernel/smp_tlb.c
@@ -234,6 +234,10 @@ void flush_tlb_range(struct vm_area_stru
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
+	if ((end - start) > (tlb_flush_all_threshold << PAGE_SHIFT)) {
+		flush_tlb_all();
+		return;
+	}
 	if (tlb_ops_need_broadcast()) {
 		struct tlb_args ta;
 		ta.ta_start = start;