[RFC PATCH 4/7] static_call: fix broken static_call_query() for non-exported keys

[Ard Biesheuvel]

static_call_query() accesses the func member of the static call key
directly, which means that it is broken for cases where it is used from
a module and the key resides elsewhere and is not exported.

Let's add a helper that returns this value, and export it from the same
module that the key resides in if the key itself is not exported. This
way, we can always get the value regardless of whether the key is
exported or not.

Note that the non-NULL check of &STATIC_CALL_KEY(...) does not typically
result in a load: for ISAs that support relocatable immediates, the
address is patched into the instruction stream.

For example, on ARM/Thumb2, we get

 14a:   f240 0300       movw    r3, #0
                        14a: R_ARM_THM_MOVW_ABS_NC      __SCK__pv_steal_clock
 14e:   f2c0 0300       movt    r3, #0
                        14e: R_ARM_THM_MOVT_ABS         __SCK__pv_steal_clock
 152:   b10b            cbz     r3, 158 <foo+0x14>
 154:   6818            ldr     r0, [r3, #0]
 156:   4770            bx      lr
 158:   f7ff bffe       b.w     0 <__SCQ__pv_steal_clock>
                        158: R_ARM_THM_JUMP24           __SCQ__pv_steal_clock

I.e., it performs a conditional branch if zero (CBZ) on a quantity that
was loaded using an MOVW/MOVT move-immediate pair, and either loads the
func member directly, or tail calls __SCQ__pv_steal_clock if they key is
not exported.

Signed-off-by: Ard Biesheuvel <ardb at kernel.org>
---
 include/linux/static_call.h             |  9 ++++++++-
 include/linux/static_call_types.h       | 18 +++++++++++++++++-
 tools/include/linux/static_call_types.h | 18 +++++++++++++++++-
 3 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/include/linux/static_call.h b/include/linux/static_call.h
index 3bba0bcba844..391f737496eb 100644
--- a/include/linux/static_call.h
+++ b/include/linux/static_call.h
@@ -149,7 +149,10 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func, bool
 			     STATIC_CALL_TRAMP_ADDR(name), __F);	\
 })
 
-#define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func))
+#define EXPORT_STATIC_CALL_QUERY(name, sfx)				\
+	typeof(STATIC_CALL_QUERY(name)()) STATIC_CALL_QUERY(name)(void)	\
+		{ return STATIC_CALL_KEY(name).func; }			\
+	EXPORT_SYMBOL ## sfx (STATIC_CALL_QUERY(name))
 
 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
 
@@ -200,9 +203,11 @@ extern long __static_call_return0(void);
 
 /* Leave the key unexported, so modules can't change static call targets: */
 #define EXPORT_STATIC_CALL_TRAMP(name)					\
+	EXPORT_STATIC_CALL_QUERY(name,);				\
 	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name));				\
 	EXPORT_STATIC_CALL_GETKEY_HELPER(name)
 #define EXPORT_STATIC_CALL_TRAMP_GPL(name)				\
+	EXPORT_STATIC_CALL_QUERY(name, _GPL);				\
 	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name));			\
 	EXPORT_STATIC_CALL_GETKEY_HELPER(name)
 
@@ -253,8 +258,10 @@ static inline long __static_call_return0(void)
 
 /* Leave the key unexported, so modules can't change static call targets: */
 #define EXPORT_STATIC_CALL_TRAMP(name)					\
+	EXPORT_STATIC_CALL_QUERY(name,);				\
 	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
 #define EXPORT_STATIC_CALL_TRAMP_GPL(name)				\
+	EXPORT_STATIC_CALL_QUERY(name, _GPL);				\
 	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
 
 #else /* Generic implementation */
diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h
index a31782909e43..2fce9aa8a995 100644
--- a/include/linux/static_call_types.h
+++ b/include/linux/static_call_types.h
@@ -23,6 +23,9 @@
 #define STATIC_CALL_GETKEY_PREFIX_LEN	(sizeof(STATIC_CALL_GETKEY_PREFIX_STR) - 1)
 #define STATIC_CALL_GETKEY(name)	__PASTE(STATIC_CALL_GETKEY_PREFIX, name)
 
+#define STATIC_CALL_QUERY_PREFIX	__SCQ__
+#define STATIC_CALL_QUERY(name)		__PASTE(STATIC_CALL_QUERY_PREFIX, name)
+
 /*
  * Flags in the low bits of static_call_site::key.
  */
@@ -43,7 +46,20 @@ struct static_call_site {
 #define DECLARE_STATIC_CALL(name, func)					\
 	extern __weak struct static_call_key STATIC_CALL_KEY(name);	\
 	extern __weak struct static_call_key *STATIC_CALL_GETKEY(name)(void);\
-	extern typeof(func) STATIC_CALL_TRAMP(name);
+	extern __weak typeof(func) *STATIC_CALL_QUERY(name)(void);	\
+	extern typeof(func) STATIC_CALL_TRAMP(name)
+
+#define __static_call_query(name)					\
+	((typeof(STATIC_CALL_QUERY(name)()))READ_ONCE(STATIC_CALL_KEY(name).func))
+
+#ifdef MODULE
+/* the key might not be exported */
+#define static_call_query(name)						\
+	(&STATIC_CALL_KEY(name) ? __static_call_query(name)		\
+				: STATIC_CALL_QUERY(name)())
+#else
+#define static_call_query(name)	__static_call_query(name)
+#endif
 
 #ifdef CONFIG_HAVE_STATIC_CALL
 
diff --git a/tools/include/linux/static_call_types.h b/tools/include/linux/static_call_types.h
index a31782909e43..2fce9aa8a995 100644
--- a/tools/include/linux/static_call_types.h
+++ b/tools/include/linux/static_call_types.h
@@ -23,6 +23,9 @@
 #define STATIC_CALL_GETKEY_PREFIX_LEN	(sizeof(STATIC_CALL_GETKEY_PREFIX_STR) - 1)
 #define STATIC_CALL_GETKEY(name)	__PASTE(STATIC_CALL_GETKEY_PREFIX, name)
 
+#define STATIC_CALL_QUERY_PREFIX	__SCQ__
+#define STATIC_CALL_QUERY(name)		__PASTE(STATIC_CALL_QUERY_PREFIX, name)
+
 /*
  * Flags in the low bits of static_call_site::key.
  */
@@ -43,7 +46,20 @@ struct static_call_site {
 #define DECLARE_STATIC_CALL(name, func)					\
 	extern __weak struct static_call_key STATIC_CALL_KEY(name);	\
 	extern __weak struct static_call_key *STATIC_CALL_GETKEY(name)(void);\
-	extern typeof(func) STATIC_CALL_TRAMP(name);
+	extern __weak typeof(func) *STATIC_CALL_QUERY(name)(void);	\
+	extern typeof(func) STATIC_CALL_TRAMP(name)
+
+#define __static_call_query(name)					\
+	((typeof(STATIC_CALL_QUERY(name)()))READ_ONCE(STATIC_CALL_KEY(name).func))
+
+#ifdef MODULE
+/* the key might not be exported */
+#define static_call_query(name)						\
+	(&STATIC_CALL_KEY(name) ? __static_call_query(name)		\
+				: STATIC_CALL_QUERY(name)())
+#else
+#define static_call_query(name)	__static_call_query(name)
+#endif
 
 #ifdef CONFIG_HAVE_STATIC_CALL
 
-- 
2.30.2