[openwrt/openwrt] lua: fix integer overflow in LNUM patch

[LEDE Commits]

hauke pushed a commit to openwrt/openwrt.git, branch openwrt-23.05:
https://git.openwrt.org/a5b03a34c346880e30f1d1554ae9d345c8dd828c

commit a5b03a34c346880e30f1d1554ae9d345c8dd828c
Author: Adam Bailey <aebailey at gmail.com>
AuthorDate: Mon Jul 3 20:16:14 2023 -0500

    lua: fix integer overflow in LNUM patch
    
    Safely detect integer overflow in try_addint() and try_subint().
    Old code relied on undefined behavior, and recent versions of GCC on x86
    optimized away the if-statements.
    This caused integer overflow in Lua code instead of falling back to
    floating-point numbers.
    
    Signed-off-by: Adam Bailey <aebailey at gmail.com>
    (cherry picked from commit 3a2e7c30d3e6a187ba1df740cdb24c8ad84dfe48)
---
 .../patches-host/010-lua-5.1.3-lnum-full-260308.patch    | 16 ++++++++--------
 .../lua/patches/010-lua-5.1.3-lnum-full-260308.patch     | 16 ++++++++--------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/package/utils/lua/patches-host/010-lua-5.1.3-lnum-full-260308.patch b/package/utils/lua/patches-host/010-lua-5.1.3-lnum-full-260308.patch
index 4530edd181..fd398c28d1 100644
--- a/package/utils/lua/patches-host/010-lua-5.1.3-lnum-full-260308.patch
+++ b/package/utils/lua/patches-host/010-lua-5.1.3-lnum-full-260308.patch
@@ -1600,18 +1600,18 @@
 + * (and doing them).
 + */
 +int try_addint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) {
-+  lua_Integer v= ib+ic; /* may overflow */
-+  if (ib>0 && ic>0)      { if (v < 0) return 0; /*overflow, use floats*/ }
-+  else if (ib<0 && ic<0) { if (v >= 0) return 0; }
-+  *r= v;
++  /* Signed int overflow is undefined behavior, so catch it without causing it. */
++  if (ic>0)  { if (ib > LUA_INTEGER_MAX - ic) return 0; /*overflow, use floats*/ }
++  else       { if (ib < LUA_INTEGER_MIN - ic) return 0; }
++  *r = ib + ic;
 +  return 1;
 +}
 +
 +int try_subint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) {
-+  lua_Integer v= ib-ic; /* may overflow */
-+  if (ib>=0 && ic<0)     { if (v < 0) return 0; /*overflow, use floats*/ }
-+  else if (ib<0 && ic>0) { if (v >= 0) return 0; }
-+  *r= v;
++  /* Signed int overflow is undefined behavior, so catch it without causing it. */
++  if (ic>0)  { if (ib < LUA_INTEGER_MIN + ic) return 0; /*overflow, use floats*/ }
++  else       { if (ib > LUA_INTEGER_MAX + ic) return 0; }
++  *r = ib - ic;
 +  return 1;
 +}
 +
diff --git a/package/utils/lua/patches/010-lua-5.1.3-lnum-full-260308.patch b/package/utils/lua/patches/010-lua-5.1.3-lnum-full-260308.patch
index ac0722c707..58cc894e1c 100644
--- a/package/utils/lua/patches/010-lua-5.1.3-lnum-full-260308.patch
+++ b/package/utils/lua/patches/010-lua-5.1.3-lnum-full-260308.patch
@@ -1589,18 +1589,18 @@
 + * (and doing them).
 + */
 +int try_addint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) {
-+  lua_Integer v= ib+ic; /* may overflow */
-+  if (ib>0 && ic>0)      { if (v < 0) return 0; /*overflow, use floats*/ }
-+  else if (ib<0 && ic<0) { if (v >= 0) return 0; }
-+  *r= v;
++  /* Signed int overflow is undefined behavior, so catch it without causing it. */
++  if (ic>0)  { if (ib > LUA_INTEGER_MAX - ic) return 0; /*overflow, use floats*/ }
++  else       { if (ib < LUA_INTEGER_MIN - ic) return 0; }
++  *r = ib + ic;
 +  return 1;
 +}
 +
 +int try_subint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) {
-+  lua_Integer v= ib-ic; /* may overflow */
-+  if (ib>=0 && ic<0)     { if (v < 0) return 0; /*overflow, use floats*/ }
-+  else if (ib<0 && ic>0) { if (v >= 0) return 0; }
-+  *r= v;
++  /* Signed int overflow is undefined behavior, so catch it without causing it. */
++  if (ic>0)  { if (ib < LUA_INTEGER_MIN + ic) return 0; /*overflow, use floats*/ }
++  else       { if (ib > LUA_INTEGER_MAX + ic) return 0; }
++  *r = ib - ic;
 +  return 1;
 +}
 +