lua: fix strange behaviour of tonumber64

Function tonumber64 has worked incorrectly with values less
than INT64_MAX.
Now it works in the interval [INT64_MAX, UINT64_MAX] returning
nil otherwise.

Closes #3466.

src/lua/init.c

@@ -222,7 +222,19 @@ lbox_tonumber64(struct lua_State *L)
 			if (argl == 0) {
 				lua_pushnil(L);
 			} else if (negative) {
-				luaL_pushint64(L, -1 * (long long )result);
+				/*
+				 * To test overflow, consider
+				 *  result > -INT64_MIN;
+				 *  result - 1 > -INT64_MIN - 1;
+				 * Assumption:
+				 *  INT64_MAX == -(INT64_MIN + 1);
+				 * Finally,
+				 *  result - 1 > INT64_MAX;
+				 */
+				if (result != 0 && result - 1 > INT64_MAX)
+					lua_pushnil(L);
+				else
+					luaL_pushint64(L, -result);
 			} else {
 				luaL_pushuint64(L, result);
 			}

test/box/misc.result

@@ -627,6 +627,33 @@ tostring(tonumber64('1234567890123456')) == '1234567890123456ULL'
 ---
 - true
 ...
+--
+-- gh-3466: Strange behaviour of tonumber64 function
+--
+tostring(tonumber64('9223372036854775807')) == '9223372036854775807ULL'
+---
+- true
+...
+tostring(tonumber64('18446744073709551615')) == '18446744073709551615ULL'
+---
+- true
+...
+tonumber64('18446744073709551616') == nil
+---
+- true
+...
+tostring(tonumber64('-9223372036854775808')) == '-9223372036854775808LL'
+---
+- true
+...
+tonumber64('-9223372036854775809') == nil
+---
+- true
+...
+tostring(tonumber64('0')) == '0'
+---
+- true
+...
 tonumber64('0x12') == 18
 ---
 - true

test/box/misc.test.lua

@@ -163,6 +163,16 @@ tostring(tonumber64('12345678901234')) == '12345678901234'
 tostring(tonumber64('123456789012345')) == '123456789012345ULL'
 tostring(tonumber64('1234567890123456')) == '1234567890123456ULL'
 
+--
+-- gh-3466: Strange behaviour of tonumber64 function
+--
+tostring(tonumber64('9223372036854775807')) == '9223372036854775807ULL'
+tostring(tonumber64('18446744073709551615')) == '18446744073709551615ULL'
+tonumber64('18446744073709551616') == nil
+tostring(tonumber64('-9223372036854775808')) == '-9223372036854775808LL'
+tonumber64('-9223372036854775809') == nil
+tostring(tonumber64('0')) == '0'
+
 tonumber64('0x12') == 18
 tonumber64('0x12', 16) == 18
 tonumber64('0x12', 17) == nil