Multiple fixes in tonumber64()

* Fix handling of negatives
* Fix handling of empty strngs
* Forbid `base` argument for non-strings

Closes #1554
Closes #1963

src/lua/init.c

@@ -159,9 +159,14 @@ lbox_tonumber64(struct lua_State *L)
 {
 	luaL_checkany(L, 1);
 	int base = luaL_optint(L, 2, -1);
-
+	luaL_argcheck(L, (2 <= base && base <= 36) || base == -1, 2,
+		      "base out of range");
 	switch (lua_type(L, 1)) {
 	case LUA_TNUMBER:
+		base = (base == -1 ? 10 : base);
+		if (base != 10)
+			return luaL_argerror(L, 1, "string expected");
+		lua_settop(L, 1); /* return original value as is */
 		return 1;
 	case LUA_TSTRING:
 	{
@@ -179,7 +184,13 @@ lbox_tonumber64(struct lua_State *L)
 		 * Check if we're parsing custom format:
 		 * 1) '0x' or '0X' trim in case of base == 16 or base == -1
 		 * 2) '0b' or '0B' trim in case of base == 2  or base == -1
+		 * 3) '-' for negative numbers
 		 */
+		char negative = 0;
+		if (arg[0] == '-') {
+			arg++; argl--;
+			negative = 1;
+		}
 		if (argl > 2 && arg[0] == '0') {
 			if ((arg[1] == 'x' || arg[1] == 'X') &&
 			    (base == 16 || base == -1)) {
@@ -191,19 +202,26 @@ lbox_tonumber64(struct lua_State *L)
 		} else if (base == -1) {
 			base = 10;
 		}
-		luaL_argcheck(L, 2 <= base && base <= 36, 2,
-			      "base out of range");
 		errno = 0;
 		char *arge;
 		unsigned long long result = strtoull(arg, &arge, base);
 		if (errno == 0 && arge == arg + argl) {
-			luaL_pushuint64(L, result);
+			if (argl == 0) {
+				lua_pushnil(L);
+			} else if (negative) {
+				luaL_pushint64(L, -1 * (long long )result);
+			} else {
+				luaL_pushuint64(L, result);
+			}
 			return 1;
 		}
 		break;
-	}
+	} /* LUA_TSTRING */
 	case LUA_TCDATA:
 	{
+		base = (base == -1 ? 10 : base);
+		if (base != 10)
+			return luaL_argerror(L, 1, "string expected");
 		uint32_t ctypeid = 0;
 		luaL_checkcdata(L, 1, &ctypeid);
 		if (ctypeid >= CTID_INT8 && ctypeid <= CTID_DOUBLE) {
@@ -211,7 +229,7 @@ lbox_tonumber64(struct lua_State *L)
 			return 1;
 		}
 		break;
-	}
+	} /* LUA_TCDATA */
 	}
 	lua_pushnil(L);
 	return 1;

test/box/misc.result

@@ -609,6 +609,86 @@ tonumber64('0b111111111111111111') == 262143
 ---
 - true
 ...
+tonumber64('20', 36)
+---
+- 72
+...
+tonumber64(20, 36)
+---
+- error: 'bad argument #1 to ''?'' (string expected)'
+...
+tonumber64(20, 10)
+---
+- 20
+...
+tonumber64(20)
+---
+- 20
+...
+tonumber64("", 10)
+---
+- null
+...
+tonumber64("", 32)
+---
+- null
+...
+tonumber64("-1")
+---
+- -1
+...
+tonumber64("-0x16")
+---
+- -22
+...
+tonumber64("-0b11")
+---
+- -3
+...
+tonumber64(" -0x16 ")
+---
+- -22
+...
+tonumber64(" -0b11 ")
+---
+- -3
+...
+tonumber64(1LL)
+---
+- 1
+...
+tonumber64(100LL)
+---
+- 100
+...
+tonumber64(100LL, 10)
+---
+- 100
+...
+tonumber64(100LL, 2)
+---
+- error: 'bad argument #1 to ''?'' (string expected)'
+...
+tonumber64(-100LL, 2)
+---
+- error: 'bad argument #1 to ''?'' (string expected)'
+...
+tonumber64(100ULL, 2)
+---
+- error: 'bad argument #1 to ''?'' (string expected)'
+...
+tonumber64(32ULL, 5)
+---
+- error: 'bad argument #1 to ''?'' (string expected)'
+...
+tonumber64(32LL, 3)
+---
+- error: 'bad argument #1 to ''?'' (string expected)'
+...
+tonumber64(32ULL, 3)
+---
+- error: 'bad argument #1 to ''?'' (string expected)'
+...
 --  dostring()
 dostring('abc')
 ---

test/box/misc.test.lua

@@ -162,6 +162,30 @@ tonumber64('12345', 123) -- big base
 tonumber64('0xfffff') == 1048575
 tonumber64('0b111111111111111111') == 262143
 
+tonumber64('20', 36)
+tonumber64(20, 36)
+tonumber64(20, 10)
+tonumber64(20)
+
+tonumber64("", 10)
+tonumber64("", 32)
+
+tonumber64("-1")
+tonumber64("-0x16")
+tonumber64("-0b11")
+tonumber64(" -0x16 ")
+tonumber64(" -0b11 ")
+
+tonumber64(1LL)
+tonumber64(100LL)
+tonumber64(100LL, 10)
+tonumber64(100LL, 2)
+tonumber64(-100LL, 2)
+tonumber64(100ULL, 2)
+tonumber64(32ULL, 5)
+tonumber64(32LL, 3)
+tonumber64(32ULL, 3)
+
 --  dostring()
 dostring('abc')
 dostring('abc=2')