trigger: fix NULL dereference and memory leak

Using LuaC API, one can create a callable number - it will be callable
and at the same time lua_topointer will return NULL if such number will
be passed. And function luaT_event_reset_trigger_with_flags relies on the
fact that lua_topointer cannot return NULL if it is called with a
callable object. The assumption is wrong, so let's rewrite the function
without relying on it.

NB: if you are using old trigger API with such exotic handlers, you can
have only one such trigger in an event - it will occupy name '0x0'.

Closes #9287

NO_CHANGELOG=bugfix for unreleased feature
NO_DOC=bugfix

src/box/lua/trigger.c

@@ -378,7 +378,11 @@ luaT_event_reset_trigger_with_flags(struct lua_State *L, int bottom,
 		lua_pushvalue(L, bottom);
 		ret_count = 1;
 	}
-	if (new_handler != NULL && old_handler != NULL) {
+	/*
+	 * Function lua_topointer can return NULL, so let's use names to check
+	 * if handlers are passed - they are assured not to be NULL in the case.
+	 */
+	if (new_name != NULL && old_name != NULL) {
 		if (old_handler == new_handler) {
 			event_reset_trigger_with_flags(event, new_name,
 						       new_trg, flags);
@@ -398,10 +402,10 @@ luaT_event_reset_trigger_with_flags(struct lua_State *L, int bottom,
 						       flags);
 			event_unref(event);
 		}
-	} else if (old_handler != NULL) {
+	} else if (old_name != NULL) {
 		event_reset_trigger(event, old_name, NULL);
 	} else {
-		assert(new_handler != NULL);
+		assert(new_name != NULL);
 		event_reset_trigger_with_flags(event, new_name, new_trg, flags);
 	}
 	return ret_count;

test/box-luatest/CMakeLists.txt

@@ -4,3 +4,4 @@ target_link_libraries(gh_4799_lib msgpuck)
 build_module(gh_6506_lib gh_6506_wakeup_writing_to_wal_fiber.c)
 build_module(gh_9131_lib gh_9131_net_box_self_call_stored_func.c)
 target_link_libraries(gh_9131_lib msgpuck)
+build_module(libcallnum libcallnum.c)

test/box-luatest/libcallnum.c

+#include <lua.h>
+#include <module.h>
+
+static int
+call_metamethod(lua_State *L)
+{
+	lua_pushnumber(L, 64);
+	return 1;
+}
+
+/**
+ * Require returns a callable number.
+ */
+LUA_API int
+luaopen_libcallnum(lua_State *L)
+{
+	lua_pushnumber(L, 42);
+	lua_createtable(L, 0, 1);
+	lua_pushstring(L, "__call");
+	lua_pushcfunction(L, call_metamethod);
+	lua_settable(L, -3);
+	lua_setmetatable(L, -2);
+	return 1;
+}

test/box-luatest/triggers_old_api_test.lua

 local server = require('luatest.server')
 local t = require('luatest')
+local pathjoin = require('fio').pathjoin
 
 local g = t.group()
 
+local libext = package.cpath:match('?.(%a+);')
+local libpath = pathjoin(os.getenv('BUILDDIR'), 'test', 'box-luatest')
+local env = {
+    LUA_CPATH = ('%s/?.%s;%s;'):format(libpath, libext, os.getenv('LUA_CPATH')),
+}
+
 g.before_all(function()
-    g.server = server:new({alias = 'master'})
+    g.server = server:new({alias = 'master', env = env})
     g.server:start()
     g.server:exec(function()
         box.schema.space.create('test', {id = 512})
@@ -220,3 +227,30 @@ g.test_associated_event = function()
         end
     end)
 end
+
+-- https://github.com/tarantool/tarantool/issues/9287
+g.test_callable_number = function()
+    g.server:exec(function()
+        local old_api_triggers = rawget(_G, 'old_api_triggers')
+        local trg = require('libcallnum')
+        t.assert_type(trg, 'number')
+        t.assert_type(trg(), 'number')
+        local function f() end
+
+        local function check_trigger(old_api_trigger)
+            old_api_trigger(trg)
+            t.assert_equals(old_api_trigger(), {trg})
+            old_api_trigger(f, trg)
+            t.assert_equals(old_api_trigger(), {f})
+            old_api_trigger(trg, f)
+            t.assert_equals(old_api_trigger(), {trg})
+            old_api_trigger(trg, trg)
+            t.assert_equals(old_api_trigger(), {trg})
+            old_api_trigger(nil, trg)
+            t.assert_equals(old_api_trigger(), {})
+        end
+        for _, trg_descr in pairs(old_api_triggers) do
+            check_trigger(trg_descr[1])
+        end
+    end)
+end