From 074fe0bfffc5f3e9582de38d0d22edb1fc7941e0 Mon Sep 17 00:00:00 2001
From: Sergey Kaplun <skaplun@tarantool.org>
Date: Mon, 27 Nov 2023 12:34:17 +0300
Subject: [PATCH] lua: prevent serialization of error for ucdata
Without checking the return value of lua_pcall()` in
`lua_field_inspect_ucdata()`, the error message itself is returned as a
serialized result. The result status of `lua_pcall()` is not ignored
now.
NO_DOC=bugfix
Closes #9396
(cherry picked from commit 98474f7012d85cffa42709d38ddaaaef9fced0fb)
---
.../gh-9396-ucdata-serialize-error.md | 4 ++++
src/lua/serializer.c | 5 ++++-
.../gh_9396_ucdata_serialize_error_test.lua | 18 ++++++++++++++++++
3 files changed, 26 insertions(+), 1 deletion(-)
create mode 100644 changelogs/unreleased/gh-9396-ucdata-serialize-error.md
create mode 100644 test/app-luatest/gh_9396_ucdata_serialize_error_test.lua
diff --git a/changelogs/unreleased/gh-9396-ucdata-serialize-error.md b/changelogs/unreleased/gh-9396-ucdata-serialize-error.md
new file mode 100644
index 0000000000..43cf1d2f6e
--- /dev/null
+++ b/changelogs/unreleased/gh-9396-ucdata-serialize-error.md
@@ -0,0 +1,4 @@
+## bugfix/lua
+
+* An error from a serializer function for cdata and userdata is not ignored now
+ (gh-9396).
diff --git a/src/lua/serializer.c b/src/lua/serializer.c
index bb8952f995..7c350a3ad4 100644
--- a/src/lua/serializer.c
+++ b/src/lua/serializer.c
@@ -274,7 +274,10 @@ lua_field_inspect_ucdata(struct lua_State *L, struct luaL_serializer *cfg,
}
/* copy object itself */
lua_pushvalue(L, idx);
- lua_pcall(L, 1, 1, 0);
+ if (lua_pcall(L, 1, 1, 0) != LUA_OK) {
+ diag_set(LuajitError, lua_tostring(L, -1));
+ return -1;
+ }
/* replace obj with the unpacked value */
lua_replace(L, idx);
if (luaL_tofield(L, cfg, idx, field) < 0)
diff --git a/test/app-luatest/gh_9396_ucdata_serialize_error_test.lua b/test/app-luatest/gh_9396_ucdata_serialize_error_test.lua
new file mode 100644
index 0000000000..1b3e216378
--- /dev/null
+++ b/test/app-luatest/gh_9396_ucdata_serialize_error_test.lua
@@ -0,0 +1,18 @@
+local t = require('luatest')
+local g = t.group()
+
+-- XXX: Use `yaml.encode()` to trigger a serialization error.
+local yaml = require('yaml')
+
+local ERRMSG = 'error in serializer'
+
+g.test_error_in_ucdata_serializer = function()
+ local ud_mt = {__serialize = function()
+ error(ERRMSG)
+ end}
+ ud_mt.__index = ud_mt
+ local ud = newproxy(true)
+ debug.setmetatable(ud, ud_mt)
+ -- XXX: This helper fails if there is no error raised.
+ t.assert_error_msg_matches('.*' .. ERRMSG, yaml.encode, ud)
+end
--
GitLab