lua-yaml: enable aliasing for objects returned by __serialize

The YAML serializer fails to detect aliases in objects returned by
the __serialize method:

tarantool> x = {}
---
...

tarantool> {a = x, b = x}
---
- a: &0 []
  b: *0
...

tarantool> setmetatable({}, {
         >     __serialize = function() return {a = x, b = x} end,
         > })
---
- a: []
  b: []
...

Fix this by scanning the object returned by the __serialize method
(called by luaL_checkfield) for references.

Closes #8240

NO_DOC=bug fix

changelogs/unreleased/gh-8240-yaml-alias-serialize-fix.md

+## bugfix/lua
+
+* Fixed alias detection in the YAML serializer in case the input contains
+  objects that implement the `__serialize` meta method (gh-8240).

src/lua/serializer.c

@@ -324,6 +324,7 @@ lua_field_try_serialize(struct lua_State *L, struct luaL_serializer *cfg,
 		if (luaL_tofield(L, cfg, -1, field) != 0)
 			return -1;
 		lua_replace(L, idx);
+		field->serialized = true;
 		return 0;
 	}
 	if (!lua_isstring(L, -1)) {
@@ -433,6 +434,11 @@ int
 luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, int index,
 	     struct luaL_field *field)
 {
+	field->type = MP_NIL;
+	field->ext_type = MP_UNKNOWN_EXTENSION;
+	field->compact = false;
+	field->serialized = false;
+
 	if (index < 0)
 		index = lua_gettop(L) + index + 1;
 
@@ -568,10 +574,7 @@ luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, int index,
 		field->type = MP_STR;
 		return 0;
 	case LUA_TTABLE:
-	{
-		field->compact = false;
 		return lua_field_inspect_table(L, cfg, index, field);
-	}
 	case LUA_TLIGHTUSERDATA:
 	case LUA_TUSERDATA:
 		field->sval.data = NULL;

src/lua/serializer.h

@@ -241,6 +241,8 @@ struct luaL_field {
 	/* subtypes of MP_EXT */
 	enum mp_extension_type ext_type;
 	bool compact;                /* a flag used by YAML serializer */
+	/** Set if __serialize metamethod was called for this field. */
+	bool serialized;
 };
 
 /**

test/app-luatest/gh_8240_yaml_alias_serialize_test.lua

@@ -27,3 +27,17 @@ g.test_objects_that_implement_serialize_are_aliased = function()
     t.assert_is(nested2[1], nested2.a[1][1])
     t.assert_is(nested2[1], nested2.a.b.c)
 end
+
+g.test_objects_returned_by_serialize_are_aliased = function()
+    local o1 = {}
+    local o2 = serialize(setmetatable({}, {
+        __serialize = function() return {o1, {a = o1}} end,
+    }))
+    local o3 = serialize(setmetatable({}, {
+        __serialize = function() return {o2, o2} end
+    }))
+    t.assert_equals(o2, {o1, {a = o1}})
+    t.assert_is(o2[1], o2[2].a)
+    t.assert_equals(o3, {{o1, {a = o1}}, {o1, {a = o1}}})
+    t.assert_is(o3[1], o3[2])
+end

third_party/lua-yaml/lyaml.cc

@@ -616,6 +616,8 @@ static int yaml_is_flow_mode(struct lua_yaml_dumper *dumper) {
    return 0;
 }
 
+static void find_references(struct lua_yaml_dumper *dumper);
+
 static int dump_node(struct lua_yaml_dumper *dumper)
 {
    size_t len = 0;
@@ -635,6 +637,8 @@ static int dump_node(struct lua_yaml_dumper *dumper)
 
    int top = lua_gettop(dumper->L);
    luaL_checkfield(dumper->L, dumper->cfg, top, &field);
+   if (field.serialized)
+      find_references(dumper);
    switch(field.type) {
    case MP_UINT:
       snprintf(buf, sizeof(buf) - 1, "%" PRIu64, field.ival);