diff --git a/changelogs/unreleased/gh-8147-tuple-format-in-iproto-responses.md b/changelogs/unreleased/gh-8147-tuple-format-in-iproto-responses.md
new file mode 100644
index 0000000000000000000000000000000000000000..9ec88ce079c1083f6bca4339b9734516724fb0ed
--- /dev/null
+++ b/changelogs/unreleased/gh-8147-tuple-format-in-iproto-responses.md
@@ -0,0 +1,5 @@
+## feature/box
+
+* Added support for sending tuple formats in IPROTO responses. Added a
+ `box_tuple_extension` backward compatibility option to disable sending
+ tuple formats in responses to IPROTO call and eval requests (gh-8146).
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 84418fc782c52f12712055fa2aeb2e91f0f7b676..07772d9b5affd0a854ce9ff36297fbe5f0ba02b8 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -260,6 +260,7 @@ set(box_sources
watcher.c
decimal.c
read_view.c
+ mp_box_ctx.c
${sql_sources}
${lua_sources}
lua/init.c
diff --git a/src/box/iproto.cc b/src/box/iproto.cc
index ebed68eba13294fddaa440857c8b1c7efed0dea6..b5dc2ffbe2246b35b8e720bb7370bfae90d663b8 100644
--- a/src/box/iproto.cc
+++ b/src/box/iproto.cc
@@ -69,12 +69,14 @@
#include "tt_static.h"
#include "trivia/util.h"
#include "salad/stailq.h"
-#include "assoc.h"
#include "txn.h"
#include "on_shutdown.h"
#include "flightrec.h"
#include "security.h"
#include "watcher.h"
+#include "box/mp_box_ctx.h"
+#include "box/tuple.h"
+#include "mpstream/mpstream.h"
enum {
IPROTO_SALT_SIZE = 32,
@@ -2172,6 +2174,14 @@ static void
tx_process1(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
+ bool box_tuple_as_ext =
+ iproto_features_test(&msg->connection->session->meta.features,
+ IPROTO_FEATURE_DML_TUPLE_EXTENSION);
+ struct tuple_format_map format_map;
+ tuple_format_map_create_empty(&format_map);
+ auto format_map_guard = make_scoped_guard([&format_map] {
+ tuple_format_map_destroy(&format_map);
+ });
if (tx_check_msg(msg) != 0)
goto error;
@@ -2185,10 +2195,25 @@ tx_process1(struct cmsg *m)
goto error;
out = msg->connection->tx.p_obuf;
iproto_prepare_select(out, &svp);
- if (tuple && tuple_to_obuf(tuple, out))
+ if (tuple != NULL) {
+ if (box_tuple_as_ext) {
+ tuple_format_map_add_format(&format_map,
+ tuple->format_id);
+ if (tuple_to_obuf_as_ext(tuple, out) != 0)
+ goto error;
+ } else if (tuple_to_obuf(tuple, out) != 0) {
+ goto error;
+ }
+ }
+ /*
+ * Even if there is no tuple, we still need to send an empty tuple
+ * format map.
+ */
+ if (box_tuple_as_ext &&
+ tuple_format_map_to_iproto_obuf(&format_map, out) != 0)
goto error;
iproto_reply_select(out, &svp, msg->header.sync, ::schema_version,
- tuple != 0);
+ tuple != 0, box_tuple_as_ext);
iproto_wpos_create(&msg->wpos, out);
tx_end_msg(msg, &svp);
return;
@@ -2203,9 +2228,24 @@ static void
tx_process_select(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
+ bool box_tuple_as_ext =
+ iproto_features_test(&msg->connection->session->meta.features,
+ IPROTO_FEATURE_DML_TUPLE_EXTENSION);
struct obuf *out;
struct obuf_svp svp;
struct port port;
+
+ struct mp_box_ctx ctx;
+ struct mp_ctx *ctx_ref = NULL;
+ if (box_tuple_as_ext) {
+ mp_box_ctx_create(&ctx, NULL, NULL);
+ ctx_ref = (struct mp_ctx *)&ctx;
+ }
+ auto ctx_guard = make_scoped_guard([ctx_ref] {
+ mp_ctx_destroy(ctx_ref);
+ });
+ ctx_guard.is_active = box_tuple_as_ext;
+
int count;
int rc;
const char *packed_pos, *packed_pos_end;
@@ -2246,19 +2286,22 @@ tx_process_select(struct cmsg *m)
/*
* SELECT output format has not changed since Tarantool 1.6
*/
- count = port_dump_msgpack_16(&port, out);
+ count = port_dump_msgpack_16_with_ctx(&port, out, ctx_ref);
port_destroy(&port);
- if (count < 0) {
+ if (count < 0 || (box_tuple_as_ext &&
+ tuple_format_map_to_iproto_obuf(&ctx.tuple_format_map,
+ out) != 0)) {
goto discard;
}
if (reply_position) {
assert(packed_pos != NULL);
iproto_reply_select_with_position(out, &svp, msg->header.sync,
::schema_version, count,
- packed_pos, packed_pos_end);
+ packed_pos, packed_pos_end,
+ box_tuple_as_ext);
} else {
iproto_reply_select(out, &svp, msg->header.sync,
- ::schema_version, count);
+ ::schema_version, count, box_tuple_as_ext);
}
region_truncate(&fiber()->gc, region_svp);
iproto_wpos_create(&msg->wpos, out);
@@ -2290,6 +2333,21 @@ static void
tx_process_call(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
+
+ bool box_tuple_as_ext =
+ iproto_features_test(&msg->connection->session->meta.features,
+ IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION);
+ struct mp_box_ctx ctx;
+ struct mp_ctx *ctx_ref = NULL;
+ if (box_tuple_as_ext) {
+ mp_box_ctx_create(&ctx, NULL, NULL);
+ ctx_ref = (struct mp_ctx *)&ctx;
+ }
+ auto ctx_guard = make_scoped_guard([ctx_ref] {
+ mp_ctx_destroy(ctx_ref);
+ });
+ ctx_guard.is_active = box_tuple_as_ext;
+
if (tx_check_msg(msg) != 0)
goto error;
@@ -2351,17 +2409,19 @@ tx_process_call(struct cmsg *m)
iproto_prepare_select(out, &svp);
if (msg->header.type == IPROTO_CALL_16)
- count = port_dump_msgpack_16(&port, out);
+ count = port_dump_msgpack_16_with_ctx(&port, out, ctx_ref);
else
- count = port_dump_msgpack(&port, out);
+ count = port_dump_msgpack_with_ctx(&port, out, ctx_ref);
+
port_destroy(&port);
- if (count < 0) {
+ if (count < 0 || (box_tuple_as_ext &&
+ tuple_format_map_to_iproto_obuf(&ctx.tuple_format_map,
+ out) != 0)) {
obuf_rollback_to_svp(out, &svp);
goto error;
}
-
iproto_reply_select(out, &svp, msg->header.sync,
- ::schema_version, count);
+ ::schema_version, count, box_tuple_as_ext);
iproto_wpos_create(&msg->wpos, out);
tx_end_msg(msg, &svp);
return;
@@ -2375,7 +2435,11 @@ tx_process_call(struct cmsg *m)
static void
tx_process_id(struct iproto_connection *con, const struct id_request *id)
{
+ extern bool box_tuple_extension;
con->session->meta.features = id->features;
+ if (!box_tuple_extension)
+ iproto_features_clear(&con->session->meta.features,
+ IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION);
}
/** Callback passed to session_watch. */
@@ -2439,7 +2503,8 @@ tx_process_misc(struct cmsg *m)
iproto_prepare_select(out, &header);
xobuf_dup(out, data, data_end - data);
iproto_reply_select(out, &header, msg->header.sync,
- ::schema_version, data != NULL ? 1 : 0);
+ ::schema_version, data != NULL ? 1 : 0,
+ /*box_tuple_as_ext=*/false);
break;
}
default:
diff --git a/src/box/iproto_constants.h b/src/box/iproto_constants.h
index ec18f273f48b805d11384008120ae55eaae56977..af7f432c77cfde2722d5d7ab24c3f258ac7c7e15 100644
--- a/src/box/iproto_constants.h
+++ b/src/box/iproto_constants.h
@@ -208,6 +208,11 @@ extern const char *iproto_flag_bit_strs[];
* when identifier is present (i.e., the identifier is ignored).
*/ \
_(INDEX_NAME, 0x5f, MP_STR) \
+ /**
+ * Mapping of format identifier to format clause consisting of field
+ * names and field types.
+ */ \
+ _(TUPLE_FORMATS, 0x60, MP_MAP)
#define IPROTO_KEY_MEMBER(s, v, ...) IPROTO_ ## s = v,
diff --git a/src/box/iproto_features.c b/src/box/iproto_features.c
index 1f2ffbde18599f6530a0039ae146cf0be6a65cdc..4aced8bb5c2a903805be00d42e24bdc0e9c13075 100644
--- a/src/box/iproto_features.c
+++ b/src/box/iproto_features.c
@@ -79,4 +79,8 @@ iproto_features_init(void)
IPROTO_FEATURE_SPACE_AND_INDEX_NAMES);
iproto_features_set(&IPROTO_CURRENT_FEATURES,
IPROTO_FEATURE_WATCH_ONCE);
+ iproto_features_set(&IPROTO_CURRENT_FEATURES,
+ IPROTO_FEATURE_DML_TUPLE_EXTENSION);
+ iproto_features_set(&IPROTO_CURRENT_FEATURES,
+ IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION);
}
diff --git a/src/box/iproto_features.h b/src/box/iproto_features.h
index 2f1e2f15de2790ed9a188c7545d0ab9679241532..1d50a94ab10c30d9bc33dccc39cce170547ae76c 100644
--- a/src/box/iproto_features.h
+++ b/src/box/iproto_features.h
@@ -59,6 +59,18 @@ extern "C" {
_(SPACE_AND_INDEX_NAMES, 5) \
/** IPROTO_WATCH_ONCE request support. */ \
_(WATCH_ONCE, 6) \
+ /**
+ * Tuple format in DML request responses support:
+ * Tuples in IPROTO_DATA response field are encoded as MP_TUPLE and
+ * tuple format is sent in IPROTO_TUPLE_FORMATS field.
+ */ \
+ _(DML_TUPLE_EXTENSION, 7) \
+ /**
+ * Tuple format in call and eval request responses support:
+ * Tuples in IPROTO_DATA response field are encoded as MP_TUPLE and
+ * tuple formats are sent in IPROTO_TUPLE_FORMATS field.
+ */ \
+ _(CALL_RET_TUPLE_EXTENSION, 8) \
#define IPROTO_FEATURE_MEMBER(s, v) IPROTO_FEATURE_ ## s = v,
diff --git a/src/box/lua/init.c b/src/box/lua/init.c
index 81bbf62b946868e3ba0cd783808bf1a804492b43..a476e8e39527cfd4a654fe55fef1fb1857841fd6 100644
--- a/src/box/lua/init.c
+++ b/src/box/lua/init.c
@@ -45,6 +45,9 @@
#include "box/txn.h"
#include "box/func.h"
#include "box/mp_error.h"
+#include "box/mp_box_ctx.h"
+#include "box/mp_tuple.h"
+#include "box/tuple.h"
#include "box/lua/error.h"
#include "box/lua/tuple.h"
@@ -711,14 +714,21 @@ static const struct luaL_Reg boxlib_backup[] = {
*/
static bool
luamp_encode_extension_box(struct lua_State *L, int idx,
- struct mpstream *stream, struct mp_ctx *ctx,
+ struct mpstream *stream, struct mp_ctx *base,
enum mp_type *type)
{
- (void)ctx;
struct tuple *tuple = luaT_istuple(L, idx);
if (tuple != NULL) {
- tuple_to_mpstream(tuple, stream);
- *type = MP_ARRAY;
+ if (base != NULL) {
+ struct mp_box_ctx *ctx = mp_box_ctx_check(base);
+ tuple_to_mpstream_as_ext(tuple, stream);
+ tuple_format_map_add_format(&ctx->tuple_format_map,
+ tuple->format_id);
+ *type = MP_EXT;
+ } else {
+ tuple_to_mpstream(tuple, stream);
+ *type = MP_ARRAY;
+ }
return true;
}
struct error *err = luaL_iserror(L, idx);
@@ -731,31 +741,44 @@ luamp_encode_extension_box(struct lua_State *L, int idx,
}
/**
- * A MsgPack extensions handler that supports errors decode.
+ * A MsgPack extensions handler that supports box extensions decode.
*/
static void
luamp_decode_extension_box(struct lua_State *L, const char **data,
struct mp_ctx *ctx)
{
- (void)ctx;
assert(mp_typeof(**data) == MP_EXT);
int8_t ext_type;
uint32_t len = mp_decode_extl(data, &ext_type);
-
- if (ext_type != MP_ERROR) {
+ switch (ext_type) {
+ case MP_ERROR: {
+ struct error *err = error_unpack(data, len);
+ if (err == NULL)
+ luaL_error(L, "Can not parse an error from MsgPack");
+ luaT_pusherror(L, err);
+ break;
+ }
+ case MP_TUPLE: {
+ struct tuple *tuple;
+ if (ctx == NULL) {
+ tuple = tuple_unpack_without_format(data);
+ } else {
+ struct tuple_format_map *tuple_format_map =
+ &mp_box_ctx_check(ctx)->tuple_format_map;
+ tuple = tuple_unpack(data, tuple_format_map);
+ }
+ if (tuple == NULL)
+ goto tuple_err;
+ luaT_pushtuple(L, tuple);
+ break;
+tuple_err:
+ luaL_error(L, "Can not parse a tuple from MsgPack");
+ break;
+ }
+ default:
luaL_error(L, "Unsupported MsgPack extension type: %d",
ext_type);
- return;
}
-
- struct error *err = error_unpack(data, len);
- if (err == NULL) {
- luaL_error(L, "Can not parse an error from MsgPack");
- return;
- }
-
- luaT_pusherror(L, err);
- return;
}
#include "say.h"
diff --git a/src/box/lua/net_box.c b/src/box/lua/net_box.c
index 1ca4241519dc6d198f6c4d289c85b906da3e5ea0..faf2dbb69309012466d0f74ace3cff0ef8842e8b 100644
--- a/src/box/lua/net_box.c
+++ b/src/box/lua/net_box.c
@@ -49,8 +49,9 @@
#include "box/execute.h"
#include "box/error.h"
#include "box/schema_def.h"
+#include "box/mp_box_ctx.h"
+#include "box/mp_tuple.h"
-#include "assoc.h"
#include "coio.h"
#include "fiber.h"
#include "fiber_cond.h"
@@ -687,20 +688,22 @@ netbox_encode_ping(lua_State *L, int idx, struct netbox_method_encode_ctx *ctx)
* Raises a Lua error on memory allocation failure.
*/
static void
-netbox_encode_id(struct lua_State *L, struct ibuf *ibuf, uint64_t sync)
+netbox_encode_id(struct lua_State *L, struct ibuf *ibuf, uint64_t sync,
+ bool fetch_schema)
{
- struct iproto_features *features = &NETBOX_IPROTO_FEATURES;
+ struct iproto_features features = NETBOX_IPROTO_FEATURES;
+ if (fetch_schema) {
+ iproto_features_clear(&features,
+ IPROTO_FEATURE_DML_TUPLE_EXTENSION);
+ }
#ifndef NDEBUG
- struct iproto_features features_value;
struct errinj *errinj = errinj(ERRINJ_NETBOX_FLIP_FEATURE, ERRINJ_INT);
if (errinj->iparam >= 0 && errinj->iparam < iproto_feature_id_MAX) {
int feature_id = errinj->iparam;
- features_value = *features;
- features = &features_value;
- if (iproto_features_test(features, feature_id))
- iproto_features_clear(features, feature_id);
+ if (iproto_features_test(&features, feature_id))
+ iproto_features_clear(&features, feature_id);
else
- iproto_features_set(features, feature_id);
+ iproto_features_set(&features, feature_id);
}
#endif
struct mpstream stream;
@@ -712,9 +715,9 @@ netbox_encode_id(struct lua_State *L, struct ibuf *ibuf, uint64_t sync)
mpstream_encode_uint(&stream, IPROTO_VERSION);
mpstream_encode_uint(&stream, NETBOX_IPROTO_VERSION);
mpstream_encode_uint(&stream, IPROTO_FEATURES);
- size_t size = mp_sizeof_iproto_features(features);
+ size_t size = mp_sizeof_iproto_features(&features);
char *data = mpstream_reserve(&stream, size);
- mp_encode_iproto_features(data, features);
+ mp_encode_iproto_features(data, &features);
mpstream_advance(&stream, size);
netbox_end_encode(&stream, svp);
@@ -1492,6 +1495,10 @@ struct response_body {
const char *pos;
/* IPROTO_POSITION length */
uint32_t pos_len;
+ /* IPROTO_TUPLE_FORMATS */
+ const char *tuple_formats;
+ /* IPROTO_TUPLE_FORMATS end. */
+ const char *tuple_formats_end;
};
/*
@@ -1522,6 +1529,11 @@ response_body_decode(struct response_body *response_body, const char **data,
mp_decode_str(&value, &response_body->pos_len);
assert(response_body->pos_len != 0);
break;
+ case IPROTO_TUPLE_FORMATS:
+ assert(mp_typeof(*value) == MP_MAP);
+ response_body->tuple_formats = value;
+ response_body->tuple_formats_end = *data;
+ break;
default:
break;
}
@@ -1561,11 +1573,17 @@ netbox_decode_table(struct lua_State *L, const char **data,
lua_pushnil(L);
return;
}
+ struct mp_box_ctx ctx;
+ mp_box_ctx_create(&ctx, NULL, response_body.tuple_formats);
if (return_raw) {
- luamp_push(L, response_body.data, response_body.data_end);
+ luamp_push_with_ctx(L, response_body.data,
+ response_body.data_end,
+ (struct mp_ctx *)&ctx);
} else {
- luamp_decode(L, cfg, &response_body.data);
+ luamp_decode_with_ctx(L, cfg, &response_body.data,
+ (struct mp_ctx *)&ctx);
}
+ mp_ctx_destroy((struct mp_ctx *)&ctx);
}
/**
@@ -1585,11 +1603,17 @@ netbox_decode_value(struct lua_State *L, const char **data,
lua_pushnil(L);
return;
}
+ struct mp_box_ctx ctx;
+ mp_box_ctx_create(&ctx, NULL, response_body.tuple_formats);
if (return_raw) {
- luamp_push(L, response_body.data, response_body.data_end);
+ luamp_push_with_ctx(L, response_body.data,
+ response_body.data_end,
+ (struct mp_ctx *)&ctx);
} else {
- luamp_decode(L, cfg, &response_body.data);
+ luamp_decode_with_ctx(L, cfg, &response_body.data,
+ (struct mp_ctx *)&ctx);
}
+ mp_ctx_destroy((struct mp_ctx *)&ctx);
}
/**
@@ -1610,17 +1634,22 @@ netbox_decode_count(struct lua_State *L, const char **data,
*/
static void
netbox_decode_data(struct lua_State *L, const char **data,
- struct tuple_format *format)
+ struct tuple_format *format, struct mp_box_ctx *ctx)
{
uint32_t count = mp_decode_array(data);
lua_createtable(L, count, 0);
for (uint32_t j = 0; j < count; ++j) {
const char *begin = *data;
mp_next(data);
- struct tuple *tuple =
- box_tuple_new(format, begin, *data);
- if (tuple == NULL)
+ struct tuple *tuple;
+ if (tuple_format_map_is_empty(&ctx->tuple_format_map))
+ tuple = box_tuple_new(format, begin, *data);
+ else
+ tuple = mp_decode_tuple(&begin, &ctx->tuple_format_map);
+ if (tuple == NULL) {
+ mp_ctx_destroy((struct mp_ctx *)ctx);
luaT_error(L);
+ }
luaT_pushtuple(L, tuple);
lua_rawseti(L, -2, j + 1);
}
@@ -1637,11 +1666,16 @@ netbox_decode_select(struct lua_State *L, const char **data,
{
struct response_body response_body;
response_body_decode(&response_body, data, data_end);
+ struct mp_box_ctx ctx;
+ mp_box_ctx_create(&ctx, NULL, response_body.tuple_formats);
if (return_raw) {
- luamp_push(L, response_body.data, response_body.data_end);
+ luamp_push_with_ctx(L, response_body.data,
+ response_body.data_end,
+ (struct mp_ctx *)&ctx);
} else {
- netbox_decode_data(L, &response_body.data, format);
+ netbox_decode_data(L, &response_body.data, format, &ctx);
}
+ mp_ctx_destroy((struct mp_ctx *)&ctx);
}
/**
@@ -1658,11 +1692,18 @@ netbox_decode_select_with_pos(struct lua_State *L, const char **data,
response_body_decode(&response_body, data, data_end);
lua_createtable(L, response_body.pos != NULL ? 2 : 1, 0);
int table_idx = lua_gettop(L);
+ struct mp_box_ctx ctx;
+ mp_box_ctx_create(&ctx, NULL, response_body.tuple_formats);
if (return_raw) {
- luamp_push(L, response_body.data, response_body.data_end);
+ luamp_push_with_ctx(L, response_body.data,
+ response_body.data_end,
+ (struct mp_ctx *)&ctx);
} else {
- netbox_decode_data(L, &response_body.data, format);
+ struct mp_box_ctx ctx;
+ mp_box_ctx_create(&ctx, NULL, response_body.tuple_formats);
+ netbox_decode_data(L, &response_body.data, format, &ctx);
}
+ mp_ctx_destroy((struct mp_ctx *)&ctx);
lua_rawseti(L, table_idx, 1);
if (response_body.pos != NULL) {
lua_pushlstring(L, response_body.pos, response_body.pos_len);
@@ -1687,13 +1728,29 @@ netbox_decode_tuple(struct lua_State *L, const char **data,
return;
}
if (return_raw) {
- luamp_push(L, response_body.data, response_body.data_end);
+ struct mp_box_ctx ctx;
+ mp_box_ctx_create(&ctx, NULL, response_body.tuple_formats);
+ luamp_push_with_ctx(L, response_body.data,
+ response_body.data_end,
+ (struct mp_ctx *)&ctx);
} else {
- struct tuple *tuple =
- box_tuple_new(format, response_body.data,
- response_body.data_end);
- if (tuple == NULL)
+ struct tuple *tuple;
+ if (response_body.tuple_formats == NULL) {
+ tuple = box_tuple_new(format, response_body.data,
+ response_body.data_end);
+ } else {
+ struct tuple_format_map tuple_format_map;
+ const char *tuple_formats = response_body.tuple_formats;
+ if (tuple_format_map_create_from_mp(&tuple_format_map,
+ tuple_formats) != 0)
+ luaT_error(L);
+ tuple = mp_decode_tuple(&response_body.data,
+ &tuple_format_map);
+ tuple_format_map_destroy(&tuple_format_map);
+ }
+ if (tuple == NULL) {
luaT_error(L);
+ }
luaT_pushtuple(L, tuple);
}
}
@@ -1833,8 +1890,11 @@ netbox_decode_execute(struct lua_State *L, const char **data,
mp_next(data);
luamp_push(L, begin, *data);
} else {
+ struct mp_box_ctx ctx;
+ mp_box_ctx_create(&ctx, NULL, NULL);
netbox_decode_data(L, data,
- tuple_format_runtime);
+ tuple_format_runtime, &ctx);
+ mp_ctx_destroy((struct mp_ctx *)&ctx);
}
rows_index = lua_gettop(L);
break;
@@ -2570,9 +2630,10 @@ netbox_transport_on_event(struct netbox_transport *transport,
}
/**
- * Argument data is the body of response, it must be an MP_MAP. Only two keys
- * are expected to appear: IPROTO_DATA (necessarily, must be the first one)
- * and IPROTO_POSITION (unnecessarily). The function writes response to
+ * Argument data is the body of response, it must be an MP_MAP. Only three keys
+ * are expected to appear: IPROTO_DATA (necessarily, must be the first one),
+ * IPROTO_TUPLE_FORMATS (optionally), and
+ * IPROTO_POSITION (optionally). The function writes response to
* passed ibuf. If skip_header flag is set, data is written without IPROTO_DATA
* header. If skip_header is true and response contains IPROTO_POSITION,
* position is not written to a buffer - the function returns a table with
@@ -2588,40 +2649,23 @@ netbox_write_response_to_buffer(const char *data, const char *data_end,
/* Copy xrow.body to user-provided buffer. */
size_t data_len = data_end - data;
bool return_table = false;
+ struct response_body response_body;
+ const char *data_ptr = data;
+ response_body_decode(&response_body, &data_ptr, data_end);
if (skip_header) {
- assert(mp_typeof(*data) == MP_MAP);
- uint32_t map_size = mp_decode_map(&data);
- uint32_t key = mp_decode_uint(&data);
- assert(key == IPROTO_DATA);
- (void)key;
- if (map_size > 1) {
- /*
- * The map has more than one element iff it
- * contains IPROTO_DATA and IPROTO_POSITION
- * and they are placed exactly in this order.
- */
- assert(map_size == 2);
- /* Find the end of IPROTO_DATA and its len. */
- const char *iproto_position = data;
- mp_next(&iproto_position);
- data_len = iproto_position - data;
+ data = response_body.data;
+ data_len = response_body.data_end - response_body.data;
+ if (response_body.pos != NULL) {
/* Create table to return 2 values. */
return_table = true;
lua_createtable(L, 2, 0);
- /* Skip IPROTO_POSITION key. */
- key = mp_decode_uint(&iproto_position);
- assert(key == IPROTO_POSITION);
/* Check position length */
- assert(mp_typeof(*iproto_position) == MP_STR);
- uint32_t str_len = mp_decode_strl(&iproto_position);
- if (str_len != 0) {
+ if (response_body.pos_len != 0) {
/* Set position to the 2nd place in table. */
- lua_pushlstring(L, iproto_position, str_len);
+ lua_pushlstring(L, response_body.pos,
+ response_body.pos_len);
lua_rawseti(L, -2, 2);
}
- } else {
- /* Update data_len if header is skipped. */
- data_len = data_end - data;
}
}
void *wpos = ibuf_alloc(buffer, data_len);
@@ -2733,7 +2777,8 @@ netbox_transport_do_id(struct netbox_transport *transport, struct lua_State *L)
ERROR_INJECT(ERRINJ_NETBOX_DISABLE_ID, goto out);
if (peer_version_id < version_id(2, 10, 0))
goto unsupported;
- netbox_encode_id(L, &transport->send_buf, transport->next_sync++);
+ netbox_encode_id(L, &transport->send_buf, transport->next_sync++,
+ transport->opts.fetch_schema);
struct xrow_header hdr;
if (netbox_transport_send_and_recv(transport, &hdr) != 0)
luaT_error(L);
@@ -3163,6 +3208,10 @@ luaopen_net_box(struct lua_State *L)
IPROTO_FEATURE_SPACE_AND_INDEX_NAMES);
iproto_features_set(&NETBOX_IPROTO_FEATURES,
IPROTO_FEATURE_WATCH_ONCE);
+ iproto_features_set(&NETBOX_IPROTO_FEATURES,
+ IPROTO_FEATURE_DML_TUPLE_EXTENSION);
+ iproto_features_set(&NETBOX_IPROTO_FEATURES,
+ IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION);
lua_pushcfunction(L, luaT_netbox_request_iterator_next);
luaT_netbox_request_iterator_next_ref = luaL_ref(L, LUA_REGISTRYINDEX);
diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c
index ab11d4cdf44f3456aa7e182050fdf2ac329b8557..1fed5050603e696114efa4e331832cf142644671 100644
--- a/src/box/lua/tuple.c
+++ b/src/box/lua/tuple.c
@@ -441,6 +441,41 @@ luamp_encode_tuple_with_ctx(struct lua_State *L, struct luaL_serializer *cfg,
return 0;
}
+ /*
+ * This snippet handles a special when a box tuple is sent over IPROTO
+ * as MP_TUPLE and is decoded by net.box using the `return_raw` option,
+ * which return a MsgPack object. This case is semantically equivalent
+ * to the case above where the MP_TUPLE should have been decoded as a
+ * box tuple. While we expect the encoded MsgPack to be an MP_ARRAY, the
+ * Lua MsgPack encoder below encodes MsgPack objects by simply copying
+ * their contents to the MsgPack stream, so we would get MP_TUPLE as the
+ * returned type.
+ *
+ * To overcome this limitation, we convert the top-level MP_TUPLE to a
+ * MsgPack array by skipping the extension header and format identifier
+ * and copying the tuple data to the MsgPack stream.
+ */
+ size_t data_len;
+ const char *data = luamp_get(L, index, &data_len);
+ if (data != NULL) {
+ if (mp_typeof(*data) == MP_EXT) {
+ const char *tuple_data = data;
+ int8_t ext_type;
+ uint32_t tuple_data_len =
+ mp_decode_extl(&tuple_data, &ext_type);
+ if (ext_type == MP_TUPLE) {
+ /* Skip the tuple format identifier. */
+ assert(mp_typeof(*tuple_data) == MP_UINT);
+ uint64_t format_id =
+ mp_decode_uint(&tuple_data);
+ tuple_data_len -= mp_sizeof_uint(format_id);
+ mpstream_memcpy(stream, tuple_data,
+ tuple_data_len);
+ return 0;
+ }
+ }
+ }
+
enum mp_type type;
if (luamp_encode_with_ctx(L, cfg, stream, index, ctx, &type) != 0)
return -1;
diff --git a/src/box/mp_box_ctx.c b/src/box/mp_box_ctx.c
new file mode 100644
index 0000000000000000000000000000000000000000..d53d337679c35a4b23f2eb2db6b86c253a64b6e1
--- /dev/null
+++ b/src/box/mp_box_ctx.c
@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2010-2023, Tarantool AUTHORS, please see AUTHORS file.
+ */
+
+#include "box/mp_box_ctx.h"
+
+void
+mp_box_ctx_destroy(struct mp_ctx *ctx)
+{
+ tuple_format_map_destroy(&mp_box_ctx_check(ctx)->tuple_format_map);
+}
+
+void
+mp_box_ctx_move(struct mp_ctx *dst, struct mp_ctx *src)
+{
+ mp_ctx_move_default(dst, src);
+ tuple_format_map_move(&mp_box_ctx_check(dst)->tuple_format_map,
+ &mp_box_ctx_check(src)->tuple_format_map);
+}
diff --git a/src/box/mp_box_ctx.h b/src/box/mp_box_ctx.h
new file mode 100644
index 0000000000000000000000000000000000000000..fd76ec23d4c3ed6cc0b923a18a4ab0508ca792af
--- /dev/null
+++ b/src/box/mp_box_ctx.h
@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2010-2023, Tarantool AUTHORS, please see AUTHORS file.
+ */
+
+#pragma once
+
+#include "core/mp_ctx.h"
+
+#include "box/tuple_format_map.h"
+
+#include <assert.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif /* defined(__cplusplus) */
+
+/**
+ * Context for MsgPack encoding/decoding of box-specific types.
+ */
+struct mp_box_ctx {
+ /** See `mp_ctx::translation`. */
+ struct mh_strnu32_t *translation;
+ /** See `mp_ctx::destroy`. */
+ void (*destroy)(struct mp_ctx *ctx);
+ /** See `mp_ctx::move`. */
+ void (*move)(struct mp_ctx *dst, struct mp_ctx *src);
+ /** Mapping of format identifiers to tuple formats. */
+ struct tuple_format_map tuple_format_map;
+};
+
+static_assert(sizeof(struct mp_box_ctx) <= sizeof(struct mp_ctx),
+ "sizeof(struct mp_box_ctx) must be <= sizeof(struct mp_ctx)");
+
+/**
+ * 'Virtual' destructor. Must not be called directly.
+ */
+void
+mp_box_ctx_destroy(struct mp_ctx *ctx);
+
+/**
+ * 'Virtual' move. Must not be called directly.
+ */
+void
+mp_box_ctx_move(struct mp_ctx *dst, struct mp_ctx *src);
+
+static inline struct mp_box_ctx *
+mp_box_ctx_check(struct mp_ctx *base)
+{
+ assert(base->destroy == mp_box_ctx_destroy &&
+ base->move == mp_box_ctx_move);
+ return (struct mp_box_ctx *)base;
+}
+
+static inline int
+mp_box_ctx_create(struct mp_box_ctx *ctx, struct mh_strnu32_t *translation,
+ const char *tuple_formats)
+{
+ mp_ctx_create((struct mp_ctx *)ctx, translation, mp_box_ctx_destroy,
+ mp_box_ctx_move);
+ if (tuple_formats == NULL) {
+ tuple_format_map_create_empty(&ctx->tuple_format_map);
+ return 0;
+ }
+ return tuple_format_map_create_from_mp(&ctx->tuple_format_map,
+ tuple_formats);
+}
+
+#if defined(__cplusplus)
+} /* extern "C" */
+#endif /* defined(__cplusplus) */
diff --git a/src/box/port.c b/src/box/port.c
index e8c23a0711ed05b03927faa15999a4a1409b1840..c77d3a1a90f52621311ff76649ca24e50dc04f7d 100644
--- a/src/box/port.c
+++ b/src/box/port.c
@@ -38,6 +38,7 @@
#include "errinj.h"
#include "mpstream/mpstream.h"
#include "tweaks.h"
+#include "box/mp_box_ctx.h"
/**
* The pool is used by port_c to allocate entries and to store
@@ -207,14 +208,22 @@ port_c_add_str(struct port *base, const char *str, uint32_t len)
static int
port_c_dump_msgpack(struct port *base, struct obuf *out, struct mp_ctx *ctx)
{
- (void)ctx;
struct port_c *port = (struct port_c *)base;
struct port_c_entry *pe;
for (pe = port->first; pe != NULL; pe = pe->next) {
uint32_t size = pe->mp_size;
if (size == 0) {
- if (tuple_to_obuf(pe->tuple, out) != 0)
+ if (ctx != NULL) {
+ if (tuple_to_obuf_as_ext(pe->tuple, out) != 0)
+ return -1;
+ struct mp_box_ctx *box_ctx =
+ mp_box_ctx_check(ctx);
+ tuple_format_map_add_format(
+ &box_ctx->tuple_format_map,
+ pe->tuple->format_id);
+ } else if (tuple_to_obuf(pe->tuple, out) != 0) {
return -1;
+ }
} else if (obuf_dup(out, pe->mp, size) != size) {
diag_set(OutOfMemory, size, "obuf_dup", "data");
return -1;
diff --git a/src/box/tuple_format_map.c b/src/box/tuple_format_map.c
index a15ac0bd632c49c0f0d0212d5e07f05395659a7e..bc50c2b6787ea86f3eaa5ceb4dc3ea95c2548c67 100644
--- a/src/box/tuple_format_map.c
+++ b/src/box/tuple_format_map.c
@@ -4,6 +4,7 @@
* Copyright 2010-2023, Tarantool AUTHORS, please see AUTHORS file.
*/
+#include "box/iproto_constants.h"
#include "box/tuple_format_map.h"
#include "box/tuple.h"
#include "diag.h"
@@ -135,6 +136,15 @@ tuple_format_map_destroy(struct tuple_format_map *map)
TRASH(map);
}
+void
+tuple_format_map_move(struct tuple_format_map *dst,
+ struct tuple_format_map *src)
+{
+ memcpy(dst, src, sizeof(*dst));
+ src->cache_last_index = -1;
+ src->hash_table = NULL;
+}
+
void
tuple_format_map_add_format(struct tuple_format_map *map, uint16_t format_id)
{
@@ -160,6 +170,31 @@ tuple_format_map_to_mpstream(struct tuple_format_map *map,
}
}
+static void
+mpstream_error(void *is_err)
+{
+ *(bool *)is_err = true;
+}
+
+int
+tuple_format_map_to_iproto_obuf(struct tuple_format_map *map,
+ struct obuf *obuf)
+{
+ struct mpstream stream;
+ bool is_error = false;
+ mpstream_init(&stream, obuf, obuf_reserve_cb, obuf_alloc_cb,
+ mpstream_error, &is_error);
+ mpstream_encode_uint(&stream, IPROTO_TUPLE_FORMATS);
+ tuple_format_map_to_mpstream(map, &stream);
+ mpstream_flush(&stream);
+ if (is_error) {
+ diag_set(OutOfMemory, stream.pos - stream.buf,
+ "mpstream_flush", "stream");
+ return -1;
+ }
+ return 0;
+}
+
struct tuple_format *
tuple_format_map_find(struct tuple_format_map *map, uint16_t format_id)
{
diff --git a/src/box/tuple_format_map.h b/src/box/tuple_format_map.h
index 1acaabd8cc65d6be4068575acb75ab8d2226ed9c..53560292c0b865963937dfe25db4872beac6fabe 100644
--- a/src/box/tuple_format_map.h
+++ b/src/box/tuple_format_map.h
@@ -61,6 +61,14 @@ tuple_format_map_create_from_mp(struct tuple_format_map *map, const char *data);
void
tuple_format_map_destroy(struct tuple_format_map *map);
+/**
+ * Move the tuple format from @a src to @a dst and destroy @a src.
+ * The destination format map must be empty or uninitialized.
+ */
+void
+tuple_format_map_move(struct tuple_format_map *dst,
+ struct tuple_format_map *src);
+
static inline bool
tuple_format_map_is_empty(struct tuple_format_map *map)
{
@@ -80,6 +88,15 @@ void
tuple_format_map_to_mpstream(struct tuple_format_map *map,
struct mpstream *stream);
+/**
+ * Serialize a tuple format map as `IPROTO_TUPLE_FORMATS` to an output buffer.
+ *
+ * Returns 0 on success, otherwise -1 and sets diagnostic.
+ */
+int
+tuple_format_map_to_iproto_obuf(struct tuple_format_map *map,
+ struct obuf *obuf);
+
/**
* Find a format in the tuple format map.
*
diff --git a/src/box/xrow.c b/src/box/xrow.c
index 18e5f4a63ddb27e0eefe7ddf7b750eaa59073d5d..f2babcecebf5a0f4f24dcbf0480c9420304047b6 100644
--- a/src/box/xrow.c
+++ b/src/box/xrow.c
@@ -46,6 +46,14 @@
#include "iproto_features.h"
#include "mpstream/mpstream.h"
#include "errinj.h"
+#include "core/tweaks.h"
+
+/**
+ * Controls whether `IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION` feature bit is set
+ * in `IPROTO_ID` request responses.
+ */
+bool box_tuple_extension;
+TWEAK_BOOL(box_tuple_extension);
/**
* Min length of the salt sent in a greeting message.
@@ -483,23 +491,22 @@ iproto_reply_id(struct obuf *out, const char *auth_type,
assert(auth_type != NULL);
uint32_t auth_type_len = strlen(auth_type);
unsigned version = IPROTO_CURRENT_VERSION;
- struct iproto_features *features = &IPROTO_CURRENT_FEATURES;
-
+ struct iproto_features features = IPROTO_CURRENT_FEATURES;
+ if (!box_tuple_extension)
+ iproto_features_clear(&features,
+ IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION);
#ifndef NDEBUG
struct errinj *errinj;
errinj = errinj(ERRINJ_IPROTO_SET_VERSION, ERRINJ_INT);
if (errinj->iparam >= 0)
version = errinj->iparam;
- struct iproto_features features_value;
errinj = errinj(ERRINJ_IPROTO_FLIP_FEATURE, ERRINJ_INT);
if (errinj->iparam >= 0 && errinj->iparam < iproto_feature_id_MAX) {
int feature_id = errinj->iparam;
- features_value = *features;
- features = &features_value;
- if (iproto_features_test(features, feature_id))
- iproto_features_clear(features, feature_id);
+ if (iproto_features_test(&features, feature_id))
+ iproto_features_clear(&features, feature_id);
else
- iproto_features_set(features, feature_id);
+ iproto_features_set(&features, feature_id);
}
#endif
@@ -508,7 +515,7 @@ iproto_reply_id(struct obuf *out, const char *auth_type,
size += mp_sizeof_uint(IPROTO_VERSION);
size += mp_sizeof_uint(version);
size += mp_sizeof_uint(IPROTO_FEATURES);
- size += mp_sizeof_iproto_features(features);
+ size += mp_sizeof_iproto_features(&features);
size += mp_sizeof_uint(IPROTO_AUTH_TYPE);
size += mp_sizeof_str(auth_type_len);
@@ -518,7 +525,7 @@ iproto_reply_id(struct obuf *out, const char *auth_type,
data = mp_encode_uint(data, IPROTO_VERSION);
data = mp_encode_uint(data, version);
data = mp_encode_uint(data, IPROTO_FEATURES);
- data = mp_encode_iproto_features(data, features);
+ data = mp_encode_iproto_features(data, &features);
data = mp_encode_uint(data, IPROTO_AUTH_TYPE);
data = mp_encode_str(data, auth_type, auth_type_len);
assert(size == (size_t)(data - buf));
@@ -713,7 +720,8 @@ iproto_prepare_header(struct obuf *buf, struct obuf_svp *svp, size_t size)
/** Reply select with IPROTO_DATA. */
void
iproto_reply_select(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
- uint64_t schema_version, uint32_t count)
+ uint64_t schema_version, uint32_t count,
+ bool box_tuple_as_ext)
{
char *pos = (char *) obuf_svp_to_ptr(buf, svp);
iproto_header_encode(pos, IPROTO_OK, sync, schema_version,
@@ -721,6 +729,7 @@ iproto_reply_select(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
IPROTO_HEADER_LEN);
struct iproto_body_bin body = iproto_body_bin;
+ body.m_body += box_tuple_as_ext;
body.v_data_len = mp_bswap_u32(count);
memcpy(pos + IPROTO_HEADER_LEN, &body, sizeof(body));
@@ -731,7 +740,8 @@ void
iproto_reply_select_with_position(struct obuf *buf, struct obuf_svp *svp,
uint64_t sync, uint32_t schema_version,
uint32_t count, const char *packed_pos,
- const char *packed_pos_end)
+ const char *packed_pos_end,
+ bool box_tuple_as_ext)
{
size_t packed_pos_size = packed_pos_end - packed_pos;
size_t key_size = mp_sizeof_uint(IPROTO_POSITION);
@@ -747,6 +757,7 @@ iproto_reply_select_with_position(struct obuf *buf, struct obuf_svp *svp,
IPROTO_HEADER_LEN);
struct iproto_body_bin body = iproto_body_bin_with_position;
+ body.m_body += box_tuple_as_ext;
body.v_data_len = mp_bswap_u32(count);
memcpy(pos + IPROTO_HEADER_LEN, &body, sizeof(body));
diff --git a/src/box/xrow.h b/src/box/xrow.h
index 5fdcbbf15e709fb2c694d9ac5ef2efce87217343..683f83bf3102a76c888e018d2ce178e9f3345fb3 100644
--- a/src/box/xrow.h
+++ b/src/box/xrow.h
@@ -779,7 +779,8 @@ iproto_prepare_select_with_position(struct obuf *buf, struct obuf_svp *svp)
*/
void
iproto_reply_select(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
- uint64_t schema_version, uint32_t count);
+ uint64_t schema_version, uint32_t count,
+ bool box_tuple_as_ext);
/**
* Write extended select header to a preallocated buffer.
@@ -788,7 +789,8 @@ void
iproto_reply_select_with_position(struct obuf *buf, struct obuf_svp *svp,
uint64_t sync, uint32_t schema_version,
uint32_t count, const char *packed_pos,
- const char *packed_pos_end);
+ const char *packed_pos_end,
+ bool box_tuple_as_ext);
/**
* Encode iproto header with IPROTO_OK response code.
diff --git a/src/lua/compat.lua b/src/lua/compat.lua
index 735f6569a7e4f064bb31744ebe3f383b6729bc01..b8429b64f7ceaed47c9f00577e631eb77f0f657e 100644
--- a/src/lua/compat.lua
+++ b/src/lua/compat.lua
@@ -96,6 +96,12 @@ additional msgpack array when returning them via iproto.
https://tarantool.io/compat/c_func_iproto_multireturn
]]
+local BOX_TUPLE_EXTENSION_BRIEF = [[
+Controls IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION feature bit.
+
+https://tarantool.io/compat/box_tuple_extension
+]]
+
-- Returns an action callback that toggles a tweak.
local function tweak_action(tweak_name, old_tweak_value, new_tweak_value)
return function(is_new)
@@ -182,6 +188,13 @@ local options = {
run_action_now = true,
action = tweak_action('c_func_iproto_multireturn', false, true),
},
+ box_tuple_extension = {
+ default = 'new',
+ obsolete = nil,
+ brief = BOX_TUPLE_EXTENSION_BRIEF,
+ run_action_now = true,
+ action = tweak_action('box_tuple_extension', false, true)
+ },
}
-- Array with option names in order of addition.
diff --git a/test/box-luatest/gh_7894_export_iproto_constants_and_features_test.lua b/test/box-luatest/gh_7894_export_iproto_constants_and_features_test.lua
index 78fdb581d2ebe6615295472452b91a984370a3c8..1c9fb190789357ad757e95e34a204cc3483a62ef 100644
--- a/test/box-luatest/gh_7894_export_iproto_constants_and_features_test.lua
+++ b/test/box-luatest/gh_7894_export_iproto_constants_and_features_test.lua
@@ -84,6 +84,7 @@ local reference_table = {
INSTANCE_NAME = 0x5d,
SPACE_NAME = 0x5e,
INDEX_NAME = 0x5f,
+ TUPLE_FORMATS = 0x60,
},
-- `iproto_metadata_key` enumeration.
@@ -175,6 +176,8 @@ local reference_table = {
pagination = true,
space_and_index_names = true,
watch_once = true,
+ dml_tuple_extension = true,
+ call_ret_tuple_extension = true,
},
feature = {
streams = 0,
@@ -184,6 +187,8 @@ local reference_table = {
pagination = 4,
space_and_index_names = 5,
watch_once = 6,
+ dml_tuple_extension = 7,
+ call_ret_tuple_extension = 8,
},
}
diff --git a/test/box-luatest/gh_8147_tuple_formats_in_iproto_responses_test.lua b/test/box-luatest/gh_8147_tuple_formats_in_iproto_responses_test.lua
new file mode 100644
index 0000000000000000000000000000000000000000..4e250aeeb382f02d632e2a5b6c17e8c4d6471f97
--- /dev/null
+++ b/test/box-luatest/gh_8147_tuple_formats_in_iproto_responses_test.lua
@@ -0,0 +1,168 @@
+local buffer = require('buffer')
+local msgpack = require('msgpack')
+local netbox = require('net.box')
+local server = require('luatest.server')
+local t = require('luatest')
+
+local g = t.group(nil, t.helpers.matrix{return_raw = {false, true}})
+
+g.before_all(function(cg)
+ cg.server = server:new()
+ cg.server:start()
+ cg.server:exec(function()
+ local s = box.schema.space.create('s')
+ s:create_index('i')
+ local sf = box.schema.space.create('sf', {format = {'field'}})
+ sf:create_index('i')
+ rawset(_G, 'f', function() return 1, 2, 3 end)
+ rawset(_G, 't', function() return {1, {t = s:get{0}}} end)
+ rawset(_G, 'tf', function() return {1, {t = sf:get{0}}} end)
+ end)
+end)
+
+g.after_all(function(cg)
+ cg.server:drop()
+end)
+
+local function res_wrapper(res, unpacked)
+ if msgpack.is_object(res) then
+ res = res:decode()
+ return unpacked and unpack(res) or res
+ end
+ return res
+end
+
+local function test_tuple_method(c, method, tuple, return_raw)
+ local res = res_wrapper(c.space.s[method](c.space.s, tuple,
+ {return_raw = return_raw}))
+ t.assert_equals(res:totable(), tuple:totable())
+ local resf = res_wrapper(c.space.sf[method](c.space.sf, tuple,
+ {return_raw = return_raw}))
+ t.assert_equals(resf:tomap{names_only = true},
+ tuple:tomap{names_only = true})
+end
+
+local function get_call_eval_res(c, method, ret_raw)
+ if method == 'call' then
+ return res_wrapper(c:call("t", {}, {return_raw = ret_raw}), true)[2].t,
+ res_wrapper(c:call("tf", {}, {return_raw = ret_raw}), true)[2].t
+ else
+ return res_wrapper(c:eval("return t()", {},
+ {return_raw = ret_raw}), true)[2].t,
+ res_wrapper(c:eval("return tf()", {},
+ {return_raw = ret_raw}), true)[2].t
+ end
+end
+
+local function test_call_eval_box_tuple_extension(c, method, tuple, ret_raw)
+ local res, resf = get_call_eval_res(c, method, ret_raw)
+ t.assert(box.tuple.is(res))
+ t.assert_equals(res:totable(), tuple:totable())
+ t.assert(box.tuple.is(resf))
+ t.assert_equals(resf:tomap{names_only = true},
+ tuple:tomap{names_only = true})
+end
+
+local function test_call_eval_old(c, method, tuple, ret_raw)
+ local res, resf = get_call_eval_res(c, method, ret_raw)
+ t.assert_not(box.tuple.is(res))
+ t.assert_equals(res, tuple:totable())
+ t.assert_not(box.tuple.is(resf))
+ t.assert_equals(resf, tuple:totable())
+end
+
+local function pack(...)
+ return { n = select("#", ...), ... }
+end
+
+-- Checks that formats in IPROTO request responses work correctly.
+g.test_net_box_formats_in_iproto_request_responses = function(cg)
+ local ret_raw = cg.params.return_raw
+ local c = netbox:connect(cg.server.net_box_uri, {fetch_schema = false})
+
+ t.assert_equals(c.space.s:get{0}, nil)
+ t.assert_equals(c.space.sf:get{0}, nil)
+ t.assert_equals(c.space.s:select{}, {})
+ t.assert_equals(c.space.sf:select{}, {})
+ local tuple_methods = {'insert', 'delete', 'replace', 'get'}
+ local fmt = box.tuple.format.new{{'field'}}
+ local tuple = box.tuple.new({0}, {format = fmt})
+ for _, method in ipairs(tuple_methods) do
+ test_tuple_method(c, method, tuple)
+ end
+ local res = res_wrapper(c.space.s:update({0}, {{'=', 2, 0}},
+ {return_raw = ret_raw}))
+ t.assert_equals(res:totable(), {0, 0})
+ c.space.s:replace{0}
+ local resf = res_wrapper(c.space.sf:update({0}, {{'=', 2, 0}},
+ {return_raw = ret_raw}))
+ t.assert_equals(resf:tomap{names_only = true},
+ tuple:tomap{names_only = true})
+ c.space.sf:replace{0}
+ res = res_wrapper(c.space.s:select({}, {return_raw = ret_raw}))[1]
+ t.assert_equals(res:totable(), tuple:totable())
+ resf = res_wrapper(c.space.sf:select({}, {return_raw = ret_raw}))[1]
+ t.assert_equals(resf:tomap{names_only = true},
+ tuple:tomap{names_only = true})
+ c.space.s:insert{1}
+ c.space.sf:insert{1}
+
+ local tuples = {box.tuple.new({0}, {format = fmt}),
+ box.tuple.new({1}, {format = fmt})}
+ res = res_wrapper(c.space.s:select({}, {return_raw = ret_raw}))
+ resf = res_wrapper(c.space.sf:select({}, {return_raw = ret_raw}))
+ for i, tpl in ipairs(tuples) do
+ t.assert_equals(res[i]:totable(), tpl:totable())
+ t.assert_equals(resf[i]:tomap{names_only = true},
+ tpl:tomap{names_only = true})
+ end
+ t.assert_equals(pack(c:call("f")), {1, 2, 3, n = 3})
+ t.assert_equals(pack(c:eval("return 1, 2, 3")), {1, 2, 3, n = 3})
+
+ test_call_eval_box_tuple_extension(c, "call", tuple, ret_raw)
+ test_call_eval_box_tuple_extension(c, "eval", tuple, ret_raw)
+end
+
+-- Checks that `box_tuple_extension` backward compatibility option works
+-- correctly.
+g.test_box_tuple_extension_compat_option = function(cg)
+ cg.server:exec(function()
+ require('compat').box_tuple_extension = 'old'
+ end)
+
+ local c = netbox:connect(cg.server.net_box_uri, {fetch_schema = false})
+
+ t.assert_equals(pack(c:call("f")), {1, 2, 3, n = 3})
+ t.assert_equals(pack(c:eval("return 1, 2, 3")), {1, 2, 3, n = 3})
+
+ local tuple = box.tuple.new{0}
+ test_call_eval_old(c, "call", tuple)
+ test_call_eval_old(c, "eval", tuple)
+
+ local ibuf = buffer.ibuf()
+ local data_size = c:call("t", {}, {buffer = ibuf})
+ local res = msgpack.object_from_raw(ibuf.rpos, data_size):decode()
+ res = res[box.iproto.key.DATA][1][2].t
+ t.assert_equals(res, tuple:totable())
+end
+
+g.before_test('test_netbox_conn_with_disabled_dml_tuple_extension_errinj',
+ function()
+ box.error.injection.set('ERRINJ_NETBOX_FLIP_FEATURE',
+ box.iproto.feature.dml_tuple_extension)
+end)
+
+-- Checks that net.box connection buffer argument works correctly with formats
+-- in IPROTO request responses.
+g.test_netbox_conn_with_disabled_dml_tuple_extension_errinj = function(cg)
+ t.tarantool.skip_if_not_debug()
+
+ local c = netbox:connect(cg.server.net_box_uri, {fetch_schema = false})
+ local res = c.space.sf:get{0}
+ t.assert_equals(res:tomap{names_only = true}, {})
+end
+
+g.after_test('test_netbox_conn_with_disabled_dml_tuple_extension_errinj',
+ function()
+ box.error.injection.set('ERRINJ_NETBOX_FLIP_FEATURE', -1)
+end)
diff --git a/test/box-py/iproto.result b/test/box-py/iproto.result
index d51578ac5b27c6307204bd7fb9427b1071e08673..3b9d4bf713b263d351f90e323c202a872e572ae7 100644
--- a/test/box-py/iproto.result
+++ b/test/box-py/iproto.result
@@ -210,11 +210,11 @@ Invalid MsgPack - request body
# Invalid auth_type
Invalid MsgPack - request body
# Empty request body
-version=6, features=[0, 1, 2, 3, 4, 5, 6], auth_type=chap-sha1
+version=6, features=[0, 1, 2, 3, 4, 5, 6, 7, 8], auth_type=chap-sha1
# Unknown version and features
-version=6, features=[0, 1, 2, 3, 4, 5, 6], auth_type=chap-sha1
+version=6, features=[0, 1, 2, 3, 4, 5, 6, 7, 8], auth_type=chap-sha1
# Unknown request key
-version=6, features=[0, 1, 2, 3, 4, 5, 6], auth_type=chap-sha1
+version=6, features=[0, 1, 2, 3, 4, 5, 6, 7, 8], auth_type=chap-sha1
#
# gh-6257 Watchers
diff --git a/test/box-tap/gh-4954-merger-via-net-box.test.lua b/test/box-tap/gh-4954-merger-via-net-box.test.lua
index e2bd6f8b9f19f6dbd54c797aad46a35c621a8ab9..7d6e9a5937696d272cc7c0ae55a967177101bd91 100755
--- a/test/box-tap/gh-4954-merger-via-net-box.test.lua
+++ b/test/box-tap/gh-4954-merger-via-net-box.test.lua
@@ -86,7 +86,6 @@ box.schema.user.grant('guest', 'execute', 'universe', nil,
{if_not_exists = true})
local test = tap.test('gh-4954-merger-via-net-box.test.lua')
-test:plan(6)
local tuples = {
{1},
@@ -94,17 +93,25 @@ local tuples = {
{3},
}
+test:plan(3 * #tuples + 3)
+
local connection = net_box.connect(box.cfg.listen)
+local function check_res(test, res, test_name)
+ for i, t in ipairs(tuples) do
+ test:is_deeply(res[i]:totable(), t, test_name)
+ end
+end
+
local res = connection:call('use_table_source', {tuples})
-test:is_deeply(res, tuples, 'verify table source')
+check_res(test, res, 'verify table source')
local res = connection:call('use_buffer_source', {tuples})
-test:is_deeply(res, tuples, 'verify buffer source')
+check_res(test, res, 'verify buffer source')
local res = connection:call('use_tuple_source', {tuples})
-test:is_deeply(res, tuples, 'verify tuple source')
+check_res(test, res, 'verify tuple source')
local function test_verify_source_async(test, func_name, request_count)
- test:plan(request_count)
+ test:plan(request_count * #tuples)
local futures = {}
for _ = 1, request_count do
@@ -113,7 +120,7 @@ local function test_verify_source_async(test, func_name, request_count)
end
for i = 1, request_count do
local res = unpack(futures[i]:wait_result())
- test:is_deeply(res, tuples, ('verify request %d'):format(i))
+ check_res(test, res, ('verify request %d'):format(i))
end
end
diff --git a/test/box/gh-4513-netbox-self-and-connect-interchangeable.result b/test/box/gh-4513-netbox-self-and-connect-interchangeable.result
index 2a4e64cd527c765c963707f5578011bdae4c6016..7369c29463a75e23496f973fef85cf1306a1ddd5 100644
--- a/test/box/gh-4513-netbox-self-and-connect-interchangeable.result
+++ b/test/box/gh-4513-netbox-self-and-connect-interchangeable.result
@@ -24,10 +24,6 @@ end
--
-- netbox:self and netbox:connect should work interchangeably
--
-type(nb:eval('return box.tuple.new{1}')) -- table
- | ---
- | - table
- | ...
type(nb:eval('return box.error.new(1, "test error")')) -- cdata
| ---
| - cdata
diff --git a/test/box/gh-4513-netbox-self-and-connect-interchangeable.test.lua b/test/box/gh-4513-netbox-self-and-connect-interchangeable.test.lua
index d05144c9e03f6108c5eb1eba198f724909f8e07c..9f04738a8497729e7ed572eb4c2bf1ec2fa0e560 100644
--- a/test/box/gh-4513-netbox-self-and-connect-interchangeable.test.lua
+++ b/test/box/gh-4513-netbox-self-and-connect-interchangeable.test.lua
@@ -13,7 +13,6 @@ end
--
-- netbox:self and netbox:connect should work interchangeably
--
-type(nb:eval('return box.tuple.new{1}')) -- table
type(nb:eval('return box.error.new(1, "test error")')) -- cdata
type(nb:eval('return box.NULL')) -- cdata
diff --git a/test/box/net.box_iproto_id.result b/test/box/net.box_iproto_id.result
index d4ca898f44c53a02282aa93b3f688ec915c456bd..c2ce4d07ffb65068e2c2e2d2d08da1f2f229db3f 100644
--- a/test/box/net.box_iproto_id.result
+++ b/test/box/net.box_iproto_id.result
@@ -22,10 +22,12 @@ c.peer_protocol_features
| - transactions: true
| watchers: true
| error_extension: true
- | streams: true
| pagination: true
| space_and_index_names: true
+ | streams: true
| watch_once: true
+ | dml_tuple_extension: true
+ | call_ret_tuple_extension: true
| ...
c:close()
| ---
@@ -52,10 +54,12 @@ c.peer_protocol_features
| - transactions: false
| watchers: false
| error_extension: false
- | streams: false
| pagination: false
| space_and_index_names: false
+ | streams: false
| watch_once: false
+ | dml_tuple_extension: false
+ | call_ret_tuple_extension: false
| ...
errinj.set('ERRINJ_IPROTO_DISABLE_ID', false)
| ---
@@ -103,10 +107,12 @@ c.peer_protocol_features
| - transactions: true
| watchers: true
| error_extension: true
- | streams: true
| pagination: true
| space_and_index_names: true
+ | streams: true
| watch_once: true
+ | dml_tuple_extension: true
+ | call_ret_tuple_extension: true
| ...
c:close()
| ---
@@ -159,10 +165,12 @@ c.peer_protocol_features
| - transactions: false
| watchers: true
| error_extension: true
- | streams: true
| pagination: true
| space_and_index_names: true
+ | streams: true
| watch_once: true
+ | dml_tuple_extension: true
+ | call_ret_tuple_extension: true
| ...
c:close()
| ---
@@ -188,10 +196,12 @@ c.peer_protocol_features
| - transactions: true
| watchers: true
| error_extension: true
- | streams: true
| pagination: true
| space_and_index_names: true
+ | streams: true
| watch_once: true
+ | dml_tuple_extension: true
+ | call_ret_tuple_extension: true
| ...
c:close()
| ---
diff --git a/test/box/net.box_msgpack_gh-2195.result b/test/box/net.box_msgpack_gh-2195.result
index 665e1c25c8a7872acdb9122e2a5c87a109289ca3..c22fe56e4ff9e3f0e0b03b9feaab13b255ca8a8a 100644
--- a/test/box/net.box_msgpack_gh-2195.result
+++ b/test/box/net.box_msgpack_gh-2195.result
@@ -265,36 +265,36 @@ function echo(...) return ... end
...
c:call("echo", {1, 2, 3}, {buffer = ibuf})
---
-- 10
+- 12
...
result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos)
---
...
result
---
-- {48: [1, 2, 3]}
+- {96: {}, 48: [1, 2, 3]}
...
c:call("echo", {}, {buffer = ibuf})
---
-- 7
+- 9
...
result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos)
---
...
result
---
-- {48: []}
+- {96: {}, 48: []}
...
c:call("echo", nil, {buffer = ibuf})
---
-- 7
+- 9
...
result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos)
---
...
result
---
-- {48: []}
+- {96: {}, 48: []}
...
-- call + skip_header
c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true})
@@ -333,36 +333,36 @@ result
-- eval
c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf})
---
-- 7
+- 9
...
result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos)
---
...
result
---
-- {48: []}
+- {96: {}, 48: []}
...
c:eval("echo(...)", {}, {buffer = ibuf})
---
-- 7
+- 9
...
result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos)
---
...
result
---
-- {48: []}
+- {96: {}, 48: []}
...
c:eval("echo(...)", nil, {buffer = ibuf})
---
-- 7
+- 9
...
result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos)
---
...
result
---
-- {48: []}
+- {96: {}, 48: []}
...
-- eval + skip_header
c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf, skip_header = true})
diff --git a/test/box/net.box_raw_response_gh-3107.result b/test/box/net.box_raw_response_gh-3107.result
index 1208341d572833910f4c6fabd56014ff63aefe07..7e9ffd53b79eca3fa215540e0bc82c0752aee3bf 100644
--- a/test/box/net.box_raw_response_gh-3107.result
+++ b/test/box/net.box_raw_response_gh-3107.result
@@ -103,14 +103,14 @@ finalize_long()
...
future:wait_result(100)
---
-- 10
+- 12
...
result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos)
---
...
result
---
-- {48: [1, 2, 3]}
+- {96: {}, 48: [1, 2, 3]}
...
box.schema.func.drop('long_function')
---
diff --git a/test/box/push.result b/test/box/push.result
index e3e4d1253e4362fae87d4248ff1071e69e62ddb9..9654b78ab50cb7be343c4b471eb9f7cbaf0d8ddd 100644
--- a/test/box/push.result
+++ b/test/box/push.result
@@ -242,7 +242,7 @@ resp_len = c:call('do_pushes', {300}, {on_push = table.insert, on_push_ctx = mes
...
resp_len
---
-- 10
+- 12
...
messages
---
@@ -274,7 +274,7 @@ r, _ = msgpack.decode_unchecked(ibuf.rpos)
...
r
---
-- {48: [300]}
+- {96: {}, 48: [300]}
...
--
-- Test error in __serialize.