diff --git a/changelogs/unreleased/gh-8633-tuple-formats-in-iproto-requests.md b/changelogs/unreleased/gh-8633-tuple-formats-in-iproto-requests.md new file mode 100644 index 0000000000000000000000000000000000000000..3dd02a0e67142fcdd5c4757ae23794b9139f0854 --- /dev/null +++ b/changelogs/unreleased/gh-8633-tuple-formats-in-iproto-requests.md @@ -0,0 +1,6 @@ +## feature/box + +* Added support for sending tuple formats in IPROTO call and eval request + arguments. `box_tuple_extension` backward compatibility option can be used to + disable receiving tuple formats in IPROTO call and eval request arguments + (gh-8633). diff --git a/src/box/call.c b/src/box/call.c index 361bfd772df3b6f6af22f97628bddbfe360eee25..c0bee2ba99ea9ee940661b5febaf9a1f56356620 100644 --- a/src/box/call.c +++ b/src/box/call.c @@ -44,7 +44,7 @@ #include "small/obuf.h" #include "small/rlist.h" #include "tt_static.h" -#include "core/mp_ctx.h" +#include "box/mp_box_ctx.h" struct rlist box_on_call = RLIST_HEAD_INITIALIZER(box_on_call); @@ -209,24 +209,39 @@ box_process_call(struct call_request *request, struct port *port) const char *name = request->name; assert(name != NULL); uint32_t name_len = mp_decode_strl(&name); + struct mp_box_ctx ctx; + if (mp_box_ctx_create(&ctx, NULL, request->tuple_formats) != 0) + return -1; struct port args; - port_msgpack_create(&args, request->args, - request->args_end - request->args); + port_msgpack_create_with_ctx(&args, request->args, + request->args_end - request->args, + (struct mp_ctx *)&ctx); struct func *func = func_by_name(name, name_len); + int rc = 0; if (func != NULL) { - if (func_access_check(func) != 0) - return -1; + if (func_access_check(func) != 0) { + rc = -1; + goto cleanup; + } box_run_on_call(IPROTO_CALL, name, name_len, request->args); - if (func_call_no_access_check(func, &args, port) != 0) - return -1; + if (func_call_no_access_check(func, &args, port) != 0) { + rc = -1; + goto cleanup; + } } else { - if (access_check_call(name, name_len) != 0) - return -1; + if (access_check_call(name, name_len) != 0) { + rc = -1; + goto cleanup; + } box_run_on_call(IPROTO_CALL, name, name_len, request->args); - if (box_lua_call(name, name_len, &args, port) != 0) - return -1; + if (box_lua_call(name, name_len, &args, port) != 0) { + rc = -1; + goto cleanup; + } } - return 0; +cleanup: + port_msgpack_destroy(&args); + return rc; } int @@ -236,13 +251,17 @@ box_process_eval(struct call_request *request, struct port *port) /* Check permissions */ if (access_check_eval() != 0) return -1; + struct mp_box_ctx ctx; + if (mp_box_ctx_create(&ctx, NULL, request->tuple_formats) != 0) + return -1; struct port args; - port_msgpack_create(&args, request->args, - request->args_end - request->args); + port_msgpack_create_with_ctx(&args, request->args, + request->args_end - request->args, + (struct mp_ctx *)&ctx); const char *expr = request->expr; uint32_t expr_len = mp_decode_strl(&expr); box_run_on_call(IPROTO_EVAL, expr, expr_len, request->args); - if (box_lua_eval(expr, expr_len, &args, port) != 0) - return -1; - return 0; + int rc = box_lua_eval(expr, expr_len, &args, port); + port_msgpack_destroy(&args); + return rc; } diff --git a/src/box/iproto_features.c b/src/box/iproto_features.c index 4aced8bb5c2a903805be00d42e24bdc0e9c13075..5ee3ad166880d2f7e7bd23e2c52e3151ae8f2688 100644 --- a/src/box/iproto_features.c +++ b/src/box/iproto_features.c @@ -83,4 +83,6 @@ iproto_features_init(void) IPROTO_FEATURE_DML_TUPLE_EXTENSION); iproto_features_set(&IPROTO_CURRENT_FEATURES, IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION); + iproto_features_set(&IPROTO_CURRENT_FEATURES, + IPROTO_FEATURE_CALL_ARG_TUPLE_EXTENSION); } diff --git a/src/box/iproto_features.h b/src/box/iproto_features.h index 1d50a94ab10c30d9bc33dccc39cce170547ae76c..8dcca268378fdef0abd799438923cb3a29536623 100644 --- a/src/box/iproto_features.h +++ b/src/box/iproto_features.h @@ -71,6 +71,12 @@ extern "C" { * tuple formats are sent in IPROTO_TUPLE_FORMATS field. */ \ _(CALL_RET_TUPLE_EXTENSION, 8) \ + /** + * Tuple format in call and eval request arguments support: + * Tuples in IPROTO_TUPLE request field are encoded as MP_TUPLE and + * tuple formats are received in IPROTO_TUPLE_FORMATS field. + */ \ + _(CALL_ARG_TUPLE_EXTENSION, 9) \ #define IPROTO_FEATURE_MEMBER(s, v) IPROTO_FEATURE_ ## s = v, @@ -95,7 +101,7 @@ struct iproto_features { * `box.iproto.protocol_version` needs to be updated correspondingly. */ enum { - IPROTO_CURRENT_VERSION = 6, + IPROTO_CURRENT_VERSION = 7, }; /** diff --git a/src/box/lua/net_box.c b/src/box/lua/net_box.c index faf2dbb69309012466d0f74ace3cff0ef8842e8b..9d67425e4ebd4a720761c76bdfd04a39296c23d0 100644 --- a/src/box/lua/net_box.c +++ b/src/box/lua/net_box.c @@ -82,7 +82,7 @@ enum { /** * IPROTO protocol version supported by the netbox connector. */ - NETBOX_IPROTO_VERSION = 6, + NETBOX_IPROTO_VERSION = 7, }; /** @@ -362,6 +362,11 @@ struct netbox_method_encode_ctx { uint64_t sync; /* Current transport's IPROTO stream identifier. */ uint64_t stream_id; + /* + * Whether box tuples should be encoded to MsgPack as MP_TUPLE + * extension. + */ + bool box_tuple_arg_as_ext; }; static const char netbox_transport_typename[] = "net.box.transport"; @@ -778,6 +783,28 @@ netbox_encode_select_all(struct lua_State *L, struct ibuf *ibuf, uint64_t sync, netbox_end_encode(&stream, svp); } +/* + * Encode the arguments of call or eval netbox methods. + */ +static int +netbox_encode_call_or_eval_args(lua_State *L, int idx, struct mpstream *stream, + bool box_tuple_arg_as_ext) +{ + mpstream_encode_uint(stream, IPROTO_TUPLE); + struct mp_box_ctx ctx; + mp_box_ctx_create(&ctx, NULL, NULL); + if (luamp_encode_tuple_with_ctx(L, cfg, stream, idx, + box_tuple_arg_as_ext ? + (struct mp_ctx *)&ctx : NULL) != 0) { + mp_ctx_destroy((struct mp_ctx *)&ctx); + return -1; + } + mpstream_encode_uint(stream, IPROTO_TUPLE_FORMATS); + tuple_format_map_to_mpstream(&ctx.tuple_format_map, stream); + mp_ctx_destroy((struct mp_ctx *)&ctx); + return 0; +} + /** * Encode an `IPROTO_CALL` request and write it to the provided MsgPack stream. */ @@ -788,7 +815,7 @@ netbox_encode_call(lua_State *L, int idx, struct netbox_method_encode_ctx *ctx) size_t svp = netbox_begin_encode(ctx->stream, ctx->sync, IPROTO_CALL, ctx->stream_id); - mpstream_encode_map(ctx->stream, 2); + mpstream_encode_map(ctx->stream, 3); /* encode proc name */ size_t name_len; @@ -796,9 +823,8 @@ netbox_encode_call(lua_State *L, int idx, struct netbox_method_encode_ctx *ctx) mpstream_encode_uint(ctx->stream, IPROTO_FUNCTION_NAME); mpstream_encode_strn(ctx->stream, name, name_len); - /* encode args */ - mpstream_encode_uint(ctx->stream, IPROTO_TUPLE); - if (luamp_encode_tuple(L, cfg, ctx->stream, idx + 1) != 0) + if (netbox_encode_call_or_eval_args(L, idx + 1, ctx->stream, + ctx->box_tuple_arg_as_ext) != 0) return -1; netbox_end_encode(ctx->stream, svp); @@ -815,7 +841,7 @@ netbox_encode_eval(lua_State *L, int idx, struct netbox_method_encode_ctx *ctx) size_t svp = netbox_begin_encode(ctx->stream, ctx->sync, IPROTO_EVAL, ctx->stream_id); - mpstream_encode_map(ctx->stream, 2); + mpstream_encode_map(ctx->stream, 3); /* encode expr */ size_t expr_len; @@ -823,9 +849,8 @@ netbox_encode_eval(lua_State *L, int idx, struct netbox_method_encode_ctx *ctx) mpstream_encode_uint(ctx->stream, IPROTO_EXPR); mpstream_encode_strn(ctx->stream, expr, expr_len); - /* encode args */ - mpstream_encode_uint(ctx->stream, IPROTO_TUPLE); - if (luamp_encode_tuple(L, cfg, ctx->stream, idx + 1) != 0) + if (netbox_encode_call_or_eval_args(L, idx + 1, ctx->stream, + ctx->box_tuple_arg_as_ext) != 0) return -1; netbox_end_encode(ctx->stream, svp); @@ -1444,7 +1469,8 @@ netbox_encode_inject(struct lua_State *L, int idx, */ static int netbox_encode_method(struct lua_State *L, int idx, enum netbox_method method, - struct ibuf *ibuf, uint64_t sync, uint64_t stream_id) + struct ibuf *ibuf, uint64_t sync, uint64_t stream_id, + bool box_tuple_arg_as_ext) { typedef int (*method_encoder_f)(struct lua_State *L, int idx, struct netbox_method_encode_ctx *ctx); @@ -1479,6 +1505,7 @@ netbox_encode_method(struct lua_State *L, int idx, enum netbox_method method, .stream = &stream, .stream_id = stream_id, .sync = sync, + .box_tuple_arg_as_ext = box_tuple_arg_as_ext, }; return method_encoder[method](L, idx, &ctx); } @@ -2441,8 +2468,11 @@ luaT_netbox_transport_make_request(struct lua_State *L, int idx, enum netbox_method method = lua_tointeger(L, arg++); assert(method < netbox_method_MAX); size_t svp = ibuf_used(&transport->send_buf); + bool box_tuple_arg_as_ext = + iproto_features_test(&transport->features, + IPROTO_FEATURE_CALL_ARG_TUPLE_EXTENSION); if (netbox_encode_method(L, arg++, method, &transport->send_buf, sync, - stream_id) != 0) { + stream_id, box_tuple_arg_as_ext) != 0) { ibuf_truncate(&transport->send_buf, svp); return -1; } @@ -3212,6 +3242,8 @@ luaopen_net_box(struct lua_State *L) IPROTO_FEATURE_DML_TUPLE_EXTENSION); iproto_features_set(&NETBOX_IPROTO_FEATURES, IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION); + iproto_features_set(&NETBOX_IPROTO_FEATURES, + IPROTO_FEATURE_CALL_ARG_TUPLE_EXTENSION); lua_pushcfunction(L, luaT_netbox_request_iterator_next); luaT_netbox_request_iterator_next_ref = luaL_ref(L, LUA_REGISTRYINDEX); diff --git a/src/box/xrow.c b/src/box/xrow.c index f2babcecebf5a0f4f24dcbf0480c9420304047b6..7f0bcd6e0a60cbf1948f5d6666ee75eabf61699b 100644 --- a/src/box/xrow.c +++ b/src/box/xrow.c @@ -492,9 +492,12 @@ iproto_reply_id(struct obuf *out, const char *auth_type, uint32_t auth_type_len = strlen(auth_type); unsigned version = IPROTO_CURRENT_VERSION; struct iproto_features features = IPROTO_CURRENT_FEATURES; - if (!box_tuple_extension) + if (!box_tuple_extension) { iproto_features_clear(&features, IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION); + iproto_features_clear(&features, + IPROTO_FEATURE_CALL_ARG_TUPLE_EXTENSION); + } #ifndef NDEBUG struct errinj *errinj; errinj = errinj(ERRINJ_IPROTO_SET_VERSION, ERRINJ_INT); @@ -1516,6 +1519,11 @@ xrow_decode_call(const struct xrow_header *row, struct call_request *request) request->args = value; request->args_end = data; break; + case IPROTO_TUPLE_FORMATS: + if (mp_typeof(*value) != MP_MAP) + goto error; + request->tuple_formats = value; + request->tuple_formats_end = data; default: continue; /* unknown key */ } diff --git a/src/box/xrow.h b/src/box/xrow.h index 683f83bf3102a76c888e018d2ce178e9f3345fb3..13a1e9a390bb3e9d62cfced97180a860f326458d 100644 --- a/src/box/xrow.h +++ b/src/box/xrow.h @@ -412,6 +412,10 @@ struct call_request { /** CALL/EVAL parameters. MessagePack Array. */ const char *args; const char *args_end; + /** Tuple formats of CALL/EVAL parameters. MessagePack Map. */ + const char *tuple_formats; + /** End of tuple formats of CALL/EVAL parameters. */ + const char *tuple_formats_end; }; /** diff --git a/src/lua/compat.lua b/src/lua/compat.lua index b8429b64f7ceaed47c9f00577e631eb77f0f657e..e739e2a398377c3263a84e3b23aeb5270a350e17 100644 --- a/src/lua/compat.lua +++ b/src/lua/compat.lua @@ -97,7 +97,8 @@ https://tarantool.io/compat/c_func_iproto_multireturn ]] local BOX_TUPLE_EXTENSION_BRIEF = [[ -Controls IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION feature bit. +Controls IPROTO_FEATURE_CALL_RET_TUPLE_EXTENSION and +IPROTO_FEATURE_CALL_ARG_TUPLE_EXTENSION feature bits. https://tarantool.io/compat/box_tuple_extension ]] diff --git a/test/box-luatest/gh_7894_export_iproto_constants_and_features_test.lua b/test/box-luatest/gh_7894_export_iproto_constants_and_features_test.lua index 1c9fb190789357ad757e95e34a204cc3483a62ef..55b56cfe8865f297cd79d26a992d008414e37b05 100644 --- a/test/box-luatest/gh_7894_export_iproto_constants_and_features_test.lua +++ b/test/box-luatest/gh_7894_export_iproto_constants_and_features_test.lua @@ -165,7 +165,7 @@ local reference_table = { }, -- `IPROTO_CURRENT_VERSION` constant - protocol_version = 6, + protocol_version = 7, -- `feature_id` enumeration protocol_features = { @@ -178,6 +178,7 @@ local reference_table = { watch_once = true, dml_tuple_extension = true, call_ret_tuple_extension = true, + call_arg_tuple_extension = true, }, feature = { streams = 0, @@ -189,6 +190,7 @@ local reference_table = { watch_once = 6, dml_tuple_extension = 7, call_ret_tuple_extension = 8, + call_arg_tuple_extension = 9, }, } diff --git a/test/box-luatest/gh_8633_tuple_formats_in_iproto_requests_test.lua b/test/box-luatest/gh_8633_tuple_formats_in_iproto_requests_test.lua new file mode 100644 index 0000000000000000000000000000000000000000..4a50975545f9f5091399ccb349d69ecdfb408382 --- /dev/null +++ b/test/box-luatest/gh_8633_tuple_formats_in_iproto_requests_test.lua @@ -0,0 +1,145 @@ +local msgpack = require('msgpack') +local netbox = require('net.box') +local server = require('luatest.server') +local t = require('luatest') + +local g = t.group() + +local tuple_return_str = [[local t1, t2 = ... + return box.tuple.is(t1) and box.tuple.is(t2) and + t1:tomap{names_only = true}.field == 0 and + next(t2:tomap{names_only = true}) == nil ]] + +local table_return_str = [[local t1, t2 = ... + return type(t1) == 'table' and type(t2) == 'table' ]] +g.before_all(function(cg) + cg.server = server:new() + cg.server:start() + cg.server:exec(function(tuple_return_str, table_return_str) + local f_tuple = function(t1, t2) + return box.tuple.is(t1) and box.tuple.is(t2) and + t1:tomap{name_only = true}.field == 0 and + next(t2:tomap{names_only = true}) == nil + end + rawset(_G, 'g_tuple', f_tuple) + rawset(_G, 'f_tuple', f_tuple) + box.schema.func.create('f_tuple') + local tuple_func_str = "function(...) " .. tuple_return_str .. "end" + box.schema.func.create('p_tuple', {body = tuple_func_str}) + + local f_table = function(t1, t2) + return type(t1) == 'table' and type(t2) == 'table' + end + rawset(_G, 'g_table', f_table) + rawset(_G, 'f_table', f_table) + box.schema.func.create('f_table') + local table_func_str = "function(...) " .. table_return_str .. "end" + box.schema.func.create('p_table', {body = table_func_str}) + end, {tuple_return_str, table_return_str}) +end) + +g.after_all(function(cg) + cg.server:drop() +end) + +-- Checks that tuple formats in IPROTO call and eval requests arguments work +-- correctly. +g.test_net_box_formats_in_iproto_call_eval_request_args = function(cg) + local c = netbox:connect(cg.server.net_box_uri) + + local t1 = box.tuple.new({0}, {format = {{'field'}}}) + local t2 = box.tuple.new{0} + t.assert(c:call('g_tuple', {t1, t2})) + t.assert(c:call('f_tuple', {t1, t2})) + t.assert(c:call('p_tuple', {t1, t2})) + t.assert(c:eval(tuple_return_str, {t1, t2})) +end + +local function inject_call_or_eval(c, request, func_or_expr, args, formats) + local header = msgpack.encode({ + [box.iproto.key.REQUEST_TYPE] = box.iproto.type[request], + [box.iproto.key.SYNC] = c:_next_sync(), + }) + local body = string.fromhex('8' .. (2 + (formats ~= nil and 1 or 0))) .. + msgpack.encode(request == 'CALL' and box.iproto.key.FUNCTION_NAME or + request == 'EVAL' and box.iproto.key.EXPR) .. + msgpack.encode(func_or_expr) .. + msgpack.encode(box.iproto.key.TUPLE) .. args + if formats ~= nil then + body = body .. msgpack.encode(box.iproto.key.TUPLE_FORMATS) .. formats + end + local size = msgpack.encode(#header + #body) + local request = size .. header .. body + return c:_inject(request) +end + +-- Checks that errors with tuple formats in IPROTO call and eval requests +-- arguments are handled correctly. +g.test_net_box_formats_in_iproto_call_eval_request_args_errors = function(cg) + local c = netbox:connect(cg.server.net_box_uri) + + local raw_data = { + {"Invalid MsgPack - packet body", + {string.fromhex('91d607') .. msgpack.encode(box.NULL)}}, + {"Invalid MsgPack - packet body", + {string.fromhex('91d407cc')}}, + {"Invalid MsgPack - packet body", + {string.fromhex('91d407') .. msgpack.encode(1)}}, + {"Invalid MsgPack - packet body", + {string.fromhex('91d507') .. msgpack.encode(1) .. msgpack.encode(1)}}, + {"Invalid MsgPack - packet body", + {string.fromhex('91d607') .. msgpack.encode(1) .. + msgpack.encode({1})}}, + {"Invalid MsgPack - packet body", + {string.fromhex('91d607') .. msgpack.encode(1) .. + string.fromhex('92') .. msgpack.encode(1)}}, + {"Can not parse a tuple from MsgPack", + {string.fromhex('91d607') .. msgpack.encode(1) .. + msgpack.encode({1, 1})}}, + {"Invalid MsgPack - packet body", + {string.fromhex('91d607') .. msgpack.encode(1) .. + msgpack.encode({1, 1}), msgpack.encode({[2] = box.NULL})}}, + {"Invalid MsgPack - packet body", + {string.fromhex('91d607') .. msgpack.encode(1) .. + msgpack.encode({1, 1}), msgpack.encode{[1] = {box.NULL}}}}, + {"Space field 'field' is duplicate", + {string.fromhex('91d607') .. msgpack.encode(1) .. + msgpack.encode({1, 1}), string.fromhex('810192') .. + msgpack.encode({name = 'field'}) .. + msgpack.encode({name = 'field'})}}, + {"Can not parse a tuple from MsgPack", + {string.fromhex('91d607') .. msgpack.encode(1) .. + msgpack.encode({1, 1}), string.fromhex('810290')}}, + } + local remote_calls = { + ['g_tuple'] = 'CALL', + ['f_tuple'] = 'CALL', + ['p_tuple'] = 'CALL', + [tuple_return_str] = 'EVAL', + } + for _, raw_datum in ipairs(raw_data) do + for func_or_expr, request in pairs(remote_calls) do + t.assert_error_msg_content_equals(raw_datum[1], function() + inject_call_or_eval(c, request, func_or_expr, + unpack(raw_datum[2])) + end) + end + end +end + +-- Checks that `box_tuple_extension` backward compatibility option works +-- correctly. +g.test_box_tuple_extension_compat_option = function(cg) + cg.server:exec(function() + require('compat').box_tuple_extension = 'old' + end) + + local c = netbox:connect(cg.server.net_box_uri) + + local t1 = box.tuple.new({0}, {format = {{'field'}}}) + local t2 = box.tuple.new{0} + t.assert(c:call('g_table', {t1, t2})) + t.assert(c:call('f_table', {t1, t2})) + t.assert(c:call('p_table', {t1, t2})) + t.assert(c:eval(table_return_str, {t1, t2})) +end diff --git a/test/box-py/iproto.result b/test/box-py/iproto.result index 3b9d4bf713b263d351f90e323c202a872e572ae7..d8224563cdb4ba7d2ba66b8444470a0d66163579 100644 --- a/test/box-py/iproto.result +++ b/test/box-py/iproto.result @@ -210,11 +210,11 @@ Invalid MsgPack - request body # Invalid auth_type Invalid MsgPack - request body # Empty request body -version=6, features=[0, 1, 2, 3, 4, 5, 6, 7, 8], auth_type=chap-sha1 +version=7, features=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], auth_type=chap-sha1 # Unknown version and features -version=6, features=[0, 1, 2, 3, 4, 5, 6, 7, 8], auth_type=chap-sha1 +version=7, features=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], auth_type=chap-sha1 # Unknown request key -version=6, features=[0, 1, 2, 3, 4, 5, 6, 7, 8], auth_type=chap-sha1 +version=7, features=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], auth_type=chap-sha1 # # gh-6257 Watchers diff --git a/test/box/net.box_iproto_id.result b/test/box/net.box_iproto_id.result index c2ce4d07ffb65068e2c2e2d2d08da1f2f229db3f..1f613a5150e8b86f85eb4c83c29d7a80e493f085 100644 --- a/test/box/net.box_iproto_id.result +++ b/test/box/net.box_iproto_id.result @@ -15,13 +15,14 @@ c = net.connect(box.cfg.listen) | ... c.peer_protocol_version | --- - | - 6 + | - 7 | ... c.peer_protocol_features | --- | - transactions: true | watchers: true | error_extension: true + | call_arg_tuple_extension: true | pagination: true | space_and_index_names: true | streams: true @@ -54,6 +55,7 @@ c.peer_protocol_features | - transactions: false | watchers: false | error_extension: false + | call_arg_tuple_extension: false | pagination: false | space_and_index_names: false | streams: false @@ -107,6 +109,7 @@ c.peer_protocol_features | - transactions: true | watchers: true | error_extension: true + | call_arg_tuple_extension: true | pagination: true | space_and_index_names: true | streams: true @@ -158,13 +161,14 @@ c.error -- error | ... c.peer_protocol_version | --- - | - 6 + | - 7 | ... c.peer_protocol_features | --- | - transactions: false | watchers: true | error_extension: true + | call_arg_tuple_extension: true | pagination: true | space_and_index_names: true | streams: true @@ -189,13 +193,14 @@ c.error -- error | ... c.peer_protocol_version | --- - | - 6 + | - 7 | ... c.peer_protocol_features | --- | - transactions: true | watchers: true | error_extension: true + | call_arg_tuple_extension: true | pagination: true | space_and_index_names: true | streams: true