diff --git a/src/box/alter.cc b/src/box/alter.cc index f1c434ba0f3a7b60fc8def6d814b6267db087baa..a936e7f2923d0d35820c0f472f97dd437260d296 100644 --- a/src/box/alter.cc +++ b/src/box/alter.cc @@ -494,11 +494,14 @@ space_def_new_from_tuple(struct tuple *tuple, uint32_t errcode, BOX_SPACE_FIELD_FORMAT, MP_ARRAY); if (format == NULL) return NULL; + const char *format_ptr = format; struct field_def *fields = NULL; uint32_t field_count; RegionGuard region_guard(&fiber()->gc); - if (field_def_array_decode(&format, &fields, &field_count, region) != 0) + if (field_def_array_decode(&format_ptr, &fields, &field_count, + region, false) != 0) return NULL; + size_t format_len = format_ptr - format; if (exact_field_count != 0 && exact_field_count < field_count) { diag_set(ClientError, errcode, tt_cstr(name, name_len), @@ -531,7 +534,7 @@ space_def_new_from_tuple(struct tuple *tuple, uint32_t errcode, struct space_def *def = space_def_new(id, uid, exact_field_count, name, name_len, engine_name, engine_name_len, &opts, fields, - field_count); + field_count, format, format_len); if (def == NULL) return NULL; auto def_guard = make_scoped_guard([=] { space_def_delete(def); }); diff --git a/src/box/field_def.c b/src/box/field_def.c index 39c0984ba95bf13dcc77afe506e529adce77111a..6b14c009204b1961304fd13bb70627052360dcf3 100644 --- a/src/box/field_def.c +++ b/src/box/field_def.c @@ -39,6 +39,7 @@ #include "tt_uuid.h" #include "tt_static.h" #include "tuple_constraint_def.h" +#include "tuple_dictionary.h" #include "tuple_format.h" #include "salad/grp_alloc.h" #include "small/region.h" @@ -208,6 +209,11 @@ static const struct opt_def field_def_reg[] = { OPT_END, }; +static const struct opt_def field_def_reg_names_only[] = { + OPT_DEF("name", OPT_STRPTR, struct field_def, name), + OPT_END, +}; + const struct field_def field_def_default = { .type = FIELD_TYPE_ANY, .name = NULL, @@ -313,10 +319,11 @@ field_def_parse_foreign_key(const char **data, void *opts, * @param data MessagePack map to decode. * @param fieldno Field number to decode. Used in error messages. * @param region Region to allocate field name. + * @param names_only Only decode 'name' field, ignore the rest. */ static int field_def_decode(struct field_def *field, const char **data, - uint32_t fieldno, struct region *region) + uint32_t fieldno, struct region *region, bool names_only) { if (mp_typeof(**data) != MP_MAP) { field_def_error(fieldno, "expected a map"); @@ -334,7 +341,9 @@ field_def_decode(struct field_def *field, const char **data, } uint32_t key_len; const char *key = mp_decode_str(data, &key_len); - if (opts_parse_key(field, field_def_reg, key, key_len, data, + const struct opt_def *reg = + names_only ? field_def_reg_names_only : field_def_reg; + if (opts_parse_key(field, reg, key, key_len, data, region, true) != 0) { field_def_error(fieldno, diag_last_error(diag_get())->errmsg); @@ -394,7 +403,8 @@ field_def_decode(struct field_def *field, const char **data, int field_def_array_decode(const char **data, struct field_def **fields, - uint32_t *field_count, struct region *region) + uint32_t *field_count, struct region *region, + bool names_only) { assert(mp_typeof(**data) == MP_ARRAY); uint32_t count = mp_decode_array(data); @@ -413,7 +423,8 @@ field_def_array_decode(const char **data, struct field_def **fields, return -1; } for (uint32_t i = 0; i < count; ++i) { - if (field_def_decode(®ion_defs[i], data, i, region) != 0) + if (field_def_decode(®ion_defs[i], data, i, region, + names_only) != 0) return -1; } *fields = region_defs; diff --git a/src/box/field_def.h b/src/box/field_def.h index d26bf3d2d9009b7df290f7c7e8d1ba0546205c36..efb5da7ee369f9b2d3fb6226c2b0fcf8d94454b4 100644 --- a/src/box/field_def.h +++ b/src/box/field_def.h @@ -213,11 +213,13 @@ action_is_nullable(enum on_conflict_action nullable_action) * @param[out] fields Array of fields. * @param[out] field_count Length of a result array. * @param region Region to allocate result array. + * @param names_only Only decode 'name' options, ignore the rest. * @retval Error code. */ int field_def_array_decode(const char **data, struct field_def **fields, - uint32_t *field_count, struct region *region); + uint32_t *field_count, struct region *region, + bool names_only); /** * Duplicates array of fields using malloc. Never fails. diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua index b677bfc747cf6fa61233d9eb009e8dc1ad570612..40cbfb46911777c0838a4f557a58476ca1f761c4 100644 --- a/src/box/lua/net_box.lua +++ b/src/box/lua/net_box.lua @@ -948,7 +948,10 @@ function remote_methods:_install_schema(schema_version, spaces, indices, s.temporary = false s.is_sync = false s._format = format - s._format_cdata = box.internal.tuple_format.new(format) + -- We pass `names_only=true` because format clauses received from IPROTO + -- can be incompatible with, for instance, the data types known on the + -- client. + s._format_cdata = box.internal.tuple_format.new(format, true) s.connection = self if #space > 5 then local opts = space[6] diff --git a/src/box/lua/tuple_format.c b/src/box/lua/tuple_format.c index 57c01d0a840b0f23fcaede7f1b1b88124afdfd24..089928ae5bd8f4fee46a2df246f165c706dbf325 100644 --- a/src/box/lua/tuple_format.c +++ b/src/box/lua/tuple_format.c @@ -9,8 +9,11 @@ #include "box/tuple.h" #include "box/tuple_format.h" +#include "lua/msgpack.h" #include "lua/utils.h" +#include "mpstream/mpstream.h" + static const char *tuple_format_typename = "box.tuple.format"; struct tuple_format * @@ -43,47 +46,39 @@ luaT_push_tuple_format(struct lua_State *L, struct tuple_format *format) } /* - * Create a tuple format using a format clause with an external source (IPROTO): - * all format clause fields except for 'name' are ignored. + * Creates a new tuple format from a format clause (can be omitted). The format + * clause is a Lua table (the same as the one passed to `format` + * method of space objects): it is encoded into MsgPack to reuse existing + * field definition decoding (see also `space_def_new_from_tuple`). Throws a Lua + * exception on failure. + * + * In some cases (formats received over IPROTO or formats for read views) we + * only need to get the 'name' field options and ignore the rest, hence the + * `names_only` flag is provided. */ static int lbox_tuple_format_new(struct lua_State *L) { int top = lua_gettop(L); - if (top == 0) - return luaT_push_tuple_format(L, tuple_format_runtime); - assert(top == 1 && lua_istable(L, 1)); - uint32_t count = lua_objlen(L, 1); + (void)top; + assert((1 <= top && 2 >= top) && lua_istable(L, 1)); struct region *region = &fiber()->gc; size_t region_svp = region_used(region); - struct field_def *fields = xregion_alloc_array(region, - struct field_def, count); - for (uint32_t i = 0; i < count; ++i) { - size_t len; - fields[i] = field_def_default; - lua_pushinteger(L, i + 1); - lua_gettable(L, 1); - lua_pushstring(L, "name"); - lua_gettable(L, -2); - assert(!lua_isnil(L, -1)); - const char *name = lua_tolstring(L, -1, &len); - fields[i].name = (char *)xregion_alloc(region, len + 1); - memcpy(fields[i].name, name, len); - fields[i].name[len] = '\0'; - lua_pop(L, 1); - lua_pop(L, 1); + struct mpstream stream; + mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb, + luamp_error, L); + if (luamp_encode(L, luaL_msgpack_default, &stream, 1) != 0) { + region_truncate(region, region_svp); + return luaT_error(L); } - struct tuple_dictionary *dict = tuple_dictionary_new(fields, count); + mpstream_flush(&stream); + size_t format_data_len = region_used(region) - region_svp; + const char *format_data = xregion_join(region, format_data_len); + bool names_only = lua_toboolean(L, 2); + struct tuple_format *format = + runtime_tuple_format_new(format_data, format_data_len, + names_only); region_truncate(region, region_svp); - if (dict == NULL) - return luaT_error(L); - struct tuple_format *format = runtime_tuple_format_new(dict); - /* - * Since dictionary reference counter is 1 from the - * beginning and after creation of the tuple_format - * increases by one, we must decrease it once. - */ - tuple_dictionary_unref(dict); if (format == NULL) return luaT_error(L); return luaT_push_tuple_format(L, format); diff --git a/src/box/schema.cc b/src/box/schema.cc index 501f5f97423a49891d7b82d4bac8ec5d009f9fff..bb1588b2f6dd29f83943a308e461b26f2da1ef80 100644 --- a/src/box/schema.cc +++ b/src/box/schema.cc @@ -203,7 +203,8 @@ sc_space_new(uint32_t id, const char *name, make_scoped_guard([=] { index_def_delete(index_def); }); struct space_def *def = space_def_new_xc(id, ADMIN, 0, name, strlen(name), "memtx", - strlen("memtx"), &space_opts_default, NULL, 0); + strlen("memtx"), &space_opts_default, NULL, 0, + NULL, 0); auto def_guard = make_scoped_guard([=] { space_def_delete(def); }); struct rlist key_list; rlist_create(&key_list); @@ -412,7 +413,7 @@ schema_init(void) struct space_def *def; def = space_def_new_xc(BOX_VINYL_DEFERRED_DELETE_ID, ADMIN, 0, name, strlen(name), engine, - strlen(engine), &opts, NULL, 0); + strlen(engine), &opts, NULL, 0, NULL, 0); auto def_guard = make_scoped_guard([=] { space_def_delete(def); }); diff --git a/src/box/space_def.c b/src/box/space_def.c index 8f9b1c8df2d702ec08f6086e904d29541790456d..df6b594ef848bc4ed74676ba2e74cae059d9f081 100644 --- a/src/box/space_def.c +++ b/src/box/space_def.c @@ -99,7 +99,8 @@ space_tuple_format_new(struct tuple_format_vtab *vtab, void *engine, def->exact_field_count, def->dict, def->opts.is_temporary, def->opts.is_ephemeral, def->opts.constraint_def, - def->opts.constraint_count); + def->opts.constraint_count, def->format_data, + def->format_data_len); } /** @@ -130,6 +131,15 @@ space_def_dup(const struct space_def *src) ret->fields = field_def_array_dup(src->fields, src->field_count); tuple_dictionary_ref(ret->dict); space_def_dup_opts(ret, &src->opts); + if (src->format_data != NULL) { + ret->format_data = xmalloc(src->format_data_len); + memcpy(ret->format_data, src->format_data, + src->format_data_len); + ret->format_data_len = src->format_data_len; + } else { + ret->format_data = NULL; + ret->format_data_len = 0; + } return ret; } @@ -138,7 +148,8 @@ space_def_new(uint32_t id, uint32_t uid, uint32_t exact_field_count, const char *name, uint32_t name_len, const char *engine_name, uint32_t engine_len, const struct space_opts *opts, const struct field_def *fields, - uint32_t field_count) + uint32_t field_count, const char *format_data, + size_t format_data_len) { size_t size = sizeof(struct space_def) + name_len + 1; struct space_def *def = xmalloc(size); @@ -161,6 +172,14 @@ space_def_new(uint32_t id, uint32_t uid, uint32_t exact_field_count, def->field_count = field_count; def->fields = field_def_array_dup(fields, field_count); space_def_dup_opts(def, opts); + if (format_data != NULL) { + def->format_data = xmalloc(format_data_len); + memcpy(def->format_data, format_data, format_data_len); + def->format_data_len = format_data_len; + } else { + def->format_data = NULL; + def->format_data_len = 0; + } return def; } @@ -179,7 +198,8 @@ space_def_new_ephemeral(uint32_t exact_field_count, struct field_def *fields) "ephemeral", strlen("ephemeral"), "memtx", strlen("memtx"), - &opts, fields, field_count); + &opts, fields, field_count, + NULL, 0); return space_def; } @@ -191,6 +211,7 @@ space_def_delete(struct space_def *def) free(def->opts.sql); free(def->opts.constraint_def); space_upgrade_def_delete(def->opts.upgrade_def); + free(def->format_data); TRASH(def); free(def); } diff --git a/src/box/space_def.h b/src/box/space_def.h index 5db266e2ad96dc1e40a4a338cbca928e0c054d10..08a5d967f12e8a7f19e35b1760336ad238fce8ed 100644 --- a/src/box/space_def.h +++ b/src/box/space_def.h @@ -130,6 +130,13 @@ struct space_def { /** Number of SQL views which refer to this space. */ uint32_t view_ref_count; struct space_opts opts; + /** + * Encoding of original (i.e., user-provided) format clause to MsgPack, + * allocated via malloc. + */ + char *format_data; + /** Length of MsgPack encoded format clause. */ + size_t format_data_len; char name[0]; }; @@ -171,7 +178,8 @@ space_def_new(uint32_t id, uint32_t uid, uint32_t exact_field_count, const char *name, uint32_t name_len, const char *engine_name, uint32_t engine_len, const struct space_opts *opts, const struct field_def *fields, - uint32_t field_count); + uint32_t field_count, const char *format_data, + size_t format_data_len); /** * Create a new ephemeral space definition. @@ -209,11 +217,13 @@ space_def_new_xc(uint32_t id, uint32_t uid, uint32_t exact_field_count, const char *name, uint32_t name_len, const char *engine_name, uint32_t engine_len, const struct space_opts *opts, const struct field_def *fields, - uint32_t field_count) + uint32_t field_count, const char *format_data, + size_t format_data_len) { struct space_def *ret = space_def_new(id, uid, exact_field_count, name, name_len, engine_name, engine_len, - opts, fields, field_count); + opts, fields, field_count, + format_data, format_data_len); if (ret == NULL) diag_raise(); return ret; diff --git a/src/box/tuple.c b/src/box/tuple.c index b52e0c0e310a7466af5be703e0b1126e975d956d..844e24402b8490a9a7ef2f58ccb8369dc5adf4ee 100644 --- a/src/box/tuple.c +++ b/src/box/tuple.c @@ -285,14 +285,48 @@ tuple_bigref_tuple_count() } struct tuple_format * -runtime_tuple_format_new(struct tuple_dictionary *dict) +runtime_tuple_format_new(const char *format_data, size_t format_data_len, + bool names_only) { - return tuple_format_new(&tuple_format_runtime_vtab, /*engine=*/NULL, - /*keys=*/NULL, /*key_count=*/0, - /*space_field_count=*/NULL, - /*exact_field_count=*/0, 0, dict, - /*is_temporary=*/false, /*is_reusable=*/true, - /*contraint_def=*/NULL, /*constraint_count=*/0); + struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); + const char *p = format_data; + struct field_def *fields = NULL; + uint32_t field_count = 0; + if (format_data != NULL && + field_def_array_decode(&p, &fields, &field_count, region, + /*names_only=*/names_only) != 0) + goto error; + struct tuple_dictionary *dict = + tuple_dictionary_new(fields, field_count); + if (dict == NULL) + goto error; + if (names_only) { + fields = NULL; + field_count = 0; + } + struct tuple_format *format = + tuple_format_new(/*vtab=*/&tuple_format_runtime_vtab, + /*engine=*/NULL, /*keys=*/NULL, + /*key_count=*/0, /*space_fields=*/fields, + /*space_field_count=*/field_count, + /*exact_field_count=*/0, /*dict=*/dict, + /*is_temporary=*/false, /*is_reusable=*/true, + /*constraint_def=*/NULL, + /*constraint_count=*/0, + /*format_data=*/format_data, + /*format_data_len=*/format_data_len); + region_truncate(region, region_svp); + /* + * Since dictionary reference counter is 1 from the + * beginning and after creation of the tuple_format + * increases by one, we must decrease it once. + */ + tuple_dictionary_unref(dict); + return format; +error: + region_truncate(region, region_svp); + return NULL; } int @@ -303,7 +337,8 @@ tuple_init(field_name_hash_f hash) /* * Create a format for runtime tuples */ - tuple_format_runtime = runtime_tuple_format_new(/*dict=*/NULL); + tuple_format_runtime = runtime_tuple_format_new(NULL, 0, + /*names_only=*/false); if (tuple_format_runtime == NULL) return -1; diff --git a/src/box/tuple.h b/src/box/tuple.h index 518507d603b4bc76ea8edf7d88a34db56d7448d7..0c1783e7309e8bf9bfe265306d32d458a797693e 100644 --- a/src/box/tuple.h +++ b/src/box/tuple.h @@ -78,13 +78,19 @@ tuple_arena_destroy(struct slab_arena *arena); /** * Creates a new format for standalone tuples. * Tuples created with the new format are allocated from the runtime arena. - * In contrast to the preallocated tuple_format_runtime, which has no field - * names, the new format uses the provided field name dictionary. + * In contrast to the preallocated tuple_format_runtime, which has no + * information about fields, the new format uses the field definitions from the + * Msgpack encoded format clause. + * + * In some cases (formats received over IPROTO or formats for read views) we + * only need to get the 'name' field options and ignore the rest, hence the + * `names_only` flag is provided. * * On success, returns the new format. On error, returns NULL and sets diag. */ struct tuple_format * -runtime_tuple_format_new(struct tuple_dictionary *dict); +runtime_tuple_format_new(const char *format_data, size_t format_data_len, + bool names_only); /** \cond public */ diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c index c91a7d5b2dfbdc5b72db67cfb1e32f884f17dfb8..a4ee36c7ba956aad581b7467b9dbce77a7164fe2 100644 --- a/src/box/tuple_format.c +++ b/src/box/tuple_format.c @@ -776,6 +776,7 @@ tuple_format_destroy(struct tuple_format *format) tuple_format_destroy_fields(format); tuple_dictionary_unref(format->dict); free(format->constraint); + free(format->data); } /** @@ -838,7 +839,8 @@ tuple_format_new(struct tuple_format_vtab *vtab, void *engine, uint32_t space_field_count, uint32_t exact_field_count, struct tuple_dictionary *dict, bool is_temporary, bool is_reusable, struct tuple_constraint_def *constraint_def, - uint32_t constraint_count) + uint32_t constraint_count, const char *format_data, + size_t format_data_len) { struct tuple_format *format = tuple_format_alloc(keys, key_count, space_field_count, dict); @@ -855,6 +857,14 @@ tuple_format_new(struct tuple_format_vtab *vtab, void *engine, format->is_compressed = false; format->exact_field_count = exact_field_count; format->epoch = ++formats_epoch; + if (format_data != NULL) { + format->data = xmalloc(format_data_len); + memcpy(format->data, format_data, format_data_len); + format->data_len = format_data_len; + } else { + format->data = NULL; + format->data_len = 0; + } if (tuple_format_create(format, keys, key_count, space_fields, space_field_count, constraint_def, constraint_count) < 0) diff --git a/src/box/tuple_format.h b/src/box/tuple_format.h index a46d8c9ad2dae6b0b801a282dbeadfa9d261f7b3..4b8feafe98a6faa75c114bbcff0b5f265f051d2a 100644 --- a/src/box/tuple_format.h +++ b/src/box/tuple_format.h @@ -298,6 +298,13 @@ struct tuple_format { struct tuple_constraint *constraint; /** Number of constraints. */ uint32_t constraint_count; + /** + * Encoding of original (i.e., user-provided) format clause to MsgPack, + * allocated via malloc. + */ + char *data; + /** Length of MsgPack encoding. */ + size_t data_len; }; /** @@ -391,6 +398,8 @@ tuple_format_unref(struct tuple_format *format) * @param is_reusable Set if format may be reused. * @param constraint_def - Array of constraint definitions. * @param constraint_count - Number of constraints above. + * @param format_data Original format clause encoded to Msgpack (may be NULL). + * @param format_data_len Length of MsgPack encoded format clause (may be 0). * * @retval not NULL Tuple format. * @retval NULL Memory error. @@ -402,7 +411,8 @@ tuple_format_new(struct tuple_format_vtab *vtab, void *engine, uint32_t space_field_count, uint32_t exact_field_count, struct tuple_dictionary *dict, bool is_temporary, bool is_reusable, struct tuple_constraint_def *constraint_def, - uint32_t constraint_count); + uint32_t constraint_count, const char *format_data, + size_t format_data_len); /** * Check, if tuple @a format is compatible with @a key_def. @@ -422,7 +432,8 @@ simple_tuple_format_new(struct tuple_format_vtab *vtab, void *engine, struct key_def * const *keys, uint16_t key_count) { return tuple_format_new(vtab, engine, keys, key_count, - NULL, 0, 0, NULL, false, false, NULL, 0); + NULL, 0, 0, NULL, false, false, NULL, 0, NULL, + 0); } /** diff --git a/test/box/misc.result b/test/box/misc.result index 5d9d26859178bcead324134b2062a976bd0def0c..68fc4622dc19a472298713103fc9751fa218b1fe 100644 --- a/test/box/misc.result +++ b/test/box/misc.result @@ -995,10 +995,6 @@ s:drop() -- We do not have a way to check cdata in Lua, but there should be -- no errors. -- --- Without argument it is equivalent to new_tuple_format({}) -tuple_format = box.internal.tuple_format.new() ---- -... -- If no type that type == "any": format = {} --- @@ -1009,18 +1005,18 @@ format[1] = {} format[1].name = 'aaa' --- ... -tuple_format = box.internal.tuple_format.new(format) +tuple_format = box.internal.tuple_format.new(format, true) --- ... -- Function space:format() without arguments returns valid format: -tuple_format = box.internal.tuple_format.new(box.space._space:format()) +tuple_format = box.internal.tuple_format.new(box.space._space:format(), true) --- ... -- Check is_nullable option fo field format[1].is_nullable = true --- ... -tuple_format = box.internal.tuple_format.new(format) +tuple_format = box.internal.tuple_format.new(format, true) --- ... -- diff --git a/test/box/misc.test.lua b/test/box/misc.test.lua index e1d9ff919102790786126bf2745cbca43ec653b7..37c004b4a9b9cbdb695b5152b218df49c36c1a10 100644 --- a/test/box/misc.test.lua +++ b/test/box/misc.test.lua @@ -325,21 +325,18 @@ s:drop() -- no errors. -- --- Without argument it is equivalent to new_tuple_format({}) -tuple_format = box.internal.tuple_format.new() - -- If no type that type == "any": format = {} format[1] = {} format[1].name = 'aaa' -tuple_format = box.internal.tuple_format.new(format) +tuple_format = box.internal.tuple_format.new(format, true) -- Function space:format() without arguments returns valid format: -tuple_format = box.internal.tuple_format.new(box.space._space:format()) +tuple_format = box.internal.tuple_format.new(box.space._space:format(), true) -- Check is_nullable option fo field format[1].is_nullable = true -tuple_format = box.internal.tuple_format.new(format) +tuple_format = box.internal.tuple_format.new(format, true) -- -- Test that calling _say using FFI w/ null filepointer doesn't