From 519bc82e4fb4250b7d337a03680cdef355555037 Mon Sep 17 00:00:00 2001 From: Vladislav Shpilevoy <v.shpilevoy@tarantool.org> Date: Wed, 13 Sep 2017 22:03:51 +0300 Subject: [PATCH] Parse and validate space formats Closes #2754 --- src/box/alter.cc | 207 ++++++++++++++++++++++++-- src/box/engine.h | 4 +- src/box/errcode.h | 1 + src/box/field_def.c | 29 +++- src/box/field_def.h | 11 +- src/box/key_def.cc | 8 +- src/box/lua/upgrade.lua | 34 +++-- src/box/memtx_engine.cc | 6 +- src/box/memtx_engine.h | 3 +- src/box/schema.cc | 2 +- src/box/space.cc | 6 +- src/box/space.h | 6 +- src/box/sysview_engine.cc | 5 +- src/box/sysview_engine.h | 3 +- src/box/tuple.c | 4 +- src/box/tuple_format.c | 84 ++++++++--- src/box/tuple_format.h | 13 +- src/box/vinyl.c | 3 +- src/box/vinyl_engine.cc | 7 +- src/box/vinyl_engine.h | 3 +- src/box/vy_index.c | 17 ++- test/box/access.result | 12 +- test/box/access.test.lua | 7 +- test/box/alter.result | 250 +++++++++++++++++++++++++++++++- test/box/alter.test.lua | 89 +++++++++++- test/box/misc.result | 1 + test/box/rtree_misc.result | 42 +----- test/box/rtree_misc.test.lua | 12 +- test/box/temp_spaces.result | 76 ---------- test/box/temp_spaces.test.lua | 27 ---- test/unit/vy_iterators_helper.c | 11 +- test/unit/vy_mem.c | 2 +- test/unit/vy_point_iterator.c | 4 +- test/xlog/legacy.result | 177 ++++++++++++++++++++++ test/xlog/legacy.test.lua | 68 +++++++++ 35 files changed, 981 insertions(+), 253 deletions(-) create mode 100644 test/xlog/legacy.result create mode 100644 test/xlog/legacy.test.lua diff --git a/src/box/alter.cc b/src/box/alter.cc index 88b785a3c6..1fabaa69ef 100644 --- a/src/box/alter.cc +++ b/src/box/alter.cc @@ -447,11 +447,140 @@ space_opts_decode(struct space_opts *opts, const char *data) } } +/** + * Get an array of named fields of @a format. Necessary to create + * a new fields array after drop of an index. In such a case some + * fields from the space format could be deleted together with + * a deleted index, and the fields array must be rebuilt from + * the named and indexed fields. + * @param format Format to get named fields. + * @param[out] out_count Count of named fields. + * @param region Region to allocate a result array. + * + * @retval Array of named fields. + */ +static struct field_def * +tuple_format_named_fields(const struct tuple_format *format, + uint32_t *out_count, struct region *region) +{ + assert(format != NULL); + uint32_t count = 0; + for (uint32_t i = 0; i < format->field_count; ++i) { + if (format->fields[i].name != NULL) + ++count; + } + *out_count = count; + if (count == 0) + return NULL; + size_t size = sizeof(struct field_def) * count; + struct field_def *ret = + (struct field_def *) region_alloc_xc(region, size); + for (uint32_t i = 0; i < format->field_count; ++i) { + if (format->fields[i].name != NULL) { + /* + * No need to copy names on a region, + * because format->fields names remains + * valid until ret is already copied into + * a new tuple_format. + */ + memcpy(&ret[i], &format->fields[i], sizeof(ret[i])); + } + } + return ret; +} + +/** + * Decode field definition from MessagePack map. Format: + * {name: <string>, type: <string>}. Type is optional. + * @param[out] field Field to decode to. + * @param data MessagePack map to decode. + * @param space_name Name of a space, from which the field is got. + * Used in error messages. + * @param errcode Error code to use for client errors. Either + * create or modify space errors. + * @param fieldno Field number to decode. Used in error messages. + * @param region Region to allocate field name. + */ +static void +field_def_decode(struct field_def *field, const char **data, + const char *space_name, uint32_t errcode, uint32_t fieldno, + struct region *region) +{ + if (mp_typeof(**data) != MP_MAP) { + tnt_raise(ClientError, errcode, space_name, + tt_sprintf("field %d is not map", + fieldno + TUPLE_INDEX_BASE)); + } + int count = mp_decode_map(data); + *field = field_def_default; + for (int i = 0; i < count; ++i) { + if (mp_typeof(**data) != MP_STR) { + tnt_raise(ClientError, errcode, space_name, + tt_sprintf("field %d format is not map"\ + " with string keys", + fieldno + TUPLE_INDEX_BASE)); + } + uint32_t key_len; + const char *key = mp_decode_str(data, &key_len); + opts_parse_key(field, field_def_reg, key, key_len, data, + ER_WRONG_SPACE_FORMAT, + fieldno + TUPLE_INDEX_BASE, region); + } + if (field->name == NULL) { + tnt_raise(ClientError, errcode, space_name, + tt_sprintf("field %d name is not specified", + fieldno + TUPLE_INDEX_BASE)); + } + if (strlen(field->name) > BOX_NAME_MAX) { + tnt_raise(ClientError, errcode, space_name, + tt_sprintf("field %d name is too long", + fieldno + TUPLE_INDEX_BASE)); + } + if (field->type == field_type_MAX) { + tnt_raise(ClientError, errcode, space_name, + tt_sprintf("field %d has unknown field type", + fieldno + TUPLE_INDEX_BASE)); + } +} + +/** + * Decode MessagePack array of fields. + * @param data MessagePack array of fields. + * @param[out] out_count Length of a result array. + * @param space_name Space name to use in error messages. + * @param errcode Errcode for client errors. + * @param region Region to allocate result array. + * + * @retval Array of fields. + */ +static struct field_def * +space_format_decode(const char *data, uint32_t *out_count, + const char *space_name, uint32_t errcode, + struct region *region) +{ + /* Type is checked by _space format. */ + assert(mp_typeof(*data) == MP_ARRAY); + uint32_t count = mp_decode_array(&data); + *out_count = count; + if (count == 0) + return NULL; + size_t size = count * sizeof(struct field_def); + struct field_def *region_defs = + (struct field_def *) region_alloc_xc(region, size); + for (uint32_t i = 0; i < count; ++i) { + field_def_decode(®ion_defs[i], &data, space_name, errcode, + i, region); + } + return region_defs; +} + /** * Fill space_def structure from struct tuple. */ static struct space_def * -space_def_new_from_tuple(struct tuple *tuple, uint32_t errcode) +space_def_new_from_tuple(struct tuple *tuple, uint32_t errcode, + struct field_def **fields, uint32_t *field_count, + struct region *region) { uint32_t name_len; const char *name = @@ -492,7 +621,30 @@ space_def_new_from_tuple(struct tuple *tuple, uint32_t errcode) memcpy(def->engine_name, engine_name, name_len); def->engine_name[name_len] = 0; identifier_check_xc(def->engine_name); - space_opts_decode(&def->opts, tuple_field(tuple, BOX_SPACE_FIELD_OPTS)); + const char *space_opts; + if (dd_version_id >= version_id(1, 7, 6)) { + /* Check space opts. */ + space_opts = + tuple_field_with_type_xc(tuple, BOX_SPACE_FIELD_OPTS, + MP_MAP); + /* Check space format */ + const char *format = + tuple_field_with_type_xc(tuple, BOX_SPACE_FIELD_FORMAT, + MP_ARRAY); + *fields = space_format_decode(format, field_count, def->name, + errcode, region); + if (def->exact_field_count != 0 && + def->exact_field_count < *field_count) { + tnt_raise(ClientError, errcode, def->name, + "exact_field_count must be either 0 or >= "\ + "formatted field count"); + } + } else { + *fields = NULL; + *field_count = 0; + space_opts = tuple_field(tuple, BOX_SPACE_FIELD_OPTS); + } + space_opts_decode(&def->opts, space_opts); Engine *engine = engine_find(def->engine_name); engine->checkSpaceDef(def); access_check_ddl(def->uid, SC_SPACE); @@ -556,16 +708,23 @@ struct alter_space { * substantially. */ struct key_def *pk_def; + /** New space format. */ + struct field_def *new_fields; + /** Length of @a new_fields. */ + uint32_t field_count; }; -struct alter_space * -alter_space_new(struct space *old_space) +static struct alter_space * +alter_space_new(struct space *old_space, struct field_def *fields, + uint32_t field_count) { struct alter_space *alter = region_calloc_object_xc(&fiber()->gc, struct alter_space); rlist_create(&alter->ops); alter->old_space = old_space; alter->space_def = space_def_dup_xc(alter->old_space->def); + alter->new_fields = fields; + alter->field_count = field_count; return alter; } @@ -701,7 +860,8 @@ alter_space_do(struct txn *txn, struct alter_space *alter) * Create a new (empty) space for the new definition. * Sic: the triggers are not moved over yet. */ - alter->new_space = space_new(alter->space_def, &alter->key_list); + alter->new_space = space_new(alter->space_def, &alter->key_list, + alter->new_fields, alter->field_count); /* * Copy the replace function, the new space is at the same recovery * phase as the old one. This hack is especially necessary for @@ -1227,6 +1387,7 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event) struct txn_stmt *stmt = txn_current_stmt(txn); struct tuple *old_tuple = stmt->old_tuple; struct tuple *new_tuple = stmt->new_tuple; + struct region *region = &fiber()->gc; /* * Things to keep in mind: * - old_tuple is set only in case of UPDATE. For INSERT @@ -1245,12 +1406,16 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event) BOX_SPACE_FIELD_ID); struct space *old_space = space_by_id(old_id); if (new_tuple != NULL && old_space == NULL) { /* INSERT */ + struct field_def *fields; + uint32_t field_count; struct space_def *def = - space_def_new_from_tuple(new_tuple, ER_CREATE_SPACE); + space_def_new_from_tuple(new_tuple, ER_CREATE_SPACE, + &fields, &field_count, region); auto def_guard = make_scoped_guard([=] { space_def_delete(def); }); RLIST_HEAD(empty_list); - struct space *space = space_new(def, &empty_list); + struct space *space = space_new(def, &empty_list, fields, + field_count); /** * The new space must be inserted in the space * cache right away to achieve linearisable @@ -1295,8 +1460,11 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event) txn_on_rollback(txn, on_rollback); } else { /* UPDATE, REPLACE */ assert(old_space != NULL && new_tuple != NULL); + struct field_def *fields; + uint32_t field_count; struct space_def *def = - space_def_new_from_tuple(new_tuple, ER_ALTER_SPACE); + space_def_new_from_tuple(new_tuple, ER_ALTER_SPACE, + &fields, &field_count, region); auto def_guard = make_scoped_guard([=] { space_def_delete(def); }); if (def->id != space_id(old_space)) @@ -1329,7 +1497,8 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event) * Allow change of space properties, but do it * in WAL-error-safe mode. */ - struct alter_space *alter = alter_space_new(old_space); + struct alter_space *alter = alter_space_new(old_space, fields, + field_count); auto alter_guard = make_scoped_guard([=] {alter_space_delete(alter);}); (void) new ModifySpace(alter, def); @@ -1422,7 +1591,16 @@ on_replace_dd_index(struct trigger * /* trigger */, void *event) "can not add a secondary key before primary"); } - struct alter_space *alter = alter_space_new(old_space); + struct alter_space *alter; + struct tuple_format *format = old_space->handler->format(); + if (format != NULL) { + uint32_t count; + struct field_def *fields = + tuple_format_named_fields(format, &count, &fiber()->gc); + alter = alter_space_new(old_space, fields, count); + } else { + alter = alter_space_new(old_space, NULL, 0); + } auto scoped_guard = make_scoped_guard([=] { alter_space_delete(alter); }); @@ -1591,7 +1769,14 @@ on_replace_dd_truncate(struct trigger * /* trigger */, void *event) /* Create an empty copy of the old space. */ struct rlist key_list; space_dump_def(old_space, &key_list); - struct space *new_space = space_new(old_space->def, &key_list); + struct space *new_space; + struct tuple_format *format = old_space->handler->format(); + if (format != NULL) { + new_space = space_new(old_space->def, &key_list, + format->fields, format->field_count); + } else { + new_space = space_new(old_space->def, &key_list, NULL, 0); + } new_space->truncate_count = truncate_count; auto space_guard = make_scoped_guard([=] { space_delete(new_space); }); diff --git a/src/box/engine.h b/src/box/engine.h index f082b6e1bd..e2ff8ddd58 100644 --- a/src/box/engine.h +++ b/src/box/engine.h @@ -45,6 +45,7 @@ engine_backup_cb(const char *path, void *arg); #if defined(__cplusplus) struct Handler; +struct field_def; /** Engine instance */ class Engine { @@ -58,7 +59,8 @@ class Engine { virtual void init(); /** Create a new engine instance for a space. */ virtual Handler *createSpace(struct rlist *key_list, - uint32_t index_count, + struct field_def *fields, + uint32_t field_count, uint32_t index_count, uint32_t exact_field_count) = 0; /** * Write statements stored in checkpoint @vclock to @stream. diff --git a/src/box/errcode.h b/src/box/errcode.h index f52a29ad1f..90f15015b3 100644 --- a/src/box/errcode.h +++ b/src/box/errcode.h @@ -193,6 +193,7 @@ struct errcode_record { /*138 */_(ER_LOAD_MODULE, "Failed to dynamically load module '%.*s': %s") \ /*139 */_(ER_VINYL_MAX_TUPLE_SIZE, "Failed to allocate %u bytes for tuple: tuple is too large. Check 'vinyl_max_tuple_size' configuration option.") \ /*140 */_(ER_WRONG_DD_VERSION, "Wrong _schema version: expected 'major.minor[.patch]'") \ + /*141 */_(ER_WRONG_SPACE_FORMAT, "Wrong space format (field %u): %s") \ /* * !IMPORTANT! Please follow instructions at start of the file diff --git a/src/box/field_def.c b/src/box/field_def.c index fa39d5a86f..3fc19918f1 100644 --- a/src/box/field_def.c +++ b/src/box/field_def.c @@ -43,18 +43,37 @@ const char *field_type_strs[] = { /* [FIELD_TYPE_MAP] = */ "map", }; +static int64_t +field_type_by_name_wrapper(const char *str, uint32_t len) +{ + return field_type_by_name(str, len); +} + +const struct opt_def field_def_reg[] = { + OPT_DEF_ENUM("type", field_type, struct field_def, type, + field_type_by_name_wrapper), + OPT_DEF("name", OPT_STRPTR, struct field_def, name), + OPT_END, +}; + +const struct field_def field_def_default = { + .type = FIELD_TYPE_ANY, + .name = NULL, +}; + enum field_type -field_type_by_name(const char *name) +field_type_by_name(const char *name, size_t len) { - enum field_type field_type = STR2ENUM(field_type, name); + enum field_type field_type = strnindex(field_type_strs, name, len, + field_type_MAX); if (field_type != field_type_MAX) return field_type; /* 'num' and 'str' in _index are deprecated since Tarantool 1.7 */ - if (strcasecmp(name, "num") == 0) + if (strncasecmp(name, "num", len) == 0) return FIELD_TYPE_UNSIGNED; - else if (strcasecmp(name, "str") == 0) + else if (strncasecmp(name, "str", len) == 0) return FIELD_TYPE_STRING; - else if (strcmp(name, "*") == 0) + else if (len == 1 && name[0] == '*') return FIELD_TYPE_ANY; return field_type_MAX; } diff --git a/src/box/field_def.h b/src/box/field_def.h index 8c20b00eea..17a336607d 100644 --- a/src/box/field_def.h +++ b/src/box/field_def.h @@ -32,6 +32,7 @@ */ #include <stdint.h> +#include "opt_def.h" #if defined(__cplusplus) extern "C" { @@ -60,8 +61,14 @@ enum field_type { extern const char *field_type_strs[]; +/** + * Get field type by name + */ enum field_type -field_type_by_name(const char *name); +field_type_by_name(const char *name, size_t len); + +extern const struct opt_def field_def_reg[]; +extern const struct field_def field_def_default; /** * @brief Field definition @@ -76,6 +83,8 @@ struct field_def { * then UNKNOWN is stored for it. */ enum field_type type; + /** 0-terminated field name. */ + char *name; /** * Offset slot in field map in tuple. Normally tuple * stores field map - offsets of all fields participating diff --git a/src/box/key_def.cc b/src/box/key_def.cc index 6132ac5a8e..d638cfa6ed 100644 --- a/src/box/key_def.cc +++ b/src/box/key_def.cc @@ -205,7 +205,6 @@ key_def_encode_parts(char *data, const struct key_def *key_def) int key_def_decode_parts(struct key_def *key_def, const char **data) { - char buf[FIELD_TYPE_NAME_MAX]; for (uint32_t i = 0; i < key_def->part_count; i++) { if (mp_typeof(**data) != MP_ARRAY) { diag_set(ClientError, ER_WRONG_INDEX_PARTS, @@ -238,8 +237,7 @@ key_def_decode_parts(struct key_def *key_def, const char **data) const char *str = mp_decode_str(data, &len); for (uint32_t j = 2; j < item_count; j++) mp_next(data); - snprintf(buf, sizeof(buf), "%.*s", len, str); - enum field_type field_type = field_type_by_name(buf); + enum field_type field_type = field_type_by_name(str, len); if (field_type == field_type_MAX) { diag_set(ClientError, ER_WRONG_INDEX_PARTS, "unknown field type"); @@ -253,13 +251,11 @@ key_def_decode_parts(struct key_def *key_def, const char **data) int key_def_decode_parts_165(struct key_def *key_def, const char **data) { - char buf[FIELD_TYPE_NAME_MAX]; for (uint32_t i = 0; i < key_def->part_count; i++) { uint32_t field_no = (uint32_t) mp_decode_uint(data); uint32_t len; const char *str = mp_decode_str(data, &len); - snprintf(buf, sizeof(buf), "%.*s", len, str); - enum field_type field_type = field_type_by_name(buf); + enum field_type field_type = field_type_by_name(str, len); if (field_type == field_type_MAX) { diag_set(ClientError, ER_WRONG_INDEX_PARTS, "unknown field type"); diff --git a/src/box/lua/upgrade.lua b/src/box/lua/upgrade.lua index eb118504f7..44beef9b5c 100644 --- a/src/box/lua/upgrade.lua +++ b/src/box/lua/upgrade.lua @@ -636,10 +636,7 @@ end -------------------------------------------------------------------------------- -local function upgrade(options) - options = options or {} - setmetatable(options, {__index = {auto = false}}) - +local function get_version() local version = box.space._schema:get{'version'} if version == nil then error('Missing "version" in box.space._schema') @@ -648,7 +645,14 @@ local function upgrade(options) local minor = version[3] local patch = version[4] or 0 - version = mkversion(major, minor, patch) + return mkversion(major, minor, patch) +end + +local function upgrade(options) + options = options or {} + setmetatable(options, {__index = {auto = false}}) + + local version = get_version() local handlers = { {version = mkversion(1, 6, 8), func = upgrade_to_1_6_8, auto = false}, @@ -680,11 +684,23 @@ end local function bootstrap() set_system_triggers(false) + local version = get_version() + + -- Initial() creates a spaces with 1.6.0 format, but 1.7.6 + -- checks space formats, that fails initial(). It is because + -- bootstrap() is called after box.cfg{}. If box.cfg{} is run + -- on 1.7.6, then spaces in the cache contains new 1.7.6 + -- formats (gh-2754). Spaces in the cache are not updated on + -- erase(), because system triggers are turned off. + if version ~= mkversion(1, 7, 6) then + -- erase current schema + erase() + -- insert initial schema + initial() + else + log.info('version is 1.7.6, do not reset schema to initial') + end - -- erase current schema - erase() - -- insert initial schema - initial() -- upgrade schema to the latest version upgrade() diff --git a/src/box/memtx_engine.cc b/src/box/memtx_engine.cc index 2432a45fcb..24f6d67167 100644 --- a/src/box/memtx_engine.cc +++ b/src/box/memtx_engine.cc @@ -283,7 +283,8 @@ MemtxEngine::endRecovery() } Handler *MemtxEngine::createSpace(struct rlist *key_list, - uint32_t index_count, + struct field_def *fields, + uint32_t field_count, uint32_t index_count, uint32_t exact_field_count) { struct index_def *index_def; @@ -296,7 +297,8 @@ Handler *MemtxEngine::createSpace(struct rlist *key_list, keys[key_no++] = index_def->key_def; struct tuple_format *format = - tuple_format_new(&memtx_tuple_format_vtab, keys, index_count, 0); + tuple_format_new(&memtx_tuple_format_vtab, keys, index_count, 0, + fields, field_count); if (format == NULL) diag_raise(); tuple_format_ref(format); diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h index 52cc144908..8ff4442866 100644 --- a/src/box/memtx_engine.h +++ b/src/box/memtx_engine.h @@ -76,7 +76,8 @@ struct MemtxEngine: public Engine { uint32_t objsize_min, float alloc_factor); ~MemtxEngine(); virtual Handler *createSpace(struct rlist *key_list, - uint32_t index_count, + struct field_def *fields, + uint32_t field_count, uint32_t index_count, uint32_t exact_field_count) override; virtual void begin(struct txn *txn) override; virtual void rollbackStatement(struct txn *, diff --git a/src/box/schema.cc b/src/box/schema.cc index 75d3cc8c20..6469451734 100644 --- a/src/box/schema.cc +++ b/src/box/schema.cc @@ -199,7 +199,7 @@ sc_space_new(uint32_t id, const char *name, struct key_def *key_def, struct rlist key_list; rlist_create(&key_list); rlist_add_entry(&key_list, index_def, link); - struct space *space = space_new(def, &key_list); + struct space *space = space_new(def, &key_list, NULL, 0); (void) space_cache_replace(space); if (replace_trigger) trigger_add(&space->on_replace, replace_trigger); diff --git a/src/box/space.cc b/src/box/space.cc index d3e5ac0061..e37b9c1c5d 100644 --- a/src/box/space.cc +++ b/src/box/space.cc @@ -78,7 +78,8 @@ space_fill_index_map(struct space *space) } struct space * -space_new(struct space_def *def, struct rlist *key_list) +space_new(struct space_def *def, struct rlist *key_list, + struct field_def *fields, uint32_t field_count) { uint32_t index_id_max = 0; uint32_t index_count = 0; @@ -110,7 +111,8 @@ space_new(struct space_def *def, struct rlist *key_list) index_count * sizeof(Index *)); Engine *engine = engine_find(def->engine_name); /* init space engine instance */ - space->handler = engine->createSpace(key_list, index_count, + space->handler = engine->createSpace(key_list, fields, field_count, + index_count, def->exact_field_count); rlist_foreach_entry(index_def, key_list, link) { space->index_map[index_def->iid] = diff --git a/src/box/space.h b/src/box/space.h index 5f299f3cd5..ca0a48727f 100644 --- a/src/box/space.h +++ b/src/box/space.h @@ -192,17 +192,21 @@ void space_noop(struct space *space); uint32_t space_size(struct space *space); +struct field_def; /** * Allocate and initialize a space. The space * needs to be loaded before it can be used * (see space->handler->recover()). * @param space_def Space definition. * @param key_list List of index_defs. + * @param fields Array of fields specified in space format. + * @param field_count Length of @a fields. * * @retval Space object. */ struct space * -space_new(struct space_def *space_def, struct rlist *key_list); +space_new(struct space_def *space_def, struct rlist *key_list, + struct field_def *fields, uint32_t field_count); /** Destroy and free a space. */ void diff --git a/src/box/sysview_engine.cc b/src/box/sysview_engine.cc index 0334249ce1..01474f33e5 100644 --- a/src/box/sysview_engine.cc +++ b/src/box/sysview_engine.cc @@ -114,10 +114,13 @@ SysviewEngine::SysviewEngine() } Handler *SysviewEngine::createSpace(struct rlist *key_list, - uint32_t index_count, + struct field_def *fields, + uint32_t field_count, uint32_t index_count, uint32_t exact_field_count) { (void) key_list; + (void) fields; + (void) field_count; (void) index_count; (void) exact_field_count; return new SysviewSpace(this); diff --git a/src/box/sysview_engine.h b/src/box/sysview_engine.h index 118e49d33b..fb5c67050c 100644 --- a/src/box/sysview_engine.h +++ b/src/box/sysview_engine.h @@ -36,7 +36,8 @@ struct SysviewEngine: public Engine { public: SysviewEngine(); virtual Handler *createSpace(struct rlist *key_list, - uint32_t index_count, + struct field_def *fields, + uint32_t field_count, uint32_t index_count, uint32_t exact_field_count) override; }; diff --git a/src/box/tuple.c b/src/box/tuple.c index ed58fcf2d8..40c79b045b 100644 --- a/src/box/tuple.c +++ b/src/box/tuple.c @@ -386,7 +386,7 @@ tuple_init(void) */ RLIST_HEAD(empty_list); tuple_format_runtime = tuple_format_new(&tuple_format_runtime_vtab, - NULL, 0, 0); + NULL, 0, 0, NULL, 0); if (tuple_format_runtime == NULL) return -1; @@ -463,7 +463,7 @@ box_tuple_format_new(struct key_def **keys, uint16_t key_count) { box_tuple_format_t *format = tuple_format_new(&tuple_format_runtime_vtab, - keys, key_count, 0); + keys, key_count, 0, NULL, 0); if (format != NULL) tuple_format_ref(format); return format; diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c index fc6d119182..7779e08cc7 100644 --- a/src/box/tuple_format.c +++ b/src/box/tuple_format.c @@ -36,20 +36,40 @@ static intptr_t recycled_format_ids = FORMAT_ID_NIL; static uint32_t formats_size = 0, formats_capacity = 0; -/** Extract all available type info from keys. */ +/** + * Extract all available type info from keys and field + * definitions. + */ static int tuple_format_create(struct tuple_format *format, struct key_def **keys, - uint16_t key_count) + uint16_t key_count, struct field_def *fields, + uint32_t field_count) { if (format->field_count == 0) { format->field_map_size = 0; return 0; } - - /* There may be fields between indexed fields (gaps). */ - for (uint32_t i = 0; i < format->field_count; i++) { + /* Initialize defined fields */ + char *name_pos = (char *)format + sizeof(struct tuple_format) + + format->field_count * sizeof(struct field_def); + for (uint32_t i = 0; i < field_count; ++i) { + format->fields[i].type = fields[i].type; + format->fields[i].offset_slot = TUPLE_OFFSET_SLOT_NIL; + if (fields[i].name != NULL) { + format->fields[i].name = name_pos; + size_t len = strlen(fields[i].name); + memcpy(name_pos, fields[i].name, len); + name_pos[len] = 0; + name_pos += len + 1; + } else { + format->fields[i].name = NULL; + } + } + /* Initialize remaining fields */ + for (uint32_t i = field_count; i < format->field_count; i++) { format->fields[i].type = FIELD_TYPE_ANY; format->fields[i].offset_slot = TUPLE_OFFSET_SLOT_NIL; + format->fields[i].name = NULL; } int current_slot = 0; @@ -70,9 +90,9 @@ tuple_format_create(struct tuple_format *format, struct key_def **keys, field->type = part->type; } else if (field->type != part->type) { /** - * Check that two different indexes do not - * put contradicting constraints on - * indexed field type. + * Check that there are no + * conflicts between index part + * types and space fields. */ diag_set(ClientError, ER_FIELD_TYPE_MISMATCH, part->fieldno + TUPLE_INDEX_BASE, @@ -153,22 +173,25 @@ tuple_format_deregister(struct tuple_format *format) } static struct tuple_format * -tuple_format_alloc(struct key_def **keys, uint16_t key_count) +tuple_format_alloc(struct key_def **keys, uint16_t key_count, + struct field_def *space_fields, uint32_t space_field_count) { - uint32_t max_fieldno = 0; - + uint32_t field_count = space_field_count; /* find max max field no */ for (uint16_t key_no = 0; key_no < key_count; ++key_no) { struct key_def *key_def = keys[key_no]; struct key_part *part = key_def->parts; struct key_part *pend = part + key_def->part_count; for (; part < pend; part++) - max_fieldno = MAX(max_fieldno, part->fieldno); + field_count = MAX(field_count, part->fieldno + 1); } - uint32_t field_count = key_count > 0 ? max_fieldno + 1 : 0; uint32_t total = sizeof(struct tuple_format) + field_count * sizeof(struct field_def); + for (uint32_t i = 0; i < space_field_count; ++i) { + if (space_fields[i].name != NULL) + total += strlen(space_fields[i].name) + 1; + } struct tuple_format *format = (struct tuple_format *) malloc(total); if (format == NULL) { @@ -193,9 +216,12 @@ tuple_format_delete(struct tuple_format *format) struct tuple_format * tuple_format_new(struct tuple_format_vtab *vtab, struct key_def **keys, - uint16_t key_count, uint16_t extra_size) + uint16_t key_count, uint16_t extra_size, + struct field_def *space_fields, uint32_t space_field_count) { - struct tuple_format *format = tuple_format_alloc(keys, key_count); + struct tuple_format *format = + tuple_format_alloc(keys, key_count, space_fields, + space_field_count); if (format == NULL) return NULL; format->vtab = *vtab; @@ -204,7 +230,8 @@ tuple_format_new(struct tuple_format_vtab *vtab, struct key_def **keys, tuple_format_delete(format); return NULL; } - if (tuple_format_create(format, keys, key_count) < 0) { + if (tuple_format_create(format, keys, key_count, space_fields, + space_field_count) < 0) { tuple_format_delete(format); return NULL; } @@ -221,6 +248,13 @@ tuple_format_eq(const struct tuple_format *a, const struct tuple_format *b) if (a->fields[i].type != b->fields[i].type || a->fields[i].offset_slot != b->fields[i].offset_slot) return false; + if (a->fields[i].name != NULL) { + assert(b->fields[i].name != NULL); + assert(strcmp(a->fields[i].name, + b->fields[i].name) == 0); + } else { + assert(b->fields[i].name == NULL); + } } return true; } @@ -230,14 +264,28 @@ tuple_format_dup(const struct tuple_format *src) { uint32_t total = sizeof(struct tuple_format) + src->field_count * sizeof(struct field_def); + uint32_t name_offset = total; + for (uint32_t i = 0; i < src->field_count; ++i) { + if (src->fields[i].name != NULL) + total += strlen(src->fields[i].name) + 1; + } struct tuple_format *format = (struct tuple_format *) malloc(total); if (format == NULL) { - diag_set(OutOfMemory, sizeof(struct tuple_format), "malloc", - "tuple format"); + diag_set(OutOfMemory, total, "malloc", "tuple format"); return NULL; } memcpy(format, src, total); + char *name_pos = (char *)format + name_offset; + for (uint32_t i = 0; i < src->field_count; ++i) { + if (src->fields[i].name != NULL) { + int len = strlen(src->fields[i].name); + format->fields[i].name = name_pos; + memcpy(name_pos, src->fields[i].name, len); + name_pos[len] = 0; + name_pos += len + 1; + } + } format->id = FORMAT_ID_NIL; format->refs = 0; if (tuple_format_register(format) != 0) { diff --git a/src/box/tuple_format.h b/src/box/tuple_format.h index ac84b28a20..667bc34ff2 100644 --- a/src/box/tuple_format.h +++ b/src/box/tuple_format.h @@ -140,17 +140,20 @@ tuple_format_unref(struct tuple_format *format) /** * Allocate, construct and register a new in-memory tuple format. - * @param vtab Virtual function table for specific engines. - * @param keys Array of key_defs of a space. - * @param key_count The number of keys in @a keys array. - * @param extra_size Extra bytes to reserve in tuples metadata. + * @param vtab Virtual function table for specific engines. + * @param keys Array of key_defs of a space. + * @param key_count The number of keys in @a keys array. + * @param extra_size Extra bytes to reserve in tuples metadata. + * @param space_fields Array of fields, defined in a space format. + * @param space_field_count Length of @a space_fields. * * @retval not NULL Tuple format. * @retval NULL Memory error. */ struct tuple_format * tuple_format_new(struct tuple_format_vtab *vtab, struct key_def **keys, - uint16_t key_count, uint16_t extra_size); + uint16_t key_count, uint16_t extra_size, + struct field_def *space_fields, uint32_t space_field_count); /** * Check that two tuple formats are identical. diff --git a/src/box/vinyl.c b/src/box/vinyl.c index ea124632ca..e9196866c2 100644 --- a/src/box/vinyl.c +++ b/src/box/vinyl.c @@ -4174,7 +4174,8 @@ vy_join_cb(const struct vy_log_record *record, void *arg) if (ctx->format != NULL) tuple_format_unref(ctx->format); ctx->format = tuple_format_new(&vy_tuple_format_vtab, - (struct key_def **)&ctx->key_def, 1, 0); + (struct key_def **)&ctx->key_def, 1, 0, + NULL, 0); if (ctx->format == NULL) return -1; tuple_format_ref(ctx->format); diff --git a/src/box/vinyl_engine.cc b/src/box/vinyl_engine.cc index 60223255e9..2d0a4d8918 100644 --- a/src/box/vinyl_engine.cc +++ b/src/box/vinyl_engine.cc @@ -112,8 +112,8 @@ VinylEngine::endRecovery() } Handler * -VinylEngine::createSpace(struct rlist *key_list, - uint32_t index_count, +VinylEngine::createSpace(struct rlist *key_list, struct field_def *fields, + uint32_t field_count, uint32_t index_count, uint32_t exact_field_count) { struct index_def *index_def; @@ -126,7 +126,8 @@ VinylEngine::createSpace(struct rlist *key_list, keys[key_no++] = index_def->key_def; struct tuple_format *format = - tuple_format_new(&vy_tuple_format_vtab, keys, index_count, 0); + tuple_format_new(&vy_tuple_format_vtab, keys, index_count, 0, + fields, field_count); if (format == NULL) diag_raise(); tuple_format_ref(format); diff --git a/src/box/vinyl_engine.h b/src/box/vinyl_engine.h index 5b80896793..aae83b876d 100644 --- a/src/box/vinyl_engine.h +++ b/src/box/vinyl_engine.h @@ -39,7 +39,8 @@ struct VinylEngine: public Engine { ~VinylEngine(); virtual void init() override; virtual Handler *createSpace(struct rlist *key_list, - uint32_t index_count, + struct field_def *fields, + uint32_t field_count, uint32_t index_count, uint32_t exact_field_count) override; virtual void beginStatement(struct txn *txn) override; virtual void begin(struct txn *txn) override; diff --git a/src/box/vy_index.c b/src/box/vy_index.c index acb052f9f2..1b038eb71d 100644 --- a/src/box/vy_index.c +++ b/src/box/vy_index.c @@ -60,7 +60,7 @@ vy_index_env_create(struct vy_index_env *env, const char *path, void *upsert_thresh_arg) { env->key_format = tuple_format_new(&vy_tuple_format_vtab, - NULL, 0, 0); + NULL, 0, 0, NULL, 0); if (env->key_format == NULL) return -1; tuple_format_ref(env->key_format); @@ -130,8 +130,19 @@ vy_index_new(struct vy_index_env *index_env, struct vy_cache_env *cache_env, index->cmp_def = cmp_def; index->key_def = key_def; - index->disk_format = tuple_format_new(&vy_tuple_format_vtab, - &cmp_def, 1, 0); + if (index_def->iid == 0) { + /* + * Disk tuples can be returned to an user from a + * primary key. And they must have field + * definitions as well as space->format tuples. + */ + index->disk_format = + tuple_format_new(&vy_tuple_format_vtab, &cmp_def, 1, 0, + format->fields, format->field_count); + } else { + index->disk_format = tuple_format_new(&vy_tuple_format_vtab, + &cmp_def, 1, 0, NULL, 0); + } if (index->disk_format == NULL) goto fail_format; tuple_format_ref(index->disk_format); diff --git a/test/box/access.result b/test/box/access.result index 5fc84b9ca7..8337bf1cd5 100644 --- a/test/box/access.result +++ b/test/box/access.result @@ -514,20 +514,14 @@ box.space._priv:select{id} -- ----------------------------------------------------------- -- Be a bit more rigorous in what is accepted in space _user -- ----------------------------------------------------------- -box.space._user:insert{10, 1, 'name'} +utils = require('utils') --- -- error: Field 4 was not found in the tuple ... -box.space._user:insert{10, 1, 'name', 'strange-object-type'} +box.space._user:insert{10, 1, 'name', 'strange-object-type', utils.setmap({})} --- - error: 'Failed to create user ''name'': unknown user type' ... -box.space._user:insert{10, 1, 'name', 'user', 'password'} ---- -- error: 'Failed to create user ''name'': invalid password format, use box.schema.user.passwd() - to reset password' -... -box.space._user:insert{10, 1, 'name', 'role', 'password'} +box.space._user:insert{10, 1, 'name', 'role', utils.setmap{'password'}} --- - error: 'Failed to create role ''name'': authentication data can not be set for a role' diff --git a/test/box/access.test.lua b/test/box/access.test.lua index 35dfa1ee70..cde4e0f9aa 100644 --- a/test/box/access.test.lua +++ b/test/box/access.test.lua @@ -211,10 +211,9 @@ box.space._priv:select{id} -- ----------------------------------------------------------- -- Be a bit more rigorous in what is accepted in space _user -- ----------------------------------------------------------- -box.space._user:insert{10, 1, 'name'} -box.space._user:insert{10, 1, 'name', 'strange-object-type'} -box.space._user:insert{10, 1, 'name', 'user', 'password'} -box.space._user:insert{10, 1, 'name', 'role', 'password'} +utils = require('utils') +box.space._user:insert{10, 1, 'name', 'strange-object-type', utils.setmap({})} +box.space._user:insert{10, 1, 'name', 'role', utils.setmap{'password'}} session = nil -- ----------------------------------------------------------- -- admin can't manage grants on not owned objects diff --git a/test/box/alter.result b/test/box/alter.result index abc38c2595..9401122600 100644 --- a/test/box/alter.result +++ b/test/box/alter.result @@ -67,9 +67,9 @@ _space:replace{_index.id, ADMIN, '_index', 'memtx', 0, EMPTY_MAP, {}} -- -- Can't change properties of a space -- -_space:replace{_space.id, ADMIN, '_space', 'memtx', 0} +_space:replace{_space.id, ADMIN, '_space', 'memtx', 0, EMPTY_MAP, {}} --- -- [280, 1, '_space', 'memtx', 0] +- [280, 1, '_space', 'memtx', 0, {}, []] ... -- -- Can't drop a system space @@ -98,7 +98,7 @@ _space:update({_space.id}, {{'-', 1, 2}}) -- -- Create a space -- -t = _space:auto_increment{ADMIN, 'hello', 'memtx', 0} +t = _space:auto_increment{ADMIN, 'hello', 'memtx', 0, EMPTY_MAP, {}} --- ... -- Check that a space exists @@ -212,9 +212,9 @@ _index:delete{_index.id, 0} --- - error: Can't drop the primary key in a system space, space '_index' ... -_space:insert{1000, ADMIN, 'hello', 'memtx', 0} +_space:insert{1000, ADMIN, 'hello', 'memtx', 0, EMPTY_MAP, {}} --- -- [1000, 1, 'hello', 'memtx', 0] +- [1000, 1, 'hello', 'memtx', 0, {}, []] ... _index:insert{1000, 0, 'primary', 'tree', 1, 1, 0, 'unsigned'} --- @@ -743,3 +743,243 @@ n ts:drop() --- ... +-- +-- gh-2652: validate space format. +-- +s = box.schema.space.create('test', { format = "format" }) +--- +- error: Illegal parameters, options parameter 'format' should be of type table +... +s = box.schema.space.create('test', { format = { "not_map" } }) +--- +- error: 'Failed to create space ''test'': field 1 is not map' +... +format = { utils.setmap({'unsigned'}) } +--- +... +s = box.schema.space.create('test', { format = format }) +--- +- error: 'Failed to create space ''test'': field 1 format is not map with string keys' +... +format = { { name = 100 } } +--- +... +s = box.schema.space.create('test', { format = format }) +--- +- error: 'Wrong space format (field 1): ''name'' must be string' +... +long = string.rep('a', box.schema.NAME_MAX + 1) +--- +... +format = { { name = long } } +--- +... +s = box.schema.space.create('test', { format = format }) +--- +- error: 'Failed to create space ''test'': field 1 name is too long' +... +format = { { name = 'id', type = '100' } } +--- +... +s = box.schema.space.create('test', { format = format }) +--- +- error: 'Failed to create space ''test'': field 1 has unknown field type' +... +format = { utils.setmap({}) } +--- +... +s = box.schema.space.create('test', { format = format }) +--- +- error: 'Failed to create space ''test'': field 1 name is not specified' +... +-- Ensure the format is updated after index drop. +format = { { name = 'id', type = 'unsigned' } } +--- +... +s = box.schema.space.create('test', { format = format }) +--- +... +pk = s:create_index('pk') +--- +... +sk = s:create_index('sk', { parts = { 2, 'string' } }) +--- +... +s:replace{1, 1} +--- +- error: 'Tuple field 2 type does not match one required by operation: expected string' +... +sk:drop() +--- +... +s:replace{1, 1} +--- +- [1, 1] +... +s:drop() +--- +... +-- Check index parts conflicting with space format. +format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } } +--- +... +s = box.schema.space.create('test', { format = format }) +--- +... +pk = s:create_index('pk') +--- +... +sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } }) +--- +- error: Ambiguous field type, field 2. Requested type is unsigned but the field has + previously been defined as string +... +sk2 = s:create_index('sk2', { parts = { 3, 'number' } }) +--- +- error: Ambiguous field type, field 3. Requested type is number but the field has + previously been defined as scalar +... +-- Check space format conflicting with index parts. +sk3 = s:create_index('sk3', { parts = { 2, 'string' } }) +--- +... +format[2].type = 'unsigned' +--- +... +s:format(format) +--- +- error: Ambiguous field type, field 2. Requested type is string but the field has + previously been defined as unsigned +... +s:format() +--- +- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'string'}, { + 'name': 'field3', 'type': 'scalar'}] +... +s.index.sk3.parts +--- +- - type: string + fieldno: 2 +... +-- Space format can be updated, if conflicted index is deleted. +sk3:drop() +--- +... +s:format(format) +--- +... +s:format() +--- +- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'unsigned'}, + {'name': 'field3', 'type': 'scalar'}] +... +-- Check deprecated field types. +format[2].type = 'num' +--- +... +format[3].type = 'str' +--- +... +format[4] = { name = 'field4', type = '*' } +--- +... +format +--- +- - name: field1 + type: unsigned + - name: field2 + type: num + - name: field3 + type: str + - name: field4 + type: '*' +... +s:format(format) +--- +... +s:format() +--- +- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'num'}, {'name': 'field3', + 'type': 'str'}, {'name': 'field4', 'type': '*'}] +... +s:replace{1, 2, '3', {4, 4, 4}} +--- +- [1, 2, '3', [4, 4, 4]] +... +-- Check not indexed fields checking. +s:truncate() +--- +... +format[2] = {name='field2', type='string'} +--- +... +format[3] = {name='field3', type='array'} +--- +... +format[4] = {name='field4', type='number'} +--- +... +format[5] = {name='field5', type='integer'} +--- +... +format[6] = {name='field6', type='scalar'} +--- +... +format[7] = {name='field7', type='map'} +--- +... +format[8] = {name='field8', type='any'} +--- +... +format[9] = {name='field9'} +--- +... +s:format(format) +--- +... +-- Check incorrect field types. +format[9] = {name='err', type='any'} +--- +... +s:format(format) +--- +... +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9} +--- +- [1, '2', [3, 3], 4.4, -5, true, {'value': 7}, 8, 9] +... +s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9} +--- +- error: 'Tuple field 2 type does not match one required by operation: expected string' +... +s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9} +--- +- error: 'Tuple field 3 type does not match one required by operation: expected array' +... +s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9} +--- +- error: 'Tuple field 4 type does not match one required by operation: expected number' +... +s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9} +--- +- error: 'Tuple field 5 type does not match one required by operation: expected integer' +... +s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9} +--- +- error: 'Tuple field 6 type does not match one required by operation: expected scalar' +... +s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9} +--- +- error: 'Tuple field 7 type does not match one required by operation: expected map' +... +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}} +--- +- error: Tuple field count 7 is less than required by a defined index (expected 9) +... +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8} +--- +- error: Tuple field count 8 is less than required by a defined index (expected 9) +... +s:drop() +--- +... diff --git a/test/box/alter.test.lua b/test/box/alter.test.lua index cfc51799ad..c6d35c0b4c 100644 --- a/test/box/alter.test.lua +++ b/test/box/alter.test.lua @@ -30,7 +30,7 @@ _space:replace{_index.id, ADMIN, '_index', 'memtx', 0, EMPTY_MAP, {}} -- -- Can't change properties of a space -- -_space:replace{_space.id, ADMIN, '_space', 'memtx', 0} +_space:replace{_space.id, ADMIN, '_space', 'memtx', 0, EMPTY_MAP, {}} -- -- Can't drop a system space -- @@ -44,7 +44,7 @@ _space:update({_space.id}, {{'-', 1, 2}}) -- -- Create a space -- -t = _space:auto_increment{ADMIN, 'hello', 'memtx', 0} +t = _space:auto_increment{ADMIN, 'hello', 'memtx', 0, EMPTY_MAP, {}} -- Check that a space exists space = box.space[t[1]] space.id @@ -69,7 +69,7 @@ _index:replace{_index.id, 0, 'primary', 'tree', 1, 2, 0, 'unsigned', 1, 'unsigne _index:select{} -- modify indexes of a system space _index:delete{_index.id, 0} -_space:insert{1000, ADMIN, 'hello', 'memtx', 0} +_space:insert{1000, ADMIN, 'hello', 'memtx', 0, EMPTY_MAP, {}} _index:insert{1000, 0, 'primary', 'tree', 1, 1, 0, 'unsigned'} box.space[1000]:insert{0, 'hello, world'} box.space[1000]:drop() @@ -286,3 +286,86 @@ o n ts:drop() + +-- +-- gh-2652: validate space format. +-- +s = box.schema.space.create('test', { format = "format" }) +s = box.schema.space.create('test', { format = { "not_map" } }) +format = { utils.setmap({'unsigned'}) } +s = box.schema.space.create('test', { format = format }) +format = { { name = 100 } } +s = box.schema.space.create('test', { format = format }) +long = string.rep('a', box.schema.NAME_MAX + 1) +format = { { name = long } } +s = box.schema.space.create('test', { format = format }) +format = { { name = 'id', type = '100' } } +s = box.schema.space.create('test', { format = format }) +format = { utils.setmap({}) } +s = box.schema.space.create('test', { format = format }) + +-- Ensure the format is updated after index drop. +format = { { name = 'id', type = 'unsigned' } } +s = box.schema.space.create('test', { format = format }) +pk = s:create_index('pk') +sk = s:create_index('sk', { parts = { 2, 'string' } }) +s:replace{1, 1} +sk:drop() +s:replace{1, 1} +s:drop() + +-- Check index parts conflicting with space format. +format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } } +s = box.schema.space.create('test', { format = format }) +pk = s:create_index('pk') +sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } }) +sk2 = s:create_index('sk2', { parts = { 3, 'number' } }) + +-- Check space format conflicting with index parts. +sk3 = s:create_index('sk3', { parts = { 2, 'string' } }) +format[2].type = 'unsigned' +s:format(format) +s:format() +s.index.sk3.parts + +-- Space format can be updated, if conflicted index is deleted. +sk3:drop() +s:format(format) +s:format() + +-- Check deprecated field types. +format[2].type = 'num' +format[3].type = 'str' +format[4] = { name = 'field4', type = '*' } +format +s:format(format) +s:format() +s:replace{1, 2, '3', {4, 4, 4}} + +-- Check not indexed fields checking. +s:truncate() +format[2] = {name='field2', type='string'} +format[3] = {name='field3', type='array'} +format[4] = {name='field4', type='number'} +format[5] = {name='field5', type='integer'} +format[6] = {name='field6', type='scalar'} +format[7] = {name='field7', type='map'} +format[8] = {name='field8', type='any'} +format[9] = {name='field9'} +s:format(format) + +-- Check incorrect field types. +format[9] = {name='err', type='any'} +s:format(format) + +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9} +s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9} +s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9} +s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9} +s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9} +s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9} +s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9} +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}} +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8} + +s:drop() diff --git a/test/box/misc.result b/test/box/misc.result index e06e0a298d..6b362c0dca 100644 --- a/test/box/misc.result +++ b/test/box/misc.result @@ -308,6 +308,7 @@ t; - 'box.error.NO_SUCH_TRIGGER : 34' - 'box.error.CHECKPOINT_IN_PROGRESS : 120' - 'box.error.FIELD_TYPE : 23' + - 'box.error.WRONG_SPACE_FORMAT : 141' - 'box.error.UNKNOWN_UPDATE_OP : 28' - 'box.error.CURSOR_NO_TRANSACTION : 80' - 'box.error.TUPLE_REF_OVERFLOW : 86' diff --git a/test/box/rtree_misc.result b/test/box/rtree_misc.result index 4ee9a54570..5c64ad4bc9 100644 --- a/test/box/rtree_misc.result +++ b/test/box/rtree_misc.result @@ -466,6 +466,9 @@ box.snapshot() - ok ... test_run:cmd("restart server default") +utils = require('utils') +--- +... s = box.space.spatial --- ... @@ -547,31 +550,12 @@ f(box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false}, {{2, 'array'} s.index.s:drop() --- ... --- support of 1.6.5 _index structure -f(box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'array'}) ---- -- [0, 2, 's', 'rtree', 0, 1, 2, 'array'] -... -s.index.s:drop() ---- -... -- with wrong args -empty_map = setmetatable({}, {__serialize = 'map'}) ---- -... box.space._index:insert{s.id, 2, 's', 'rtree', nil, {{2, 'array'}}} --- -- error: 'Wrong record in _index space: got {number, number, string, string, unknown, - array}, expected {space id (number), index id (number), name (string), type (string), - options (map), parts (array)}' +- error: 'Tuple field 5 type does not match one required by operation: expected map' ... -box.space._index:insert{s.id, 2, 's', 'rtree', {}, {{2, 'array'}}} ---- -- error: 'Wrong record in _index space: got {number, number, string, string, array, - array}, expected {space id (number), index id (number), name (string), type (string), - options (map), parts (array)}' -... -box.space._index:insert{s.id, 2, 's', 'rtree', empty_map, {{2, 'array'}}} +box.space._index:insert{s.id, 2, 's', 'rtree', utils.setmap({}), {{2, 'array'}}} --- - error: 'Can''t create or modify index ''s'' in space ''s'': RTREE index can not be unique' @@ -613,22 +597,6 @@ box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false}, {{}}} - error: 'Wrong index parts: expected a non-empty array; expected field1 id (number), field1 type (string), ...' ... -box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'thing'} ---- -- error: 'Wrong index parts: unknown field type; expected field1 id (number), field1 - type (string), ...' -... -box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'array', 'wtf'} ---- -- error: 'Wrong record in _index space: got {number, number, string, string, number, - number, number, string, string}, expected {space id (number), index id (number), - name (string), type (string), is_unique (number), part count (number) part0 field - no (number), part0 field type (string), ...}' -... -box.space._index:insert{s.id, 2, 's', 'rtree', 0, 0} ---- -- error: 'Can''t create or modify index ''s'' in space ''s'': part count must be positive' -... -- unknown args checked f(box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false, holy = 'cow'}, {{2, 'array'}}}) --- diff --git a/test/box/rtree_misc.test.lua b/test/box/rtree_misc.test.lua index 7f84d58949..1a2cdd0f5f 100644 --- a/test/box/rtree_misc.test.lua +++ b/test/box/rtree_misc.test.lua @@ -164,6 +164,8 @@ box.snapshot() test_run:cmd("restart server default") +utils = require('utils') + s = box.space.spatial i = s.index.spatial s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) @@ -199,15 +201,10 @@ function f(t) local r = {} for i, v in ipairs(t) do r[i] = v end r[1] = 0 return -- new index through inserting to _index space f(box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false}, {{2, 'array'}}}) s.index.s:drop() --- support of 1.6.5 _index structure -f(box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'array'}) -s.index.s:drop() -- with wrong args -empty_map = setmetatable({}, {__serialize = 'map'}) box.space._index:insert{s.id, 2, 's', 'rtree', nil, {{2, 'array'}}} -box.space._index:insert{s.id, 2, 's', 'rtree', {}, {{2, 'array'}}} -box.space._index:insert{s.id, 2, 's', 'rtree', empty_map, {{2, 'array'}}} +box.space._index:insert{s.id, 2, 's', 'rtree', utils.setmap({}), {{2, 'array'}}} box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false, dimension = 22}, {{2, 'array'}}} box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false, dimension = 'dimension'}, {{2, 'array'}}} box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false}, {{2, 'unsigned'}}} @@ -216,9 +213,6 @@ box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false}, {{'no','time'}} box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false, distance = 'lobachevsky'}, {{2, 'array'}}} box.space._index:insert{s.id, 2, 's', 'rtee', {unique = false}, {{2, 'array'}}} box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false}, {{}}} -box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'thing'} -box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'array', 'wtf'} -box.space._index:insert{s.id, 2, 's', 'rtree', 0, 0} -- unknown args checked f(box.space._index:insert{s.id, 2, 's', 'rtree', {unique = false, holy = 'cow'}, {{2, 'array'}}}) diff --git a/test/box/temp_spaces.result b/test/box/temp_spaces.result index f0d9a684ef..d939f65639 100644 --- a/test/box/temp_spaces.result +++ b/test/box/temp_spaces.result @@ -56,22 +56,6 @@ s:len() --- - 1 ... -_ = _space:update(s.id, {{'=', FLAGS, 'temporary'}}) ---- -... -s.temporary ---- -- true -... -_ = _space:update(s.id, {{'=', FLAGS, ''}}) ---- -- error: 'Can''t modify space ''t'': can not switch temporary flag on a non-empty - space' -... -s.temporary ---- -- true -... -- check that temporary space can be modified in read-only mode (gh-1378) box.cfg{read_only=true} --- @@ -123,66 +107,6 @@ s.temporary --- - true ... --- <!-- Tarantool < 1.7.0 compatibility -_ = _space:update(s.id, {{'=', FLAGS, 'no-temporary'}}) ---- -... -s.temporary ---- -- false -... -_ = _space:update(s.id, {{'=', FLAGS, ',:asfda:temporary'}}) ---- -... -s.temporary ---- -- false -... -_ = _space:update(s.id, {{'=', FLAGS, 'a,b,c,d,e'}}) ---- -... -s.temporary ---- -- false -... -_ = _space:update(s.id, {{'=', FLAGS, 'temporary'}}) ---- -... -s.temporary ---- -- true -... -s:get{1} ---- -... -s:insert{1, 2, 3} ---- -- [1, 2, 3] -... -_ = _space:update(s.id, {{'=', FLAGS, 'temporary'}}) ---- -... -s.temporary ---- -- true -... -_ = _space:update(s.id, {{'=', FLAGS, 'no-temporary'}}) ---- -- error: 'Can''t modify space ''t'': can not switch temporary flag on a non-empty - space' -... -s.temporary ---- -- true -... -s:delete{1} ---- -- [1, 2, 3] -... -_ = _space:update(s.id, {{'=', FLAGS, 'no-temporary'}}) ---- -... --- Tarantool < 1.7.0 compatibility //--> s:drop() --- ... diff --git a/test/box/temp_spaces.test.lua b/test/box/temp_spaces.test.lua index c485b42f1f..47d8b415bc 100644 --- a/test/box/temp_spaces.test.lua +++ b/test/box/temp_spaces.test.lua @@ -23,11 +23,6 @@ s:insert{1, 2, 3} s:get{1} s:len() -_ = _space:update(s.id, {{'=', FLAGS, 'temporary'}}) -s.temporary -_ = _space:update(s.id, {{'=', FLAGS, ''}}) -s.temporary - -- check that temporary space can be modified in read-only mode (gh-1378) box.cfg{read_only=true} box.cfg.read_only @@ -48,27 +43,5 @@ s = box.space.t s:len() s.temporary --- <!-- Tarantool < 1.7.0 compatibility -_ = _space:update(s.id, {{'=', FLAGS, 'no-temporary'}}) -s.temporary -_ = _space:update(s.id, {{'=', FLAGS, ',:asfda:temporary'}}) -s.temporary -_ = _space:update(s.id, {{'=', FLAGS, 'a,b,c,d,e'}}) -s.temporary -_ = _space:update(s.id, {{'=', FLAGS, 'temporary'}}) -s.temporary - -s:get{1} -s:insert{1, 2, 3} - -_ = _space:update(s.id, {{'=', FLAGS, 'temporary'}}) -s.temporary -_ = _space:update(s.id, {{'=', FLAGS, 'no-temporary'}}) -s.temporary - -s:delete{1} -_ = _space:update(s.id, {{'=', FLAGS, 'no-temporary'}}) --- Tarantool < 1.7.0 compatibility //--> - s:drop() s = nil diff --git a/test/unit/vy_iterators_helper.c b/test/unit/vy_iterators_helper.c index 1fee0474d2..ec24e3ae0e 100644 --- a/test/unit/vy_iterators_helper.c +++ b/test/unit/vy_iterators_helper.c @@ -12,7 +12,8 @@ vy_iterator_C_test_init(size_t cache_size) fiber_init(fiber_c_invoke); tuple_init(); vy_cache_env_create(&cache_env, cord_slab_cache(), cache_size); - vy_key_format = tuple_format_new(&vy_tuple_format_vtab, NULL, 0, 0); + vy_key_format = tuple_format_new(&vy_tuple_format_vtab, NULL, 0, 0, + NULL, 0); tuple_format_ref(vy_key_format); } @@ -189,9 +190,9 @@ create_test_mem(struct lsregion *region, struct key_def *def) { /* Create format */ struct key_def *defs[] = { def }; - struct tuple_format *format = tuple_format_new(&vy_tuple_format_vtab, - defs, def->part_count, - 0); + struct tuple_format *format = + tuple_format_new(&vy_tuple_format_vtab, defs, def->part_count, + 0, NULL, 0); fail_if(format == NULL); /* Create format with column mask */ @@ -219,7 +220,7 @@ create_test_cache(uint32_t *fields, uint32_t *types, *def = box_key_def_new(fields, types, key_cnt); assert(*def != NULL); vy_cache_create(cache, &cache_env, *def); - *format = tuple_format_new(&vy_tuple_format_vtab, def, 1, 0); + *format = tuple_format_new(&vy_tuple_format_vtab, def, 1, 0, NULL, 0); tuple_format_ref(*format); } diff --git a/test/unit/vy_mem.c b/test/unit/vy_mem.c index 1fd01a1dc2..debac5cfa8 100644 --- a/test/unit/vy_mem.c +++ b/test/unit/vy_mem.c @@ -85,7 +85,7 @@ test_iterator_restore_after_insertion() /* Create format */ struct tuple_format *format = tuple_format_new(&vy_tuple_format_vtab, - &key_def, 1, 0); + &key_def, 1, 0, NULL, 0); assert(format != NULL); tuple_format_ref(format); diff --git a/test/unit/vy_point_iterator.c b/test/unit/vy_point_iterator.c index d9c763018e..c10c8a1b61 100644 --- a/test/unit/vy_point_iterator.c +++ b/test/unit/vy_point_iterator.c @@ -46,8 +46,8 @@ test_basic() vy_cache_create(&cache, &cache_env, key_def); - struct tuple_format *format = - tuple_format_new(&vy_tuple_format_vtab, &key_def, 1, 0); + struct tuple_format *format = tuple_format_new(&vy_tuple_format_vtab, + &key_def, 1, 0, NULL, 0); isnt(format, NULL, "tuple_format_new is not NULL"); tuple_format_ref(format); diff --git a/test/xlog/legacy.result b/test/xlog/legacy.result new file mode 100644 index 0000000000..683c8b650b --- /dev/null +++ b/test/xlog/legacy.result @@ -0,0 +1,177 @@ +test_run = require('test_run').new() +--- +... +version = test_run:get_cfg('version') +--- +... +-- Use 1.7.5 snapshot to check that space formats are not checked. +-- It allows to use >= 1.6.5 format versions. +test_run:cmd('create server legacy with script="xlog/upgrade.lua", workdir="xlog/upgrade/1.7.5"') +--- +- true +... +test_run:cmd("start server legacy") +--- +- true +... +test_run:switch('legacy') +--- +- true +... +box.space._schema:get({'version'}) +--- +- ['version', 1, 7, 5] +... +_space = box.space._space +--- +... +-- +-- Check _space 1.7.5 format. +-- +_space:replace{600, 1, 'test', 'memtx', 0} +--- +- [600, 1, 'test', 'memtx', 0] +... +box.space.test:drop() +--- +... +-- +-- Check _index 1.6.5 format. +-- +s = box.schema.space.create('s') +--- +... +pk = s:create_index('pk') +--- +... +sk = box.space._index:insert{s.id, 2, 'sk', 'rtree', 0, 1, 2, 'array'} +--- +... +s.index.sk.parts +--- +- - type: array + fieldno: 3 +... +s.index.sk:drop() +--- +... +box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'thing'} +--- +- error: 'Wrong index parts: unknown field type; expected field1 id (number), field1 + type (string), ...' +... +box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'array', 'wtf'} +--- +- error: 'Wrong record in _index space: got {number, number, string, string, number, + number, number, string, string}, expected {space id (number), index id (number), + name (string), type (string), is_unique (number), part count (number) part0 field + no (number), part0 field type (string), ...}' +... +box.space._index:insert{s.id, 2, 's', 'rtree', 0, 0} +--- +- error: 'Can''t create or modify index ''s'' in space ''s'': part count must be positive' +... +s:drop() +--- +... +-- +-- Check 1.6.5 space flags. +-- +s = box.schema.space.create('t', { temporary = true }) +--- +... +index = s:create_index('primary', { type = 'hash' }) +--- +... +s:insert{1, 2, 3} +--- +- [1, 2, 3] +... +_ = _space:update(s.id, {{'=', 6, 'temporary'}}) +--- +... +s.temporary +--- +- true +... +_ = _space:update(s.id, {{'=', 6, ''}}) +--- +- error: 'Can''t modify space ''t'': can not switch temporary flag on a non-empty + space' +... +s.temporary +--- +- true +... +s:truncate() +--- +... +_ = _space:update(s.id, {{'=', 6, 'no-temporary'}}) +--- +... +s.temporary +--- +- false +... +_ = _space:update(s.id, {{'=', 6, ',:asfda:temporary'}}) +--- +... +s.temporary +--- +- false +... +_ = _space:update(s.id, {{'=', 6, 'a,b,c,d,e'}}) +--- +... +s.temporary +--- +- false +... +_ = _space:update(s.id, {{'=', 6, 'temporary'}}) +--- +... +s.temporary +--- +- true +... +s:get{1} +--- +... +s:insert{1, 2, 3} +--- +- [1, 2, 3] +... +_ = _space:update(s.id, {{'=', 6, 'temporary'}}) +--- +... +s.temporary +--- +- true +... +_ = _space:update(s.id, {{'=', 6, 'no-temporary'}}) +--- +- error: 'Can''t modify space ''t'': can not switch temporary flag on a non-empty + space' +... +s.temporary +--- +- true +... +s:delete{1} +--- +- [1, 2, 3] +... +_ = _space:update(s.id, {{'=', 6, 'no-temporary'}}) +--- +... +s:drop() +--- +... +test_run:switch('default') +--- +- true +... +test_run:cmd('stop server legacy') +--- +- true +... diff --git a/test/xlog/legacy.test.lua b/test/xlog/legacy.test.lua new file mode 100644 index 0000000000..74569d3ce5 --- /dev/null +++ b/test/xlog/legacy.test.lua @@ -0,0 +1,68 @@ +test_run = require('test_run').new() + +version = test_run:get_cfg('version') +-- Use 1.7.5 snapshot to check that space formats are not checked. +-- It allows to use >= 1.6.5 format versions. +test_run:cmd('create server legacy with script="xlog/upgrade.lua", workdir="xlog/upgrade/1.7.5"') +test_run:cmd("start server legacy") + +test_run:switch('legacy') + +box.space._schema:get({'version'}) +_space = box.space._space + +-- +-- Check _space 1.7.5 format. +-- +_space:replace{600, 1, 'test', 'memtx', 0} +box.space.test:drop() + +-- +-- Check _index 1.6.5 format. +-- +s = box.schema.space.create('s') +pk = s:create_index('pk') +sk = box.space._index:insert{s.id, 2, 'sk', 'rtree', 0, 1, 2, 'array'} +s.index.sk.parts +s.index.sk:drop() +box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'thing'} +box.space._index:insert{s.id, 2, 's', 'rtree', 0, 1, 2, 'array', 'wtf'} +box.space._index:insert{s.id, 2, 's', 'rtree', 0, 0} +s:drop() + +-- +-- Check 1.6.5 space flags. +-- +s = box.schema.space.create('t', { temporary = true }) +index = s:create_index('primary', { type = 'hash' }) +s:insert{1, 2, 3} +_ = _space:update(s.id, {{'=', 6, 'temporary'}}) +s.temporary +_ = _space:update(s.id, {{'=', 6, ''}}) +s.temporary +s:truncate() + +_ = _space:update(s.id, {{'=', 6, 'no-temporary'}}) +s.temporary +_ = _space:update(s.id, {{'=', 6, ',:asfda:temporary'}}) +s.temporary +_ = _space:update(s.id, {{'=', 6, 'a,b,c,d,e'}}) +s.temporary +_ = _space:update(s.id, {{'=', 6, 'temporary'}}) +s.temporary + +s:get{1} +s:insert{1, 2, 3} + +_ = _space:update(s.id, {{'=', 6, 'temporary'}}) +s.temporary +_ = _space:update(s.id, {{'=', 6, 'no-temporary'}}) +s.temporary + +s:delete{1} +_ = _space:update(s.id, {{'=', 6, 'no-temporary'}}) + +s:drop() + +test_run:switch('default') +test_run:cmd('stop server legacy') -- GitLab