diff --git a/src/box/errcode.h b/src/box/errcode.h index 94381f9f74f89993c0d715e4e9e1322ad61bc149..f7dbb948e3c9e0a0d47e214f2856b864d44d5d0a 100644 --- a/src/box/errcode.h +++ b/src/box/errcode.h @@ -91,7 +91,7 @@ struct errcode_record { /* 36 */_(ER_NO_SUCH_SPACE, "Space '%s' does not exist") \ /* 37 */_(ER_NO_SUCH_FIELD, "Field %d was not found in the tuple") \ /* 38 */_(ER_EXACT_FIELD_COUNT, "Tuple field count %u does not match space field count %u") \ - /* 39 */_(ER_MIN_FIELD_COUNT, "Tuple field count %u is less than required by space format or defined indexes (expected at least %u)") \ + /* 39 */_(ER_FIELD_MISSING, "Tuple field %s required by space format is missing") \ /* 40 */_(ER_WAL_IO, "Failed to write to disk") \ /* 41 */_(ER_MORE_THAN_ONE_TUPLE, "Get() doesn't support partial keys and non-unique indexes") \ /* 42 */_(ER_ACCESS_DENIED, "%s access to %s '%s' is denied for user '%s'") \ diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c index d7489dcd0ccdbcd9fc96282860216de086fc6c2f..e11b4e6f32d20c4c29e6eaa04e456a144f79d7dc 100644 --- a/src/box/tuple_format.c +++ b/src/box/tuple_format.c @@ -28,6 +28,8 @@ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ +#include "bit/bit.h" +#include "fiber.h" #include "json/json.h" #include "tuple_format.h" #include "coll_id_cache.h" @@ -47,6 +49,7 @@ tuple_field_new(void) "tuple field"); return NULL; } + field->id = UINT32_MAX; field->token.type = JSON_TOKEN_END; field->type = FIELD_TYPE_ANY; field->offset_slot = TUPLE_OFFSET_SLOT_NIL; @@ -71,6 +74,24 @@ tuple_field_path(const struct tuple_field *field) return int2str(field->token.num + TUPLE_INDEX_BASE); } +/** + * Look up field metadata by identifier. + * + * Used only for error reporing so we can afford full field + * tree traversal here. + */ +static struct tuple_field * +tuple_format_field_by_id(struct tuple_format *format, uint32_t id) +{ + struct tuple_field *field; + json_tree_foreach_entry_preorder(field, &format->fields.root, + struct tuple_field, token) { + if (field->id == id) + return field; + } + return NULL; +} + static int tuple_format_use_key_part(struct tuple_format *format, uint32_t field_count, const struct key_part *part, bool is_sequential, @@ -207,6 +228,26 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys, return -1; } format->field_map_size = field_map_size; + + size_t required_fields_sz = bitmap_size(format->total_field_count); + format->required_fields = calloc(1, required_fields_sz); + if (format->required_fields == NULL) { + diag_set(OutOfMemory, required_fields_sz, + "malloc", "required field bitmap"); + return -1; + } + struct tuple_field *field; + json_tree_foreach_entry_preorder(field, &format->fields.root, + struct tuple_field, token) { + /* + * Mark all leaf non-nullable fields as required + * by setting the corresponding bit in the bitmap + * of required fields. + */ + if (json_token_is_leaf(&field->token) && + !tuple_field_is_nullable(field)) + bit_set(format->required_fields, field->id); + } return 0; } @@ -305,6 +346,7 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, struct tuple_field *field = tuple_field_new(); if (field == NULL) goto error; + field->id = fieldno; field->token.num = fieldno; field->token.type = JSON_TOKEN_NUM; if (json_tree_add(&format->fields, &format->fields.root, @@ -324,6 +366,8 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, format->dict = dict; tuple_dictionary_ref(dict); } + format->total_field_count = field_count; + format->required_fields = NULL; format->refs = 0; format->id = FORMAT_ID_NIL; format->index_field_count = index_field_count; @@ -340,6 +384,7 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, static inline void tuple_format_destroy(struct tuple_format *format) { + free(format->required_fields); tuple_format_destroy_fields(format); tuple_dictionary_unref(format->dict); } @@ -429,7 +474,10 @@ tuple_init_field_map(struct tuple_format *format, uint32_t *field_map, if (tuple_format_field_count(format) == 0) return 0; /* Nothing to initialize */ + struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); const char *pos = tuple; + int rc = 0; /* Check to see if the tuple has a sufficient number of fields. */ uint32_t field_count = mp_decode_array(&pos); @@ -438,15 +486,32 @@ tuple_init_field_map(struct tuple_format *format, uint32_t *field_map, diag_set(ClientError, ER_EXACT_FIELD_COUNT, (unsigned) field_count, (unsigned) format->exact_field_count); - return -1; + goto error; } - if (validate && field_count < format->min_field_count) { - diag_set(ClientError, ER_MIN_FIELD_COUNT, - (unsigned) field_count, - (unsigned) format->min_field_count); - return -1; + /* + * Allocate a field bitmap that will be used for checking + * that all mandatory fields are present. + */ + void *required_fields = NULL; + size_t required_fields_sz = 0; + if (validate) { + required_fields_sz = bitmap_size(format->total_field_count); + required_fields = region_alloc(region, required_fields_sz); + if (required_fields == NULL) { + diag_set(OutOfMemory, required_fields_sz, + "region", "required field bitmap"); + goto error; + } + memcpy(required_fields, format->required_fields, + required_fields_sz); + } + /* + * Initialize the tuple field map and validate field types. + */ + if (field_count == 0) { + /* Empty tuple, nothing to do. */ + goto skip; } - /* first field is simply accessible, so we do not store offset to it */ struct tuple_field *field = tuple_format_field(format, 0); if (validate && @@ -454,8 +519,10 @@ tuple_init_field_map(struct tuple_format *format, uint32_t *field_map, tuple_field_is_nullable(field))) { diag_set(ClientError, ER_FIELD_TYPE, tuple_field_path(field), field_type_strs[field->type]); - return -1; + goto error; } + if (required_fields != NULL) + bit_clear(required_fields, field->id); mp_next(&pos); /* other fields...*/ uint32_t i = 1; @@ -478,15 +545,40 @@ tuple_init_field_map(struct tuple_format *format, uint32_t *field_map, diag_set(ClientError, ER_FIELD_TYPE, tuple_field_path(field), field_type_strs[field->type]); - return -1; + goto error; } if (field->offset_slot != TUPLE_OFFSET_SLOT_NIL) { field_map[field->offset_slot] = (uint32_t) (pos - tuple); } + if (required_fields != NULL) + bit_clear(required_fields, field->id); mp_next(&pos); } - return 0; +skip: + /* + * Check the required field bitmap for missing fields. + */ + if (required_fields != NULL) { + struct bit_iterator it; + bit_iterator_init(&it, required_fields, + required_fields_sz, true); + size_t id = bit_iterator_next(&it); + if (id < SIZE_MAX) { + /* A field is missing, report an error. */ + field = tuple_format_field_by_id(format, id); + assert(field != NULL); + diag_set(ClientError, ER_FIELD_MISSING, + tuple_field_path(field)); + goto error; + } + } +out: + region_truncate(region, region_svp); + return rc; +error: + rc = -1; + goto out; } uint32_t diff --git a/src/box/tuple_format.h b/src/box/tuple_format.h index 21f3141263000a5875b6d468f5aef1f085568842..30b93b6108fa9a32d1551453c4f84d7a7c84ca14 100644 --- a/src/box/tuple_format.h +++ b/src/box/tuple_format.h @@ -85,6 +85,8 @@ struct tuple_format_vtab { /** Tuple field meta information for tuple_format. */ struct tuple_field { + /** Unique field identifier. */ + uint32_t id; /** * Field type of an indexed field. * If a field participates in at least one of space indexes @@ -169,6 +171,16 @@ struct tuple_format { * index_field_count <= min_field_count <= field_count. */ uint32_t min_field_count; + /** + * Total number of formatted fields, including JSON + * path fields. See also tuple_format::fields. + */ + uint32_t total_field_count; + /** + * Bitmap of fields that must be present in a tuple + * conforming to the format. Indexed by tuple_field::id. + */ + void *required_fields; /** * Shared names storage used by all formats of a space. */ diff --git a/test/box/alter_limits.result b/test/box/alter_limits.result index 4fd80a374f21818b55120e93f8b62ffbea4ac028..4fb33dd67307dfc93876e150c3c989fe50eb66f1 100644 --- a/test/box/alter_limits.result +++ b/test/box/alter_limits.result @@ -842,8 +842,7 @@ index = s:create_index('string', { type = 'tree', unique = false, parts = { 2, -- create index on a non-existing field index = s:create_index('nosuchfield', { type = 'tree', unique = true, parts = { 3, 'string'}}) --- -- error: Tuple field count 2 is less than required by space format or defined indexes - (expected at least 3) +- error: Tuple field 3 required by space format is missing ... s.index.year:drop() --- @@ -864,8 +863,7 @@ s:replace{'Der Baader Meinhof Komplex'} ... index = s:create_index('year', { type = 'tree', unique = false, parts = { 2, 'unsigned'}}) --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... s:drop() --- diff --git a/test/box/ddl.result b/test/box/ddl.result index d3b0d1e0ef400ae2b085584a3d5e9944861efaa2..3d6d07f437a67d5533342f7b1efc463c0c38d0b4 100644 --- a/test/box/ddl.result +++ b/test/box/ddl.result @@ -326,33 +326,27 @@ box.internal.collation.drop('test') ... box.space._collation:auto_increment{'test'} --- -- error: Tuple field count 2 is less than required by space format or defined indexes - (expected at least 6) +- error: Tuple field 3 required by space format is missing ... box.space._collation:auto_increment{'test', 0, 'ICU'} --- -- error: Tuple field count 4 is less than required by space format or defined indexes - (expected at least 6) +- error: Tuple field 5 required by space format is missing ... box.space._collation:auto_increment{'test', 'ADMIN', 'ICU', 'ru_RU'} --- -- error: Tuple field count 5 is less than required by space format or defined indexes - (expected at least 6) +- error: 'Tuple field 3 type does not match one required by operation: expected unsigned' ... box.space._collation:auto_increment{42, 0, 'ICU', 'ru_RU'} --- -- error: Tuple field count 5 is less than required by space format or defined indexes - (expected at least 6) +- error: 'Tuple field 2 type does not match one required by operation: expected string' ... box.space._collation:auto_increment{'test', 0, 42, 'ru_RU'} --- -- error: Tuple field count 5 is less than required by space format or defined indexes - (expected at least 6) +- error: 'Tuple field 4 type does not match one required by operation: expected string' ... box.space._collation:auto_increment{'test', 0, 'ICU', 42} --- -- error: Tuple field count 5 is less than required by space format or defined indexes - (expected at least 6) +- error: 'Tuple field 5 type does not match one required by operation: expected string' ... box.space._collation:auto_increment{'test', 0, 'ICU', 'ru_RU', setmap{}} --ok --- diff --git a/test/box/misc.result b/test/box/misc.result index 9fecbce760b6c8ee013ffa10c5020c3cfa9d7572..c3cabcc8aa3a6104f0f3681d5009647d399b2171 100644 --- a/test/box/misc.result +++ b/test/box/misc.result @@ -369,7 +369,7 @@ t; 36: box.error.NO_SUCH_SPACE 37: box.error.NO_SUCH_FIELD 38: box.error.EXACT_FIELD_COUNT - 39: box.error.MIN_FIELD_COUNT + 39: box.error.FIELD_MISSING 40: box.error.WAL_IO 41: box.error.MORE_THAN_ONE_TUPLE 42: box.error.ACCESS_DENIED diff --git a/test/box/sql.result b/test/box/sql.result index 1818b294dc36eb9b346974d44f7864eafd9ddd8a..78dc471676299f578d5e37f162439b9d59eed67f 100644 --- a/test/box/sql.result +++ b/test/box/sql.result @@ -299,8 +299,7 @@ s:truncate() -- get away with it. space:insert{'Britney'} --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... sorted(space.index.secondary:select('Anything')) --- @@ -308,8 +307,7 @@ sorted(space.index.secondary:select('Anything')) ... space:insert{'Stephanie'} --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... sorted(space.index.secondary:select('Anything')) --- @@ -638,8 +636,7 @@ sorted(space.index.secondary:select('Britney')) -- try to insert the incoplete tuple space:replace{'Spears'} --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... -- check that nothing has been updated space:select{'Spears'} diff --git a/test/box/tree_pk_multipart.result b/test/box/tree_pk_multipart.result index 28cab3f94e4b70a285813d14e98acbcad55d74d0..93219f6663ec7176b9f6247f5d30b0fdff196a31 100644 --- a/test/box/tree_pk_multipart.result +++ b/test/box/tree_pk_multipart.result @@ -490,13 +490,11 @@ i1 = space:create_index('primary', { type = 'tree', parts = {1, 'unsigned', 3, ' ... space:insert{1, 1} --- -- error: Tuple field count 2 is less than required by space format or defined indexes - (expected at least 3) +- error: Tuple field 3 required by space format is missing ... space:replace{1, 1} --- -- error: Tuple field count 2 is less than required by space format or defined indexes - (expected at least 3) +- error: Tuple field 3 required by space format is missing ... space:drop() --- diff --git a/test/engine/ddl.result b/test/engine/ddl.result index 272ff761871e1cbf45ff73fef4117b3321a6bfc8..8d34d5ef447d0d4c37e5d196dd282839144c3b3f 100644 --- a/test/engine/ddl.result +++ b/test/engine/ddl.result @@ -77,8 +77,7 @@ index = space:create_index('primary', {type = 'tree', parts = {1, 'unsigned', 2, ... space:insert({13}) --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... space:drop() --- @@ -844,13 +843,11 @@ s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9} ... s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}} --- -- error: Tuple field count 7 is less than required by space format or defined indexes - (expected at least 9) +- error: Tuple field 8 required by space format is missing ... s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8} --- -- error: Tuple field count 8 is less than required by space format or defined indexes - (expected at least 9) +- error: Tuple field 9 required by space format is missing ... s:truncate() --- @@ -1334,8 +1331,7 @@ s:format(format) -- Fail, not enough fields. s:replace{2, 2, 2, 2, 2} --- -- error: Tuple field count 5 is less than required by space format or defined indexes - (expected at least 6) +- error: Tuple field 6 required by space format is missing ... s:replace{2, 2, 2, 2, 2, 2, 2} --- @@ -1347,8 +1343,7 @@ format[7] = {name = 'field7', type = 'unsigned'} -- Fail, the tuple {1, ... 1} is invalid for a new format. s:format(format) --- -- error: Tuple field count 6 is less than required by space format or defined indexes - (expected at least 7) +- error: Tuple field 7 required by space format is missing ... s:drop() --- @@ -2012,8 +2007,7 @@ s:create_index('sk', {parts = {4, 'unsigned'}}) -- error: field type ... s:create_index('sk', {parts = {4, 'integer', 5, 'string'}}) -- error: field missing --- -- error: Tuple field count 4 is less than required by space format or defined indexes - (expected at least 5) +- error: Tuple field 5 required by space format is missing ... i1 = s:create_index('i1', {parts = {2, 'string'}, unique = false}) --- @@ -2065,8 +2059,7 @@ i3:alter{parts = {4, 'unsigned'}} -- error: field type ... i3:alter{parts = {4, 'integer', 5, 'string'}} -- error: field missing --- -- error: Tuple field count 4 is less than required by space format or defined indexes - (expected at least 5) +- error: Tuple field 5 required by space format is missing ... i3:alter{parts = {2, 'string', 4, 'integer'}} -- ok --- diff --git a/test/engine/null.result b/test/engine/null.result index 757e63185a49a919412a68e8b82efd009b93b68e..d55bc05bd7f80938b8274c07b281503c3f998dd8 100644 --- a/test/engine/null.result +++ b/test/engine/null.result @@ -458,8 +458,7 @@ sk = s:create_index('sk', {parts = {2, 'unsigned'}}) ... s:replace{1, 2} -- error --- -- error: Tuple field count 2 is less than required by space format or defined indexes - (expected at least 3) +- error: Tuple field 3 required by space format is missing ... t1 = s:replace{2, 3, 4} --- @@ -530,18 +529,15 @@ sk = s:create_index('sk', {parts = {2, 'unsigned'}}) ... s:replace{1, 2} -- error --- -- error: Tuple field count 2 is less than required by space format or defined indexes - (expected at least 5) +- error: Tuple field 3 required by space format is missing ... s:replace{2, 3, 4} -- error --- -- error: Tuple field count 3 is less than required by space format or defined indexes - (expected at least 5) +- error: Tuple field 5 required by space format is missing ... s:replace{3, 4, 5, 6} -- error --- -- error: Tuple field count 4 is less than required by space format or defined indexes - (expected at least 5) +- error: Tuple field 5 required by space format is missing ... t1 = s:replace{4, 5, 6, 7, 8} --- @@ -1069,8 +1065,7 @@ sk = s:create_index('sk', {parts = {{2, 'unsigned', is_nullable = true}}}) -- Test tuple_compare_slowpath, tuple_compare_with_key_slowpath. s:replace{} -- Fail --- -- error: Tuple field count 0 is less than required by space format or defined indexes - (expected at least 1) +- error: Tuple field 1 required by space format is missing ... -- Compare full vs not full. s:replace{2} @@ -1769,8 +1764,7 @@ s:format(format) -- Field 2 is not nullable. s:insert{5} --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... s:insert{5, box.NULL} --- @@ -1786,8 +1780,7 @@ s:insert{5, box.NULL} ... s:insert{5} --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... s.index.secondary:alter{parts={{2, 'unsigned', is_nullable=false}}} --- @@ -1805,8 +1798,7 @@ s:insert{5, box.NULL} ... s:insert{5} --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... s.index.secondary:alter{parts={{2, 'unsigned', is_nullable=true}}} --- @@ -1849,13 +1841,11 @@ _ = s:delete{5} ... s:format(format) -- Still fail. --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... s.index.secondary:alter{parts={{2, 'unsigned', is_nullable=false}}} -- Still fail. --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... -- Now check we can set nullability to false step by step. _ = s:delete{6} @@ -1873,8 +1863,7 @@ s:insert{5, box.NULL} -- Fail. ... s:insert{5} -- Fail. --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... format[2].is_nullable = true --- @@ -1891,8 +1880,7 @@ s:insert{5, box.NULL} -- Fail. ... s:insert{5} -- Fail. --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... format[2].is_nullable = false --- @@ -1906,8 +1894,7 @@ s:select{} ... s:insert{5} -- Fail. --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... s:insert{9, 10} -- Success. --- diff --git a/test/vinyl/constraint.result b/test/vinyl/constraint.result index 46ed1c9ebffb4c90d258cd5c25bb77dda0bc3fd7..520a0f8aae3c4096d5ba2466bb6b2750a93aa2f7 100644 --- a/test/vinyl/constraint.result +++ b/test/vinyl/constraint.result @@ -83,13 +83,11 @@ index = space:create_index('primary', { type = 'tree', parts = {1,'unsigned',2,' ... space:insert{1} --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... space:replace{1} --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... space:delete{1} --- @@ -101,8 +99,7 @@ space:update(1, {{'=', 1, 101}}) ... space:upsert({1}, {{'+', 1, 10}}) --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... space:get{1} --- diff --git a/test/vinyl/errinj.result b/test/vinyl/errinj.result index a081575be8cc9c135ac4532c841ddf190532eef3..23ab845b3f087988bcfe8811d52e529e25580558 100644 --- a/test/vinyl/errinj.result +++ b/test/vinyl/errinj.result @@ -1634,8 +1634,7 @@ errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0.001) ... s:create_index('sk', {parts = {2, 'unsigned'}}) -- must fail --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0) --- @@ -2087,8 +2086,7 @@ fiber.sleep(0) ... s:format{{'key', 'unsigned'}, {'value', 'unsigned'}} -- must fail --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... s:select() --- @@ -2112,8 +2110,7 @@ fiber.sleep(0) ... s:create_index('sk', {parts = {2, 'unsigned'}}) --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 2) +- error: Tuple field 2 required by space format is missing ... s:select() --- diff --git a/test/vinyl/savepoint.result b/test/vinyl/savepoint.result index d7b57a7759b86cb427b6c7bd95425e1f107c3421..a62f2ea800ab3bb19503621f757b674089090ea5 100644 --- a/test/vinyl/savepoint.result +++ b/test/vinyl/savepoint.result @@ -124,8 +124,7 @@ index2 = space:create_index('secondary', { parts = {2, 'int', 3, 'str'} }) ... space:insert({1}) --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 3) +- error: Tuple field 2 required by space format is missing ... space:insert({1, 1, 'a'}) --- @@ -623,8 +622,7 @@ space:insert({4, 2, 'b'}) ... space:upsert({2}, {{'=', 4, 1000}}) --- -- error: Tuple field count 1 is less than required by space format or defined indexes - (expected at least 3) +- error: Tuple field 2 required by space format is missing ... index3:delete({3, 'a'}) ---