memtx: check for new optional fields on format update

When a space format is updated, a new min field count must be
calculated before a new format construction to check that some
of fields became optional.

Part of #3229

src/box/alter.cc

@@ -1578,6 +1578,25 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event)
 		struct alter_space *alter = alter_space_new(old_space);
 		auto alter_guard =
 			make_scoped_guard([=] {alter_space_delete(alter);});
+		/*
+		 * Calculate a new min_field_count. It can be
+		 * changed by resetting space:format(), if an old
+		 * format covers some nullable indexed fields in
+		 * the format tail. And when the format is reset,
+		 * these fields become optional - index
+		 * comparators must be updated.
+		 */
+		struct key_def **keys;
+		size_t bsize = old_space->index_count * sizeof(keys[0]);
+		keys = (struct key_def **) region_alloc_xc(&fiber()->gc,
+							   bsize);
+		for (uint32_t i = 0; i < old_space->index_count; ++i)
+			keys[i] = old_space->index[i]->def->key_def;
+		alter->new_min_field_count =
+			tuple_format_min_field_count(keys,
+						     old_space->index_count,
+						     def->fields,
+						     def->field_count);
 		(void) new ModifySpaceFormat(alter, def);
 		(void) new ModifySpace(alter, def);
 		def_guard.is_active = false;

test/engine/ddl.result

@@ -387,3 +387,82 @@ pk.parts[1].type
 s:drop()
 ---
 ...
+--
+-- gh-3229: update optionality if a space format is changed too,
+-- not only when indexes are updated.
+--
+box.cfg{}
+---
+...
+s = box.schema.create_space('test', {engine = 'memtx'})
+---
+...
+format = {}
+---
+...
+format[1] = {'field1', 'unsigned'}
+---
+...
+format[2] = {'field2', 'unsigned', is_nullable = true}
+---
+...
+format[3] = {'field3', 'unsigned'}
+---
+...
+s:format(format)
+---
+...
+pk = s:create_index('pk')
+---
+...
+sk = s:create_index('sk', {parts = {{2, 'unsigned', is_nullable = true}}})
+---
+...
+s:replace{2, 3, 4}
+---
+- [2, 3, 4]
+...
+s:format({})
+---
+...
+s:insert({1})
+---
+- [1]
+...
+s:insert({4, 5})
+---
+- [4, 5]
+...
+s:insert({3, 4})
+---
+- [3, 4]
+...
+s:insert({0})
+---
+- [0]
+...
+_ = s:delete({1})
+---
+...
+s:select({})
+---
+- - [0]
+  - [2, 3, 4]
+  - [3, 4]
+  - [4, 5]
+...
+pk:get({4})
+---
+- [4, 5]
+...
+sk:select({box.NULL})
+---
+- - [0]
+...
+sk:get({5})
+---
+- [4, 5]
+...
+s:drop()
+---
+...

test/engine/ddl.test.lua

@@ -136,3 +136,29 @@ s = box.schema.create_space('test', {format = format})
 pk = s:create_index('pk')
 pk.parts[1].type
 s:drop()
+
+--
+-- gh-3229: update optionality if a space format is changed too,
+-- not only when indexes are updated.
+--
+box.cfg{}
+s = box.schema.create_space('test', {engine = 'memtx'})
+format = {}
+format[1] = {'field1', 'unsigned'}
+format[2] = {'field2', 'unsigned', is_nullable = true}
+format[3] = {'field3', 'unsigned'}
+s:format(format)
+pk = s:create_index('pk')
+sk = s:create_index('sk', {parts = {{2, 'unsigned', is_nullable = true}}})
+s:replace{2, 3, 4}
+s:format({})
+s:insert({1})
+s:insert({4, 5})
+s:insert({3, 4})
+s:insert({0})
+_ = s:delete({1})
+s:select({})
+pk:get({4})
+sk:select({box.NULL})
+sk:get({5})
+s:drop()