memtx: fix definition of old story in MVCC insert statement preparation

During preparation of insert statements in MVCC, we define an old story and
abort all transactions that delete this story.

If there exists an older
story in the history chain, but the story is deleted by a prepared (not
necessarily committed) transaction, we consider that it de-facto does not
exist anymore — this logic is consistent, since during preparation of the
transaction deleting this story, the conflict resolution described above
was already done.

In this manner, there can be no more than one prepared statement deleting
a story at any point in time.

Closes #8104

NO_DOC=bugfix

changelogs/unreleased/gh-8104-memtx-mvcc-assertion-failure-during-stmt-preparation.md

+## bugfix/memtx
+
+* Fixed assertion failure in MVCC during statement preparation (gh-8104).

src/box/memtx_tx.c

@@ -2409,14 +2409,6 @@ memtx_tx_history_prepare_insert_stmt(struct txn_stmt *stmt)
 	uint32_t index_count = story->index_count;
 	memtx_tx_history_sink_story(story);
 
-	struct memtx_story *old_story = story->link[0].older_story;
-	if (stmt->del_story == NULL)
-		assert(old_story == NULL || old_story->rollbacked ||
-		       old_story->del_psn != 0);
-	else
-		assert(old_story != NULL &&
-		       (old_story->rollbacked || stmt->del_story == old_story));
-
 	if (stmt->del_story == NULL) {
 		/*
 		 * This statement replaced nothing. That means that before
@@ -2449,15 +2441,26 @@ memtx_tx_history_prepare_insert_stmt(struct txn_stmt *stmt)
 		}
 	}
 
-	if (old_story != NULL && old_story->rollbacked) {
-		assert(old_story->link[0].older_story == NULL);
-		old_story = NULL;
+	struct memtx_story *old_story = story->link[0].older_story;
+	if (stmt->del_story == NULL)
+		assert(old_story == NULL || old_story->rollbacked ||
+		       old_story->del_psn != 0);
+	else
+		assert(old_story != NULL &&
+		       (old_story->rollbacked || stmt->del_story == old_story));
+	if (old_story != NULL) {
+		if (old_story->rollbacked) {
+			assert(old_story->link[0].older_story == NULL);
+			old_story = NULL;
+		} else if (old_story->del_psn != 0) {
+			assert(stmt->del_story == NULL);
+			old_story = NULL;
+		}
 	}
-
 	if (old_story != NULL) {
 		/*
 		 * There can be some transactions that want to delete old_story.
-		 * It can be this transaction, or some other prepared TX.
+		 * It can be this transaction.
 		 * All other transactions must be aborted or relinked to delete
 		 * this tuple.
 		 */

test/box-luatest/gh_8104_memtx_mvcc_assertion_failure_during_stmt_preparation_test.lua

+local server = require('luatest.server')
+local t = require('luatest')
+
+local g = t.group()
+
+g.before_all(function(cg)
+    cg.server = server:new {
+        alias   = 'dflt',
+        box_cfg = {memtx_use_mvcc_engine = true}
+    }
+    cg.server:start()
+    cg.server:exec(function()
+        local s = box.schema.create_space('s')
+        s:create_index('pk')
+    end)
+end)
+
+g.after_all(function(cg)
+    cg.server:drop()
+end)
+
+-- Checks that preparation of an insert statement with an older story deleted by
+-- a prepared transaction does not fail assertion.
+g.test_preparation_with_deleted_older_story_assertion = function(cg)
+    cg.server:exec(function()
+        local fiber = require('fiber')
+        local t = require('luatest')
+
+        box.space.s:replace{0}
+
+        local first_replace = fiber.create(function()
+            fiber.self():set_joinable(true)
+            box.atomic(function()
+                box.space.s:delete{0}
+            end)
+        end)
+        local second_replace = fiber.create(function()
+            fiber.self():set_joinable(true)
+            box.atomic(function()
+                box.space.s:insert{0}
+            end)
+        end)
+
+        first_replace:join()
+        second_replace:join()
+
+        t.assert_equals(box.space.s:select{}, {{0}})
+    end)
+end