From eff7d524b539319fe061f7a2ba2216a882885286 Mon Sep 17 00:00:00 2001
From: Dmitry Ivanov <ivadmi5@gmail.com>
Date: Tue, 23 Jul 2024 05:45:39 +0300
Subject: [PATCH] test(qsync): fix tests and drop the ones incompatible with
 new RAFT_PROMOTE

NO_DOC=internal
NO_TEST=internal
NO_CHANGELOG=internal
---
 src/box/box.cc                                |  10 ++
 src/lib/core/errinj.h                         |   1 +
 test/box-luatest/builtin_events_test.lua      |   4 +-
 test/box/errinj.result                        |   1 +
 .../gh_5568_read_only_reason_test.lua         |   6 +-
 .../gh_6033_box_promote_demote_test.lua       |  50 +-----
 .../gh_6842_qsync_applier_order_test.lua      |  10 +-
 ...h_7286_split_brain_false_positive_test.lua |   2 +-
 .../gh_7592_local_write_with_syncro_test.lua  |   2 +-
 .../gh_8996_synchro_filter_enable_test.lua    |  78 ---------
 .../replication-luatest/linearizable_test.lua |   8 +-
 test/replication/election_basic.result        |   2 +-
 test/replication/election_basic.test.lua      |   2 +-
 .../gh-5140-qsync-casc-rollback.result        |   4 +-
 .../gh-5140-qsync-casc-rollback.test.lua      |   4 +-
 .../gh-5144-qsync-dup-confirm.result          |   2 +-
 .../gh-5144-qsync-dup-confirm.test.lua        |   2 +-
 .../gh-5167-qsync-rollback-snap.result        |   2 +-
 .../gh-5167-qsync-rollback-snap.test.lua      |   2 +-
 .../gh-5195-qsync-replica-write.result        |   2 +-
 .../gh-5195-qsync-replica-write.test.lua      |   2 +-
 .../gh-5213-qsync-applier-order-3.result      |  15 +-
 .../gh-5213-qsync-applier-order-3.test.lua    |  10 +-
 .../gh-5430-qsync-promote-crash.result        | 159 ------------------
 .../gh-5430-qsync-promote-crash.test.lua      |  76 ---------
 .../gh-6034-qsync-limbo-ownership.result      |   6 +-
 .../gh-6034-qsync-limbo-ownership.test.lua    |   6 +-
 .../gh-6035-election-filter.result            |   2 +-
 .../gh-6035-election-filter.test.lua          |   2 +-
 test/replication/qsync_advanced.result        |   6 +-
 test/replication/qsync_advanced.test.lua      |   6 +-
 test/replication/qsync_basic.result           |   4 +-
 test/replication/qsync_basic.test.lua         |   4 +-
 test/replication/qsync_errinj.result          |   2 +-
 test/replication/qsync_errinj.test.lua        |   2 +-
 test/replication/qsync_snapshots.result       |   2 +-
 test/replication/qsync_snapshots.test.lua     |   2 +-
 37 files changed, 86 insertions(+), 414 deletions(-)
 delete mode 100644 test/replication-luatest/gh_8996_synchro_filter_enable_test.lua
 delete mode 100644 test/replication/gh-5430-qsync-promote-crash.result
 delete mode 100644 test/replication/gh-5430-qsync-promote-crash.test.lua

diff --git a/src/box/box.cc b/src/box/box.cc
index b48a21fb94..d4aaffc16d 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -2686,6 +2686,16 @@ box_promote(void)
 		is_in_box_promote = false;
 		if (box_raft_try_promote() != 0)
 			return -1;
+		/*
+		 * XXX: this restores the previous buggy behavior: when
+		 * the function exits, it forcibly resets is_in_box_promote
+		 * even if the raft machine fiber is stuck on a write, which
+		 * should be protected by this flag.
+		 * This is needed for gh_6033_box_promote_demote_test.lua.
+		 */
+		ERROR_INJECT(ERRINJ_LIMBO_WRITE_PROMOTE_FAST_EXIT, {
+			return 0;
+		});
 		txn_limbo_wait_promote_attempts(&txn_limbo,
 						promote_attempts + 1,
 						TIMEOUT_INFINITY);
diff --git a/src/lib/core/errinj.h b/src/lib/core/errinj.h
index ecc236821d..dcbaef23c0 100644
--- a/src/lib/core/errinj.h
+++ b/src/lib/core/errinj.h
@@ -103,6 +103,7 @@ struct errinj {
 	_(ERRINJ_IPROTO_TX_DELAY, ERRINJ_BOOL, {.bparam = false}) \
 	_(ERRINJ_IPROTO_WRITE_ERROR_DELAY, ERRINJ_BOOL, {.bparam = false})\
 	_(ERRINJ_LIMBO_PROMOTE_ACK_SLEEP, ERRINJ_BOOL, {.bparam = false})\
+	_(ERRINJ_LIMBO_WRITE_PROMOTE_FAST_EXIT, ERRINJ_BOOL, {.bparam = false})\
 	_(ERRINJ_LIMBO_WRITE_PROMOTE_SLEEP, ERRINJ_BOOL, {.bparam = false})\
 	_(ERRINJ_LOG_ROTATE, ERRINJ_BOOL, {.bparam = false}) \
 	_(ERRINJ_MAIN_MAKE_FILE_ON_RETURN, ERRINJ_BOOL, {.bparam = false}) \
diff --git a/test/box-luatest/builtin_events_test.lua b/test/box-luatest/builtin_events_test.lua
index 1d9173fb0f..5b104feaf0 100644
--- a/test/box-luatest/builtin_events_test.lua
+++ b/test/box-luatest/builtin_events_test.lua
@@ -129,8 +129,8 @@ g.test_box_status = function(cg)
                     {is_ro = true, is_ro_cfg = false, status = 'running'})
 
     -- promotion should turn rm
-    cg.master:exec(function() box.ctl.promote() end)
-    t.helpers.retrying({}, function() t.assert_equals(result_no, 7) end)
+    cg.master:exec(function() box.ctl.promote(); box.ctl.wait_rw() end)
+    t.helpers.retrying({}, function() t.assert_ge(result_no, 7) end)
     t.assert_equals(result,
                     {is_ro = false, is_ro_cfg = false, status = 'running'})
 
diff --git a/test/box/errinj.result b/test/box/errinj.result
index edd5022067..78ecb10ca7 100644
--- a/test/box/errinj.result
+++ b/test/box/errinj.result
@@ -77,6 +77,7 @@ evals
   - ERRINJ_IPROTO_TX_DELAY: false
   - ERRINJ_IPROTO_WRITE_ERROR_DELAY: false
   - ERRINJ_LIMBO_PROMOTE_ACK_SLEEP: false
+  - ERRINJ_LIMBO_WRITE_PROMOTE_FAST_EXIT: false
   - ERRINJ_LIMBO_WRITE_PROMOTE_SLEEP: false
   - ERRINJ_LOG_ROTATE: false
   - ERRINJ_MAIN_MAKE_FILE_ON_RETURN: false
diff --git a/test/replication-luatest/gh_5568_read_only_reason_test.lua b/test/replication-luatest/gh_5568_read_only_reason_test.lua
index 0777ab9ae3..6558f5abfd 100644
--- a/test/replication-luatest/gh_5568_read_only_reason_test.lua
+++ b/test/replication-luatest/gh_5568_read_only_reason_test.lua
@@ -177,6 +177,7 @@ g.test_read_only_reason_election_has_leader = function(g)
         box.cfg{election_mode = 'voter'}
     end)
     g.master:wait_for_election_leader()
+    g.master:wait_for_synchro_queue_term(g.master:get_election_term())
     g.replica:wait_until_election_leader_found()
 
     local ok, err = g.replica:exec(function()
@@ -220,7 +221,7 @@ g.test_read_only_reason_synchro = function(g)
             replication_synchro_quorum = 2,
             replication_synchro_timeout = 1000000,
         }
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
     end)
 
     t.helpers.retrying({}, function()
@@ -279,6 +280,7 @@ g.test_read_only_reason_election_has_leader_no_uuid = function(g)
         }
     end)
     g.master:wait_for_election_leader()
+    g.master:wait_for_synchro_queue_term(g.master:get_election_term())
     g.replica:wait_until_election_leader_found()
     local leader_id = g.master:get_instance_id()
 
@@ -324,7 +326,7 @@ g.test_read_only_reason_synchro_no_uuid = function(g)
             replication_synchro_quorum = 2,
             replication_synchro_timeout = 1000000,
         }
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
         box.space._cluster:run_triggers(false)
         box.space._cluster:delete{box.info.id}
     end)
diff --git a/test/replication-luatest/gh_6033_box_promote_demote_test.lua b/test/replication-luatest/gh_6033_box_promote_demote_test.lua
index fa1d0455b6..2730e29cfb 100644
--- a/test/replication-luatest/gh_6033_box_promote_demote_test.lua
+++ b/test/replication-luatest/gh_6033_box_promote_demote_test.lua
@@ -57,6 +57,9 @@ local function fiber_join(server, fiber)
 end
 
 local function wal_delay_start(server, countdown)
+    server:exec(function()
+        box.error.injection.set('ERRINJ_LIMBO_WRITE_PROMOTE_FAST_EXIT', true)
+    end)
     if countdown == nil then
         server:exec(function()
             box.error.injection.set('ERRINJ_WAL_DELAY', true)
@@ -434,48 +437,6 @@ g_common.test_limbo_full_interfering_promote = function(g)
     wait_sync(g.cluster.servers)
 end
 
--- Demoting should fail if it is interrupted from another server
--- while waiting for synchro queue being emptied.
-g_common.test_limbo_full_interfering_demote = function(g)
-    promote(g.server_1)
-    wait_sync(g.cluster.servers)
-
-    g.server_1:exec(function()
-        box.schema.create_space('test', {is_sync = true}):create_index('pk')
-    end)
-
-    box_cfg_update({g.server_1}, {
-        replication_synchro_quorum = 3,
-        replication_synchro_timeout = 1000,
-    })
-
-    box_cfg_update({g.server_2}, {
-        replication_synchro_timeout = 0.1,
-    })
-
-    g.server_1:exec(function()
-        local s = box.space.test
-        require('fiber').create(s.replace, s, {1})
-    end)
-    wait_sync(g.cluster.servers)
-
-    -- Start demoting server_1 and interrupt it from server_2
-    local f = demote_start(g.server_1)
-    local term = g.server_1:get_synchro_queue_term()
-    g.server_2:exec(function() pcall(box.ctl.promote) end)
-    g.server_1:wait_for_synchro_queue_term(term + 1)
-
-    local ok, err = fiber_join(g.server_1, f)
-    luatest.assert(not ok and err.code == box.error.INTERFERING_PROMOTE,
-        'Interfering demote fails')
-
-    wait_sync(g.cluster.servers)
-    promote(g.server_1)
-    g.server_1:exec(function() box.space.test:drop() end)
-    demote(g.server_1)
-    wait_sync(g.cluster.servers)
-end
-
 -- Promoting should fail if synchro queue replication timeouts during it
 g_common.test_fail_limbo_ack_promote = function(g)
     box_cfg_update({g.server_1}, {
@@ -484,12 +445,13 @@ g_common.test_fail_limbo_ack_promote = function(g)
     })
 
     box_cfg_update({g.server_2}, {
-        replication_synchro_quorum = 3,
+        replication_synchro_quorum = 1,
         replication_synchro_timeout = 1000,
     })
+    promote(g.server_2)
+    box_cfg_update({g.server_2}, {replication_synchro_quorum = 3})
 
     -- fill synchro queue on server_1
-    promote(g.server_2)
     g.server_2:exec(function()
         local s = box.schema.create_space('test', {is_sync = true})
         s:create_index('pk')
diff --git a/test/replication-luatest/gh_6842_qsync_applier_order_test.lua b/test/replication-luatest/gh_6842_qsync_applier_order_test.lua
index b3dfa82dab..bd8518d0d4 100644
--- a/test/replication-luatest/gh_6842_qsync_applier_order_test.lua
+++ b/test/replication-luatest/gh_6842_qsync_applier_order_test.lua
@@ -124,7 +124,7 @@ end
 g.test_local_txn_during_remote_promote = function(g)
     -- Server 1 takes the synchro queue.
     g.server1:exec(function()
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
         box.cfg{
             -- To hang own transactions in the synchro queue.
             replication_synchro_quorum = 3,
@@ -198,7 +198,7 @@ end
 g.test_remote_promote_during_local_txn_including_it = function(g)
     -- Start synchro txns on server 1.
     local fids = g.server1:exec(function()
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
         local s = box.schema.create_space('test', {is_sync = true})
         s:create_index('pk')
         box.cfg{
@@ -226,7 +226,7 @@ g.test_remote_promote_during_local_txn_including_it = function(g)
             -- the promotion ASAP.
             replication_synchro_timeout = 0.001,
         }
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
         box.cfg{
             replication_synchro_quorum = 2,
             replication_synchro_timeout = 1000,
@@ -287,7 +287,7 @@ end
 g.test_remote_promote_during_local_txn_not_including_it = function(g)
     -- Start a synchro txn on server 1.
     local fids = g.server1:exec(function()
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
         local s = box.schema.create_space('test', {is_sync = true})
         s:create_index('pk')
         box.cfg{
@@ -311,7 +311,7 @@ g.test_remote_promote_during_local_txn_not_including_it = function(g)
         box.cfg{
             replication_synchro_quorum = 1,
         }
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
     end)
 
     -- Server 1 receives the PROMOTE.
diff --git a/test/replication-luatest/gh_7286_split_brain_false_positive_test.lua b/test/replication-luatest/gh_7286_split_brain_false_positive_test.lua
index 8e7ef421ac..d75dcff52c 100644
--- a/test/replication-luatest/gh_7286_split_brain_false_positive_test.lua
+++ b/test/replication-luatest/gh_7286_split_brain_false_positive_test.lua
@@ -35,7 +35,7 @@ end)
 
 g.test_false_positive_split_brain = function(cg)
     cg.node1:exec(function()
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
         box.ctl.demote()
     end)
     cg.node2:wait_for_vclock_of(cg.node1)
diff --git a/test/replication-luatest/gh_7592_local_write_with_syncro_test.lua b/test/replication-luatest/gh_7592_local_write_with_syncro_test.lua
index c3ae21e32a..a54dbbe186 100644
--- a/test/replication-luatest/gh_7592_local_write_with_syncro_test.lua
+++ b/test/replication-luatest/gh_7592_local_write_with_syncro_test.lua
@@ -27,7 +27,7 @@ g.before_each(function(cg)
     }
     cg.cluster:start()
     cg.master:exec(function()
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
         box.schema.space.create('sync', {is_sync = true})
         box.space.sync:create_index('pk')
         box.schema.space.create('loc', {is_local = true})
diff --git a/test/replication-luatest/gh_8996_synchro_filter_enable_test.lua b/test/replication-luatest/gh_8996_synchro_filter_enable_test.lua
deleted file mode 100644
index dacc0bb2fb..0000000000
--- a/test/replication-luatest/gh_8996_synchro_filter_enable_test.lua
+++ /dev/null
@@ -1,78 +0,0 @@
-local t = require('luatest')
-local replica_set = require('luatest.replica_set')
-local server = require('luatest.server')
-
-local g = t.group('synchro-filter-enable-by-version')
-
-g.before_each(function(cg)
-    cg.replica_set = replica_set:new{}
-    cg.box_cfg = {
-        replication = {
-            server.build_listen_uri('server1', cg.replica_set.id),
-            server.build_listen_uri('server2', cg.replica_set.id),
-        },
-        replication_timeout = 0.1,
-    }
-    for i = 1,2 do
-        cg['server' .. i] = cg.replica_set:build_and_add_server{
-            alias = 'server' .. i,
-            box_cfg = cg.box_cfg,
-        }
-    end
-end)
-
-g.after_each(function(cg)
-    cg.replica_set:drop()
-end)
-
--- Check that split-brain detection does not work with schema version <=
--- 2.10.1, and is re-enabled back after a schema upgrade.
-g.test_filter_enable_disable = function(cg)
-    cg.replica_set:start()
-    cg.server1:exec(function()
-        box.ctl.wait_rw()
-        box.schema.downgrade('2.10.1')
-        t.assert_equals(box.space._schema:get{'version'},
-                        {'version', 2, 10, 1})
-    end)
-    cg.server2:wait_for_vclock_of(cg.server1)
-
-    cg.server1:update_box_cfg({replication = ""})
-    cg.server2:update_box_cfg({replication = ""})
-
-    cg.server1:exec(function()
-        box.ctl.promote()
-    end)
-    cg.server2:exec(function()
-        box.ctl.promote()
-    end)
-
-    cg.server1:update_box_cfg(cg.box_cfg)
-    cg.server2:update_box_cfg(cg.box_cfg)
-    cg.server1:wait_for_vclock_of(cg.server2)
-    cg.server2:wait_for_vclock_of(cg.server1)
-    cg.server1:assert_follows_upstream(cg.server2:get_instance_id())
-    cg.server2:assert_follows_upstream(cg.server1:get_instance_id())
-
-    cg.server1:update_box_cfg({replication = ""})
-    cg.server2:update_box_cfg({replication = ""})
-
-    for i = 1,2 do
-        cg['server' .. i]:exec(function()
-            box.ctl.promote()
-            box.schema.upgrade()
-        end)
-    end
-
-    t.helpers.retrying({}, function()
-        for i = 1,2 do
-            cg['server' .. i]:update_box_cfg(cg.box_cfg)
-            cg['server' .. i]:exec(function(id)
-                t.assert_equals(box.info.replication[id].upstream.status,
-                                'stopped')
-                t.assert_str_contains(box.info.replication[id].upstream.message,
-                                      'Split-Brain discovered')
-            end, {cg['server' .. 3 - i]:get_instance_id()})
-        end
-    end)
-end
diff --git a/test/replication-luatest/linearizable_test.lua b/test/replication-luatest/linearizable_test.lua
index 1eeadac04d..aaf46c1dd6 100644
--- a/test/replication-luatest/linearizable_test.lua
+++ b/test/replication-luatest/linearizable_test.lua
@@ -108,7 +108,7 @@ g.before_test('test_no_dirty_reads', function(cg)
     end)
     cg.servers[2]:wait_for_vclock_of(cg.servers[1])
     cg.servers[2]:exec(function()
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
     end)
     cg.servers[1]:wait_for_vclock_of(cg.servers[2])
     cg.servers[3]:wait_for_vclock_of(cg.servers[2])
@@ -189,7 +189,7 @@ end
 
 g.test_leader_change = function(cg)
     cg.servers[1]:exec(function()
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
     end)
     cg.servers[2]:wait_for_vclock_of(cg.servers[1])
     cg.servers[3]:wait_for_vclock_of(cg.servers[1])
@@ -201,7 +201,7 @@ g.test_leader_change = function(cg)
         cg.proxies[i]:pause()
     end
     cg.servers[2]:exec(function()
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
         box.space.sync:insert{2}
     end)
     local fid = cg.servers[1]:exec(function()
@@ -250,7 +250,7 @@ g_basic.before_all(function(cg)
         box.schema.space.create('vinyl', {engine = 'vinyl', is_sync = true})
         box.space.vinyl:create_index('pk')
         -- For the sake of writes to the sync space.
-        box.ctl.promote()
+        box.ctl.promote(); box.ctl.wait_rw()
     end)
 end)
 
diff --git a/test/replication/election_basic.result b/test/replication/election_basic.result
index 1fd69caf1c..3e8a602b7f 100644
--- a/test/replication/election_basic.result
+++ b/test/replication/election_basic.result
@@ -399,7 +399,7 @@ assert(election_tbl[6].state == 'follower')
  | - true
  | ...
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 
diff --git a/test/replication/election_basic.test.lua b/test/replication/election_basic.test.lua
index 8fb0f6fe5a..7a3eeb1f9e 100644
--- a/test/replication/election_basic.test.lua
+++ b/test/replication/election_basic.test.lua
@@ -163,7 +163,7 @@ box.cfg{election_mode='manual'}
 test_run:wait_cond(function() return #election_tbl == 6 end)
 assert(election_tbl[6].state == 'follower')
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 
 test_run:wait_cond(function() return #election_tbl == 8 end)
 assert(election_tbl[7].state == 'candidate')
diff --git a/test/replication/gh-5140-qsync-casc-rollback.result b/test/replication/gh-5140-qsync-casc-rollback.result
index a71d8f9818..7e78d40a32 100644
--- a/test/replication/gh-5140-qsync-casc-rollback.result
+++ b/test/replication/gh-5140-qsync-casc-rollback.result
@@ -73,7 +73,7 @@ _ = box.schema.space.create('async', {is_sync=false, engine = engine})
 _ = _:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 -- Write something to flush the master state to replica.
@@ -206,7 +206,7 @@ box.space.sync:select{}
  |   - [4]
  | ...
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 
diff --git a/test/replication/gh-5140-qsync-casc-rollback.test.lua b/test/replication/gh-5140-qsync-casc-rollback.test.lua
index 15732e3324..74f4539049 100644
--- a/test/replication/gh-5140-qsync-casc-rollback.test.lua
+++ b/test/replication/gh-5140-qsync-casc-rollback.test.lua
@@ -48,7 +48,7 @@ _ = box.schema.space.create('sync', {is_sync = true, engine = engine})
 _ = _:create_index('pk')
 _ = box.schema.space.create('async', {is_sync=false, engine = engine})
 _ = _:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 -- Write something to flush the master state to replica.
 box.space.sync:replace{1}
 
@@ -97,7 +97,7 @@ test_run:switch('default')
 box.space.async:select{}
 box.space.sync:select{}
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 
 box.space.sync:drop()
 box.space.async:drop()
diff --git a/test/replication/gh-5144-qsync-dup-confirm.result b/test/replication/gh-5144-qsync-dup-confirm.result
index 21571fba87..6496cb1b5a 100644
--- a/test/replication/gh-5144-qsync-dup-confirm.result
+++ b/test/replication/gh-5144-qsync-dup-confirm.result
@@ -54,7 +54,7 @@ _ = box.schema.space.create('sync', {is_sync = true, engine = engine})
 _ = _:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 
diff --git a/test/replication/gh-5144-qsync-dup-confirm.test.lua b/test/replication/gh-5144-qsync-dup-confirm.test.lua
index b3e5b2ab2f..d3fd5ea031 100644
--- a/test/replication/gh-5144-qsync-dup-confirm.test.lua
+++ b/test/replication/gh-5144-qsync-dup-confirm.test.lua
@@ -23,7 +23,7 @@ box.cfg{replication_synchro_quorum = 2, replication_synchro_timeout = 1000}
 
 _ = box.schema.space.create('sync', {is_sync = true, engine = engine})
 _ = _:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 
 -- Remember the current LSN. In the end, when the following synchronous
 -- transaction is committed, result LSN should be this value +2: for the
diff --git a/test/replication/gh-5167-qsync-rollback-snap.result b/test/replication/gh-5167-qsync-rollback-snap.result
index 85ef58612e..5246030b5c 100644
--- a/test/replication/gh-5167-qsync-rollback-snap.result
+++ b/test/replication/gh-5167-qsync-rollback-snap.result
@@ -41,7 +41,7 @@ _ = box.schema.space.create('sync', {is_sync = true, engine = engine})
 _ = box.space.sync:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 -- Write something to flush the current master's state to replica.
diff --git a/test/replication/gh-5167-qsync-rollback-snap.test.lua b/test/replication/gh-5167-qsync-rollback-snap.test.lua
index 5e8950eb91..ecb80c6a78 100644
--- a/test/replication/gh-5167-qsync-rollback-snap.test.lua
+++ b/test/replication/gh-5167-qsync-rollback-snap.test.lua
@@ -16,7 +16,7 @@ fiber = require('fiber')
 box.cfg{replication_synchro_quorum = 2, replication_synchro_timeout = 1000}
 _ = box.schema.space.create('sync', {is_sync = true, engine = engine})
 _ = box.space.sync:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 -- Write something to flush the current master's state to replica.
 _ = box.space.sync:insert{1}
 _ = box.space.sync:delete{1}
diff --git a/test/replication/gh-5195-qsync-replica-write.result b/test/replication/gh-5195-qsync-replica-write.result
index 26555b731d..6980d70417 100644
--- a/test/replication/gh-5195-qsync-replica-write.result
+++ b/test/replication/gh-5195-qsync-replica-write.result
@@ -43,7 +43,7 @@ _ = box.schema.space.create('sync', {engine = engine, is_sync = true})
 _ = box.space.sync:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 
diff --git a/test/replication/gh-5195-qsync-replica-write.test.lua b/test/replication/gh-5195-qsync-replica-write.test.lua
index 8753dce6cd..ead7ef508a 100644
--- a/test/replication/gh-5195-qsync-replica-write.test.lua
+++ b/test/replication/gh-5195-qsync-replica-write.test.lua
@@ -18,7 +18,7 @@ test_run:cmd('start server replica with wait=True, wait_load=True')
 --
 _ = box.schema.space.create('sync', {engine = engine, is_sync = true})
 _ = box.space.sync:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 
 box.cfg{                                                                        \
     replication_synchro_timeout = 1000,                                         \
diff --git a/test/replication/gh-5213-qsync-applier-order-3.result b/test/replication/gh-5213-qsync-applier-order-3.result
index 2fc9e4e7f0..9c72dea8c7 100644
--- a/test/replication/gh-5213-qsync-applier-order-3.result
+++ b/test/replication/gh-5213-qsync-applier-order-3.result
@@ -45,7 +45,7 @@ s = box.schema.space.create('test', {is_sync = true})
 _ = s:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 
@@ -174,7 +174,7 @@ test_run:wait_lsn('replica2', 'default')
  | ---
  | ...
 box.cfg{                                                                        \
-    replication_synchro_quorum = 2,                                             \
+    replication_synchro_quorum = 1,                                             \
     replication_synchro_timeout = 1000,                                         \
 }
  | ---
@@ -182,7 +182,14 @@ box.cfg{
 -- Replica2 takes the limbo ownership and sends the transaction to the replica1.
 -- Along with the CONFIRM from the default node, which is still not applied
 -- on the replica1.
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
+ | ---
+ | ...
+box.info.id == box.info.synchro.queue.owner -- promote should've been applied
+ | ---
+ | - true
+ | ...
+box.cfg{replication_synchro_quorum = 2}
  | ---
  | ...
 fiber = require('fiber')
@@ -259,7 +266,7 @@ test_run:cmd('delete server replica2')
 box.cfg{replication_synchro_quorum = 1}
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 s:drop()
diff --git a/test/replication/gh-5213-qsync-applier-order-3.test.lua b/test/replication/gh-5213-qsync-applier-order-3.test.lua
index 18f3a194dd..996a49e42e 100644
--- a/test/replication/gh-5213-qsync-applier-order-3.test.lua
+++ b/test/replication/gh-5213-qsync-applier-order-3.test.lua
@@ -30,7 +30,7 @@ box.schema.user.grant('guest', 'super')
 
 s = box.schema.space.create('test', {is_sync = true})
 _ = s:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 
 test_run:cmd('create server replica1 with rpl_master=default,\
               script="replication/replica1.lua"')
@@ -85,13 +85,15 @@ end)
 test_run:switch('replica2')
 test_run:wait_lsn('replica2', 'default')
 box.cfg{                                                                        \
-    replication_synchro_quorum = 2,                                             \
+    replication_synchro_quorum = 1,                                             \
     replication_synchro_timeout = 1000,                                         \
 }
 -- Replica2 takes the limbo ownership and sends the transaction to the replica1.
 -- Along with the CONFIRM from the default node, which is still not applied
 -- on the replica1.
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
+box.info.id == box.info.synchro.queue.owner -- promote should've been applied
+box.cfg{replication_synchro_quorum = 2}
 fiber = require('fiber')
 f = fiber.new(function() box.space.test:replace{2} end)
 
@@ -121,7 +123,7 @@ test_run:cmd('stop server replica2')
 test_run:cmd('delete server replica2')
 -- Restore leadership to make the default instance writable.
 box.cfg{replication_synchro_quorum = 1}
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 s:drop()
 box.schema.user.revoke('guest', 'super')
 box.cfg{                                                                        \
diff --git a/test/replication/gh-5430-qsync-promote-crash.result b/test/replication/gh-5430-qsync-promote-crash.result
deleted file mode 100644
index 1204c625a0..0000000000
--- a/test/replication/gh-5430-qsync-promote-crash.result
+++ /dev/null
@@ -1,159 +0,0 @@
--- test-run result file version 2
---
--- gh-5430: box.ctl.promote() could assert if one of replicas didn't receive
--- anything from the old leader. Internally box.ctl.promote() collected a quorum
--- using vclock_follow() from all connected relays by the old leader ID and in
--- that case one of such replicas led to vclock_follow(0) which is always a
--- crash.
---
-test_run = require('test_run').new()
- | ---
- | ...
-
---
--- Start 2 fullmesh nodes working normally.
---
-test_run:cmd('create server master1 with '..                                    \
-             'script="replication/gh-5430-master1.lua"')
- | ---
- | - true
- | ...
-test_run:cmd('start server master1 with wait=False')
- | ---
- | - true
- | ...
-
-test_run:cmd('create server master2 with '..                                    \
-             'script="replication/gh-5430-master2.lua"')
- | ---
- | - true
- | ...
-test_run:cmd('start server master2 with wait=True')
- | ---
- | - true
- | ...
-
---
--- One of them won't write to WAL anything from now on. If a new instance is
--- added and it will write something, master2 won't apply it.
---
-test_run:switch('master2')
- | ---
- | - true
- | ...
-box.error.injection.set('ERRINJ_WAL_DELAY', true)
- | ---
- | - ok
- | ...
-
---
--- Third node is the future 'old leader', by which master2 has
--- vclock[master3] == 0.
---
-test_run:cmd('create server master3 with '..                                    \
-             'script="replication/gh-5430-master3.lua"')
- | ---
- | - true
- | ...
-test_run:cmd('start server master3 with wait=True')
- | ---
- | - true
- | ...
-
---
--- Make master1 fetch data from master3 so as it could receive sync data and
--- confirm it later.
---
-test_run:switch('master1')
- | ---
- | - true
- | ...
--- Can't keep master2 in there because it hangs with ER_CFG about a duplicate
--- connection. Even reset via replication = {} does not help for 100%. But for
--- the test it does not matter.
-box.cfg{replication = {test_run:eval('master3', 'return box.cfg.listen')[1]}}
- | ---
- | ...
-
---
--- Master3 fills the limbo and dies.
---
-test_run:switch('master3')
- | ---
- | - true
- | ...
-box.ctl.promote()
- | ---
- | ...
-s = box.schema.create_space('test', {is_sync = true})
- | ---
- | ...
-_ = s:create_index('pk')
- | ---
- | ...
-_ = require('fiber').create(s.replace, s, {1})
- | ---
- | ...
-test_run:wait_lsn('master1', 'master3')
- | ---
- | ...
-
-test_run:switch('master1')
- | ---
- | - true
- | ...
-test_run:cmd('stop server master3')
- | ---
- | - true
- | ...
-test_run:cmd('delete server master3')
- | ---
- | - true
- | ...
-
---
--- Master1 tries to promote self. In the meantime master2 has
--- vclock[master3] == 0. It is still blocked in the WAL thread. Master1 should
--- be ready to seeing 0 LSN by the old leader's component in some replicas.
---
-box.cfg{replication_synchro_timeout = 0.1}
- | ---
- | ...
-assert(box.info.synchro.queue.len > 0)
- | ---
- | - true
- | ...
-assert(not pcall(box.ctl.promote))
- | ---
- | - true
- | ...
-
-test_run:switch('master2')
- | ---
- | - true
- | ...
-box.error.injection.set('ERRINJ_WAL_DELAY', false)
- | ---
- | - ok
- | ...
-
-test_run:switch('default')
- | ---
- | - true
- | ...
-test_run:cmd('stop server master2')
- | ---
- | - true
- | ...
-test_run:cmd('delete server master2')
- | ---
- | - true
- | ...
-test_run:cmd('stop server master1')
- | ---
- | - true
- | ...
-test_run:cmd('delete server master1')
- | ---
- | - true
- | ...
diff --git a/test/replication/gh-5430-qsync-promote-crash.test.lua b/test/replication/gh-5430-qsync-promote-crash.test.lua
deleted file mode 100644
index 7ef8860e79..0000000000
--- a/test/replication/gh-5430-qsync-promote-crash.test.lua
+++ /dev/null
@@ -1,76 +0,0 @@
---
--- gh-5430: box.ctl.promote() could assert if one of replicas didn't receive
--- anything from the old leader. Internally box.ctl.promote() collected a quorum
--- using vclock_follow() from all connected relays by the old leader ID and in
--- that case one of such replicas led to vclock_follow(0) which is always a
--- crash.
---
-test_run = require('test_run').new()
-
---
--- Start 2 fullmesh nodes working normally.
---
-test_run:cmd('create server master1 with '..                                    \
-             'script="replication/gh-5430-master1.lua"')
-test_run:cmd('start server master1 with wait=False')
-
-test_run:cmd('create server master2 with '..                                    \
-             'script="replication/gh-5430-master2.lua"')
-test_run:cmd('start server master2 with wait=True')
-
---
--- One of them won't write to WAL anything from now on. If a new instance is
--- added and it will write something, master2 won't apply it.
---
-test_run:switch('master2')
-box.error.injection.set('ERRINJ_WAL_DELAY', true)
-
---
--- Third node is the future 'old leader', by which master2 has
--- vclock[master3] == 0.
---
-test_run:cmd('create server master3 with '..                                    \
-             'script="replication/gh-5430-master3.lua"')
-test_run:cmd('start server master3 with wait=True')
-
---
--- Make master1 fetch data from master3 so as it could receive sync data and
--- confirm it later.
---
-test_run:switch('master1')
--- Can't keep master2 in there because it hangs with ER_CFG about a duplicate
--- connection. Even reset via replication = {} does not help for 100%. But for
--- the test it does not matter.
-box.cfg{replication = {test_run:eval('master3', 'return box.cfg.listen')[1]}}
-
---
--- Master3 fills the limbo and dies.
---
-test_run:switch('master3')
-box.ctl.promote()
-s = box.schema.create_space('test', {is_sync = true})
-_ = s:create_index('pk')
-_ = require('fiber').create(s.replace, s, {1})
-test_run:wait_lsn('master1', 'master3')
-
-test_run:switch('master1')
-test_run:cmd('stop server master3')
-test_run:cmd('delete server master3')
-
---
--- Master1 tries to promote self. In the meantime master2 has
--- vclock[master3] == 0. It is still blocked in the WAL thread. Master1 should
--- be ready to seeing 0 LSN by the old leader's component in some replicas.
---
-box.cfg{replication_synchro_timeout = 0.1}
-assert(box.info.synchro.queue.len > 0)
-assert(not pcall(box.ctl.promote))
-
-test_run:switch('master2')
-box.error.injection.set('ERRINJ_WAL_DELAY', false)
-
-test_run:switch('default')
-test_run:cmd('stop server master2')
-test_run:cmd('delete server master2')
-test_run:cmd('stop server master1')
-test_run:cmd('delete server master1')
diff --git a/test/replication/gh-6034-qsync-limbo-ownership.result b/test/replication/gh-6034-qsync-limbo-ownership.result
index 58acf7db28..4f558fa1ae 100644
--- a/test/replication/gh-6034-qsync-limbo-ownership.result
+++ b/test/replication/gh-6034-qsync-limbo-ownership.result
@@ -42,7 +42,7 @@ box.space.sync:insert{1} -- error.
  | - error: The synchronous transaction queue doesn't belong to any instance
  | ...
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 assert(not box.info.ro)
@@ -107,7 +107,7 @@ assert(err.code == box.error.READONLY)
  | ...
 
 -- Promotion on the other node. Default should become ro.
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 assert(not box.info.ro)
@@ -150,7 +150,7 @@ assert(err.code == box.error.READONLY)
  | - true
  | ...
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 box.ctl.demote()
diff --git a/test/replication/gh-6034-qsync-limbo-ownership.test.lua b/test/replication/gh-6034-qsync-limbo-ownership.test.lua
index 0f62ba6a40..3b690f8a35 100644
--- a/test/replication/gh-6034-qsync-limbo-ownership.test.lua
+++ b/test/replication/gh-6034-qsync-limbo-ownership.test.lua
@@ -17,7 +17,7 @@ box.space.async:insert{1} -- success.
 -- Synchro spaces aren't writeable
 box.space.sync:insert{1} -- error.
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 assert(not box.info.ro)
 assert(box.info.synchro.queue.owner == box.info.id)
 box.space.sync:insert{1} -- success.
@@ -39,7 +39,7 @@ assert(not ok)
 assert(err.code == box.error.READONLY)
 
 -- Promotion on the other node. Default should become ro.
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 assert(not box.info.ro)
 assert(box.info.synchro.queue.owner == box.info.id)
 box.space.sync:insert{2} -- success.
@@ -52,7 +52,7 @@ ok, err = pcall(box.space.sync.insert, box.space.sync, {3})
 assert(not ok)
 assert(err.code == box.error.READONLY)
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 box.ctl.demote()
 assert(not box.info.ro)
 box.space.sync:insert{3} -- still fails.
diff --git a/test/replication/gh-6035-election-filter.result b/test/replication/gh-6035-election-filter.result
index 2fa593267b..6c18acba7b 100644
--- a/test/replication/gh-6035-election-filter.result
+++ b/test/replication/gh-6035-election-filter.result
@@ -69,7 +69,7 @@ box.cfg({                                       \
  | ---
  | ...
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 _ = box.schema.space.create("sync", {is_sync = true})
diff --git a/test/replication/gh-6035-election-filter.test.lua b/test/replication/gh-6035-election-filter.test.lua
index 716c84bb6e..8d04218a1c 100644
--- a/test/replication/gh-6035-election-filter.test.lua
+++ b/test/replication/gh-6035-election-filter.test.lua
@@ -38,7 +38,7 @@ box.cfg({                                       \
     election_mode = 'manual',                   \
 })
 
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 _ = box.schema.space.create("sync", {is_sync = true})
 _ = box.space.sync:create_index("pk")
 box.space.sync:insert{1}
diff --git a/test/replication/qsync_advanced.result b/test/replication/qsync_advanced.result
index 067abac0cf..57124441ac 100644
--- a/test/replication/qsync_advanced.result
+++ b/test/replication/qsync_advanced.result
@@ -79,7 +79,7 @@ _ = box.schema.space.create('sync', {is_sync=true, engine=engine})
 _ = box.space.sync:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 -- Testcase body.
@@ -493,7 +493,7 @@ box.space.sync:select{} -- 1
 box.cfg{read_only=false} -- promote replica to master
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 test_run:switch('default')
@@ -539,7 +539,7 @@ test_run:switch('default')
 box.cfg{read_only=false}
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 test_run:switch('replica')
diff --git a/test/replication/qsync_advanced.test.lua b/test/replication/qsync_advanced.test.lua
index 179cf3dd0e..5b8370544c 100644
--- a/test/replication/qsync_advanced.test.lua
+++ b/test/replication/qsync_advanced.test.lua
@@ -35,7 +35,7 @@ test_run:switch('default')
 box.cfg{replication_synchro_quorum=NUM_INSTANCES, replication_synchro_timeout=1000}
 _ = box.schema.space.create('sync', {is_sync=true, engine=engine})
 _ = box.space.sync:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 -- Testcase body.
 box.space.sync:insert{1} -- success
 test_run:switch('replica')
@@ -181,7 +181,7 @@ box.space.sync:select{} -- 1
 test_run:switch('replica')
 box.space.sync:select{} -- 1
 box.cfg{read_only=false} -- promote replica to master
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 test_run:switch('default')
 box.cfg{read_only=true} -- demote master to replica
 test_run:switch('replica')
@@ -194,7 +194,7 @@ t
 -- Revert cluster configuration.
 test_run:switch('default')
 box.cfg{read_only=false}
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 test_run:switch('replica')
 box.cfg{read_only=true}
 -- Testcase cleanup.
diff --git a/test/replication/qsync_basic.result b/test/replication/qsync_basic.result
index 8eae516cd8..14c15c0164 100644
--- a/test/replication/qsync_basic.result
+++ b/test/replication/qsync_basic.result
@@ -14,7 +14,7 @@ s1.is_sync
 pk = s1:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 box.begin() s1:insert({1}) s1:insert({2}) box.commit()
@@ -712,7 +712,7 @@ assert(box.info.lsn == old_lsn + 1)
  | ---
  | - true
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 
diff --git a/test/replication/qsync_basic.test.lua b/test/replication/qsync_basic.test.lua
index a1dfc9e9cf..fddc943e50 100644
--- a/test/replication/qsync_basic.test.lua
+++ b/test/replication/qsync_basic.test.lua
@@ -6,7 +6,7 @@
 s1 = box.schema.create_space('test1', {is_sync = true})
 s1.is_sync
 pk = s1:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 box.begin() s1:insert({1}) s1:insert({2}) box.commit()
 s1:select{}
 
@@ -274,7 +274,7 @@ box.space.sync:before_replace(nil, skip_row)
 assert(box.space.sync:get{1} == nil)
 assert(box.space.sync:get{2} == nil)
 assert(box.info.lsn == old_lsn + 1)
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 
 box.space.sync:truncate()
 
diff --git a/test/replication/qsync_errinj.result b/test/replication/qsync_errinj.result
index cf1e30a907..9bab38cc96 100644
--- a/test/replication/qsync_errinj.result
+++ b/test/replication/qsync_errinj.result
@@ -35,7 +35,7 @@ _ = box.schema.space.create('sync', {is_sync = true, engine = engine})
 _ = box.space.sync:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 
diff --git a/test/replication/qsync_errinj.test.lua b/test/replication/qsync_errinj.test.lua
index e7c85c58ce..548c78ca4f 100644
--- a/test/replication/qsync_errinj.test.lua
+++ b/test/replication/qsync_errinj.test.lua
@@ -12,7 +12,7 @@ test_run:cmd('start server replica with wait=True, wait_load=True')
 
 _ = box.schema.space.create('sync', {is_sync = true, engine = engine})
 _ = box.space.sync:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 
 --
 -- gh-5100: slow ACK sending shouldn't stun replica for the
diff --git a/test/replication/qsync_snapshots.result b/test/replication/qsync_snapshots.result
index 742a56fce0..ecdb7f9f9a 100644
--- a/test/replication/qsync_snapshots.result
+++ b/test/replication/qsync_snapshots.result
@@ -57,7 +57,7 @@ _ = box.schema.space.create('sync', {is_sync=true, engine=engine})
 _ = box.space.sync:create_index('pk')
  | ---
  | ...
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
  | ---
  | ...
 -- Testcase body.
diff --git a/test/replication/qsync_snapshots.test.lua b/test/replication/qsync_snapshots.test.lua
index 5208c26d4e..c16f4dc06a 100644
--- a/test/replication/qsync_snapshots.test.lua
+++ b/test/replication/qsync_snapshots.test.lua
@@ -23,7 +23,7 @@ test_run:switch('default')
 box.cfg{replication_synchro_quorum=NUM_INSTANCES, replication_synchro_timeout=1000}
 _ = box.schema.space.create('sync', {is_sync=true, engine=engine})
 _ = box.space.sync:create_index('pk')
-box.ctl.promote()
+box.ctl.promote(); box.ctl.wait_rw()
 -- Testcase body.
 box.space.sync:insert{1}
 box.space.sync:select{} -- 1
-- 
GitLab