test: fix and split flaky join_vclock test

join_vclock test is assumed to verify that changes are not being lost
on the replica. Due to this the test is changed to explicitly check
that all changes on master are applied on replica.
Previously this test was also indirectly verifying that changes are
being applied in the correct order. Now there is separate test for
this, called replica_apply_order.
As far as changed join_vclock test might fail due to #4669, we are now
creating cluster out of fresh instances instead of using default
instance. Considering mentioned fixes it is not fragile anymore.

Closes #4160

test/replication/join_vclock.result

-fiber = require('fiber')
----
-...
-env = require('test_run')
+test_run = require('test_run').new()
 ---
 ...
-replica_set = require('fast_replica')
+test_run:cmd("create server master with script='replication/master1.lua'")
 ---
+- true
 ...
-test_run = env.new()
+test_run:cmd('start server master')
 ---
+- true
 ...
-engine = test_run:get_cfg('engine')
+test_run:cmd("switch master")
 ---
+- true
 ...
 errinj = box.error.injection
 ---
@@ -20,6 +20,9 @@ errinj.set("ERRINJ_RELAY_FINAL_SLEEP", true)
 ---
 - ok
 ...
+engine = test_run:get_cfg('engine')
+---
+...
 box.schema.user.grant('guest', 'replication')
 ---
 ...
@@ -29,61 +32,79 @@ s = box.schema.space.create('test', {engine = engine});
 index = s:create_index('primary')
 ---
 ...
+fiber = require('fiber')
+---
+...
 ch = fiber.channel(1)
 ---
 ...
 done = false
 ---
 ...
-function repl_f() local i = 0 while not done do s:replace({i, i}) fiber.sleep(0.001) i = i + 1 end ch:put(true) end
+function repl_f() local i = 0 while not done do s:replace({i, i}) fiber.sleep(0.001) i = i + 1 end ch:put(i) end
 ---
 ...
 _ = fiber.create(repl_f)
 ---
 ...
-replica_set.join(test_run, 1)
+test_run:cmd("create server replica with rpl_master=master, script='replication/replica.lua'")
+---
+- true
+...
+test_run:cmd("start server replica")
 ---
+- true
 ...
-test_run:cmd("switch replica1")
+test_run:cmd("switch replica")
 ---
 - true
 ...
-test_run:cmd("switch default")
+test_run:cmd("switch master")
 ---
 - true
 ...
 done = true
 ---
 ...
-ch:get()
+count = ch:get()
 ---
-- true
 ...
 errinj.set("ERRINJ_RELAY_FINAL_SLEEP", false)
 ---
 - ok
 ...
-test_run:cmd("switch replica1")
+test_run:cmd("switch replica")
 ---
 - true
 ...
-cnt = box.space.test.index[0]:count()
+test_run:cmd("setopt delimiter ';'")
 ---
+- true
 ...
-box.space.test.index.primary:max()[1] == cnt - 1
+-- Wait for all tuples to be inserted on replica
+test_run:wait_cond(function()
+    return box.space.test.index.primary:max()[1] == test_run:eval('master', 'count')[1] - 1
+end);
 ---
 - true
 ...
-test_run:cmd("switch default")
+test_run:cmd("setopt delimiter ''");
 ---
 - true
 ...
-replica_set.drop_all(test_run)
+replica_count = box.space.test.index.primary:count()  master_count = test_run:eval('master', 'count')[1]
 ---
 ...
-box.space.test:drop()
+-- Verify that there are the same amount of tuples on master and replica
+replica_count == master_count or {replica_count, master_count}
 ---
+- true
+...
+-- Cleanup.
+test_run:cmd('switch default')
+---
+- true
 ...
-box.schema.user.revoke('guest', 'replication')
+test_run:drop_cluster({'master', 'replica'})
 ---
 ...

test/replication/join_vclock.test.lua

-fiber = require('fiber')
-env = require('test_run')
-replica_set = require('fast_replica')
-test_run = env.new()
-engine = test_run:get_cfg('engine')
+test_run = require('test_run').new()
+
+test_run:cmd("create server master with script='replication/master1.lua'")
+test_run:cmd('start server master')
+test_run:cmd("switch master")
 
 errinj = box.error.injection
 errinj.set("ERRINJ_RELAY_FINAL_SLEEP", true)
 
+engine = test_run:get_cfg('engine')
 box.schema.user.grant('guest', 'replication')
 s = box.schema.space.create('test', {engine = engine});
 index = s:create_index('primary')
 
+fiber = require('fiber')
 ch = fiber.channel(1)
 done = false
 
-function repl_f() local i = 0 while not done do s:replace({i, i}) fiber.sleep(0.001) i = i + 1 end ch:put(true) end
+function repl_f() local i = 0 while not done do s:replace({i, i}) fiber.sleep(0.001) i = i + 1 end ch:put(i) end
 _ = fiber.create(repl_f)
 
-replica_set.join(test_run, 1)
-test_run:cmd("switch replica1")
+test_run:cmd("create server replica with rpl_master=master, script='replication/replica.lua'")
+test_run:cmd("start server replica")
+test_run:cmd("switch replica")
 
-test_run:cmd("switch default")
+test_run:cmd("switch master")
 done = true
-ch:get()
+count = ch:get()
 
 errinj.set("ERRINJ_RELAY_FINAL_SLEEP", false)
-test_run:cmd("switch replica1")
-cnt = box.space.test.index[0]:count()
-box.space.test.index.primary:max()[1] == cnt - 1
-test_run:cmd("switch default")
-
-replica_set.drop_all(test_run)
-box.space.test:drop()
-box.schema.user.revoke('guest', 'replication')
+test_run:cmd("switch replica")
+test_run:cmd("setopt delimiter ';'")
+-- Wait for all tuples to be inserted on replica
+test_run:wait_cond(function()
+    return box.space.test.index.primary:max()[1] == test_run:eval('master', 'count')[1] - 1
+end);
+test_run:cmd("setopt delimiter ''");
+replica_count = box.space.test.index.primary:count()  master_count = test_run:eval('master', 'count')[1]
+-- Verify that there are the same amount of tuples on master and replica
+replica_count == master_count or {replica_count, master_count}
+
+-- Cleanup.
+test_run:cmd('switch default')
+test_run:drop_cluster({'master', 'replica'})

test/replication/replica_apply_order.result

+-- test-run result file version 2
+fiber = require('fiber')
+ | ---
+ | ...
+env = require('test_run')
+ | ---
+ | ...
+replica_set = require('fast_replica')
+ | ---
+ | ...
+test_run = env.new()
+ | ---
+ | ...
+engine = test_run:get_cfg('engine')
+ | ---
+ | ...
+
+errinj = box.error.injection
+ | ---
+ | ...
+errinj.set("ERRINJ_RELAY_FINAL_SLEEP", true)
+ | ---
+ | - ok
+ | ...
+
+box.schema.user.grant('guest', 'replication')
+ | ---
+ | ...
+s = box.schema.space.create('test', {engine = engine});
+ | ---
+ | ...
+index = s:create_index('primary')
+ | ---
+ | ...
+
+ch = fiber.channel(1)
+ | ---
+ | ...
+done = false
+ | ---
+ | ...
+
+function repl_f() local i = 0 while not done do s:replace({i, i}) fiber.sleep(0.001) i = i + 1 end ch:put(true) end
+ | ---
+ | ...
+_ = fiber.create(repl_f)
+ | ---
+ | ...
+
+replica_set.join(test_run, 1)
+ | ---
+ | ...
+test_run:cmd("switch replica1")
+ | ---
+ | - true
+ | ...
+
+test_run:cmd("switch default")
+ | ---
+ | - true
+ | ...
+done = true
+ | ---
+ | ...
+ch:get()
+ | ---
+ | - true
+ | ...
+
+errinj.set("ERRINJ_RELAY_FINAL_SLEEP", false)
+ | ---
+ | - ok
+ | ...
+test_run:cmd("switch replica1")
+ | ---
+ | - true
+ | ...
+test_run:cmd("setopt delimiter ';'")
+ | ---
+ | - true
+ | ...
+function get_max_index_and_count()
+    return box.space.test.index.primary:max()[1], box.space.test.index.primary:count()
+end;
+ | ---
+ | ...
+max, count = 0, 0;
+ | ---
+ | ...
+for i = 1, 100 do
+    max, count = box.atomic(get_max_index_and_count)
+    if max ~= count - 1 then
+        break
+    end
+end;
+ | ---
+ | ...
+-- Verify that at any moment max index is corresponding to amount of tuples,
+-- which means that changes apply order is correct
+max == count - 1 or {max, count - 1};
+ | ---
+ | - true
+ | ...
+test_run:cmd("setopt delimiter ''");
+ | ---
+ | - true
+ | ...
+test_run:cmd("switch default")
+ | ---
+ | - true
+ | ...
+
+replica_set.drop_all(test_run)
+ | ---
+ | ...
+box.space.test:drop()
+ | ---
+ | ...
+box.schema.user.revoke('guest', 'replication')
+ | ---
+ | ...

test/replication/replica_apply_order.test.lua

+fiber = require('fiber')
+env = require('test_run')
+replica_set = require('fast_replica')
+test_run = env.new()
+engine = test_run:get_cfg('engine')
+
+errinj = box.error.injection
+errinj.set("ERRINJ_RELAY_FINAL_SLEEP", true)
+
+box.schema.user.grant('guest', 'replication')
+s = box.schema.space.create('test', {engine = engine});
+index = s:create_index('primary')
+
+ch = fiber.channel(1)
+done = false
+
+function repl_f() local i = 0 while not done do s:replace({i, i}) fiber.sleep(0.001) i = i + 1 end ch:put(true) end
+_ = fiber.create(repl_f)
+
+replica_set.join(test_run, 1)
+test_run:cmd("switch replica1")
+
+test_run:cmd("switch default")
+done = true
+ch:get()
+
+errinj.set("ERRINJ_RELAY_FINAL_SLEEP", false)
+test_run:cmd("switch replica1")
+test_run:cmd("setopt delimiter ';'")
+function get_max_index_and_count()
+    return box.space.test.index.primary:max()[1], box.space.test.index.primary:count()
+end;
+max, count = 0, 0;
+for i = 1, 100 do
+    max, count = box.atomic(get_max_index_and_count)
+    if max ~= count - 1 then
+        break
+    end
+end;
+-- Verify that at any moment max index is corresponding to amount of tuples,
+-- which means that changes apply order is correct
+max == count - 1 or {max, count - 1};
+test_run:cmd("setopt delimiter ''");
+test_run:cmd("switch default")
+
+replica_set.drop_all(test_run)
+box.space.test:drop()
+box.schema.user.revoke('guest', 'replication')

test/replication/suite.ini

@@ -12,7 +12,6 @@ long_run = prune.test.lua
 is_parallel = True
 pretest_clean = True
 fragile = errinj.test.lua            ; gh-3870
-          join_vclock.test.lua       ; gh-4160
           long_row_timeout.test.lua  ; gh-4351
           skip_conflict_row.test.lua ; gh-4457
           sync.test.lua              ; gh-3835 gh-3877