From 7f5876faeb504bd48ac092e62062d438c460ef1e Mon Sep 17 00:00:00 2001
From: Serge Petrenko <sergepetrenko@tarantool.org>
Date: Thu, 22 Jul 2021 00:43:11 +0200
Subject: [PATCH] box: introduce on_election triggers
On_election triggers are fired asynchronously after any Raft event with
a broadcast, they are run in a worker fiber, so it's allowed to yield
inside them, unlike Raft's on_update triggers we already had.
Closes #5819
@TarantoolBot document
Title: document triggers on election state change
A new function to register triggers is added, `box.ctl.on_election()`.
Triggers registered via this function are run asynchronously every time
a visible change in `box.info.election` table appears.
No parameters are passed to the trigger, it may check what's changed by
looking at `box.info.election` and `box.info.synchro`.
---
.../unreleased/gh-5819-election-triggers.md | 5 +
src/box/lua/ctl.c | 8 +
src/box/raft.c | 4 +
src/box/raft.h | 7 +
test/replication/election_basic.result | 144 ++++++++++++++++++
test/replication/election_basic.test.lua | 53 +++++++
6 files changed, 221 insertions(+)
create mode 100644 changelogs/unreleased/gh-5819-election-triggers.md
diff --git a/changelogs/unreleased/gh-5819-election-triggers.md b/changelogs/unreleased/gh-5819-election-triggers.md
new file mode 100644
index 0000000000..edb7be8ac2
--- /dev/null
+++ b/changelogs/unreleased/gh-5819-election-triggers.md
@@ -0,0 +1,5 @@
+## feature/replication
+
+* Introduce on_election triggers. The triggers may be registered via
+`box.ctl.on_election()` interface and are run asynchronously each time
+`box.info.election` changes (gh-5819).
diff --git a/src/box/lua/ctl.c b/src/box/lua/ctl.c
index 368b9ab609..4a9212f333 100644
--- a/src/box/lua/ctl.c
+++ b/src/box/lua/ctl.c
@@ -43,6 +43,7 @@
#include "box/schema.h"
#include "box/engine.h"
#include "box/memtx_engine.h"
+#include "box/raft.h"
static int
lbox_ctl_wait_ro(struct lua_State *L)
@@ -81,6 +82,12 @@ lbox_ctl_on_schema_init(struct lua_State *L)
return lbox_trigger_reset(L, 2, &on_schema_init, NULL, NULL);
}
+static int
+lbox_ctl_on_election(struct lua_State *L)
+{
+ return lbox_trigger_reset(L, 2, &box_raft_on_broadcast, NULL, NULL);
+}
+
static int
lbox_ctl_promote(struct lua_State *L)
{
@@ -124,6 +131,7 @@ static const struct luaL_Reg lbox_ctl_lib[] = {
{"wait_rw", lbox_ctl_wait_rw},
{"on_shutdown", lbox_ctl_on_shutdown},
{"on_schema_init", lbox_ctl_on_schema_init},
+ {"on_election", lbox_ctl_on_election},
{"promote", lbox_ctl_promote},
/* An old alias. */
{"clear_synchro_queue", lbox_ctl_promote},
diff --git a/src/box/raft.c b/src/box/raft.c
index eb62e96300..563ed89c4a 100644
--- a/src/box/raft.c
+++ b/src/box/raft.c
@@ -52,6 +52,9 @@ enum election_mode box_election_mode = ELECTION_MODE_INVALID;
*/
static struct trigger box_raft_on_update;
+struct rlist box_raft_on_broadcast =
+ RLIST_HEAD_INITIALIZER(box_raft_on_broadcast);
+
/**
* Worker fiber does all the asynchronous work, which may need yields and can be
* long. These are WAL writes, network broadcasts. That allows not to block the
@@ -276,6 +279,7 @@ box_raft_broadcast(struct raft *raft, const struct raft_msg *msg)
box_raft_msg_to_request(msg, &req);
replicaset_foreach(replica)
relay_push_raft(replica->relay, &req);
+ trigger_run(&box_raft_on_broadcast, NULL);
}
static void
diff --git a/src/box/raft.h b/src/box/raft.h
index d55e90d2fa..b4a97444af 100644
--- a/src/box/raft.h
+++ b/src/box/raft.h
@@ -30,11 +30,18 @@
* SUCH DAMAGE.
*/
#include "raft/raft.h"
+#include "small/rlist.h"
#if defined(__cplusplus)
extern "C" {
#endif
+/**
+ * A public trigger fired on Raft state change, i.e. on a broadcast.
+ * It's allowed to yield inside it, and it's run asynchronously.
+ */
+extern struct rlist box_raft_on_broadcast;
+
enum election_mode {
ELECTION_MODE_INVALID = -1,
ELECTION_MODE_OFF = 0,
diff --git a/test/replication/election_basic.result b/test/replication/election_basic.result
index b64028c602..fc7121d7f8 100644
--- a/test/replication/election_basic.result
+++ b/test/replication/election_basic.result
@@ -275,3 +275,147 @@ test_run:cmd(string.format('start server %s', leader_name))
test_run:drop_cluster(SERVERS)
| ---
| ...
+
+-- gh-5819: on_election triggers, that are filed on every visible state change.
+box.schema.user.grant('guest', 'replication')
+ | ---
+ | ...
+test_run:cmd('create server replica with rpl_master=default,\
+ script="replication/replica.lua"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('start server replica')
+ | ---
+ | - true
+ | ...
+
+repl = test_run:eval('replica', 'return box.cfg.listen')[1]
+ | ---
+ | ...
+box.cfg{replication = repl}
+ | ---
+ | ...
+test_run:switch('replica')
+ | ---
+ | - true
+ | ...
+box.cfg{election_mode='voter'}
+ | ---
+ | ...
+test_run:switch('default')
+ | ---
+ | - true
+ | ...
+
+election_tbl = {}
+ | ---
+ | ...
+
+function trig()\
+ election_tbl[#election_tbl+1] = box.info.election\
+end
+ | ---
+ | ...
+
+_ = box.ctl.on_election(trig)
+ | ---
+ | ...
+
+box.cfg{replication_synchro_quorum=2}
+ | ---
+ | ...
+box.cfg{election_mode='candidate'}
+ | ---
+ | ...
+
+test_run:wait_cond(function() return #election_tbl == 3 end)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[1].state == 'follower')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[2].state == 'candidate')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[2].vote == 1)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[3].state == 'leader')
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='voter'}
+ | ---
+ | ...
+test_run:wait_cond(function() return #election_tbl == 4 end)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[4].state == 'follower')
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='off'}
+ | ---
+ | ...
+test_run:wait_cond(function() return #election_tbl == 5 end)
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='manual'}
+ | ---
+ | ...
+test_run:wait_cond(function() return #election_tbl == 6 end)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[6].state == 'follower')
+ | ---
+ | - true
+ | ...
+
+box.ctl.promote()
+ | ---
+ | ...
+
+test_run:wait_cond(function() return #election_tbl == 9 end)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[7].state == 'follower')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[7].term == election_tbl[6].term + 1)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[8].state == 'candidate')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[9].state == 'leader')
+ | ---
+ | - true
+ | ...
+
+test_run:cmd('stop server replica')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server replica')
+ | ---
+ | - true
+ | ...
+
+box.schema.user.revoke('guest', 'replication')
+ | ---
+ | ...
diff --git a/test/replication/election_basic.test.lua b/test/replication/election_basic.test.lua
index 77fdf6340e..f1330d2329 100644
--- a/test/replication/election_basic.test.lua
+++ b/test/replication/election_basic.test.lua
@@ -116,3 +116,56 @@ assert(r1_leader == r2_leader)
test_run:cmd(string.format('start server %s', leader_name))
test_run:drop_cluster(SERVERS)
+
+-- gh-5819: on_election triggers, that are filed on every visible state change.
+box.schema.user.grant('guest', 'replication')
+test_run:cmd('create server replica with rpl_master=default,\
+ script="replication/replica.lua"')
+test_run:cmd('start server replica')
+
+repl = test_run:eval('replica', 'return box.cfg.listen')[1]
+box.cfg{replication = repl}
+test_run:switch('replica')
+box.cfg{election_mode='voter'}
+test_run:switch('default')
+
+election_tbl = {}
+
+function trig()\
+ election_tbl[#election_tbl+1] = box.info.election\
+end
+
+_ = box.ctl.on_election(trig)
+
+box.cfg{replication_synchro_quorum=2}
+box.cfg{election_mode='candidate'}
+
+test_run:wait_cond(function() return #election_tbl == 3 end)
+assert(election_tbl[1].state == 'follower')
+assert(election_tbl[2].state == 'candidate')
+assert(election_tbl[2].vote == 1)
+assert(election_tbl[3].state == 'leader')
+
+box.cfg{election_mode='voter'}
+test_run:wait_cond(function() return #election_tbl == 4 end)
+assert(election_tbl[4].state == 'follower')
+
+box.cfg{election_mode='off'}
+test_run:wait_cond(function() return #election_tbl == 5 end)
+
+box.cfg{election_mode='manual'}
+test_run:wait_cond(function() return #election_tbl == 6 end)
+assert(election_tbl[6].state == 'follower')
+
+box.ctl.promote()
+
+test_run:wait_cond(function() return #election_tbl == 9 end)
+assert(election_tbl[7].state == 'follower')
+assert(election_tbl[7].term == election_tbl[6].term + 1)
+assert(election_tbl[8].state == 'candidate')
+assert(election_tbl[9].state == 'leader')
+
+test_run:cmd('stop server replica')
+test_run:cmd('delete server replica')
+
+box.schema.user.revoke('guest', 'replication')
--
GitLab