box: introduce on_election triggers

On_election triggers are fired asynchronously after any Raft event with
a broadcast, they are run in a worker fiber, so it's allowed to yield
inside them, unlike Raft's on_update triggers we already had.

Closes #5819

@TarantoolBot document
Title: document triggers on election state change

A new function to register triggers is added, `box.ctl.on_election()`.
Triggers registered via this function are run asynchronously every time
a visible change in `box.info.election` table appears.
No parameters are passed to the trigger, it may check what's changed by
looking at `box.info.election` and `box.info.synchro`.

changelogs/unreleased/gh-5819-election-triggers.md

+## feature/replication
+
+* Introduce on_election triggers. The triggers may be registered via
+`box.ctl.on_election()` interface and are run asynchronously each time
+`box.info.election` changes (gh-5819).

src/box/lua/ctl.c

@@ -43,6 +43,7 @@
 #include "box/schema.h"
 #include "box/engine.h"
 #include "box/memtx_engine.h"
+#include "box/raft.h"
 
 static int
 lbox_ctl_wait_ro(struct lua_State *L)
@@ -81,6 +82,12 @@ lbox_ctl_on_schema_init(struct lua_State *L)
 	return lbox_trigger_reset(L, 2, &on_schema_init, NULL, NULL);
 }
 
+static int
+lbox_ctl_on_election(struct lua_State *L)
+{
+	return lbox_trigger_reset(L, 2, &box_raft_on_broadcast, NULL, NULL);
+}
+
 static int
 lbox_ctl_promote(struct lua_State *L)
 {
@@ -124,6 +131,7 @@ static const struct luaL_Reg lbox_ctl_lib[] = {
 	{"wait_rw", lbox_ctl_wait_rw},
 	{"on_shutdown", lbox_ctl_on_shutdown},
 	{"on_schema_init", lbox_ctl_on_schema_init},
+	{"on_election", lbox_ctl_on_election},
 	{"promote", lbox_ctl_promote},
 	/* An old alias. */
 	{"clear_synchro_queue", lbox_ctl_promote},

src/box/raft.c

@@ -52,6 +52,9 @@ enum election_mode box_election_mode = ELECTION_MODE_INVALID;
  */
 static struct trigger box_raft_on_update;
 
+struct rlist box_raft_on_broadcast =
+	RLIST_HEAD_INITIALIZER(box_raft_on_broadcast);
+
 /**
  * Worker fiber does all the asynchronous work, which may need yields and can be
  * long. These are WAL writes, network broadcasts. That allows not to block the
@@ -276,6 +279,7 @@ box_raft_broadcast(struct raft *raft, const struct raft_msg *msg)
 	box_raft_msg_to_request(msg, &req);
 	replicaset_foreach(replica)
 		relay_push_raft(replica->relay, &req);
+	trigger_run(&box_raft_on_broadcast, NULL);
 }
 
 static void

src/box/raft.h

@@ -30,11 +30,18 @@
  * SUCH DAMAGE.
  */
 #include "raft/raft.h"
+#include "small/rlist.h"
 
 #if defined(__cplusplus)
 extern "C" {
 #endif
 
+/**
+ * A public trigger fired on Raft state change, i.e. on a broadcast.
+ * It's allowed to yield inside it, and it's run asynchronously.
+ */
+extern struct rlist box_raft_on_broadcast;
+
 enum election_mode {
 	ELECTION_MODE_INVALID = -1,
 	ELECTION_MODE_OFF = 0,

test/replication/election_basic.result

@@ -275,3 +275,147 @@ test_run:cmd(string.format('start server %s', leader_name))
 test_run:drop_cluster(SERVERS)
  | ---
  | ...
+
+-- gh-5819: on_election triggers, that are filed on every visible state change.
+box.schema.user.grant('guest', 'replication')
+ | ---
+ | ...
+test_run:cmd('create server replica with rpl_master=default,\
+	      script="replication/replica.lua"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('start server replica')
+ | ---
+ | - true
+ | ...
+
+repl = test_run:eval('replica', 'return box.cfg.listen')[1]
+ | ---
+ | ...
+box.cfg{replication = repl}
+ | ---
+ | ...
+test_run:switch('replica')
+ | ---
+ | - true
+ | ...
+box.cfg{election_mode='voter'}
+ | ---
+ | ...
+test_run:switch('default')
+ | ---
+ | - true
+ | ...
+
+election_tbl = {}
+ | ---
+ | ...
+
+function trig()\
+    election_tbl[#election_tbl+1] = box.info.election\
+end
+ | ---
+ | ...
+
+_ = box.ctl.on_election(trig)
+ | ---
+ | ...
+
+box.cfg{replication_synchro_quorum=2}
+ | ---
+ | ...
+box.cfg{election_mode='candidate'}
+ | ---
+ | ...
+
+test_run:wait_cond(function() return #election_tbl == 3 end)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[1].state == 'follower')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[2].state == 'candidate')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[2].vote == 1)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[3].state == 'leader')
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='voter'}
+ | ---
+ | ...
+test_run:wait_cond(function() return #election_tbl == 4 end)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[4].state == 'follower')
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='off'}
+ | ---
+ | ...
+test_run:wait_cond(function() return #election_tbl == 5 end)
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='manual'}
+ | ---
+ | ...
+test_run:wait_cond(function() return #election_tbl == 6 end)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[6].state == 'follower')
+ | ---
+ | - true
+ | ...
+
+box.ctl.promote()
+ | ---
+ | ...
+
+test_run:wait_cond(function() return #election_tbl == 9 end)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[7].state == 'follower')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[7].term == election_tbl[6].term + 1)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[8].state == 'candidate')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[9].state == 'leader')
+ | ---
+ | - true
+ | ...
+
+test_run:cmd('stop server replica')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server replica')
+ | ---
+ | - true
+ | ...
+
+box.schema.user.revoke('guest', 'replication')
+ | ---
+ | ...

test/replication/election_basic.test.lua

@@ -116,3 +116,56 @@ assert(r1_leader == r2_leader)
 test_run:cmd(string.format('start server %s', leader_name))
 
 test_run:drop_cluster(SERVERS)
+
+-- gh-5819: on_election triggers, that are filed on every visible state change.
+box.schema.user.grant('guest', 'replication')
+test_run:cmd('create server replica with rpl_master=default,\
+	      script="replication/replica.lua"')
+test_run:cmd('start server replica')
+
+repl = test_run:eval('replica', 'return box.cfg.listen')[1]
+box.cfg{replication = repl}
+test_run:switch('replica')
+box.cfg{election_mode='voter'}
+test_run:switch('default')
+
+election_tbl = {}
+
+function trig()\
+    election_tbl[#election_tbl+1] = box.info.election\
+end
+
+_ = box.ctl.on_election(trig)
+
+box.cfg{replication_synchro_quorum=2}
+box.cfg{election_mode='candidate'}
+
+test_run:wait_cond(function() return #election_tbl == 3 end)
+assert(election_tbl[1].state == 'follower')
+assert(election_tbl[2].state == 'candidate')
+assert(election_tbl[2].vote == 1)
+assert(election_tbl[3].state == 'leader')
+
+box.cfg{election_mode='voter'}
+test_run:wait_cond(function() return #election_tbl == 4 end)
+assert(election_tbl[4].state == 'follower')
+
+box.cfg{election_mode='off'}
+test_run:wait_cond(function() return #election_tbl == 5 end)
+
+box.cfg{election_mode='manual'}
+test_run:wait_cond(function() return #election_tbl == 6 end)
+assert(election_tbl[6].state == 'follower')
+
+box.ctl.promote()
+
+test_run:wait_cond(function() return #election_tbl == 9 end)
+assert(election_tbl[7].state == 'follower')
+assert(election_tbl[7].term == election_tbl[6].term + 1)
+assert(election_tbl[8].state == 'candidate')
+assert(election_tbl[9].state == 'leader')
+
+test_run:cmd('stop server replica')
+test_run:cmd('delete server replica')
+
+box.schema.user.revoke('guest', 'replication')