diff --git a/src/main.rs b/src/main.rs index a6272a6c36c99e3978a2b36bf99357bc842cb633..c4640d65a1376145c623480d6b62fb7d891354c8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -76,6 +76,7 @@ fn picolib_setup(args: args::Run) { // Export public API luamod.set("run", tlua::function0(move || start(&args))); luamod.set("raft_status", tlua::function0(raft_status)); + luamod.set("raft_tick", tlua::function1(raft_tick)); luamod.set( "raft_propose_info", tlua::function1(|x: String| raft_propose(Message::Info { msg: x })), @@ -185,6 +186,15 @@ fn start(args: &args::Run) { ); } +fn raft_tick(n_times: u32) { + let stash = Stash::access(); + let raft_ref = stash.raft_node(); + let raft_node = raft_ref.as_ref().expect("Picodata not running yet"); + for _ in 0..n_times { + raft_node.tick(); + } +} + fn raft_status() -> traft::Status { let stash = Stash::access(); let raft_ref = stash.raft_node(); diff --git a/src/traft/node.rs b/src/traft/node.rs index 09751a771c124d7f86e788573275d7a22b789a08..b7add2eb34290206827db87291a30af2d2f6d856 100644 --- a/src/traft/node.rs +++ b/src/traft/node.rs @@ -37,6 +37,7 @@ enum Request { Propose { data: Vec<u8> }, ProposeWaitApplied { data: Vec<u8>, notify: Notify }, Step(raft::Message), + Tick, } impl Node { @@ -100,6 +101,11 @@ impl Node { let req = Request::Step(msg); self.inbox.send(req).unwrap(); } + + pub fn tick(&self) { + let req = Request::Tick; + self.inbox.send(req).unwrap(); + } } fn raft_main( @@ -152,6 +158,9 @@ fn raft_main( tlog!(Error, "{e}"); } } + Ok(Request::Tick) => { + raw_node.tick(); + } Err(fiber::RecvError::Timeout) => (), Err(fiber::RecvError::Disconnected) => unreachable!(), } diff --git a/test/couple_test.lua b/test/couple_test.lua index 5ac01d63a955092aea31aded5a427390fb665e13..6f344ec8d1616f360271a5f5dbff6dc0ab99a51f 100644 --- a/test/couple_test.lua +++ b/test/couple_test.lua @@ -37,7 +37,7 @@ g.after_all(function() fio.rmtree(g.data_dir) end) -g.test = function() +g.test_follower_proposal = function() -- Speed up node election g.cluster.i1:try_promote() @@ -54,3 +54,22 @@ g.test = function() '127.0.0.1:13302' ) end + +g.test_failover = function() + g.cluster.i1:try_promote() + h.retrying({}, function() + g.cluster.i2:assert_raft_status("Follower", 1) + end) + + -- Speed up election timeout + g.cluster.i2:connect():eval([[ + while picolib.raft_status().raft_state == 'Follower' do + picolib.raft_tick(1) + end + ]]) + + h.retrying({}, function() + g.cluster.i1:assert_raft_status("Follower", 2) + g.cluster.i2:assert_raft_status("Leader") + end) +end diff --git a/test/threesome_test.lua b/test/threesome_test.lua index 50cfb0ea89eaa3d95adbeadfee83614986f278a7..faf6c443e70b85ff0a4464e1e5e430082e49ee1b 100644 --- a/test/threesome_test.lua +++ b/test/threesome_test.lua @@ -57,7 +57,7 @@ local function propose_state_change(srv, value) return srv:raft_propose_eval(0.1, code) end -g.test = function() +g.test_log_rollback = function() -- Speed up node election g.cluster.i1:try_promote() h.retrying({}, function() @@ -106,3 +106,45 @@ g.test = function() true ) end + +g.test_leader_disruption = function() + g.cluster.i1:try_promote() + h.retrying({}, function() + g.cluster.i2:assert_raft_status("Follower", 1) + g.cluster.i3:assert_raft_status("Follower", 1) + end) + + -- Simulate asymmetric network failure. + -- Node i3 doesn't receive any messages, + -- including the heartbeat from the leader. + -- Then it starts a new election. + g.cluster.i3:connect():call( + 'box.schema.func.drop', + {'.raft_interact'} + ) + + -- Speed up election timeout + g.cluster.i3:connect():eval([[ + while picolib.raft_status().raft_state == 'Follower' do + picolib.raft_tick(1) + end + ]]) + g.cluster.i3:assert_raft_status("PreCandidate", 0) + + -- Advance the raft log. It makes i1 and i2 to reject the RequestPreVote. + g.cluster.i1:raft_propose_eval(1, 'return') + + -- Restore normal network operation + g.cluster.i3:connect():call( + 'box.schema.func.create', + {'.raft_interact', { + language = "C", + if_not_exists = true + }} + ) + + -- i3 should become the follower again without disrupting i1 + h.retrying({}, function() + g.cluster.i3:assert_raft_status("Follower", 1) + end) +end