fix: speed up instance join

src/rpc/join.rs

@@ -66,6 +66,7 @@ pub fn handle_join_request_and_wait(req: Request, timeout: Duration) -> Result<R
     let cluster_id = node.raft_storage.cluster_id()?;
     let storage = &node.storage;
     let raft_storage = &node.raft_storage;
+    let guard = node.instances_update.lock();
 
     if req.cluster_id != cluster_id {
         return Err(Error::ClusterIdMismatch {
@@ -147,6 +148,7 @@ pub fn handle_join_request_and_wait(req: Request, timeout: Duration) -> Result<R
         // A joined instance needs to communicate with other nodes.
         // TODO: limit the number of entries sent to reduce response size.
         let peer_addresses = node.storage.peer_addresses.iter()?.collect();
+        drop(guard);
         return Ok(Response {
             instance: instance.into(),
             peer_addresses,

src/rpc/update_instance.rs

@@ -87,6 +87,7 @@ pub fn handle_update_instance_request_and_wait(req: Request, timeout: Duration)
     let cluster_id = node.raft_storage.cluster_id()?;
     let storage = &node.storage;
     let raft_storage = &node.raft_storage;
+    let guard = node.instances_update.lock();
 
     if req.cluster_id != cluster_id {
         return Err(Error::ClusterIdMismatch {
@@ -136,6 +137,7 @@ pub fn handle_update_instance_request_and_wait(req: Request, timeout: Duration)
             }
         }
         node.main_loop.wakeup();
+        drop(guard);
         return Ok(());
     }
 }

src/traft/node.rs

@@ -143,6 +143,10 @@ pub struct Node {
     pub(crate) governor_loop: governor::Loop,
     status: watch::Receiver<Status>,
     watchers: Rc<Mutex<StorageWatchers>>,
+
+    /// Should be locked during join and update instance request
+    /// to avoid costly cas conflicts during concurrent requests.
+    pub instances_update: Mutex<()>,
 }
 
 impl std::fmt::Debug for Node {
@@ -187,6 +191,7 @@ impl Node {
             raft_storage,
             status,
             watchers,
+            instances_update: Mutex::new(()),
         };
 
         // Wait for the node to enter the main loop

test/manual/test_scaling.py

+# mypy: disable-error-code="import"
 import funcy  # type: ignore
 import time
-from matplotlib import pyplot  # type: ignore
-import matplotlib  # type: ignore
+from matplotlib import pyplot
+import matplotlib
 
 from conftest import (
     Cluster,