diff --git a/src/traft/network.rs b/src/traft/network.rs
index 3a53b625738f12430dd8989956b5b6325a8ef41d..7cc08867f45a8d56c43dafde111c04a4ff40a6c8 100644
--- a/src/traft/network.rs
+++ b/src/traft/network.rs
@@ -162,9 +162,8 @@ impl PoolWorker {
                             "peer" => address,
                             "raft_id" => raft_id,
                         );
-                        cond.wait_timeout(
-                            closest_promise_timeout_or(opts.connect_timeout - iter_start.elapsed())
-                        );
+                        let timeout = opts.connect_timeout.saturating_sub(iter_start.elapsed());
+                        cond.wait_timeout(closest_promise_timeout_or(timeout));
                         continue
                     }
                 );