From 4797999c05a9cabfcfb67727a5d29beba05064c9 Mon Sep 17 00:00:00 2001
From: Yaroslav Dynnikov <yaroslav.dynnikov@gmail.com>
Date: Tue, 8 Feb 2022 13:13:55 +0300
Subject: [PATCH] feature: pass peers in env vars
Now it's possible to set up initial topology via command-line arguments.
Start one of several nodes (for example, two):
```bash
picodata run --peer 127.0.0.1:3301 --peer 127.0.0.1:3302
```
Single-instance configuration is started as before:
```bash
picodata run
```
---
picodata/main.rs | 2 ++
picolib/lib.rs | 23 ++++++++++++++++++-----
picolib/traft.rs | 2 ++
picolib/traft/row/peer.rs | 5 +++++
picolib/traft/storage.rs | 33 +++++++++++++++++++++++++++------
tests/cli.rs | 1 +
6 files changed, 55 insertions(+), 11 deletions(-)
create mode 100644 picolib/traft/row/peer.rs
diff --git a/picodata/main.rs b/picodata/main.rs
index 3b2a738de3..f499bf4101 100644
--- a/picodata/main.rs
+++ b/picodata/main.rs
@@ -66,6 +66,8 @@ fn main_run(matches: &clap::ArgMatches) {
envp.insert("PICODATA_PEER".into(), peer);
}
+ envp.entry("PICODATA_PEER".into())
+ .or_insert_with(|| "127.0.0.1:3301".into());
envp.entry("PICODATA_LISTEN".into())
.or_insert_with(|| "3301".into());
envp.entry("PICODATA_DATA_DIR".into())
diff --git a/picolib/lib.rs b/picolib/lib.rs
index 286156be7c..ea17e80d1d 100644
--- a/picolib/lib.rs
+++ b/picolib/lib.rs
@@ -90,7 +90,8 @@ pub unsafe extern "C" fn luaopen_picolib(l: *mut std::ffi::c_void) -> c_int {
function inspect()
return
{raft_log = box.space.raft_log:fselect()},
- {raft_state = box.space.raft_state:fselect()}
+ {raft_state = box.space.raft_state:fselect()},
+ {raft_group = box.space.raft_group:fselect()}
end
"#,
)
@@ -115,12 +116,11 @@ fn main_run() {
..Default::default()
};
- std::env::var("PICODATA_DATA_DIR").ok().map(|v| {
+ if let Ok(v) = std::env::var("PICODATA_DATA_DIR") {
std::fs::create_dir_all(&v).unwrap();
cfg.wal_dir = v.clone();
- cfg.memtx_dir = v.clone();
- Some(v)
- });
+ cfg.memtx_dir = v;
+ };
tarantool::set_cfg(&cfg);
tarantool::eval(
@@ -166,6 +166,19 @@ fn main_run() {
}
};
+ if let Ok(peers) = std::env::var("PICODATA_PEER") {
+ // This is a temporary hack until fair joining is implemented
+ let peers: Vec<_> = peers
+ .split(',')
+ .enumerate()
+ .map(|(i, x)| traft::row::Peer {
+ raft_id: (i + 1) as u64,
+ uri: x.to_owned(),
+ })
+ .collect();
+ traft::Storage::persist_peers(&peers);
+ }
+
let raft_cfg = raft::Config {
id: raft_id,
applied: traft::Storage::applied().unwrap().unwrap_or_default(),
diff --git a/picolib/traft.rs b/picolib/traft.rs
index 7dabd3699e..06cbf8b4e2 100644
--- a/picolib/traft.rs
+++ b/picolib/traft.rs
@@ -8,9 +8,11 @@ pub use storage::Storage;
pub mod row {
mod entry;
mod message;
+ mod peer;
pub use entry::Entry;
pub use message::Message;
+ pub use peer::Peer;
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, Hash, PartialEq, Eq)]
diff --git a/picolib/traft/row/peer.rs b/picolib/traft/row/peer.rs
new file mode 100644
index 0000000000..276589889c
--- /dev/null
+++ b/picolib/traft/row/peer.rs
@@ -0,0 +1,5 @@
+#[derive(serde::Deserialize)]
+pub struct Peer {
+ pub raft_id: u64,
+ pub uri: String,
+}
diff --git a/picolib/traft/storage.rs b/picolib/traft/storage.rs
index d83fb181bc..b58a21601b 100644
--- a/picolib/traft/storage.rs
+++ b/picolib/traft/storage.rs
@@ -15,6 +15,7 @@ use crate::traft::row;
pub struct Storage;
+const RAFT_GROUP: &str = "raft_group";
const RAFT_STATE: &str = "raft_state";
const RAFT_LOG: &str = "raft_log";
@@ -69,6 +70,7 @@ impl Storage {
is_local = true,
format = {
{name = 'raft_id', type = 'unsigned', is_nullable = false},
+ {name = 'peer_uri', type = 'string', is_nullable = false},
-- {name = 'raft_role', type = 'string', is_nullable = false},
-- {name = 'instance_id', type = 'string', is_nullable = false},
-- {name = 'instance_uuid', type = 'string', is_nullable = false},
@@ -164,6 +166,14 @@ impl Storage {
Ok(())
}
+ pub fn persist_peers(peers: &[row::Peer]) {
+ let mut space = Space::find(RAFT_GROUP).unwrap();
+ space.truncate().unwrap();
+ for peer in peers {
+ space.insert(&(peer.raft_id, &peer.uri)).unwrap();
+ }
+ }
+
pub fn entries(low: u64, high: u64) -> Result<Vec<raft::Entry>, StorageError> {
let mut ret: Vec<raft::Entry> = vec![];
let iter = Storage::space(RAFT_LOG)?
@@ -192,6 +202,21 @@ impl Storage {
Ok(())
}
+ pub fn conf_state() -> Result<raft::ConfState, StorageError> {
+ let mut ret = raft::ConfState::default();
+
+ let iter = Storage::space(RAFT_GROUP)?
+ .select(IteratorType::All, &())
+ .map_err(box_err!())?;
+
+ for tuple in iter {
+ let peer: row::Peer = tuple.into_struct().map_err(box_err!())?;
+ ret.mut_voters().push(peer.raft_id);
+ }
+
+ Ok(ret)
+ }
+
pub fn hard_state() -> Result<raft::HardState, StorageError> {
let mut ret = raft::HardState::default();
if let Some(term) = Storage::term()? {
@@ -217,13 +242,9 @@ impl Storage {
impl raft::Storage for Storage {
fn initial_state(&self) -> Result<raft::RaftState, RaftError> {
- let hs = Storage::hard_state()?;
-
// See also: https://github.com/etcd-io/etcd/blob/main/raft/raftpb/raft.pb.go
- let cs = raft::ConfState {
- voters: vec![1],
- ..Default::default()
- };
+ let hs = Storage::hard_state()?;
+ let cs = Storage::conf_state()?;
let ret = raft::RaftState::new(hs, cs);
Ok(ret)
diff --git a/tests/cli.rs b/tests/cli.rs
index b3caf0862c..8746da8611 100644
--- a/tests/cli.rs
+++ b/tests/cli.rs
@@ -19,6 +19,7 @@ fn positive() {
error(('Assertion failed: %q ~= %q'):format(l, r), 2)
end
end
+ assert_eq(os.environ()['PICODATA_PEER'], "127.0.0.1:3301")
assert_eq(os.environ()['PICODATA_LISTEN'], "3301")
assert_eq(os.environ()['PICODATA_DATA_DIR'], ".")
"#,
--
GitLab