From 43480d10e8304cdfed33f450d778ef39084247ba Mon Sep 17 00:00:00 2001 From: Anatoly Popov <a.popov@picodata.io> Date: Tue, 18 Feb 2025 13:10:57 +0300 Subject: [PATCH] chore: rename `bucket_count` to `shard_count` in public interface Closes #1391. It would be breaking change but `bucket_count` isn't released yet, so we're safe. Signed-off-by: Anatoly Popov <a.popov@picodata.io> --- src/config.rs | 40 +++++++++---------- src/http_server.rs | 4 +- src/info.rs | 2 +- src/luamod.rs | 2 +- src/rpc/sharding.rs | 2 +- src/sql.rs | 2 +- src/sql/router.rs | 12 +++--- src/sql/storage.rs | 8 ++-- src/tier.rs | 8 ++-- src/vshard.rs | 8 ++-- test/int/test_basics.py | 8 ++-- test/int/test_config_file.py | 2 +- test/int/test_http_server.py | 4 +- test/int/test_sharding.py | 20 +++++----- test/int/test_sql.py | 4 +- .../NodesContent/TierCard/TierCard.tsx | 2 +- .../intl/translations/en/pages/instances.ts | 4 +- .../intl/translations/ru/pages/instances.ts | 4 +- 18 files changed, 67 insertions(+), 69 deletions(-) diff --git a/src/config.rs b/src/config.rs index 4d1544d18e..2e1b0a0a0f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -197,7 +197,7 @@ Using configuration file '{args_path}'."); TierConfig { replication_factor: Some(1), can_vote: true, - bucket_count: Some(sql::DEFAULT_BUCKET_COUNT), + shard_count: Some(sql::DEFAULT_SHARD_COUNT), ..Default::default() }, )])); @@ -1044,10 +1044,10 @@ pub struct ClusterConfig { #[introspection(config_default = 1)] pub default_replication_factor: Option<u8>, - /// Bucket count which is used for tiers which didn't specify one - /// explicitly. For default value see [`Self::default_bucket_count()`]. - #[introspection(config_default = sql::DEFAULT_BUCKET_COUNT)] - pub default_bucket_count: Option<u64>, + /// Shard count which is used for tiers which didn't specify one + /// explicitly. For default value see [`Self::default_shard_count()`]. + #[introspection(config_default = sql::DEFAULT_SHARD_COUNT)] + pub default_shard_count: Option<u64>, #[serde(flatten)] #[introspection(ignore)] @@ -1063,7 +1063,7 @@ impl ClusterConfig { Tier { name: DEFAULT_TIER.into(), replication_factor: self.default_replication_factor(), - bucket_count: self.default_bucket_count(), + shard_count: self.default_shard_count(), can_vote: true, ..Default::default() }, @@ -1076,15 +1076,15 @@ impl ClusterConfig { .replication_factor .unwrap_or_else(|| self.default_replication_factor()); - let bucket_count = info - .bucket_count - .unwrap_or_else(|| self.default_bucket_count()); + let shard_count = info + .shard_count + .unwrap_or_else(|| self.default_shard_count()); let tier_def = Tier { name: name.into(), replication_factor, can_vote: info.can_vote, - bucket_count, + shard_count, ..Default::default() }; tier_defs.insert(name.into(), tier_def); @@ -1099,8 +1099,8 @@ impl ClusterConfig { } #[inline] - pub fn default_bucket_count(&self) -> u64 { - self.default_bucket_count + pub fn default_shard_count(&self) -> u64 { + self.default_shard_count .expect("is set in PicodataConfig::set_defaults_explicitly") } } @@ -2019,7 +2019,7 @@ cluster: can_vote: false radix: - bucket_count: 16384 + shard_count: 16384 instance: instance_name: voter1 @@ -2110,7 +2110,7 @@ cluster: let yaml = r###" cluster: default_replication_factor: 3 - default_bucket_count: 5000 + default_shard_count: 5000 name: test "###; let config = PicodataConfig::read_yaml_contents(&yaml.trim()).unwrap(); @@ -2123,7 +2123,7 @@ cluster: .get("default") .expect("default replication factor should applied to default tier configuration"); assert_eq!(default_tier.replication_factor, 3); - assert_eq!(default_tier.bucket_count, 5000); + assert_eq!(default_tier.shard_count, 5000); } #[test] @@ -2168,20 +2168,20 @@ instance: } #[test] - fn default_bucket_count_replication_factor() { + fn default_shard_count_replication_factor() { let yaml = r###" cluster: tier: non-default: replication_factor: 2 - bucket_count: 1000 + shard_count: 1000 default: default-rf: - bucket_count: 10000 + shard_count: 10000 default-bc: replication_factor: 4 default_replication_factor: 3 - default_bucket_count: 5000 + default_shard_count: 5000 name: test "###; let config = PicodataConfig::read_yaml_contents(&yaml.trim()).unwrap(); @@ -2196,7 +2196,7 @@ cluster: .expect("default replication factor should applied to default tier configuration"); assert_eq!(tier.replication_factor, rf); - assert_eq!(tier.bucket_count, bc); + assert_eq!(tier.shard_count, bc); }; assert_tier("default", 3, 5000); diff --git a/src/http_server.rs b/src/http_server.rs index 4409416a87..78690d4b9a 100644 --- a/src/http_server.rs +++ b/src/http_server.rs @@ -114,7 +114,7 @@ pub(crate) struct TierInfo { replicasets: Vec<ReplicasetInfo>, replicaset_count: usize, rf: u8, - bucket_count: u64, + shard_count: u64, instance_count: usize, #[serde(rename = "can_vote")] // for compatibility with lua version can_vote: bool, @@ -383,7 +383,7 @@ pub(crate) fn http_api_tiers() -> Result<Vec<TierInfo>> { replicasets: Vec::new(), replicaset_count: 0, rf: item.replication_factor, - bucket_count: item.bucket_count, + shard_count: item.shard_count, instance_count: 0, can_vote: true, name: item.name.clone(), diff --git a/src/info.rs b/src/info.rs index 7fd1e30422..5c1519d1d9 100644 --- a/src/info.rs +++ b/src/info.rs @@ -357,7 +357,7 @@ pub fn proc_get_vshard_config(tier_name: Option<String>) -> Result<RawByteBuf, E return Err(Error::NoSuchTier(tier_name)); }; - let config = VshardConfig::from_storage(&node.storage, &tier.name, tier.bucket_count)?; + let config = VshardConfig::from_storage(&node.storage, &tier.name, tier.shard_count)?; let data = rmp_serde::to_vec_named(&config).map_err(Error::other)?; Ok(RawByteBuf::from(data)) } diff --git a/src/luamod.rs b/src/luamod.rs index af7e63920f..028e64d2a1 100644 --- a/src/luamod.rs +++ b/src/luamod.rs @@ -166,7 +166,7 @@ pub(crate) fn setup() { let config = crate::vshard::VshardConfig::from_storage( &node.storage, &tier.name, - tier.bucket_count, + tier.shard_count, )?; Ok(config) }), diff --git a/src/rpc/sharding.rs b/src/rpc/sharding.rs index 0529546b7f..1e77b74104 100644 --- a/src/rpc/sharding.rs +++ b/src/rpc/sharding.rs @@ -66,7 +66,7 @@ crate::define_rpc_request! { continue; } - let mut config = VshardConfig::from_storage(storage, &tier.name, tier.bucket_count)?; + let mut config = VshardConfig::from_storage(storage, &tier.name, tier.shard_count)?; config.listen = Some(lua.eval("return box.info.listen")?); config.set_password_in_uris(); diff --git a/src/sql.rs b/src/sql.rs index 24d08a0357..5bc269c4bc 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -83,7 +83,7 @@ pub mod storage; use self::router::DEFAULT_QUERY_TIMEOUT; use serde::Serialize; -pub const DEFAULT_BUCKET_COUNT: u64 = 3000; +pub const DEFAULT_SHARD_COUNT: u64 = 3000; enum Privileges { Read, diff --git a/src/sql/router.rs b/src/sql/router.rs index 3cf14d425d..3642a44783 100644 --- a/src/sql/router.rs +++ b/src/sql/router.rs @@ -72,7 +72,7 @@ pub fn get_tier_info(tier_name: &str) -> Result<Tier, SbroadError> { })??; Ok(Tier { - bucket_count: tier.bucket_count, + shard_count: tier.shard_count, name: tier.name, }) } @@ -99,7 +99,7 @@ fn get_current_tier_name() -> Result<String, SbroadError> { #[derive(Default)] pub struct Tier { - bucket_count: u64, + shard_count: u64, name: String, } @@ -329,7 +329,7 @@ impl Router for RouterRuntime { } } -pub(crate) fn calculate_bucket_id(tuple: &[&Value], bucket_count: u64) -> Result<u64, SbroadError> { +pub(crate) fn calculate_bucket_id(tuple: &[&Value], shard_count: u64) -> Result<u64, SbroadError> { let wrapped_tuple = tuple .iter() .map(|v| MsgPackValue::from(*v)) @@ -359,7 +359,7 @@ pub(crate) fn calculate_bucket_id(tuple: &[&Value], bucket_count: u64) -> Result format_smolstr!("{e:?}"), ) })?; - Ok(u64::from(key.hash(&tnt_tuple)) % bucket_count + 1) + Ok(u64::from(key.hash(&tnt_tuple)) % shard_count + 1) } impl Vshard for Tier { @@ -381,7 +381,7 @@ impl Vshard for Tier { } fn bucket_count(&self) -> u64 { - self.bucket_count + self.shard_count } fn get_random_bucket(&self) -> Buckets { @@ -404,7 +404,7 @@ impl Vshard for Tier { impl Vshard for &Tier { fn bucket_count(&self) -> u64 { - self.bucket_count + self.shard_count } fn get_random_bucket(&self) -> Buckets { diff --git a/src/sql/storage.rs b/src/sql/storage.rs index e0bf863d12..2631dd90e8 100644 --- a/src/sql/storage.rs +++ b/src/sql/storage.rs @@ -31,7 +31,7 @@ use smol_str::{format_smolstr, SmolStr}; use std::collections::HashMap; use std::{any::Any, cell::RefCell, rc::Rc}; -use super::{router::calculate_bucket_id, DEFAULT_BUCKET_COUNT}; +use super::{router::calculate_bucket_id, DEFAULT_SHARD_COUNT}; thread_local!( static STATEMENT_CACHE: Rc<Mutex<PicoStorageCache>> = Rc::new( @@ -42,7 +42,7 @@ thread_local!( #[allow(clippy::module_name_repetitions)] pub struct StorageRuntime { pub metadata: RefCell<StorageMetadata>, - bucket_count: u64, + shard_count: u64, cache: Rc<Mutex<PicoStorageCache>>, } @@ -209,7 +209,7 @@ impl PlanInfo for LocalExecutionQueryInfo<'_> { impl Vshard for StorageRuntime { fn bucket_count(&self) -> u64 { - self.bucket_count + self.shard_count } fn get_random_bucket(&self) -> Buckets { @@ -297,7 +297,7 @@ impl StorageRuntime { pub fn new() -> Result<Self, SbroadError> { let runtime = STATEMENT_CACHE.with(|cache| StorageRuntime { metadata: RefCell::new(StorageMetadata::new()), - bucket_count: DEFAULT_BUCKET_COUNT, + shard_count: DEFAULT_SHARD_COUNT, cache: cache.clone(), }); Ok(runtime) diff --git a/src/tier.rs b/src/tier.rs index e0b1a94482..6ebc3bf5ff 100644 --- a/src/tier.rs +++ b/src/tier.rs @@ -23,7 +23,7 @@ pub struct Tier { pub current_vshard_config_version: u64, pub target_vshard_config_version: u64, pub vshard_bootstrapped: bool, - pub bucket_count: u64, + pub shard_count: u64, } impl Encode for Tier {} @@ -40,7 +40,7 @@ impl Tier { Field::from(("current_vshard_config_version", FieldType::Unsigned)), Field::from(("target_vshard_config_version", FieldType::Unsigned)), Field::from(("vshard_bootstrapped", FieldType::Boolean)), - Field::from(("bucket_count", FieldType::Unsigned)), + Field::from(("shard_count", FieldType::Unsigned)), ] } @@ -68,7 +68,7 @@ impl Default for Tier { name: DEFAULT_TIER.into(), replication_factor: 1, can_vote: true, - bucket_count: sql::DEFAULT_BUCKET_COUNT, + shard_count: sql::DEFAULT_SHARD_COUNT, current_vshard_config_version: 0, target_vshard_config_version: 0, vshard_bootstrapped: false, @@ -96,7 +96,7 @@ pub struct TierConfig { pub replication_factor: Option<u8>, #[serde(skip_serializing_if = "Option::is_none")] - pub bucket_count: Option<u64>, + pub shard_count: Option<u64>, #[serde(default = "default_can_vote")] pub can_vote: bool, diff --git a/src/vshard.rs b/src/vshard.rs index c18195274c..40481b939c 100644 --- a/src/vshard.rs +++ b/src/vshard.rs @@ -133,7 +133,7 @@ impl VshardConfig { pub fn from_storage( storage: &Clusterwide, tier_name: &str, - bucket_count: u64, + shard_count: u64, ) -> Result<Self, Error> { let instances = storage.instances.all_instances()?; let peer_addresses: HashMap<_, _> = storage @@ -149,7 +149,7 @@ impl VshardConfig { &peer_addresses, &replicasets, tier_name, - bucket_count, + shard_count, ); Ok(result) } @@ -159,7 +159,7 @@ impl VshardConfig { peer_addresses: &HashMap<RaftId, String>, replicasets: &HashMap<&ReplicasetName, &Replicaset>, tier_name: &str, - bucket_count: u64, + shard_count: u64, ) -> Self { let mut sharding: HashMap<String, ReplicasetSpec> = HashMap::new(); for peer in instances { @@ -201,7 +201,7 @@ impl VshardConfig { sharding, discovery_mode: DiscoveryMode::On, space_bucket_id: TABLE_ID_BUCKET, - bucket_count, + bucket_count: shard_count, } } diff --git a/test/int/test_basics.py b/test/int/test_basics.py index f6605b1038..403c29a2c6 100644 --- a/test/int/test_basics.py +++ b/test/int/test_basics.py @@ -417,7 +417,7 @@ Insert(_pico_index, [{_pico_user},2,"_pico_user_owner_id","tree",[{{"unique":fal Insert(_pico_table, [{_pico_privilege},"_pico_privilege",{{"Global":null}},[{{"field_type":"unsigned","is_nullable":false,"name":"grantor_id"}},{{"field_type":"unsigned","is_nullable":false,"name":"grantee_id"}},{{"field_type":"string","is_nullable":false,"name":"privilege"}},{{"field_type":"string","is_nullable":false,"name":"object_type"}},{{"field_type":"integer","is_nullable":false,"name":"object_id"}},{{"field_type":"unsigned","is_nullable":false,"name":"schema_version"}}],0,true,"memtx",1,""]), Insert(_pico_index, [{_pico_privilege},0,"_pico_privilege_primary","tree",[{{"unique":true}}],[["grantee_id","unsigned",null,false,null],["object_type","string",null,false,null],["object_id","integer",null,false,null],["privilege","string",null,false,null]],true,0]), Insert(_pico_index, [{_pico_privilege},1,"_pico_privilege_object","tree",[{{"unique":false}}],[["object_type","string",null,false,null],["object_id","integer",null,false,null]],true,0]), -Insert(_pico_table, [{_pico_tier},"_pico_tier",{{"Global":null}},[{{"field_type":"string","is_nullable":false,"name":"name"}},{{"field_type":"unsigned","is_nullable":false,"name":"replication_factor"}},{{"field_type":"boolean","is_nullable":false,"name":"can_vote"}},{{"field_type":"unsigned","is_nullable":false,"name":"current_vshard_config_version"}},{{"field_type":"unsigned","is_nullable":false,"name":"target_vshard_config_version"}},{{"field_type":"boolean","is_nullable":false,"name":"vshard_bootstrapped"}},{{"field_type":"unsigned","is_nullable":false,"name":"bucket_count"}}],0,true,"memtx",1,""]), +Insert(_pico_table, [{_pico_tier},"_pico_tier",{{"Global":null}},[{{"field_type":"string","is_nullable":false,"name":"name"}},{{"field_type":"unsigned","is_nullable":false,"name":"replication_factor"}},{{"field_type":"boolean","is_nullable":false,"name":"can_vote"}},{{"field_type":"unsigned","is_nullable":false,"name":"current_vshard_config_version"}},{{"field_type":"unsigned","is_nullable":false,"name":"target_vshard_config_version"}},{{"field_type":"boolean","is_nullable":false,"name":"vshard_bootstrapped"}},{{"field_type":"unsigned","is_nullable":false,"name":"shard_count"}}],0,true,"memtx",1,""]), Insert(_pico_index, [{_pico_tier},0,"_pico_tier_name","tree",[{{"unique":true}}],[["name","string",null,false,null]],true,0]), Insert(_pico_table, [{_pico_routine},"_pico_routine",{{"Global":null}},[{{"field_type":"unsigned","is_nullable":false,"name":"id"}},{{"field_type":"string","is_nullable":false,"name":"name"}},{{"field_type":"string","is_nullable":false,"name":"kind"}},{{"field_type":"array","is_nullable":false,"name":"params"}},{{"field_type":"array","is_nullable":false,"name":"returns"}},{{"field_type":"string","is_nullable":false,"name":"language"}},{{"field_type":"string","is_nullable":false,"name":"body"}},{{"field_type":"string","is_nullable":false,"name":"security"}},{{"field_type":"boolean","is_nullable":false,"name":"operable"}},{{"field_type":"unsigned","is_nullable":false,"name":"schema_version"}},{{"field_type":"unsigned","is_nullable":false,"name":"owner"}}],0,true,"memtx",1,""]), Insert(_pico_index, [{_pico_routine},0,"_pico_routine_id","tree",[{{"unique":true}}],[["id","unsigned",null,false,null]],true,0]), @@ -639,14 +639,14 @@ cluster: } space_bucket_id = storage_instance.eval("return box.space._bucket.id") - total_bucket_count = 3000 + total_shard_count = 3000 storage_vshard_config_explicit = storage_instance.call(".proc_get_vshard_config", "storage") assert storage_vshard_config_explicit == dict( discovery_mode="on", sharding=storage_sharding, space_bucket_id=space_bucket_id, - bucket_count=total_bucket_count, + bucket_count=total_shard_count, ) storage_vshard_config_implicit = storage_instance.call(".proc_get_vshard_config", None) @@ -657,7 +657,7 @@ cluster: discovery_mode="on", sharding=router_sharding, space_bucket_id=space_bucket_id, - bucket_count=total_bucket_count, + bucket_count=total_shard_count, ) router_vshard_config_implicit = router_instance_1.call(".proc_get_vshard_config", None) diff --git a/test/int/test_config_file.py b/test/int/test_config_file.py index 1b138f8147..5c422f99cf 100644 --- a/test/int/test_config_file.py +++ b/test/int/test_config_file.py @@ -82,7 +82,7 @@ instance: value=dict(deluxe=dict(can_vote=True)), source="config_file", ), - default_bucket_count=dict(value=3000, source="default"), + default_shard_count=dict(value=3000, source="default"), default_replication_factor=dict(value=1, source="default"), ), instance=dict( diff --git a/test/int/test_http_server.py b/test/int/test_http_server.py index a3da0f6a17..7a63fe7749 100644 --- a/test/int/test_http_server.py +++ b/test/int/test_http_server.py @@ -67,7 +67,7 @@ def test_webui_basic(instance: Instance): ], "replicasetCount": 1, "rf": 1, - "bucketCount": 3000, + "shardCount": 3000, "instanceCount": 1, "can_vote": True, "name": "default", @@ -203,7 +203,7 @@ def test_webui_with_plugin(cluster: Cluster): tier_template = { "replicasetCount": 1, "rf": 1, - "bucketCount": 3000, + "shardCount": 3000, "instanceCount": 1, "can_vote": True, } diff --git a/test/int/test_sharding.py b/test/int/test_sharding.py index ebd82982f2..3a18ca0ed5 100644 --- a/test/int/test_sharding.py +++ b/test/int/test_sharding.py @@ -390,32 +390,30 @@ def test_expel_blocked_by_bucket_rebalancing(cluster: Cluster): cluster.wait_until_instance_has_this_many_active_buckets(i2, 1500) -def assert_tier_bucket_count(cluster: Cluster, name: str, bucket_count: int, *instances: Instance): +def assert_tier_shard_count(cluster: Cluster, name: str, shard_count: int, *instances: Instance): assert len(instances) > 0 i1 = instances[0] - # default `bucket_count` is 3000 - rows = i1.sql(""" SELECT name, bucket_count FROM _pico_tier WHERE name = ?""", name) + rows = i1.sql(""" SELECT name, shard_count FROM _pico_tier WHERE name = ?""", name) assert rows == [ - [name, bucket_count], + [name, shard_count], ] - # 3000 bucket counts, 3 replicasets for x in instances: - cluster.wait_until_instance_has_this_many_active_buckets(x, int(bucket_count / len(instances))) + cluster.wait_until_instance_has_this_many_active_buckets(x, int(shard_count / len(instances))) -def test_bucket_count_custom_and_default(cluster: Cluster): +def test_shard_count_custom_and_default(cluster: Cluster): cluster.set_service_password("secret") cluster.set_config_file( yaml=""" cluster: name: test - default_bucket_count: 6000 + default_shard_count: 6000 tier: radix: replication_factor: 1 - bucket_count: 16384 + shard_count: 16384 storage: replication_factor: 1 """ @@ -427,5 +425,5 @@ cluster: i5 = cluster.add_instance(tier="radix", wait_online=False) cluster.wait_online() - assert_tier_bucket_count(cluster, "storage", 6000, i1, i2, i3) - assert_tier_bucket_count(cluster, "radix", 16384, i4, i5) + assert_tier_shard_count(cluster, "storage", 6000, i1, i2, i3) + assert_tier_shard_count(cluster, "radix", 16384, i4, i5) diff --git a/test/int/test_sql.py b/test/int/test_sql.py index c7dc5de54a..7f721b5d4d 100644 --- a/test/int/test_sql.py +++ b/test/int/test_sql.py @@ -1582,13 +1582,13 @@ def test_hash(cluster: Cluster): tup = (1,) key_def = KeyDef([KeyPart(1, "integer", True)]) lua_hash = i1.hash(tup, key_def) - bucket_count = 3000 + shard_count = 3000 # Compare SQL and Lua bucket_id data = i1.sql("""insert into t values(?);""", 1) assert data["row_count"] == 1 data = i1.sql(""" select "bucket_id" from t where a = ?""", 1) - assert data == [[lua_hash % bucket_count + 1]] + assert data == [[lua_hash % shard_count + 1]] def test_select_lowercase_name(cluster: Cluster): diff --git a/webui/src/modules/nodes/nodesPage/NodesContent/TierCard/TierCard.tsx b/webui/src/modules/nodes/nodesPage/NodesContent/TierCard/TierCard.tsx index 851965e1ab..035ccd9ca0 100644 --- a/webui/src/modules/nodes/nodesPage/NodesContent/TierCard/TierCard.tsx +++ b/webui/src/modules/nodes/nodesPage/NodesContent/TierCard/TierCard.tsx @@ -92,7 +92,7 @@ export const TierCard: FC<TierCardProps> = React.memo(({ tier }) => { </div> <div className={cn(styles.infoColumn, styles.bucketCountColumn)}> <div className={styles.label}> - {tierTranslations.bucket_count.label} + {tierTranslations.shard_count.label} </div> <div className={styles.infoValue}>{tier.bucketCount}</div> </div> diff --git a/webui/src/shared/intl/translations/en/pages/instances.ts b/webui/src/shared/intl/translations/en/pages/instances.ts index 0e5d9395d6..e438a0faf8 100644 --- a/webui/src/shared/intl/translations/en/pages/instances.ts +++ b/webui/src/shared/intl/translations/en/pages/instances.ts @@ -72,8 +72,8 @@ export const instances = { rf: { label: "RF", }, - bucket_count: { - label: "Buckets", + shard_count: { + label: "Shards", }, canVote: { label: "Can vote", diff --git a/webui/src/shared/intl/translations/ru/pages/instances.ts b/webui/src/shared/intl/translations/ru/pages/instances.ts index 973fc82f31..d1d13b6da1 100644 --- a/webui/src/shared/intl/translations/ru/pages/instances.ts +++ b/webui/src/shared/intl/translations/ru/pages/instances.ts @@ -74,8 +74,8 @@ export const instances: TPages["instances"] = { rf: { label: "Фактор репликации", }, - bucket_count: { - label: "Бакеты", + shard_count: { + label: "Сегменты", }, canVote: { label: "ГолоÑует?", -- GitLab