connection balancing skew
Тестировалось на версии драйвера - 3.2.1
instance 1001 active connection: 9
instance 1002 active connection: 2
instance 1003 active connection: 0
instance 1004 active connection: 3
instance 1005 active connection: 0instance 2001 active connection: 2
instance 2002 active connection: 3
instance 2003 active connection: 0
instance 2004 active connection: 1
instance 2005 active connection: 0instance 3001 active connection: 2
instance 3002 active connection: 0
instance 3003 active connection: 0
instance 3004 active connection: 1
instance 3005 active connection: 0admin=> select * from _pico_replicaset;
name | uuid | current_master_name | target_master_name | tier | weight | weight_origin | state | current_config_version | target_config_version | promotion_vclock
-----------+--------------------------------------+---------------------+--------------------+---------+--------+---------------+-------+------------------------+-----------------------+------------------
arbiter_1 | bd47041b-7caa-4e73-93c9-cc97a3334ea7 | arbiter-1000 | arbiter-1000 | arbiter | 1 | auto | ready | 20 | 20 | {}
arbiter_2 | ba9fb9a5-fb63-4135-a17b-6e05dc516820 | arbiter-3000 | arbiter-3000 | arbiter | 1 | auto | ready | 7 | 7 | {}
arbiter_3 | ce28aa45-21b1-44a8-91ee-a01de3e26f73 | arbiter-2000 | arbiter-2000 | arbiter | 1 | auto | ready | 2 | 2 | {}
default_1 | d7056b01-8134-47eb-9eda-de9ef4f7bd6b | default-3001 | default-3001 | default | 1 | auto | ready | 4 | 4 | {}
default_2 | 74db1048-7f9e-4434-a7e1-a29b286326be | default-1001 | default-1001 | default | 1 | auto | ready | 4 | 4 | {}
default_3 | 8de8608a-0aed-4eb8-920e-e658bf7c4494 | default-2002 | default-2002 | default | 1 | auto | ready | 2 | 2 | {}
default_4 | 258bb294-c521-4540-bea0-855862c4ee04 | default-3003 | default-3003 | default | 1 | auto | ready | 4 | 4 | {}
default_5 | 88b0d192-129d-4840-a420-43c36c120862 | default-1003 | default-1003 | default | 1 | auto | ready | 2 | 2 | {}
default_6 | c1a09893-b9c7-4b3a-b0b2-032784c45468 | default-2004 | default-2004 | default | 1 | auto | ready | 2 | 2 | {}
(9 rows)
admin=> select * from _pico_instance;
name | uuid | raft_id | replicaset_name | replicaset_uuid | current_state | target_state | failure_domain | tier | picodata_version
--------------+--------------------------------------+---------+-----------------+--------------------------------------+---------------+---------------+------------------------------------+---------+-----------------------
arbiter-1000 | 0ab2365e-4e58-488e-bd4b-2f6360ed337c | 1 | arbiter_1 | bd47041b-7caa-4e73-93c9-cc97a3334ea7 | ["Online",11] | ["Online",11] | {"DC":"['DC1']","HOST":"SERVER-1"} | arbiter | 25.3.0-195-g569299da2
arbiter-2000 | b8e9f8da-6a55-4db1-b6cd-bdaa9a14dc22 | 3 | arbiter_3 | ce28aa45-21b1-44a8-91ee-a01de3e26f73 | ["Online",2] | ["Online",2] | {"HOST":"SERVER-2","DC":"['DC2']"} | arbiter | 25.3.0-195-g569299da2
arbiter-3000 | 6bb268e9-ad2d-4567-8ce3-cfe37becc7df | 2 | arbiter_2 | ba9fb9a5-fb63-4135-a17b-6e05dc516820 | ["Online",3] | ["Online",3] | {"HOST":"SERVER-3","DC":"['DC3']"} | arbiter | 25.3.0-195-g569299da2
default-1001 | ca4ae280-fc59-417a-ad81-fbac77dc2f5c | 6 | default_2 | 74db1048-7f9e-4434-a7e1-a29b286326be | ["Online",1] | ["Online",1] | {"HOST":"SERVER-1","DC":"['DC1']"} | default | 25.3.0-195-g569299da2
default-1002 | 1c3d596a-eb47-4817-85f1-e7c7d5816d22 | 9 | default_3 | 8de8608a-0aed-4eb8-920e-e658bf7c4494 | ["Online",1] | ["Online",1] | {"DC":"['DC1']","HOST":"SERVER-1"} | default | 25.3.0-195-g569299da2
default-1003 | f1079801-1a64-45fd-8f0d-44061b664550 | 12 | default_5 | 88b0d192-129d-4840-a420-43c36c120862 | ["Online",1] | ["Online",1] | {"DC":"['DC1']","HOST":"SERVER-1"} | default | 25.3.0-195-g569299da2
default-1004 | 27d227b4-83c7-4037-b2c1-5ee81438f22f | 15 | default_6 | c1a09893-b9c7-4b3a-b0b2-032784c45468 | ["Online",1] | ["Online",1] | {"HOST":"SERVER-1","DC":"['DC1']"} | default | 25.3.0-195-g569299da2
default-2001 | 1e1e84e4-dfb4-47d1-8971-30f43d4bb646 | 5 | default_1 | d7056b01-8134-47eb-9eda-de9ef4f7bd6b | ["Online",2] | ["Online",2] | {"DC":"['DC2']","HOST":"SERVER-2"} | default | 25.3.0-195-g569299da2
default-2002 | 9e705b9d-97d4-4059-be0e-9e99061a88be | 8 | default_3 | 8de8608a-0aed-4eb8-920e-e658bf7c4494 | ["Online",1] | ["Online",1] | {"DC":"['DC2']","HOST":"SERVER-2"} | default | 25.3.0-195-g569299da2
default-2003 | 0b767ced-9901-4f4a-bd30-8c300b8b38d6 | 11 | default_4 | 258bb294-c521-4540-bea0-855862c4ee04 | ["Online",2] | ["Online",2] | {"DC":"['DC2']","HOST":"SERVER-2"} | default | 25.3.0-195-g569299da2
default-2004 | 70837d91-04f3-49bb-922a-831b4fdf1c95 | 14 | default_6 | c1a09893-b9c7-4b3a-b0b2-032784c45468 | ["Online",1] | ["Online",1] | {"DC":"['DC2']","HOST":"SERVER-2"} | default | 25.3.0-195-g569299da2
default-3001 | 0068e4bc-a871-472f-93ad-432f54a213d2 | 4 | default_1 | d7056b01-8134-47eb-9eda-de9ef4f7bd6b | ["Online",1] | ["Online",1] | {"HOST":"SERVER-3","DC":"['DC3']"} | default | 25.3.0-195-g569299da2
default-3002 | 53eae7f6-79cc-401c-a67a-e614ed61246d | 7 | default_2 | 74db1048-7f9e-4434-a7e1-a29b286326be | ["Online",2] | ["Online",2] | {"DC":"['DC3']","HOST":"SERVER-3"} | default | 25.3.0-195-g569299da2
default-3003 | c875115b-459c-4441-b2f0-518dfefb865b | 10 | default_4 | 258bb294-c521-4540-bea0-855862c4ee04 | ["Online",1] | ["Online",1] | {"DC":"['DC3']","HOST":"SERVER-3"} | default | 25.3.0-195-g569299da2
default-3004 | a3b6b622-10b6-4985-9512-7661d7298dee | 13 | default_5 | 88b0d192-129d-4840-a420-43c36c120862 | ["Online",1] | ["Online",1] | {"DC":"['DC3']","HOST":"SERVER-3"} | default | 25.3.0-195-g569299da2
(15 rows)
Конфигурация спарка
.config("spark.driver.cores", "2")
.config("spark.driver.memory", "1G")
.config("spark.executor.cores", "4")
.config("spark.executor.memory", "10G")
.config("spark.executor.memoryOverhead", "1G")
.config("spark.executor.instances", "6")
.config("spark.default.parallelism", "24")
.config("spark.sql.shuffle.partitions", "100")