tests are failing on mac m1
The following tests fail when running on mac m1 on the master branch at hash 4706bcef
FAILED test/int/test_cli_connect.py::test_connect_unix_ok_via_default_sock - pexpect.exceptions.TIMEOUT: Timeout exceeded.
FAILED test/int/test_config_file.py::test_config_file_box_cfg_parameters - conftest.ProcessDead: process exited unexpectedly, exit_code=1
FAILED test/int/test_plugin.py::test_plugin_rpc_sdk_send_request - conftest.ProcessDead: process exited unexpectedly, exit_code=-6
FAILED test/int/test_expelling.py::test_expel_leader - AssertionError: assert ('Online', 'Expelled') == ('Expelled', 'Expelled')
FAILED test/int/test_plugin.py::test_plugin_rpc_sdk_single_instance - conftest.ProcessDead: process exited unexpectedly, exit_code=-6
FAILED test/int/test_plugin.py::test_sdk_internal - conftest.ProcessDead: process exited unexpectedly, exit_code=-6
FAILED test/int/test_plugin.py::test_sdk_sql - conftest.ProcessDead: process exited unexpectedly, exit_code=-6
FAILED test/int/test_plugin.py::test_set_string_values_in_config - conftest.ProcessDead: process exited unexpectedly, exit_code=-6
FAILED test/int/test_replication.py::test_expel_blocked_by_replicaset_master_switchover_to_online_replica - TimeoutError
FAILED test/int/test_expelling.py::test_expel_by_follower - AssertionError: assert ('Online', 'Expelled') == ('Expelled', 'Expelled')Also, the following tests are flaky:
FAILED test/int/test_gostech_audit.py::test_gostech_user - AssertionError: assert 'guest' == 'ymir'
FAILED test/int/test_gostech_audit.py::test_gostech_join_expel_instance - AssertionError: assert 'transfer buc...om replicaset' == 'idle'
FAILED test/int/test_sharding.py::test_expel_blocked_by_bucket_rebalancing - assert FalseDescription:
FAILED test/int/test_cli_connect.py::test_connect_unix_ok_via_default_sock - pexpect.exceptions.TIMEOUT: Timeout exceeded. :
def test_connect_unix_ok_via_default_sock(cluster: Cluster):
i1 = cluster.add_instance(wait_online=False)
i1.start()
i1.wait_online()
cli = pexpect.spawn(
# For some uninvestigated reason, readline trims the propmt in CI
# Instead of
# unix/:/some/path/to/admin.sock>
# it prints
# </path/to/admin.sock>
#
# We were unable to debug it quickly and used cwd as a workaround
cwd=i1.data_dir,
command=i1.binary_path,
args=["admin", "./admin.sock"],
encoding="utf-8",
timeout=CLI_TIMEOUT,
)
cli.logfile = sys.stdout
cli.expect_exact("picodata> ")
# Change language to SQL works
cli.sendline("\\sql;")
cli.sendline("CREATE ROLE CHANGE_TO_SQL_WORKS;")
cli.expect_exact("1")
cli.sendline("\\lua;")
> cli.expect_exact("Language switched to Lua")
cli = <pexpect.pty_spawn.spawn object at 0x107cf06e0>
cluster = Cluster("127.0.0.1", n=1)
i1 = Instance(i1, listen=127.0.0.1:3300 cluster=cluster-0-0, process.pid=20223)err = TIMEOUT('Timeout exceeded.')
exc = TIMEOUT('Timeout exceeded.\n<pexpect.pty_spawn.spawn object at 0x107cf06e0>\ncommand: /Users/diana/picodata/target/deb...0.05\ndelayafterclose: 0.1\ndelayafterterminate: 0.1\nsearcher: searcher_string:\n 0: \'Language switched to Lua\'')
index = -1
msg = "Timeout exceeded.\n<pexpect.pty_spawn.spawn object at 0x107cf06e0>\ncommand: /Users/diana/picodata/target/debug/picod...d: 0.05\ndelayafterclose: 0.1\ndelayafterterminate: 0.1\nsearcher: searcher_string:\n 0: 'Language switched to Lua'"
self = <pexpect.expect.Expecter object at 0x107cedd10>
spawn = <pexpect.pty_spawn.spawn object at 0x107cf06e0>FAILED test/int/test_config_file.py::test_config_file_box_cfg_parameters - conftest.ProcessDead: process exited unexpectedly, exit_code=1:
def check_parses_as_json(line: bytes):
nonlocal json_line_count
nonlocal non_json_lines
try:
json.loads(line)
json_line_count += 1
except json.JSONDecodeError:
non_json_lines.append(line)
i1.on_output_line(check_parses_as_json)
i1.start()
> i1.wait_online()
box_cfg = {'auth_type': 'chap-sha1', 'background': False, 'bootstrap_strategy': 'legacy', 'checkpoint_count': 2, ...}
check_parses_as_json = <function test_config_file_box_cfg_parameters.<locals>.check_parses_as_json at 0x105eb9300>
cluster = Cluster("127.0.0.1", n=1)
i1 = Instance(i1, listen=127.0.0.1:3300 cluster=cluster-0-0, process.pid=21892)
json_line_count = 0
non_json_lines = [b"SystemError: can't open log file: /proc/self/fd/2: No such file or directory\n", b'failed to initialize logging subsystem\n']FAILED test/int/test_plugin.py::test_plugin_rpc_sdk_send_request - conftest.ProcessDead: process exited unexpectedly, exit_code=-6:
args = (<conftest.Connection object at 0x106ad6990>, '.proc_rpc_dispatch', ('/proxy', b'\x83\xa4path\xa5/ping\xadinstance_nam...', {1: UUID('4168fb51-d5d3-48f9-80c2-bd461bfe3cd5'), 2: 'testplug_sdk', 3: 'service_with_rpc_tests', 4: '0.1.0', ...}))
kwargs = {}, strerror = 'Connection reset by peer'
def inner(*args, **kwargs):
try:
> result = func(*args, **kwargs)
args = (<conftest.Connection object at 0x106ad6990>, '.proc_rpc_dispatch', ('/proxy', b'\x83\xa4path\xa5/ping\xadinstance_nam...', {1: UUID('4168fb51-d5d3-48f9-80c2-bd461bfe3cd5'), 2: 'testplug_sdk', 3: 'service_with_rpc_tests', 4: '0.1.0', ...}))
func = <function Connection.call at 0x105245d00>
kwargs = {}
strerror = 'Connection reset by peer'FAILED test/int/test_expelling.py::test_expel_leader - AssertionError: assert ('Online', 'Expelled') == ('Expelled', 'Expelled'):
def test_expel_leader(cluster3: Cluster):
# Scenario: expel a Leader instance by command to itself
# Given a cluster
# When a Leader instance expelled from the cluster
# Then the instance marked as expelled in the instances table
# And excluded from the voters list
i1, i2, i3 = cluster3.instances
i1.promote_or_fail()
i1.assert_raft_status("Leader")
cluster3.expel(i1)
> Retriable(timeout=30).call(lambda: assert_instance_expelled(i1, i2))
cluster3 = Cluster("127.0.0.1", n=3)
i1 = Instance(i1, listen=127.0.0.1:3300 cluster=cluster-0-0, process.pid=25601)
i2 = Instance(i2, listen=127.0.0.1:3302 cluster=cluster-0-0, process.pid=25602)
i3 = Instance(i3, listen=127.0.0.1:3303 cluster=cluster-0-0, process.pid=25603) def assert_instance_expelled(expelled_instance: Instance, instance: Instance):
info = instance.call(".proc_instance_info", expelled_instance.name)
states = (info["current_state"]["variant"], info["target_state"]["variant"])
> assert states == ("Expelled", "Expelled")
E AssertionError: assert ('Online', 'Expelled') == ('Expelled', 'Expelled')
E
E At index 0 diff: 'Online' != 'Expelled'
E
E Full diff:
E (
E - 'Expelled',
E + 'Online',
E 'Expelled',
E )
expelled_instance = Instance(i1, listen=127.0.0.1:3300 cluster=cluster-0-0, process.pid=25601)
info = {'advertise_address': '127.0.0.1:3300', 'cluster_name': 'cluster-0-0', 'current_state': {'incarnation': 1, 'variant': 'Online'}, 'name': 'i1', ...}
instance = Instance(i2, listen=127.0.0.1:3302 cluster=cluster-0-0, process.pid=25602)
states = ('Online', 'Expelled')
test/int/test_expelling.py:16: AssertionErrorFAILED test/int/test_plugin.py::test_plugin_rpc_sdk_single_instance - conftest.ProcessDead: process exited unexpectedly, exit_code=-6:
args = (<conftest.Connection object at 0x107932210>, '.proc_rpc_dispatch', ('/proxy', b'\x85\xacservice_info\x93\xactestplug_...', {1: UUID('ed93bbec-9cab-464d-a94e-019b441bca75'), 2: 'testplug_sdk', 3: 'service_with_rpc_tests', 4: '0.1.0', ...}))
kwargs = {}, strerror = 'Connection reset by peer'
def inner(*args, **kwargs):
try:
> result = func(*args, **kwargs)
args = (<conftest.Connection object at 0x107932210>, '.proc_rpc_dispatch', ('/proxy', b'\x85\xacservice_info\x93\xactestplug_...', {1: UUID('ed93bbec-9cab-464d-a94e-019b441bca75'), 2: 'testplug_sdk', 3: 'service_with_rpc_tests', 4: '0.1.0', ...}))
func = <function Connection.call at 0x1056e1d00>
kwargs = {}
strerror = 'Connection reset by peer'FAILED test/int/test_plugin.py::test_sdk_internal - conftest.ProcessDead: process exited unexpectedly, exit_code=-6:
args = (<conftest.Connection object at 0x105e25f90>, 'pico.enable_plugin', ('testplug_sdk', '0.1.0')), kwargs = {}, strerror = 'Connection reset by peer'
def inner(*args, **kwargs):
try:
> result = func(*args, **kwargs)
args = (<conftest.Connection object at 0x105e25f90>, 'pico.enable_plugin', ('testplug_sdk', '0.1.0'))
func = <function Connection.call at 0x1047edd00>
kwargs = {}
strerror = 'Connection reset by peer'FAILED test/int/test_plugin.py::test_sdk_sql - conftest.ProcessDead: process exited unexpectedly, exit_code=-6:
args = (<conftest.Connection object at 0x103cf5f90>, 'pico.enable_plugin', ('testplug_sdk', '0.1.0')), kwargs = {}, strerror = 'Connection reset by peer'
def inner(*args, **kwargs):
try:
> result = func(*args, **kwargs)
args = (<conftest.Connection object at 0x103cf5f90>, 'pico.enable_plugin', ('testplug_sdk', '0.1.0'))
func = <function Connection.call at 0x1026c1d00>
kwargs = {}
strerror = 'Connection reset by peer'FAILED test/int/test_plugin.py::test_set_string_values_in_config - conftest.ProcessDead: process exited unexpectedly, exit_code=-6:
args = (<conftest.Connection object at 0x105231f90>, 'pico.enable_plugin', ('testplug_sdk', '0.1.0')), kwargs = {}, strerror = 'Connection reset by peer'
def inner(*args, **kwargs):
try:
> result = func(*args, **kwargs)
args = (<conftest.Connection object at 0x105231f90>, 'pico.enable_plugin', ('testplug_sdk', '0.1.0'))
func = <function Connection.call at 0x1038e1d00>
kwargs = {}
strerror = 'Connection reset by peer'FAILED test/int/test_replication.py::test_expel_blocked_by_replicaset_master_switchover_to_online_replica - TimeoutError:
self = <conftest.Connection object at 0x103ea6e90>, to_read = 5
def _recv(self, to_read):
"""
Receive binary data from connection socket.
:param to_read: Amount of data to read, in bytes.
:type to_read: :obj:`int`
:return: Buffer with read data
:rtype: :obj:`bytes`
:meta private:
"""
buf = b""
while to_read > 0:
try:
> tmp = self._socket.recv(to_read)
E TimeoutError: timed out
buf = b''
err = ConnectionResetError(54, 'Lost connection to server during query')
self = <conftest.Connection object at 0x103ea6e90>
to_read = 5
../Library/Caches/pypoetry/virtualenvs/picodata-D_lcLBcu-py3.13/lib/python3.13/site-packages/tarantool/connection.py:1176: TimeoutError
The above exception was the direct cause of the following exception:
args = (<conftest.Connection object at 0x103ea6e90>, '.proc_sql_dispatch', ' SELECT * FROM mytable ORDER BY id ', ()), kwargs = {}
def inner(*args, **kwargs):
try:
> result = func(*args, **kwargs)
args = (<conftest.Connection object at 0x103ea6e90>, '.proc_sql_dispatch', ' SELECT * FROM mytable ORDER BY id ', ())
func = <function Connection.call at 0x10286dd00>
kwargs = {}---------------------------------------------------------------------------- Captured stdout teardown ----------------------------------------------------------------------------
2024-11-22 17:08:33.962 [27667] main/114/raft_main_loop V> Sending from 1 to 5, msg: msg_type: MsgHeartbeat to: 5 commit: 59, to: 5, from: 1, raft_id: 1
i3 | 2024-11-22 17:08:33.963 [27669] main/131/.proc_raft_interact V> Sending from 2 to 1, msg: msg_type: MsgHeartbeatResponse to: 1 commit: 59, to: 1, from: 2, raft_id: 2
i5 | 2024-11-22 17:08:33.963 [27683] main/112/.proc_raft_interact V> Sending from 5 to 1, msg: msg_type: MsgHeartbeatResponse to: 1 commit: 59, to: 1, from: 5, raft_id: 5
i2 | 2024-11-22 17:08:33.963 [27668] main/112/.proc_raft_interact V> Sending from 3 to 1, msg: msg_type: MsgHeartbeatResponse to: 1 commit: 59, to: 1, from: 3, raft_id: 3
i1 | 2024-11-22 17:08:33.962 [27667] main/114/raft_main_loop V> Sending from 1 to 2, msg: msg_type: MsgHeartbeat to: 2 commit: 59, to: 2, from: 1, raft_id: 1
i1 | 2024-11-22 17:08:33.962 [27667] main/114/raft_main_loop V> Sending from 1 to 3, msg: msg_type: MsgHeartbeat to: 3 commit: 59, to: 3, from: 1, raft_id: 1
i1 | 2024-11-22 17:08:33.962 [27667] main/114/raft_main_loop V> main_loop_status = 'sending raft messages'
i1 | 2024-11-22 17:08:33.963 [27667] main/114/raft_main_loop V> done sending messages, sent: 3, skipped: 0
i1 | 2024-11-22 17:08:33.963 [27667] main/114/raft_main_loop V> main_loop_status = 'idle'
i5 | 2024-11-22 17:08:33.964 [27683] main/121/raft_main_loop V> main_loop_status = 'sending raft messages'
i5 | 2024-11-22 17:08:33.964 [27683] main/121/raft_main_loop V> done sending messages, sent: 1, skipped: 0
i5 | 2024-11-22 17:08:33.964 [27683] main/121/raft_main_loop V> main_loop_status = 'idle'
i2 | 2024-11-22 17:08:33.964 [27668] main/119/raft_main_loop V> main_loop_status = 'sending raft messages'
i2 | 2024-11-22 17:08:33.964 [27668] main/119/raft_main_loop V> done sending messages, sent: 1, skipped: 0
i2 | 2024-11-22 17:08:33.964 [27668] main/119/raft_main_loop V> main_loop_status = 'idle'
i3 | 2024-11-22 17:08:33.964 [27669] main/119/raft_main_loop V> main_loop_status = 'sending raft messages'
i3 | 2024-11-22 17:08:33.964 [27669] main/119/raft_main_loop V> done sending messages, sent: 1, skipped: 0
i3 | 2024-11-22 17:08:33.964 [27669] main/119/raft_main_loop V> main_loop_status = 'idle'
i1 | 2024-11-22 17:08:33.982 [27667] main/174/vshard.rebalancer_worker_1/vshard.storage init.lua:2622 E> Error during rebalancer routes applying: receiver c5d99c8f-8e1a-411b-9ef2-fb8c862a8cb4, error {"code":78,"base_type":"ClientError","type":"ClientError","message":"Timeout exceeded","trace":[{"file":"vshard\/error.lua","line":312}]}
i1 | 2024-11-22 17:08:33.982 [27667] main/174/vshard.rebalancer_worker_1/vshard.storage I> Can not finish transfers to c5d99c8f-8e1a-411b-9ef2-fb8c862a8cb4, skip to next round
i2 | 2024-11-22 17:08:34.063 [27668] main/144/vshard.failover.default/vshard.router V> Failovering step is finished. Schedule next after 1.000000 seconds
i5 | 2024-11-22 17:08:34.121 [27683] main/152/127.0.0.1:3300 (net.box)/vshard.replicaset I> disconnected from 127.0.0.1:3300
i5 | 2024-11-22 17:08:34.121 [27683] main/152/127.0.0.1:3300 (net.box)/box.net_box net_box.lua:352 W> 127.0.0.1:3300: Peer closedFAILED test/int/test_expelling.py::test_expel_by_follower - AssertionError: assert ('Online', 'Expelled') == ('Expelled', 'Expelled'):
def test_expel_by_follower(cluster3: Cluster):
# Scenario: expel an instance by command to a Follower
# Given a cluster
# When instance which is not a Leader receives expel CLI command
# Then expelling instance is expelled
i1, i2, i3 = cluster3.instances
i1.promote_or_fail()
i2.assert_raft_status("Follower", leader_id=i1.raft_id)
i3.assert_raft_status("Follower", leader_id=i1.raft_id)
cluster3.expel(i3, i2)
> Retriable(timeout=30).call(lambda: assert_instance_expelled(i3, i1))
cluster3 = Cluster("127.0.0.1", n=3)
i1 = Instance(i1, listen=127.0.0.1:3300 cluster=cluster-0-0, process.pid=28340)
i2 = Instance(i2, listen=127.0.0.1:3302 cluster=cluster-0-0, process.pid=28341)
i3 = Instance(i3, listen=127.0.0.1:3303 cluster=cluster-0-0, process.pid=28342) def assert_instance_expelled(expelled_instance: Instance, instance: Instance):
info = instance.call(".proc_instance_info", expelled_instance.name)
states = (info["current_state"]["variant"], info["target_state"]["variant"])
> assert states == ("Expelled", "Expelled")
E AssertionError: assert ('Online', 'Expelled') == ('Expelled', 'Expelled')
E
E At index 0 diff: 'Online' != 'Expelled'
E
E Full diff:
E (
E - 'Expelled',
E + 'Online',
E 'Expelled',
E )
expelled_instance = Instance(i3, listen=127.0.0.1:3303 cluster=cluster-0-0, process.pid=28342)
info = {'advertise_address': '127.0.0.1:3303', 'cluster_name': 'cluster-0-0', 'current_state': {'incarnation': 1, 'variant': 'Online'}, 'name': 'i3', ...}
instance = Instance(i1, listen=127.0.0.1:3300 cluster=cluster-0-0, process.pid=28340)
states = ('Online', 'Expelled')
test/int/test_expelling.py:16: AssertionErrorEdited by Diana Tikhonova