From f4c17f516c3284a287571ba419a87659f92d8163 Mon Sep 17 00:00:00 2001
From: Georgy Moshkin <gmoshkin@picodata.io>
Date: Mon, 19 Aug 2024 17:06:04 +0300
Subject: [PATCH] test: do not move PluginReflection to conftest.py
---
test/conftest.py | 246 ---------------------------------
test/int/test_http_server.py | 41 +++---
test/int/test_plugin.py | 258 +++++++++++++++++++++++++++++++++--
3 files changed, 269 insertions(+), 276 deletions(-)
diff --git a/test/conftest.py b/test/conftest.py
index 3af04bb1bd..780308d8df 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -2253,252 +2253,6 @@ instance:
return self.cluster.instances[0]
-_PLUGIN = "testplug"
-_PLUGIN_SERVICES = ["testservice_1", "testservice_2"]
-_PLUGIN_SMALL = "testplug_small"
-_PLUGIN_SMALL_SERVICES = ["testservice_1"]
-_PLUGIN_SMALL_SERVICES_SVC2 = ["testservice_2"]
-_PLUGIN_VERSION_1 = "0.1.0"
-_PLUGIN_VERSION_2 = "0.2.0"
-_DEFAULT_TIER = "default"
-
-
-@dataclass
-class PluginReflection:
- """PluginReflection used to describe the expected state of the plugin"""
-
- # plugin name
- name: str
- # plugin version
- version: str
- # list of plugin services
- services: List[str]
- # instances in cluster
- instances: List[Instance]
- # plugin topology
- topology: Dict[Instance, List[str]] = field(default_factory=dict)
- # if True - assert_synced checks that plugin are installed
- installed: bool = False
- # if True - assert_synced checks that plugin are enabled
- enabled: bool = False
- # plugin data [table -> tuples] map
- data: Dict[str, Optional[List[Any]]] = field(default_factory=dict)
-
- def __post__init__(self):
- for i in self.instances:
- self.topology[i] = []
-
- @staticmethod
- def default(*instances):
- """Create reflection for default plugin with default topology"""
- topology = {}
- for i in instances:
- topology[i] = _PLUGIN_SERVICES
- return PluginReflection(
- name=_PLUGIN,
- version="0.1.0",
- services=_PLUGIN_SERVICES,
- instances=list(instances),
- ).set_topology(topology)
-
- def install(self, installed: bool):
- self.installed = installed
- return self
-
- def enable(self, enabled: bool):
- self.enabled = enabled
- return self
-
- def set_topology(self, topology: dict[Instance, list[str]]):
- self.topology = topology
- return self
-
- def add_instance(self, i):
- self.instances.append(i)
- return self
-
- def set_data(self, data: dict[str, Optional[list[Any]]]):
- self.data = data
- return self
-
- def assert_synced(self):
- """Assert that plugin reflection and plugin state in cluster are synchronized.
- This means that system tables `_pico_plugin`, `_pico_service` and `_pico_service_route`
- contain necessary plugin information."""
- for i in self.instances:
- plugins = i.eval(
- "return box.space._pico_plugin:select({...})", self.name, self.version
- )
- if self.installed:
- assert len(plugins) == 1
- assert plugins[0][1] == self.enabled
- else:
- assert len(plugins) == 0
-
- for service in self.services:
- svcs = i.eval(
- "return box.space._pico_service:select({...})",
- [self.name, service, self.version],
- )
- if self.installed:
- assert len(svcs) == 1
- else:
- assert len(svcs) == 0
-
- for i in self.topology:
- expected_services = []
- for service in self.topology[i]:
- expected_services.append(
- [i.instance_id, self.name, self.version, service, False]
- )
-
- for neighboring_i in self.topology:
- routes = neighboring_i.eval(
- 'return box.space._pico_service_route:pairs({...}, {iterator="EQ"}):totable()',
- i.instance_id,
- self.name,
- self.version,
- )
- assert routes == expected_services
-
- def assert_data_synced(self):
- for table in self.data:
- data = []
-
- for i in self.instances:
- if self.data[table] is None:
- with pytest.raises(TarantoolError, match="attempt to index field"):
- i.eval(f"return box.space.{table}:select()")
- else:
- data += i.eval(f"return box.space.{table}:select()")
-
- if self.data[table] is not None:
- assert data.sort() == self.data[table].sort()
-
- @staticmethod
- def assert_cb_called(service, callback, called_times, *instances):
- for i in instances:
- cb_calls_number = i.eval(
- f"if _G['plugin_state'] == nil then _G['plugin_state'] = {{}} end "
- f"if _G['plugin_state']['{service}'] == nil then _G['plugin_state']['{service}']"
- f" = {{}} end "
- f"if _G['plugin_state']['{service}']['{callback}'] == nil then _G['plugin_state']"
- f"['{service}']['{callback}'] = 0 end "
- f"return _G['plugin_state']['{service}']['{callback}']"
- )
- assert cb_calls_number == called_times
-
- @staticmethod
- def assert_persisted_data_exists(data, *instances):
- for i in instances:
- data_exists = i.eval(
- f"return box.space.persisted_data:get({{'{data}'}}) ~= box.NULL"
- )
- assert data_exists
-
- @staticmethod
- def clear_persisted_data(data, *instances):
- for i in instances:
- i.eval("return box.space.persisted_data:drop()")
-
- @staticmethod
- def inject_error(service, error, value, instance):
- instance.eval("if _G['err_inj'] == nil then _G['err_inj'] = {} end")
- instance.eval(
- f"if _G['err_inj']['{service}'] == nil then _G['err_inj']['{service}'] "
- "= {{}} end"
- )
- instance.eval(f"_G['err_inj']['{service}']['{error}'] = ...", (value,))
-
- @staticmethod
- def remove_error(service, error, instance):
- instance.eval("if _G['err_inj'] == nil then _G['err_inj'] = {} end")
- instance.eval(
- f"if _G['err_inj']['{service}'] == nil then _G['err_inj']['{service}'] "
- "= {{}} end"
- )
- instance.eval(f"_G['err_inj']['{service}']['{error}'] = nil")
-
- @staticmethod
- def assert_last_seen_ctx(service, expected_ctx, *instances):
- for i in instances:
- ctx = i.eval(f"return _G['plugin_state']['{service}']['last_seen_ctx']")
- assert ctx == expected_ctx
-
- def get_config(self, service, instance):
- config = dict()
- records = instance.eval(
- "return box.space._pico_plugin_config:select({...})",
- [self.name, self.version, service],
- )
- for record in records:
- config[record[3]] = record[4]
- return config
-
- @staticmethod
- def get_seen_config(service, instance):
- return instance.eval(
- f"return _G['plugin_state']['{service}']['current_config']"
- )
-
- def assert_config(self, service, expected_cfg, *instances):
- for i in instances:
- cfg_space = self.get_config(service, i)
- assert cfg_space == expected_cfg
- cfg_seen = self.get_seen_config(service, i)
- assert cfg_seen == expected_cfg
-
- def assert_route_poisoned(self, poison_instance_id, service, poisoned=True):
- for i in self.instances:
- route_poisoned = i.eval(
- "return box.space._pico_service_route:get({...}).poison",
- poison_instance_id,
- self.name,
- self.version,
- service,
- )
- assert route_poisoned == poisoned
-
- @staticmethod
- def assert_data_eq(instance, key, expected):
- val = instance.eval(f"return _G['plugin_state']['data']['{key}']")
- assert val == expected
-
- @staticmethod
- def assert_int_data_le(instance, key, expected):
- val = instance.eval(f"return _G['plugin_state']['data']['{key}']")
- assert int(val) <= expected
-
-
-def install_and_enable_plugin(
- instance,
- plugin,
- services,
- version="0.1.0",
- migrate=False,
- timeout=3,
- default_config=None,
- if_not_exist=False,
-):
- instance.call(
- "pico.install_plugin",
- plugin,
- version,
- {"migrate": migrate, "if_not_exist": if_not_exist},
- timeout=timeout,
- )
- for s in services:
- if default_config is not None:
- for key in default_config:
- instance.eval(
- f"box.space._pico_plugin_config:replace"
- f"({{'{plugin}', '0.1.0', '{s}', '{key}', ...}})",
- default_config[key],
- )
- instance.call("pico.service_append_tier", plugin, version, s, _DEFAULT_TIER)
- instance.call("pico.enable_plugin", plugin, version, timeout=timeout)
-
-
@pytest.fixture
def postgres(cluster: Cluster):
return Postgres(cluster).install()
diff --git a/test/int/test_http_server.py b/test/int/test_http_server.py
index 856b0e7463..38c7505335 100644
--- a/test/int/test_http_server.py
+++ b/test/int/test_http_server.py
@@ -1,11 +1,6 @@
from conftest import (
Cluster,
Instance,
- _PLUGIN,
- _PLUGIN_SERVICES,
- _PLUGIN_SMALL,
- _PLUGIN_SMALL_SERVICES,
- _PLUGIN_VERSION_1,
)
from urllib.request import urlopen
import pytest
@@ -106,31 +101,37 @@ def test_webui_with_plugin(cluster: Cluster):
"""
cluster.set_config_file(yaml=cluster_cfg)
+ plugin_1 = "testplug"
+ plugin_1_services = ["testservice_1", "testservice_2"]
+ plugin_2 = "testplug_small"
+ plugin_2_service = "testservice_1"
+ version_1 = "0.1.0"
+
i1 = cluster.add_instance(wait_online=True, tier="red", enable_http=True)
i2 = cluster.add_instance(wait_online=True, tier="blue")
i3 = cluster.add_instance(wait_online=True, tier="green")
- i1.call("pico.install_plugin", _PLUGIN, _PLUGIN_VERSION_1)
- i1.call("pico.install_plugin", _PLUGIN_SMALL, _PLUGIN_VERSION_1)
+ i1.call("pico.install_plugin", plugin_1, version_1)
+ i1.call("pico.install_plugin", plugin_2, version_1)
i1.call(
"pico.service_append_tier",
- _PLUGIN,
- _PLUGIN_VERSION_1,
- _PLUGIN_SERVICES[0],
+ plugin_1,
+ version_1,
+ plugin_1_services[0],
"red",
)
i1.call(
"pico.service_append_tier",
- _PLUGIN,
- _PLUGIN_VERSION_1,
- _PLUGIN_SERVICES[1],
+ plugin_1,
+ version_1,
+ plugin_1_services[1],
"blue",
)
i1.call(
"pico.service_append_tier",
- _PLUGIN_SMALL,
- _PLUGIN_VERSION_1,
- _PLUGIN_SMALL_SERVICES[0],
+ plugin_2,
+ version_1,
+ plugin_2_service,
"blue",
)
@@ -207,13 +208,13 @@ def test_webui_with_plugin(cluster: Cluster):
tier_red = {
**tier_template,
"name": "red",
- "services": [_PLUGIN_SERVICES[0]],
+ "services": [plugin_1_services[0]],
"replicasets": [r1],
}
tier_blue = {
**tier_template,
"name": "blue",
- "services": [_PLUGIN_SERVICES[1], _PLUGIN_SMALL_SERVICES[0]],
+ "services": [plugin_1_services[1], plugin_2_service],
"replicasets": [r2],
}
tier_green = {**tier_template, "name": "green", "services": [], "replicasets": [r3]}
@@ -236,8 +237,8 @@ def test_webui_with_plugin(cluster: Cluster):
"currentInstaceVersion": instance_version,
"memory": {"usable": 201326592, "used": 100663296},
"plugins": [
- _PLUGIN + " " + _PLUGIN_VERSION_1,
- _PLUGIN_SMALL + " " + _PLUGIN_VERSION_1,
+ plugin_1 + " " + version_1,
+ plugin_2 + " " + version_1,
],
}
diff --git a/test/int/test_plugin.py b/test/int/test_plugin.py
index 92eb7b6aa9..58eb0b8a09 100644
--- a/test/int/test_plugin.py
+++ b/test/int/test_plugin.py
@@ -1,5 +1,6 @@
+from dataclasses import dataclass, field
import time
-from typing import Any
+from typing import Any, Dict, List, Optional
import pytest
import uuid
import msgpack # type: ignore
@@ -12,15 +13,6 @@ from conftest import (
Instance,
TarantoolError,
log_crawler,
- PluginReflection,
- _PLUGIN,
- _PLUGIN_SERVICES,
- _PLUGIN_SMALL,
- _PLUGIN_SMALL_SERVICES,
- _PLUGIN_VERSION_1,
- _PLUGIN_VERSION_2,
- _DEFAULT_TIER,
- install_and_enable_plugin,
)
from decimal import Decimal
import requests # type: ignore
@@ -33,6 +25,14 @@ _DEFAULT_CFG = {"foo": True, "bar": 101, "baz": ["one", "two", "three"]}
_NEW_CFG = {"foo": True, "bar": 102, "baz": ["a", "b"]}
_NEW_CFG_2 = {"foo": False, "bar": 102, "baz": ["a", "b"]}
+_PLUGIN = "testplug"
+_PLUGIN_SERVICES = ["testservice_1", "testservice_2"]
+_PLUGIN_SMALL = "testplug_small"
+_PLUGIN_SMALL_SERVICES = ["testservice_1"]
+_PLUGIN_SMALL_SERVICES_SVC2 = ["testservice_2"]
+_PLUGIN_VERSION_1 = "0.1.0"
+_PLUGIN_VERSION_2 = "0.2.0"
+_DEFAULT_TIER = "default"
_PLUGIN_WITH_MIGRATION = "testplug_w_migration"
_PLUGIN_WITH_MIGRATION_SERVICES = ["testservice_2"]
_PLUGIN_W_SDK = "testplug_sdk"
@@ -44,6 +44,244 @@ PLUGIN_NAME = 2
SERVICE_NAME = 3
PLUGIN_VERSION = 4
+# ---------------------------------- Test helper classes {-----------------------------------------
+
+
+@dataclass
+class PluginReflection:
+ """PluginReflection used to describe the expected state of the plugin"""
+
+ # plugin name
+ name: str
+ # plugin version
+ version: str
+ # list of plugin services
+ services: List[str]
+ # instances in cluster
+ instances: List[Instance]
+ # plugin topology
+ topology: Dict[Instance, List[str]] = field(default_factory=dict)
+ # if True - assert_synced checks that plugin are installed
+ installed: bool = False
+ # if True - assert_synced checks that plugin are enabled
+ enabled: bool = False
+ # plugin data [table -> tuples] map
+ data: Dict[str, Optional[List[Any]]] = field(default_factory=dict)
+
+ def __post__init__(self):
+ for i in self.instances:
+ self.topology[i] = []
+
+ @staticmethod
+ def default(*instances):
+ """Create reflection for default plugin with default topology"""
+ topology = {}
+ for i in instances:
+ topology[i] = _PLUGIN_SERVICES
+ return PluginReflection(
+ name=_PLUGIN,
+ version="0.1.0",
+ services=_PLUGIN_SERVICES,
+ instances=list(instances),
+ ).set_topology(topology)
+
+ def install(self, installed: bool):
+ self.installed = installed
+ return self
+
+ def enable(self, enabled: bool):
+ self.enabled = enabled
+ return self
+
+ def set_topology(self, topology: dict[Instance, list[str]]):
+ self.topology = topology
+ return self
+
+ def add_instance(self, i):
+ self.instances.append(i)
+ return self
+
+ def set_data(self, data: dict[str, Optional[list[Any]]]):
+ self.data = data
+ return self
+
+ def assert_synced(self):
+ """Assert that plugin reflection and plugin state in cluster are synchronized.
+ This means that system tables `_pico_plugin`, `_pico_service` and `_pico_service_route`
+ contain necessary plugin information."""
+ for i in self.instances:
+ plugins = i.eval(
+ "return box.space._pico_plugin:select({...})", self.name, self.version
+ )
+ if self.installed:
+ assert len(plugins) == 1
+ assert plugins[0][1] == self.enabled
+ else:
+ assert len(plugins) == 0
+
+ for service in self.services:
+ svcs = i.eval(
+ "return box.space._pico_service:select({...})",
+ [self.name, service, self.version],
+ )
+ if self.installed:
+ assert len(svcs) == 1
+ else:
+ assert len(svcs) == 0
+
+ for i in self.topology:
+ expected_services = []
+ for service in self.topology[i]:
+ expected_services.append(
+ [i.instance_id, self.name, self.version, service, False]
+ )
+
+ for neighboring_i in self.topology:
+ routes = neighboring_i.eval(
+ 'return box.space._pico_service_route:pairs({...}, {iterator="EQ"}):totable()',
+ i.instance_id,
+ self.name,
+ self.version,
+ )
+ assert routes == expected_services
+
+ def assert_data_synced(self):
+ for table in self.data:
+ data = []
+
+ for i in self.instances:
+ if self.data[table] is None:
+ with pytest.raises(TarantoolError, match="attempt to index field"):
+ i.eval(f"return box.space.{table}:select()")
+ else:
+ data += i.eval(f"return box.space.{table}:select()")
+
+ if self.data[table] is not None:
+ assert data.sort() == self.data[table].sort()
+
+ @staticmethod
+ def assert_cb_called(service, callback, called_times, *instances):
+ for i in instances:
+ cb_calls_number = i.eval(
+ f"if _G['plugin_state'] == nil then _G['plugin_state'] = {{}} end "
+ f"if _G['plugin_state']['{service}'] == nil then _G['plugin_state']['{service}']"
+ f" = {{}} end "
+ f"if _G['plugin_state']['{service}']['{callback}'] == nil then _G['plugin_state']"
+ f"['{service}']['{callback}'] = 0 end "
+ f"return _G['plugin_state']['{service}']['{callback}']"
+ )
+ assert cb_calls_number == called_times
+
+ @staticmethod
+ def assert_persisted_data_exists(data, *instances):
+ for i in instances:
+ data_exists = i.eval(
+ f"return box.space.persisted_data:get({{'{data}'}}) ~= box.NULL"
+ )
+ assert data_exists
+
+ @staticmethod
+ def clear_persisted_data(data, *instances):
+ for i in instances:
+ i.eval("return box.space.persisted_data:drop()")
+
+ @staticmethod
+ def inject_error(service, error, value, instance):
+ instance.eval("if _G['err_inj'] == nil then _G['err_inj'] = {} end")
+ instance.eval(
+ f"if _G['err_inj']['{service}'] == nil then _G['err_inj']['{service}'] "
+ "= {{}} end"
+ )
+ instance.eval(f"_G['err_inj']['{service}']['{error}'] = ...", (value,))
+
+ @staticmethod
+ def remove_error(service, error, instance):
+ instance.eval("if _G['err_inj'] == nil then _G['err_inj'] = {} end")
+ instance.eval(
+ f"if _G['err_inj']['{service}'] == nil then _G['err_inj']['{service}'] "
+ "= {{}} end"
+ )
+ instance.eval(f"_G['err_inj']['{service}']['{error}'] = nil")
+
+ @staticmethod
+ def assert_last_seen_ctx(service, expected_ctx, *instances):
+ for i in instances:
+ ctx = i.eval(f"return _G['plugin_state']['{service}']['last_seen_ctx']")
+ assert ctx == expected_ctx
+
+ def get_config(self, service, instance):
+ config = dict()
+ records = instance.eval(
+ "return box.space._pico_plugin_config:select({...})",
+ [self.name, self.version, service],
+ )
+ for record in records:
+ config[record[3]] = record[4]
+ return config
+
+ @staticmethod
+ def get_seen_config(service, instance):
+ return instance.eval(
+ f"return _G['plugin_state']['{service}']['current_config']"
+ )
+
+ def assert_config(self, service, expected_cfg, *instances):
+ for i in instances:
+ cfg_space = self.get_config(service, i)
+ assert cfg_space == expected_cfg
+ cfg_seen = self.get_seen_config(service, i)
+ assert cfg_seen == expected_cfg
+
+ def assert_route_poisoned(self, poison_instance_id, service, poisoned=True):
+ for i in self.instances:
+ route_poisoned = i.eval(
+ "return box.space._pico_service_route:get({...}).poison",
+ poison_instance_id,
+ self.name,
+ self.version,
+ service,
+ )
+ assert route_poisoned == poisoned
+
+ @staticmethod
+ def assert_data_eq(instance, key, expected):
+ val = instance.eval(f"return _G['plugin_state']['data']['{key}']")
+ assert val == expected
+
+ @staticmethod
+ def assert_int_data_le(instance, key, expected):
+ val = instance.eval(f"return _G['plugin_state']['data']['{key}']")
+ assert int(val) <= expected
+
+
+def install_and_enable_plugin(
+ instance,
+ plugin,
+ services,
+ version="0.1.0",
+ migrate=False,
+ timeout=3,
+ default_config=None,
+ if_not_exist=False,
+):
+ instance.call(
+ "pico.install_plugin",
+ plugin,
+ version,
+ {"migrate": migrate, "if_not_exist": if_not_exist},
+ timeout=timeout,
+ )
+ for s in services:
+ if default_config is not None:
+ for key in default_config:
+ instance.eval(
+ f"box.space._pico_plugin_config:replace"
+ f"({{'{plugin}', '0.1.0', '{s}', '{key}', ...}})",
+ default_config[key],
+ )
+ instance.call("pico.service_append_tier", plugin, version, s, _DEFAULT_TIER)
+ instance.call("pico.enable_plugin", plugin, version, timeout=timeout)
+
def test_invalid_manifest_plugin(cluster: Cluster):
i1, i2 = cluster.deploy(instance_count=2)
--
GitLab