From fae597ad9d2f06bd835ce0434e8ec9bfc65265e3 Mon Sep 17 00:00:00 2001
From: Valentin Syrovatskiy <v.syrovatskiy@picodata.io>
Date: Tue, 26 Jul 2022 11:33:58 +0300
Subject: [PATCH] test: randomized testing
---
test/conftest.py | 11 ++++++
test/rand/test_randomized.py | 65 ++++++++++++++++++++++++------------
2 files changed, 54 insertions(+), 22 deletions(-)
diff --git a/test/conftest.py b/test/conftest.py
index ac1dda25ef..90945ac8a2 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -35,6 +35,12 @@ def pytest_addoption(parser):
parser.addoption(
"--seed", action="store", default=None, help="Seed for randomized tests"
)
+ parser.addoption(
+ "--delay",
+ action="store",
+ default=None,
+ help="Delay between steps for fandomized tests",
+ )
@pytest.fixture(scope="session")
@@ -42,6 +48,11 @@ def seed(pytestconfig):
return pytestconfig.getoption("seed")
+@pytest.fixture(scope="session")
+def delay(pytestconfig):
+ return pytestconfig.getoption("delay")
+
+
@pytest.fixture(scope="session")
def xdist_worker_number(worker_id: str) -> int:
"""
diff --git a/test/rand/test_randomized.py b/test/rand/test_randomized.py
index 546415ed59..f6a9ad0f5d 100644
--- a/test/rand/test_randomized.py
+++ b/test/rand/test_randomized.py
@@ -1,12 +1,14 @@
-import sys
+import math
import time
import random
import os
-import signal
-from conftest import Cluster, Instance, ReturnError
+from conftest import Cluster, Instance
import pathlib
+SEED_CAP = 14 # 19 max
+
+
def int_to_base_62(num: int):
base_62 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
BASE = len(base_62)
@@ -24,10 +26,11 @@ def create(c: Cluster, istate):
def stop(i: Instance, istate):
- assert i.process, f"Instance {i.instance_id} expected has process"
- assert i.process.pid, f"Instance {i.instance_id} expected process to be alive"
+ # assert i.process, f"Instance {i.instance_id} expected has process"
+ # assert i.process.pid, f"Instance {i.instance_id} expected process to be alive"
istate[i.instance_id]["started"] = False
- return os.kill(i.process.pid, signal.SIGTERM), istate
+ return i.terminate(), istate
+ # return os.kill(i.process.pid, signal.SIGTERM), istate
def start(i: Instance, istate):
@@ -64,14 +67,22 @@ SEEDLOG = "seeds.txt"
def possible_actions(c: Cluster, istate):
actions = []
actions.append([ADD, None])
+ stopping_allowed = stop_allowed(istate)
for i in c.instances:
- if istate[i.instance_id]["started"]:
- actions.append([STOP, i])
- else:
- actions.append([START, i])
+ if istate[i.instance_id]["started"] and i.process is not None:
+ if stopping_allowed:
+ actions.append([STOP, i]) # type: ignore
+ elif not istate[i.instance_id]["started"] and i.process is None:
+ actions.append([START, i]) # type: ignore
return actions
+def stop_allowed(istate):
+ return len(list(filter(lambda i: i[1]["started"], istate.items()))) > math.trunc(
+ len(istate) / 2 + 1
+ )
+
+
def choose_action(c: Cluster, istate):
actions = possible_actions(c, istate)
a = actions[random.randint(0, len(actions) - 1)]
@@ -83,32 +94,42 @@ def step_msg(step: int, action, i: Instance):
return f"Step {step}: {msg}"
-def get_seed():
- return int_to_base_62(time.time_ns())
+def generate_seed():
+ return int_to_base_62(time.time_ns() % pow(10, SEED_CAP))
-def log_seed(seed):
+def log_params(seed, delay):
dir = pathlib.Path(__file__).parent.absolute()
with open(os.path.join(dir, SEEDLOG), "a") as f:
- f.write(seed + "\n")
+ f.write(f"{seed} {delay}\n")
+
+
+def initial_istate(cluster: Cluster):
+ istate = {}
+ for i in cluster.instances:
+ istate[i.instance_id] = {"instance": i, "started": True}
+ return istate
-def test_randomized(cluster: Cluster, seed: int, capsys):
+def test_randomized(cluster: Cluster, seed: int, delay: int, capsys):
cluster.deploy(instance_count=3)
- seed = seed if seed else get_seed()
- log_seed(seed)
+ seed = seed if seed else generate_seed()
+
+ # delay should be generated by random even if given as CLI param
+ generated_delay = random.randint(100, 500)
+ delay = int(delay) if delay else generated_delay
+
+ log_params(seed, delay)
with capsys.disabled():
- print(f"Seed: {seed}")
+ print(f"Seed: {seed} , step delay: {delay} ms")
random.seed(seed)
steps_count = random.randint(5, 10)
print(f"Do {steps_count} steps...")
- istate = {}
- for i in cluster.instances:
- istate[i.instance_id] = {"instance": i, "started": True}
+ istate = initial_istate(cluster)
for step in range(steps_count):
a, i = choose_action(cluster, istate)
@@ -116,7 +137,7 @@ def test_randomized(cluster: Cluster, seed: int, capsys):
i, istate = a["pre_fn"](cluster, istate)
print(step_msg(step + 1, a, i))
_, istate = a["exec_fn"](i, istate)
- time.sleep(0.1)
+ time.sleep(delay / 1000)
for instance_id in istate:
ist = istate[instance_id]
--
GitLab