From edcd30fe184259259d85c722cb4e5e06e2fbf148 Mon Sep 17 00:00:00 2001
From: Alexandr Lyapunov <a.lyapunov@corp.mail.ru>
Date: Tue, 9 Jun 2015 12:03:31 +0300
Subject: [PATCH] *) added cwd() function to fio package *) added stress test
for snapshotting *) moved finalizers.test.py to long_run
---
src/lua/fio.cc | 21 ++
test/{box => long_run}/finalizers.result | 0
test/{box => long_run}/finalizers.test.py | 2 +-
test/{box => long_run}/lua/finalizers.lua | 0
test/wal_off/snapshot_stress.result | 362 ++++++++++++++++++++++
test/wal_off/snapshot_stress.test.lua | 222 +++++++++++++
6 files changed, 606 insertions(+), 1 deletion(-)
rename test/{box => long_run}/finalizers.result (100%)
rename test/{box => long_run}/finalizers.test.py (87%)
rename test/{box => long_run}/lua/finalizers.lua (100%)
create mode 100644 test/wal_off/snapshot_stress.result
create mode 100644 test/wal_off/snapshot_stress.test.lua
diff --git a/src/lua/fio.cc b/src/lua/fio.cc
index 5dfb85b130..736be7548c 100644
--- a/src/lua/fio.cc
+++ b/src/lua/fio.cc
@@ -622,6 +622,26 @@ lbox_fio_tempdir(struct lua_State *L)
return 1;
}
+static int
+lbox_fio_cwd(struct lua_State *L)
+{
+ char *buf = (char *)lua_newuserdata(L, PATH_MAX);
+ if (!buf) {
+ errno = ENOMEM;
+ lua_pushnil(L);
+ return 1;
+ }
+
+
+ if (getcwd(buf, PATH_MAX)) {
+ lua_pushstring(L, buf);
+ lua_remove(L, -2);
+ } else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
static int
lbox_fio_fsync(struct lua_State *L)
{
@@ -674,6 +694,7 @@ tarantool_lua_fio_init(struct lua_State *L)
{ "chmod", lbox_fio_chmod },
{ "truncate", lbox_fio_truncate },
{ "tempdir", lbox_fio_tempdir },
+ { "cwd", lbox_fio_cwd },
{ "sync", lbox_fio_sync },
{ NULL, NULL }
};
diff --git a/test/box/finalizers.result b/test/long_run/finalizers.result
similarity index 100%
rename from test/box/finalizers.result
rename to test/long_run/finalizers.result
diff --git a/test/box/finalizers.test.py b/test/long_run/finalizers.test.py
similarity index 87%
rename from test/box/finalizers.test.py
rename to test/long_run/finalizers.test.py
index ca20abac92..c471892f17 100644
--- a/test/box/finalizers.test.py
+++ b/test/long_run/finalizers.test.py
@@ -5,7 +5,7 @@ import yaml
from lib.tarantool_server import TarantoolServer
server = TarantoolServer(server.ini)
-server.script = 'box/lua/finalizers.lua'
+server.script = 'long_run/lua/finalizers.lua'
server.vardir = os.path.join(server.vardir, 'finalizers')
try:
server.deploy()
diff --git a/test/box/lua/finalizers.lua b/test/long_run/lua/finalizers.lua
similarity index 100%
rename from test/box/lua/finalizers.lua
rename to test/long_run/lua/finalizers.lua
diff --git a/test/wal_off/snapshot_stress.result b/test/wal_off/snapshot_stress.result
new file mode 100644
index 0000000000..0646564ea9
--- /dev/null
+++ b/test/wal_off/snapshot_stress.result
@@ -0,0 +1,362 @@
+-- The test emulates account system. There are increasing number or accounts
+-- and a lot of double entry transactions are made that moving random
+-- ammount from random account to another random accont.
+-- Snapshots are made every snapshot_interval seconds and then checked for consistency
+-- Settings: You may increase theese value to make test longer
+-- number of worker fibers:
+workers_count = 80
+---
+...
+-- number of iterations per fiber (operations + add new account + add space)
+iteration_count = 8
+---
+...
+-- number of operations per iterations
+operation_count = 8
+---
+...
+-- limit of random string length in every account
+string_max_size = 128
+---
+...
+-- initial number of accounts
+accounts_start = 5
+---
+...
+-- delay between snapshots
+snapshot_interval = 0.005
+---
+...
+fiber = require('fiber')
+---
+...
+fio = require('fio')
+---
+...
+tarantool_bin_path = arg[-1]
+---
+...
+work_dir = fio.cwd()
+---
+...
+snap_dir_as_expected = false
+---
+...
+snap_dir_as_expected = snap_dir_as_expected or box.cfg.snap_dir == nil
+---
+...
+snap_dir_as_expected = snap_dir_as_expected or box.cfg.snap_dir == ''
+---
+...
+snap_dir_as_expected = snap_dir_as_expected or box.cfg.snap_dir == '.'
+---
+...
+snap_dir_as_expected = snap_dir_as_expected or box.cfg.snap_dir == work_dir
+---
+...
+snap_dir_as_expected
+---
+- true
+...
+box.cfg.snap_dir
+---
+- .
+...
+script_path = fio.pathjoin(work_dir, 'snap_script.lua')
+---
+...
+cmd_template = [[/bin/sh -c 'cd "%s" && "%s" ./snap_script.lua 2> /dev/null']]
+---
+...
+cmd = string.format(cmd_template, work_dir, tarantool_bin_path)
+---
+...
+open_flags = {'O_CREAT', 'O_WRONLY', 'O_TRUNC'}
+---
+...
+script = fio.open(script_path, open_flags, tonumber('0777', 8))
+---
+...
+script:write("os.exit(-1)")
+---
+- true
+...
+script:close()
+---
+- true
+...
+res = os.execute(cmd)
+---
+...
+str_res = 'precheck ' .. (res ~= 0 and ' ok(1)' or 'failed(1)')
+---
+...
+str_res
+---
+- precheck ok(1)
+...
+script = fio.open(script_path, open_flags, tonumber('0777', 8))
+---
+...
+script:write("os.exit(0)")
+---
+- true
+...
+script:close()
+---
+- true
+...
+res = os.execute(cmd)
+---
+...
+str_res = 'precheck ' .. (res == 0 and ' ok(2)' or 'failed(2)')
+---
+...
+str_res
+---
+- precheck ok(2)
+...
+snap_search_wildcard = fio.pathjoin(box.cfg.snap_dir, '*.snap');
+---
+...
+snaps = fio.glob(snap_search_wildcard);
+---
+...
+initial_snap_count = #snaps
+---
+...
+s1 = box.schema.create_space("accounts")
+---
+...
+i1 = s1:create_index('primary', { type = 'HASH', parts = {1, 'num'} })
+---
+...
+s2 = box.schema.create_space("operations")
+---
+...
+i2 = s2:create_index('primary', { type = 'HASH', parts = {1, 'num'} })
+---
+...
+n_accs = 0
+---
+...
+n_ops = 0
+---
+...
+n_spaces = 0
+---
+...
+workers_done = 0
+---
+...
+--# setopt delimiter ';'
+garbage = {};
+---
+...
+str = ""
+for i = 1,string_max_size do
+ str = str .. '-' garbage[i - 1] = str
+end;
+---
+...
+function get_new_space_name()
+ n_spaces = n_spaces + 1
+ return "test" .. tostring(n_spaces - 1)
+end;
+---
+...
+function get_rnd_acc()
+ return math.floor(math.random() * n_accs)
+end;
+---
+...
+function get_rnd_val()
+ return math.floor(math.random() * 10)
+end;
+---
+...
+function get_rnd_str()
+ return garbage[math.floor(math.random() * string_max_size)]
+end;
+---
+...
+function add_space()
+ box.schema.create_space(get_new_space_name()):create_index('test')
+ n_spaces = n_spaces + 1
+end;
+---
+...
+function add_acc()
+ s1:insert{n_accs, 0} n_accs = n_accs + 1
+end;
+---
+...
+function add_op(n1, n2, v)
+ s2:insert{n_ops, n1, n2, v}
+ n_ops = n_ops + 1
+end;
+---
+...
+function acc_add(n, v)
+ s1:update({n}, {{'+', 2, v}, {'=', 3, get_rnd_str()}})
+end;
+---
+...
+function do_op(n1, n2, v)
+ box.begin()
+ add_op(n1, n2, v)
+ acc_add(n1, v)
+ acc_add(n2, -v)
+ box.commit()
+end;
+---
+...
+function do_rand_op()
+ do_op(get_rnd_acc(), get_rnd_acc(), get_rnd_val())
+end;
+---
+...
+function init()
+ for i = 1,accounts_start do
+ add_acc()
+ end
+end;
+---
+...
+function work_itr()
+ for j = 1,operation_count do
+ do_rand_op()
+ fiber.sleep(0)
+ end
+ add_acc()
+ add_space()
+end;
+---
+...
+function work()
+ for i = 1,iteration_count do
+ work_itr()
+ end
+ workers_done = workers_done + 1
+end;
+---
+...
+function snaps()
+ while (workers_done ~= workers_count) do
+ pcall(box.snapshot)
+ fiber.sleep(snapshot_interval)
+ end snaps_done = true
+end;
+---
+...
+function wait()
+ while (not snaps_done) do
+ fiber.sleep(1)
+ end
+end;
+---
+...
+init();
+---
+...
+print('creating snapshot start');
+---
+...
+for i = 1,workers_count do
+ fiber.create(work)
+end;
+---
+...
+local tmp_fib = fiber.create(snaps);
+---
+...
+wait();
+---
+...
+print('creating snapshot done');
+---
+...
+#s1:select{};
+---
+- 645
+...
+#s2:select{};
+---
+- 5120
+...
+s1:drop();
+---
+...
+s2:drop();
+---
+...
+script_code = [[
+box.cfg{ slab_alloc_arena = 0.5, snap_dir = ".", wal_mode = "none" }
+
+s1 = box.space.accounts
+s2 = box.space.operations
+
+total_sum = 0
+t1 = {}
+for k,v in s1:pairs() do t1[ v[1] ] = v[2] total_sum = total_sum + v[2] end
+if total_sum ~= 0 then print('error: total sum mismatch') os.exit(-1) end
+
+t2 = {}
+function acc_inc(n1, v) t2[n1] = (t2[n1] and t2[n1] or 0) + v end
+for k,v in s2:pairs() do acc_inc(v[2], v[4]) acc_inc(v[3], -v[4]) end
+
+bad = false
+for k,v in pairs(t1) do if (t2[k] and t2[k] or 0) ~= v then bad = true end end
+for k,v in pairs(t2) do if (t1[k] and t1[k] or 0) ~= v then bad = true end end
+if bad then print('error: operation apply mismatch') os.exit(-1) end
+
+print('success: snapshot is ok')
+os.exit(0)
+]];
+---
+...
+script = fio.open(script_path, open_flags, tonumber('0777', 8))
+script:write(script_code)
+script:close()
+
+print('checking snapshot start');
+---
+...
+snaps = fio.glob(snap_search_wildcard);
+---
+...
+snaps_find_status = #snaps <= initial_snap_count and 'where are my snapshots?' or 'snaps found';
+---
+...
+snaps_find_status;
+---
+- snaps found
+...
+snapshot_check_status = "snap check ok";
+---
+...
+while #snaps > initial_snap_count do
+ if os.execute(cmd) ~= 0 then
+ snapshot_check_status = "snap check failed!"
+ break
+ end
+ max_snap = nil
+ for k,v in pairs(snaps) do
+ if max_snap == nil or v > max_snap then
+ max_snap = v
+ max_snap_k = k
+ end
+ end
+ fio.unlink(fio.pathjoin(box.cfg.snap_dir, max_snap))
+ snaps[max_snap_k] = nil
+end;
+---
+...
+snapshot_check_status;
+---
+- snap check ok
+...
+print('checking snapshot done');
+---
+...
+--# setopt delimiter ''
diff --git a/test/wal_off/snapshot_stress.test.lua b/test/wal_off/snapshot_stress.test.lua
new file mode 100644
index 0000000000..afb9eea80a
--- /dev/null
+++ b/test/wal_off/snapshot_stress.test.lua
@@ -0,0 +1,222 @@
+-- The test emulates account system. There are increasing number or accounts
+-- and a lot of double entry transactions are made that moving random
+-- ammount from random account to another random accont.
+-- Snapshots are made every snapshot_interval seconds and then checked for consistency
+-- Settings: You may increase theese value to make test longer
+-- number of worker fibers:
+workers_count = 80
+-- number of iterations per fiber (operations + add new account + add space)
+iteration_count = 8
+-- number of operations per iterations
+operation_count = 8
+-- limit of random string length in every account
+string_max_size = 128
+-- initial number of accounts
+accounts_start = 5
+-- delay between snapshots
+snapshot_interval = 0.005
+
+fiber = require('fiber')
+fio = require('fio')
+
+tarantool_bin_path = arg[-1]
+work_dir = fio.cwd()
+snap_dir_as_expected = false
+snap_dir_as_expected = snap_dir_as_expected or box.cfg.snap_dir == nil
+snap_dir_as_expected = snap_dir_as_expected or box.cfg.snap_dir == ''
+snap_dir_as_expected = snap_dir_as_expected or box.cfg.snap_dir == '.'
+snap_dir_as_expected = snap_dir_as_expected or box.cfg.snap_dir == work_dir
+snap_dir_as_expected
+box.cfg.snap_dir
+script_path = fio.pathjoin(work_dir, 'snap_script.lua')
+cmd_template = [[/bin/sh -c 'cd "%s" && "%s" ./snap_script.lua 2> /dev/null']]
+cmd = string.format(cmd_template, work_dir, tarantool_bin_path)
+
+open_flags = {'O_CREAT', 'O_WRONLY', 'O_TRUNC'}
+script = fio.open(script_path, open_flags, tonumber('0777', 8))
+script:write("os.exit(-1)")
+script:close()
+res = os.execute(cmd)
+str_res = 'precheck ' .. (res ~= 0 and ' ok(1)' or 'failed(1)')
+str_res
+
+script = fio.open(script_path, open_flags, tonumber('0777', 8))
+script:write("os.exit(0)")
+script:close()
+res = os.execute(cmd)
+str_res = 'precheck ' .. (res == 0 and ' ok(2)' or 'failed(2)')
+str_res
+
+snap_search_wildcard = fio.pathjoin(box.cfg.snap_dir, '*.snap');
+snaps = fio.glob(snap_search_wildcard);
+initial_snap_count = #snaps
+
+s1 = box.schema.create_space("accounts")
+i1 = s1:create_index('primary', { type = 'HASH', parts = {1, 'num'} })
+s2 = box.schema.create_space("operations")
+i2 = s2:create_index('primary', { type = 'HASH', parts = {1, 'num'} })
+
+n_accs = 0
+n_ops = 0
+n_spaces = 0
+workers_done = 0
+
+--# setopt delimiter ';'
+garbage = {};
+str = ""
+for i = 1,string_max_size do
+ str = str .. '-' garbage[i - 1] = str
+end;
+
+function get_new_space_name()
+ n_spaces = n_spaces + 1
+ return "test" .. tostring(n_spaces - 1)
+end;
+
+function get_rnd_acc()
+ return math.floor(math.random() * n_accs)
+end;
+
+function get_rnd_val()
+ return math.floor(math.random() * 10)
+end;
+
+function get_rnd_str()
+ return garbage[math.floor(math.random() * string_max_size)]
+end;
+
+function add_space()
+ box.schema.create_space(get_new_space_name()):create_index('test')
+ n_spaces = n_spaces + 1
+end;
+
+function add_acc()
+ s1:insert{n_accs, 0} n_accs = n_accs + 1
+end;
+
+function add_op(n1, n2, v)
+ s2:insert{n_ops, n1, n2, v}
+ n_ops = n_ops + 1
+end;
+
+function acc_add(n, v)
+ s1:update({n}, {{'+', 2, v}, {'=', 3, get_rnd_str()}})
+end;
+
+function do_op(n1, n2, v)
+ box.begin()
+ add_op(n1, n2, v)
+ acc_add(n1, v)
+ acc_add(n2, -v)
+ box.commit()
+end;
+
+function do_rand_op()
+ do_op(get_rnd_acc(), get_rnd_acc(), get_rnd_val())
+end;
+
+function init()
+ for i = 1,accounts_start do
+ add_acc()
+ end
+end;
+
+function work_itr()
+ for j = 1,operation_count do
+ do_rand_op()
+ fiber.sleep(0)
+ end
+ add_acc()
+ add_space()
+end;
+
+function work()
+ for i = 1,iteration_count do
+ work_itr()
+ end
+ workers_done = workers_done + 1
+end;
+
+function snaps()
+ while (workers_done ~= workers_count) do
+ pcall(box.snapshot)
+ fiber.sleep(snapshot_interval)
+ end snaps_done = true
+end;
+
+function wait()
+ while (not snaps_done) do
+ fiber.sleep(1)
+ end
+end;
+
+init();
+
+print('creating snapshot start');
+for i = 1,workers_count do
+ fiber.create(work)
+end;
+local tmp_fib = fiber.create(snaps);
+wait();
+print('creating snapshot done');
+
+#s1:select{};
+#s2:select{};
+
+s1:drop();
+s2:drop();
+
+script_code = [[
+box.cfg{ slab_alloc_arena = 0.5, snap_dir = ".", wal_mode = "none" }
+
+s1 = box.space.accounts
+s2 = box.space.operations
+
+total_sum = 0
+t1 = {}
+for k,v in s1:pairs() do t1[ v[1] ] = v[2] total_sum = total_sum + v[2] end
+if total_sum ~= 0 then print('error: total sum mismatch') os.exit(-1) end
+
+t2 = {}
+function acc_inc(n1, v) t2[n1] = (t2[n1] and t2[n1] or 0) + v end
+for k,v in s2:pairs() do acc_inc(v[2], v[4]) acc_inc(v[3], -v[4]) end
+
+bad = false
+for k,v in pairs(t1) do if (t2[k] and t2[k] or 0) ~= v then bad = true end end
+for k,v in pairs(t2) do if (t1[k] and t1[k] or 0) ~= v then bad = true end end
+if bad then print('error: operation apply mismatch') os.exit(-1) end
+
+print('success: snapshot is ok')
+os.exit(0)
+]];
+script = fio.open(script_path, open_flags, tonumber('0777', 8))
+script:write(script_code)
+script:close()
+
+print('checking snapshot start');
+
+snaps = fio.glob(snap_search_wildcard);
+snaps_find_status = #snaps <= initial_snap_count and 'where are my snapshots?' or 'snaps found';
+snaps_find_status;
+snapshot_check_status = "snap check ok";
+while #snaps > initial_snap_count do
+ if os.execute(cmd) ~= 0 then
+ snapshot_check_status = "snap check failed!"
+ break
+ end
+ max_snap = nil
+ for k,v in pairs(snaps) do
+ if max_snap == nil or v > max_snap then
+ max_snap = v
+ max_snap_k = k
+ end
+ end
+ fio.unlink(fio.pathjoin(box.cfg.snap_dir, max_snap))
+ snaps[max_snap_k] = nil
+end;
+snapshot_check_status;
+print('checking snapshot done');
+
+--# setopt delimiter ''
+
+
--
GitLab