From e88073ac427fff1cffea3d51a8201586742d9f95 Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov.dev@gmail.com>
Date: Wed, 19 Jul 2017 12:00:17 +0300
Subject: [PATCH] test: vinyl/recovery_quota: check recovery with smaller limit
Check that vinyl can recover if vinyl_memory is lowered after restart.
In this case it should forcefully use quota and schedule dump after
recovery is complete.
Needed for #2397
---
test/vinyl/dump_stress.result | 2 +-
test/vinyl/dump_stress.test.lua | 2 +-
test/vinyl/errinj.result | 2 +-
test/vinyl/errinj.test.lua | 2 +-
test/vinyl/low_quota.lua | 4 +-
test/vinyl/low_quota_1.lua | 1 +
test/vinyl/low_quota_2.lua | 1 +
test/vinyl/quota_timeout.result | 2 +-
test/vinyl/quota_timeout.test.lua | 2 +-
test/vinyl/recovery_quota.result | 71 ++++++++++++++++++++++++++++--
test/vinyl/recovery_quota.test.lua | 31 +++++++++++--
test/vinyl/suite.ini | 2 +-
12 files changed, 106 insertions(+), 16 deletions(-)
create mode 120000 test/vinyl/low_quota_1.lua
create mode 120000 test/vinyl/low_quota_2.lua
diff --git a/test/vinyl/dump_stress.result b/test/vinyl/dump_stress.result
index b16e93b4ac..5c1dd9a512 100644
--- a/test/vinyl/dump_stress.result
+++ b/test/vinyl/dump_stress.result
@@ -1,7 +1,7 @@
test_run = require('test_run').new()
---
...
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
---
- true
...
diff --git a/test/vinyl/dump_stress.test.lua b/test/vinyl/dump_stress.test.lua
index 823d4c2bb8..e7f20d2d65 100644
--- a/test/vinyl/dump_stress.test.lua
+++ b/test/vinyl/dump_stress.test.lua
@@ -1,6 +1,6 @@
test_run = require('test_run').new()
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
test_run:cmd("start server test")
test_run:cmd('switch test')
diff --git a/test/vinyl/errinj.result b/test/vinyl/errinj.result
index 1752bc220b..242d583ad6 100644
--- a/test/vinyl/errinj.result
+++ b/test/vinyl/errinj.result
@@ -780,7 +780,7 @@ space:drop()
--
-- Space drop in the middle of dump.
--
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
---
- true
...
diff --git a/test/vinyl/errinj.test.lua b/test/vinyl/errinj.test.lua
index 3a599218f5..457fad6281 100644
--- a/test/vinyl/errinj.test.lua
+++ b/test/vinyl/errinj.test.lua
@@ -314,7 +314,7 @@ space:drop()
--
-- Space drop in the middle of dump.
--
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
test_run:cmd("start server test")
test_run:cmd('switch test')
fiber = require 'fiber'
diff --git a/test/vinyl/low_quota.lua b/test/vinyl/low_quota.lua
index a0870d5f64..20269c8c26 100644
--- a/test/vinyl/low_quota.lua
+++ b/test/vinyl/low_quota.lua
@@ -1,7 +1,9 @@
#!/usr/bin/env tarantool
+local LIMIT = string.match(arg[0], "%d")
+
box.cfg{
- vinyl_memory = 1024 * 1024,
+ vinyl_memory = LIMIT * 1024 * 1024,
}
require('console').listen(os.getenv('ADMIN'))
diff --git a/test/vinyl/low_quota_1.lua b/test/vinyl/low_quota_1.lua
new file mode 120000
index 0000000000..b5b834c7c3
--- /dev/null
+++ b/test/vinyl/low_quota_1.lua
@@ -0,0 +1 @@
+low_quota.lua
\ No newline at end of file
diff --git a/test/vinyl/low_quota_2.lua b/test/vinyl/low_quota_2.lua
new file mode 120000
index 0000000000..b5b834c7c3
--- /dev/null
+++ b/test/vinyl/low_quota_2.lua
@@ -0,0 +1 @@
+low_quota.lua
\ No newline at end of file
diff --git a/test/vinyl/quota_timeout.result b/test/vinyl/quota_timeout.result
index ddd0221564..7a38e706b9 100644
--- a/test/vinyl/quota_timeout.result
+++ b/test/vinyl/quota_timeout.result
@@ -1,7 +1,7 @@
test_run = require('test_run').new()
---
...
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
---
- true
...
diff --git a/test/vinyl/quota_timeout.test.lua b/test/vinyl/quota_timeout.test.lua
index 5db1cd648a..32bcf2cb49 100644
--- a/test/vinyl/quota_timeout.test.lua
+++ b/test/vinyl/quota_timeout.test.lua
@@ -1,6 +1,6 @@
test_run = require('test_run').new()
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
test_run:cmd("start server test")
test_run:cmd('switch test')
diff --git a/test/vinyl/recovery_quota.result b/test/vinyl/recovery_quota.result
index eadb156d78..ca8fd0a6e1 100644
--- a/test/vinyl/recovery_quota.result
+++ b/test/vinyl/recovery_quota.result
@@ -1,12 +1,15 @@
test_run = require('test_run').new()
---
...
+fio = require 'fio'
+---
+...
-- Upon start the test server creates a space and populates it with
-- more tuples than can be stored in memory, which results in dumping
-- some of them to disk. If on restart, during recovery from WAL,
-- it replayed the dumped statements, it would exceed memory quota.
-- Check that it does not.
-test_run:cmd('create server test with script = "vinyl/low_quota.lua"')
+test_run:cmd('create server test with script = "vinyl/low_quota_2.lua"')
---
- true
...
@@ -18,17 +21,20 @@ test_run:cmd('switch test')
---
- true
...
--- Create a vinyl space and trigger dump by exceeding memory quota (1 MB).
+-- Create a vinyl space and trigger dump by exceeding memory quota.
s = box.schema.space.create('test', {engine = 'vinyl'})
---
...
_ = s:create_index('pk', {run_count_per_level = 10})
---
...
-pad = string.rep('x', 1000)
+pad_size = 1000
+---
+...
+pad = string.rep('x', pad_size)
---
...
-for i = 1, 2000 do s:insert{i, pad} end
+for i = 1, 2 * box.cfg.vinyl_memory / pad_size do s:insert{i, pad} end
---
...
-- Save the total number of committed and dumped statements.
@@ -94,6 +100,28 @@ put_before - dump_before == put_after or {dump_before, dump_after, put_before, p
---
- true
...
+-- Disable dump and use all memory up to the limit.
+box.error.injection.set('ERRINJ_VY_RUN_WRITE', false)
+---
+- ok
+...
+box.cfg{vinyl_timeout=0.001}
+---
+...
+pad_size = 1000
+---
+...
+pad = string.rep('x', pad_size)
+---
+...
+for i = 1, box.cfg.vinyl_memory / pad_size do box.space.test:replace{i, pad} end
+---
+- error: Timed out waiting for Vinyl memory quota
+...
+box.info.vinyl().memory.used > 1024 * 1024
+---
+- true
+...
test_run:cmd('switch default')
---
- true
@@ -102,7 +130,42 @@ test_run:cmd('stop server test')
---
- true
...
+-- Check that tarantool can recover with a smaller memory limit.
+_ = test_run:cmd(string.format('create server test2 with script = "vinyl/low_quota_1.lua", workdir = "%s"', fio.pathjoin(fio.cwd(), 'low_quota_2')))
+---
+...
+_ = test_run:cmd('start server test2')
+---
+...
+_ = test_run:cmd('switch test2')
+---
+...
+fiber = require 'fiber'
+---
+...
+-- All memory above the limit must be dumped after recovery.
+while box.space.test.index.pk:info().disk.dump.count == 0 do fiber.sleep(0.001) end
+---
+...
+stat = box.info.vinyl()
+---
+...
+stat.memory.used <= stat.memory.limit or {stat.memory.used, stat.memory.limit}
+---
+- true
+...
+_ = test_run:cmd('switch default')
+---
+...
+test_run:cmd('stop server test2')
+---
+- true
+...
test_run:cmd('cleanup server test')
---
- true
...
+test_run:cmd('cleanup server test2')
+---
+- true
+...
diff --git a/test/vinyl/recovery_quota.test.lua b/test/vinyl/recovery_quota.test.lua
index cf0c1eac02..07483815f1 100644
--- a/test/vinyl/recovery_quota.test.lua
+++ b/test/vinyl/recovery_quota.test.lua
@@ -1,4 +1,5 @@
test_run = require('test_run').new()
+fio = require 'fio'
-- Upon start the test server creates a space and populates it with
-- more tuples than can be stored in memory, which results in dumping
@@ -6,15 +7,16 @@ test_run = require('test_run').new()
-- it replayed the dumped statements, it would exceed memory quota.
-- Check that it does not.
-test_run:cmd('create server test with script = "vinyl/low_quota.lua"')
+test_run:cmd('create server test with script = "vinyl/low_quota_2.lua"')
test_run:cmd('start server test')
test_run:cmd('switch test')
--- Create a vinyl space and trigger dump by exceeding memory quota (1 MB).
+-- Create a vinyl space and trigger dump by exceeding memory quota.
s = box.schema.space.create('test', {engine = 'vinyl'})
_ = s:create_index('pk', {run_count_per_level = 10})
-pad = string.rep('x', 1000)
-for i = 1, 2000 do s:insert{i, pad} end
+pad_size = 1000
+pad = string.rep('x', pad_size)
+for i = 1, 2 * box.cfg.vinyl_memory / pad_size do s:insert{i, pad} end
-- Save the total number of committed and dumped statements.
var = box.schema.space.create('var')
_ = var:create_index('pk', {parts = {1, 'string'}})
@@ -38,7 +40,28 @@ put_before = var:get('put')[2]
put_after = stat.put.rows
dump_after == 0 or dump_after
put_before - dump_before == put_after or {dump_before, dump_after, put_before, put_after}
+-- Disable dump and use all memory up to the limit.
+box.error.injection.set('ERRINJ_VY_RUN_WRITE', false)
+box.cfg{vinyl_timeout=0.001}
+pad_size = 1000
+pad = string.rep('x', pad_size)
+for i = 1, box.cfg.vinyl_memory / pad_size do box.space.test:replace{i, pad} end
+box.info.vinyl().memory.used > 1024 * 1024
test_run:cmd('switch default')
test_run:cmd('stop server test')
+
+-- Check that tarantool can recover with a smaller memory limit.
+_ = test_run:cmd(string.format('create server test2 with script = "vinyl/low_quota_1.lua", workdir = "%s"', fio.pathjoin(fio.cwd(), 'low_quota_2')))
+_ = test_run:cmd('start server test2')
+_ = test_run:cmd('switch test2')
+fiber = require 'fiber'
+-- All memory above the limit must be dumped after recovery.
+while box.space.test.index.pk:info().disk.dump.count == 0 do fiber.sleep(0.001) end
+stat = box.info.vinyl()
+stat.memory.used <= stat.memory.limit or {stat.memory.used, stat.memory.limit}
+_ = test_run:cmd('switch default')
+test_run:cmd('stop server test2')
+
test_run:cmd('cleanup server test')
+test_run:cmd('cleanup server test2')
diff --git a/test/vinyl/suite.ini b/test/vinyl/suite.ini
index 3466a57c4f..e6ba9e41c5 100644
--- a/test/vinyl/suite.ini
+++ b/test/vinyl/suite.ini
@@ -2,7 +2,7 @@
core = tarantool
description = vinyl integration tests
script = vinyl.lua
-release_disabled = errinj.test.lua errinj_gc.test.lua errinj_vylog.test.lua partial_dump.test.lua quota_timeout.test.lua
+release_disabled = errinj.test.lua errinj_gc.test.lua errinj_vylog.test.lua partial_dump.test.lua quota_timeout.test.lua recovery_quota.test.lua
config = suite.cfg
lua_libs = suite.lua stress.lua large.lua txn_proxy.lua ../box/lua/utils.lua
use_unix_sockets = True
--
GitLab