test: vinyl/recovery_quota: check recovery with smaller limit

Check that vinyl can recover if vinyl_memory is lowered after restart.
In this case it should forcefully use quota and schedule dump after
recovery is complete.

Needed for #2397

test/vinyl/dump_stress.result

 test_run = require('test_run').new()
 ---
 ...
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
 ---
 - true
 ...

test/vinyl/dump_stress.test.lua

 test_run = require('test_run').new()
 
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
 test_run:cmd("start server test")
 test_run:cmd('switch test')
 

test/vinyl/errinj.result

@@ -780,7 +780,7 @@ space:drop()
 --
 -- Space drop in the middle of dump.
 --
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
 ---
 - true
 ...

test/vinyl/errinj.test.lua

@@ -314,7 +314,7 @@ space:drop()
 --
 -- Space drop in the middle of dump.
 --
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
 test_run:cmd("start server test")
 test_run:cmd('switch test')
 fiber = require 'fiber'

test/vinyl/low_quota.lua

 #!/usr/bin/env tarantool
 
+local LIMIT = string.match(arg[0], "%d")
+
 box.cfg{
-    vinyl_memory = 1024 * 1024,
+    vinyl_memory = LIMIT * 1024 * 1024,
 }
 
 require('console').listen(os.getenv('ADMIN'))

test/vinyl/low_quota_1.lua

+low_quota.lua
\ No newline at end of file

test/vinyl/low_quota_2.lua

+low_quota.lua
\ No newline at end of file

test/vinyl/quota_timeout.result

 test_run = require('test_run').new()
 ---
 ...
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
 ---
 - true
 ...

test/vinyl/quota_timeout.test.lua

 test_run = require('test_run').new()
 
-test_run:cmd("create server test with script='vinyl/low_quota.lua'")
+test_run:cmd("create server test with script='vinyl/low_quota_1.lua'")
 test_run:cmd("start server test")
 test_run:cmd('switch test')
 

test/vinyl/recovery_quota.result

 test_run = require('test_run').new()
 ---
 ...
+fio = require 'fio'
+---
+...
 -- Upon start the test server creates a space and populates it with
 -- more tuples than can be stored in memory, which results in dumping
 -- some of them to disk. If on restart, during recovery from WAL,
 -- it replayed the dumped statements, it would exceed memory quota.
 -- Check that it does not.
-test_run:cmd('create server test with script = "vinyl/low_quota.lua"')
+test_run:cmd('create server test with script = "vinyl/low_quota_2.lua"')
 ---
 - true
 ...
@@ -18,17 +21,20 @@ test_run:cmd('switch test')
 ---
 - true
 ...
--- Create a vinyl space and trigger dump by exceeding memory quota (1 MB).
+-- Create a vinyl space and trigger dump by exceeding memory quota.
 s = box.schema.space.create('test', {engine = 'vinyl'})
 ---
 ...
 _ = s:create_index('pk', {run_count_per_level = 10})
 ---
 ...
-pad = string.rep('x', 1000)
+pad_size = 1000
+---
+...
+pad = string.rep('x', pad_size)
 ---
 ...
-for i = 1, 2000 do s:insert{i, pad} end
+for i = 1, 2 * box.cfg.vinyl_memory / pad_size do s:insert{i, pad} end
 ---
 ...
 -- Save the total number of committed and dumped statements.
@@ -94,6 +100,28 @@ put_before - dump_before == put_after or {dump_before, dump_after, put_before, p
 ---
 - true
 ...
+-- Disable dump and use all memory up to the limit.
+box.error.injection.set('ERRINJ_VY_RUN_WRITE', false)
+---
+- ok
+...
+box.cfg{vinyl_timeout=0.001}
+---
+...
+pad_size = 1000
+---
+...
+pad = string.rep('x', pad_size)
+---
+...
+for i = 1, box.cfg.vinyl_memory / pad_size do box.space.test:replace{i, pad} end
+---
+- error: Timed out waiting for Vinyl memory quota
+...
+box.info.vinyl().memory.used > 1024 * 1024
+---
+- true
+...
 test_run:cmd('switch default')
 ---
 - true
@@ -102,7 +130,42 @@ test_run:cmd('stop server test')
 ---
 - true
 ...
+-- Check that tarantool can recover with a smaller memory limit.
+_ = test_run:cmd(string.format('create server test2 with script = "vinyl/low_quota_1.lua", workdir = "%s"', fio.pathjoin(fio.cwd(), 'low_quota_2')))
+---
+...
+_ = test_run:cmd('start server test2')
+---
+...
+_ = test_run:cmd('switch test2')
+---
+...
+fiber = require 'fiber'
+---
+...
+-- All memory above the limit must be dumped after recovery.
+while box.space.test.index.pk:info().disk.dump.count == 0 do fiber.sleep(0.001) end
+---
+...
+stat = box.info.vinyl()
+---
+...
+stat.memory.used <= stat.memory.limit or {stat.memory.used, stat.memory.limit}
+---
+- true
+...
+_ = test_run:cmd('switch default')
+---
+...
+test_run:cmd('stop server test2')
+---
+- true
+...
 test_run:cmd('cleanup server test')
 ---
 - true
 ...
+test_run:cmd('cleanup server test2')
+---
+- true
+...

test/vinyl/recovery_quota.test.lua

 test_run = require('test_run').new()
+fio = require 'fio'
 
 -- Upon start the test server creates a space and populates it with
 -- more tuples than can be stored in memory, which results in dumping
@@ -6,15 +7,16 @@ test_run = require('test_run').new()
 -- it replayed the dumped statements, it would exceed memory quota.
 -- Check that it does not.
 
-test_run:cmd('create server test with script = "vinyl/low_quota.lua"')
+test_run:cmd('create server test with script = "vinyl/low_quota_2.lua"')
 test_run:cmd('start server test')
 
 test_run:cmd('switch test')
--- Create a vinyl space and trigger dump by exceeding memory quota (1 MB).
+-- Create a vinyl space and trigger dump by exceeding memory quota.
 s = box.schema.space.create('test', {engine = 'vinyl'})
 _ = s:create_index('pk', {run_count_per_level = 10})
-pad = string.rep('x', 1000)
-for i = 1, 2000 do s:insert{i, pad} end
+pad_size = 1000
+pad = string.rep('x', pad_size)
+for i = 1, 2 * box.cfg.vinyl_memory / pad_size do s:insert{i, pad} end
 -- Save the total number of committed and dumped statements.
 var = box.schema.space.create('var')
 _ = var:create_index('pk', {parts = {1, 'string'}})
@@ -38,7 +40,28 @@ put_before = var:get('put')[2]
 put_after = stat.put.rows
 dump_after == 0 or dump_after
 put_before - dump_before == put_after or {dump_before, dump_after, put_before, put_after}
+-- Disable dump and use all memory up to the limit.
+box.error.injection.set('ERRINJ_VY_RUN_WRITE', false)
+box.cfg{vinyl_timeout=0.001}
+pad_size = 1000
+pad = string.rep('x', pad_size)
+for i = 1, box.cfg.vinyl_memory / pad_size do box.space.test:replace{i, pad} end
+box.info.vinyl().memory.used > 1024 * 1024
 test_run:cmd('switch default')
 
 test_run:cmd('stop server test')
+
+-- Check that tarantool can recover with a smaller memory limit.
+_ = test_run:cmd(string.format('create server test2 with script = "vinyl/low_quota_1.lua", workdir = "%s"', fio.pathjoin(fio.cwd(), 'low_quota_2')))
+_ = test_run:cmd('start server test2')
+_ = test_run:cmd('switch test2')
+fiber = require 'fiber'
+-- All memory above the limit must be dumped after recovery.
+while box.space.test.index.pk:info().disk.dump.count == 0 do fiber.sleep(0.001) end
+stat = box.info.vinyl()
+stat.memory.used <= stat.memory.limit or {stat.memory.used, stat.memory.limit}
+_ = test_run:cmd('switch default')
+test_run:cmd('stop server test2')
+
 test_run:cmd('cleanup server test')
+test_run:cmd('cleanup server test2')

test/vinyl/suite.ini

@@ -2,7 +2,7 @@
 core = tarantool
 description = vinyl integration tests
 script = vinyl.lua
-release_disabled = errinj.test.lua errinj_gc.test.lua errinj_vylog.test.lua partial_dump.test.lua quota_timeout.test.lua
+release_disabled = errinj.test.lua errinj_gc.test.lua errinj_vylog.test.lua partial_dump.test.lua quota_timeout.test.lua recovery_quota.test.lua
 config = suite.cfg
 lua_libs = suite.lua stress.lua large.lua txn_proxy.lua ../box/lua/utils.lua
 use_unix_sockets = True