From f1862699a5802afe14ba53dd91a7cfabdbcc2e01 Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov.dev@gmail.com>
Date: Thu, 15 Jun 2017 18:23:33 +0300
Subject: [PATCH] vinyl: test vylog error handling
Check that:
- Error writing to vylog does not break vinyl permanently.
- An index drop/truncate/create record that was not written to vylog
due to a write error is buffered and flushed along with the next
write.
- A buffered index drop/truncate/create record that hasn't been flushed
before restart is replayed on WAL recovery.
Closes #2490
---
src/box/vy_log.c | 6 +
src/errinj.h | 1 +
test/vinyl/errinj_vylog.result | 249 +++++++++++++++++++++++++++++++
test/vinyl/errinj_vylog.test.lua | 121 +++++++++++++++
test/vinyl/suite.ini | 2 +-
5 files changed, 378 insertions(+), 1 deletion(-)
create mode 100644 test/vinyl/errinj_vylog.result
create mode 100644 test/vinyl/errinj_vylog.test.lua
diff --git a/src/box/vy_log.c b/src/box/vy_log.c
index 3ff9e048d3..147e30af42 100644
--- a/src/box/vy_log.c
+++ b/src/box/vy_log.c
@@ -49,6 +49,7 @@
#include "coio_task.h"
#include "diag.h"
#include "errcode.h"
+#include "errinj.h"
#include "fiber.h"
#include "iproto_constants.h" /* IPROTO_INSERT */
#include "key_def.h"
@@ -683,6 +684,11 @@ vy_log_flush(void)
if (vy_log.tx_size == 0)
return 0; /* nothing to do */
+ ERROR_INJECT(ERRINJ_VY_LOG_FLUSH, {
+ diag_set(ClientError, ER_INJECTION, "vinyl log flush");
+ return -1;
+ });
+
struct journal_entry *entry = journal_entry_new(vy_log.tx_size);
if (entry == NULL)
return -1;
diff --git a/src/errinj.h b/src/errinj.h
index 94a79d9233..75cafd92b1 100644
--- a/src/errinj.h
+++ b/src/errinj.h
@@ -91,6 +91,7 @@ struct errinj {
_(ERRINJ_VY_SQUASH_TIMEOUT, ERRINJ_DOUBLE, {.dparam = 0}) \
_(ERRINJ_VY_SCHED_TIMEOUT, ERRINJ_DOUBLE, {.dparam = 0}) \
_(ERRINJ_VY_GC, ERRINJ_BOOL, {.bparam = false}) \
+ _(ERRINJ_VY_LOG_FLUSH, ERRINJ_BOOL, {.bparam = false}) \
_(ERRINJ_RELAY_TIMEOUT, ERRINJ_DOUBLE, {.dparam = 0}) \
_(ERRINJ_RELAY_REPORT_INTERVAL, ERRINJ_DOUBLE, {.dparam = 0}) \
_(ERRINJ_RELAY_FINAL_SLEEP, ERRINJ_BOOL, {.bparam = false}) \
diff --git a/test/vinyl/errinj_vylog.result b/test/vinyl/errinj_vylog.result
new file mode 100644
index 0000000000..f9ae3250e7
--- /dev/null
+++ b/test/vinyl/errinj_vylog.result
@@ -0,0 +1,249 @@
+test_run = require('test_run').new()
+---
+...
+fiber = require('fiber')
+---
+...
+--
+-- Check that an error to commit a new run to vylog does not
+-- break vinyl permanently.
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+---
+...
+_ = s:create_index('pk')
+---
+...
+_ = s:insert{1, 'x'}
+---
+...
+SCHED_TIMEOUT = 0.05
+---
+...
+box.error.injection.set('ERRINJ_VY_SCHED_TIMEOUT', SCHED_TIMEOUT)
+---
+- ok
+...
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', true);
+---
+- ok
+...
+box.snapshot()
+---
+- error: Error injection 'vinyl log flush'
+...
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', false);
+---
+- ok
+...
+fiber.sleep(2 * SCHED_TIMEOUT)
+---
+...
+box.error.injection.set('ERRINJ_VY_SCHED_TIMEOUT', 0)
+---
+- ok
+...
+_ = s:insert{2, 'y'}
+---
+...
+box.snapshot()
+---
+- ok
+...
+_ = s:insert{3, 'z'}
+---
+...
+test_run:cmd('restart server default')
+s = box.space.test
+---
+...
+s:select()
+---
+- - [1, 'x']
+ - [2, 'y']
+ - [3, 'z']
+...
+s:drop()
+---
+...
+--
+-- Check that an index drop/truncate/create record we failed to
+-- write to vylog is flushed along with the next record.
+--
+s1 = box.schema.space.create('test1', {engine = 'vinyl'})
+---
+...
+_ = s1:create_index('pk')
+---
+...
+_ = s1:insert{1, 'a'}
+---
+...
+s2 = box.schema.space.create('test2', {engine = 'vinyl'})
+---
+...
+_ = s2:create_index('pk')
+---
+...
+_ = s2:insert{2, 'b'}
+---
+...
+box.snapshot()
+---
+- ok
+...
+_ = s1:insert{3, 'c'}
+---
+...
+_ = s2:insert{4, 'd'}
+---
+...
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', true);
+---
+- ok
+...
+s1:drop()
+---
+...
+s2:truncate()
+---
+...
+_ = s2:insert{5, 'e'}
+---
+...
+s3 = box.schema.space.create('test3', {engine = 'vinyl'})
+---
+...
+_ = s3:create_index('pk')
+---
+...
+_ = s3:insert{6, 'f'}
+---
+...
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', false);
+---
+- ok
+...
+box.snapshot()
+---
+- ok
+...
+_ = s2:insert{7, 'g'}
+---
+...
+_ = s3:insert{8, 'h'}
+---
+...
+test_run:cmd('restart server default')
+s1 = box.space.test1
+---
+...
+s1 == nil
+---
+- true
+...
+s2 = box.space.test2
+---
+...
+s2:select()
+---
+- - [5, 'e']
+ - [7, 'g']
+...
+s2:drop()
+---
+...
+s3 = box.space.test3
+---
+...
+s3:select()
+---
+- - [6, 'f']
+ - [8, 'h']
+...
+s3:drop()
+---
+...
+--
+-- Check that if a buffered index drop/truncate/create record
+-- does not make it to the vylog before restart, it will be
+-- replayed on recovery.
+--
+s1 = box.schema.space.create('test1', {engine = 'vinyl'})
+---
+...
+_ = s1:create_index('pk')
+---
+...
+_ = s1:insert{111, 'aaa'}
+---
+...
+s2 = box.schema.space.create('test2', {engine = 'vinyl'})
+---
+...
+_ = s2:create_index('pk')
+---
+...
+_ = s2:insert{222, 'bbb'}
+---
+...
+box.snapshot()
+---
+- ok
+...
+_ = s1:insert{333, 'ccc'}
+---
+...
+_ = s2:insert{444, 'ddd'}
+---
+...
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', true);
+---
+- ok
+...
+s1:drop()
+---
+...
+s2:truncate()
+---
+...
+_ = s2:insert{555, 'eee'}
+---
+...
+s3 = box.schema.space.create('test3', {engine = 'vinyl'})
+---
+...
+_ = s3:create_index('pk')
+---
+...
+_ = s3:insert{666, 'fff'}
+---
+...
+test_run:cmd('restart server default')
+s1 = box.space.test1
+---
+...
+s1 == nil
+---
+- true
+...
+s2 = box.space.test2
+---
+...
+s2:select()
+---
+- - [555, 'eee']
+...
+s2:drop()
+---
+...
+s3 = box.space.test3
+---
+...
+s3:select()
+---
+- - [666, 'fff']
+...
+s3:drop()
+---
+...
diff --git a/test/vinyl/errinj_vylog.test.lua b/test/vinyl/errinj_vylog.test.lua
new file mode 100644
index 0000000000..ee0073d09c
--- /dev/null
+++ b/test/vinyl/errinj_vylog.test.lua
@@ -0,0 +1,121 @@
+test_run = require('test_run').new()
+fiber = require('fiber')
+
+--
+-- Check that an error to commit a new run to vylog does not
+-- break vinyl permanently.
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+_ = s:create_index('pk')
+_ = s:insert{1, 'x'}
+
+SCHED_TIMEOUT = 0.05
+box.error.injection.set('ERRINJ_VY_SCHED_TIMEOUT', SCHED_TIMEOUT)
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', true);
+
+box.snapshot()
+
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', false);
+fiber.sleep(2 * SCHED_TIMEOUT)
+box.error.injection.set('ERRINJ_VY_SCHED_TIMEOUT', 0)
+
+_ = s:insert{2, 'y'}
+
+box.snapshot()
+
+_ = s:insert{3, 'z'}
+
+test_run:cmd('restart server default')
+
+s = box.space.test
+s:select()
+s:drop()
+
+--
+-- Check that an index drop/truncate/create record we failed to
+-- write to vylog is flushed along with the next record.
+--
+s1 = box.schema.space.create('test1', {engine = 'vinyl'})
+_ = s1:create_index('pk')
+_ = s1:insert{1, 'a'}
+
+s2 = box.schema.space.create('test2', {engine = 'vinyl'})
+_ = s2:create_index('pk')
+_ = s2:insert{2, 'b'}
+
+box.snapshot()
+
+_ = s1:insert{3, 'c'}
+_ = s2:insert{4, 'd'}
+
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', true);
+
+s1:drop()
+s2:truncate()
+_ = s2:insert{5, 'e'}
+
+s3 = box.schema.space.create('test3', {engine = 'vinyl'})
+_ = s3:create_index('pk')
+_ = s3:insert{6, 'f'}
+
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', false);
+
+box.snapshot()
+
+_ = s2:insert{7, 'g'}
+_ = s3:insert{8, 'h'}
+
+test_run:cmd('restart server default')
+
+s1 = box.space.test1
+s1 == nil
+
+s2 = box.space.test2
+s2:select()
+s2:drop()
+
+s3 = box.space.test3
+s3:select()
+s3:drop()
+
+--
+-- Check that if a buffered index drop/truncate/create record
+-- does not make it to the vylog before restart, it will be
+-- replayed on recovery.
+--
+
+s1 = box.schema.space.create('test1', {engine = 'vinyl'})
+_ = s1:create_index('pk')
+_ = s1:insert{111, 'aaa'}
+
+s2 = box.schema.space.create('test2', {engine = 'vinyl'})
+_ = s2:create_index('pk')
+_ = s2:insert{222, 'bbb'}
+
+box.snapshot()
+
+_ = s1:insert{333, 'ccc'}
+_ = s2:insert{444, 'ddd'}
+
+box.error.injection.set('ERRINJ_VY_LOG_FLUSH', true);
+
+s1:drop()
+s2:truncate()
+_ = s2:insert{555, 'eee'}
+
+s3 = box.schema.space.create('test3', {engine = 'vinyl'})
+_ = s3:create_index('pk')
+_ = s3:insert{666, 'fff'}
+
+test_run:cmd('restart server default')
+
+s1 = box.space.test1
+s1 == nil
+
+s2 = box.space.test2
+s2:select()
+s2:drop()
+
+s3 = box.space.test3
+s3:select()
+s3:drop()
diff --git a/test/vinyl/suite.ini b/test/vinyl/suite.ini
index 29fbc8d78b..3466a57c4f 100644
--- a/test/vinyl/suite.ini
+++ b/test/vinyl/suite.ini
@@ -2,7 +2,7 @@
core = tarantool
description = vinyl integration tests
script = vinyl.lua
-release_disabled = errinj.test.lua errinj_gc.test.lua partial_dump.test.lua quota_timeout.test.lua
+release_disabled = errinj.test.lua errinj_gc.test.lua errinj_vylog.test.lua partial_dump.test.lua quota_timeout.test.lua
config = suite.cfg
lua_libs = suite.lua stress.lua large.lua txn_proxy.lua ../box/lua/utils.lua
use_unix_sockets = True
--
GitLab