From 2acb733e2a6062e97fbf466d45b66ff1fc2b378a Mon Sep 17 00:00:00 2001
From: Konstantin Osipov <kostja@tarantool.org>
Date: Thu, 19 Sep 2013 21:31:57 +0400
Subject: [PATCH] When gaps in LSN fall on xlog boundaries, recovery could
fail.
When looking for the next xlog to recover from, take
into account gaps in LSNs which may occur due to wal_write()
failures.
Add test coverage for create/drop space and error injections.
Add cleanup to alter_limits.test
---
src/recovery.cc | 11 ++-
test/box/alter_limits.result | 35 +++------
test/box/alter_limits.test.lua | 15 ++--
test/box/errinj.result | 135 +++++++++++++++++++++++++++++++++
test/box/errinj.test.lua | 39 ++++++++++
5 files changed, 204 insertions(+), 31 deletions(-)
diff --git a/src/recovery.cc b/src/recovery.cc
index 2b04353c6e..1c7e1ded86 100644
--- a/src/recovery.cc
+++ b/src/recovery.cc
@@ -445,7 +445,9 @@ recover_remaining_wals(struct recovery_state *r)
}
/* TODO: find a better way of finding the next xlog */
- current_lsn = r->confirmed_lsn + 1;
+ current_lsn = r->confirmed_lsn;
+find_next_wal:
+ current_lsn++;
/*
* For the last WAL, first try to open .inprogress
* file: if it doesn't exist, we can safely try an
@@ -465,6 +467,13 @@ recover_remaining_wals(struct recovery_state *r)
filename = format_filename(r->wal_dir,
current_lsn, suffix);
f = fopen(filename, "r");
+ /*
+ * Try finding wal for the next lsn if there is a
+ * gap in LSNs.
+ */
+ if (f == NULL && errno == ENOENT &&
+ current_lsn < wal_greatest_lsn)
+ goto find_next_wal;
}
next_wal = log_io_open(r->wal_dir, LOG_READ, filename, suffix, f);
/*
diff --git a/test/box/alter_limits.result b/test/box/alter_limits.result
index dd0d16af27..d6ef571759 100644
--- a/test/box/alter_limits.result
+++ b/test/box/alter_limits.result
@@ -882,6 +882,9 @@ primary:select()
---
- error: 'No index #0 is defined in space 512'
...
+s:drop()
+---
+...
-- @todo: add a test case for dangling iterator (currently no checks
-- for a dangling iterator in the code
-- ----------------------------------------------------------------
@@ -1166,39 +1169,25 @@ s_nil.index.secondary:count(1)
---
- 0
...
-s_empty.index.primary:select()
+-- cleanup
+s_empty:drop()
---
...
-s_full.index.primary:select()
+s_full:drop()
---
-- [1, 1, 'a']
-- [2, 2, 'b']
-- [3, 3, 'c']
-- [4, 4, 'd']
-- [5, 5, 'e']
-- [6, 6, 'f']
-- [7, 7, 'g']
-- [8, 8, 'h']
...
-r_empty.index.primary:select()
+r_empty:drop()
---
...
-r_full.index.primary:select()
+r_full:drop()
---
-- [1, 1, 'a']
-- [2, 2, 'b']
-- [3, 3, 'c']
-- [4, 4, 'd']
-- [5, 5, 'e']
...
-s_nil.index.secondary:select()
+s_nil:drop()
+---
+...
+r_disabled:drop()
---
-- [1, 2, 3, 4, 5, 6]
-- [7, 8, 9, 10, 11, 12]
-- [13, 14, 15, 16, 17]
...
--- -- inject error at various stages of commit and see that
--- the alter has no effects
--
-- @todo usability
-- ---------
diff --git a/test/box/alter_limits.test.lua b/test/box/alter_limits.test.lua
index 07ac0a66a4..9a3eb57bcd 100644
--- a/test/box/alter_limits.test.lua
+++ b/test/box/alter_limits.test.lua
@@ -304,6 +304,7 @@ primary = s1.index.primary
s.index.primary:drop()
primary.id
primary:select()
+s:drop()
-- @todo: add a test case for dangling iterator (currently no checks
-- for a dangling iterator in the code
-- ----------------------------------------------------------------
@@ -411,14 +412,14 @@ r_empty.index.secondary:count(1)
r_full.index.secondary:count(1)
s_nil.index.secondary:count(1)
-s_empty.index.primary:select()
-s_full.index.primary:select()
-r_empty.index.primary:select()
-r_full.index.primary:select()
-s_nil.index.secondary:select()
+-- cleanup
+s_empty:drop()
+s_full:drop()
+r_empty:drop()
+r_full:drop()
+s_nil:drop()
+r_disabled:drop()
--- -- inject error at various stages of commit and see that
--- the alter has no effects
--
-- @todo usability
-- ---------
diff --git a/test/box/errinj.result b/test/box/errinj.result
index f07602205d..b36bbd2b0e 100644
--- a/test/box/errinj.result
+++ b/test/box/errinj.result
@@ -148,3 +148,138 @@ space:truncate()
space:drop()
---
...
+-- Check how well we handle a failed log write in DDL
+s_disabled = box.schema.create_space('disabled')
+---
+...
+s_withindex = box.schema.create_space('withindex')
+---
+...
+s_withindex:create_index('primary', 'hash')
+---
+...
+s_withdata = box.schema.create_space('withdata')
+---
+...
+s_withdata:create_index('primary', 'tree')
+---
+...
+s_withdata:insert(1, 2, 3, 4, 5)
+---
+- [1, 2, 3, 4, 5]
+...
+s_withdata:insert(4, 5, 6, 7, 8)
+---
+- [4, 5, 6, 7, 8]
+...
+s_withdata:create_index('secondary', 'hash', {parts = {1, 'num', 2, 'num' }})
+---
+...
+box.errinj.set("ERRINJ_WAL_IO", true)
+---
+- ok
+...
+test = box.schema.create_space('test')
+---
+- error: Failed to write to disk
+...
+s_disabled:create_index('primary', 'hash')
+---
+- error: Failed to write to disk
+...
+s_disabled.enabled
+---
+- false
+...
+s_disabled:insert(0)
+---
+- error: 'No index #0 is defined in space 512'
+...
+s_withindex:create_index('secondary', 'tree', { parts = { 1, 'num'} })
+---
+- error: Failed to write to disk
+...
+s_withindex.index.secondary
+---
+- null
+...
+s_withdata.index.secondary:drop()
+---
+- error: Failed to write to disk
+...
+s_withdata.index.secondary.unique
+---
+- true
+...
+s_withdata:drop()
+---
+- error: Failed to write to disk
+...
+box.space['withdata'].enabled
+---
+- true
+...
+s_withdata:create_index('another', 'tree', {parts = { 4, 'num' }, unique = false})
+---
+- error: Failed to write to disk
+...
+s_withdata.index.another
+---
+- null
+...
+box.errinj.set("ERRINJ_WAL_IO", false)
+---
+- ok
+...
+test = box.schema.create_space('test')
+---
+...
+s_disabled:create_index('primary', 'hash')
+---
+...
+s_disabled.enabled
+---
+- true
+...
+s_disabled:insert(0)
+---
+- [0]
+...
+s_withindex:create_index('secondary', 'tree', { parts = { 1, 'num'} })
+---
+...
+s_withindex.index.secondary.unique
+---
+- true
+...
+s_withdata.index.secondary:drop()
+---
+...
+s_withdata.index.secondary
+---
+- null
+...
+s_withdata:drop()
+---
+...
+box.space['withdata']
+---
+- null
+...
+s_withdata:create_index('another', 'tree', {parts = { 4, 'num' }, unique = false})
+---
+- error: Space 514 does not exist
+...
+s_withdata.index.another
+---
+- null
+...
+test:drop()
+---
+...
+s_disabled:drop()
+---
+...
+s_withindex:drop()
+---
+...
diff --git a/test/box/errinj.test.lua b/test/box/errinj.test.lua
index ba5a1d4f99..dfbf63d33d 100644
--- a/test/box/errinj.test.lua
+++ b/test/box/errinj.test.lua
@@ -42,3 +42,42 @@ box.errinj.set("ERRINJ_WAL_ROTATE", false)
space:truncate()
space:drop()
+
+-- Check how well we handle a failed log write in DDL
+s_disabled = box.schema.create_space('disabled')
+s_withindex = box.schema.create_space('withindex')
+s_withindex:create_index('primary', 'hash')
+s_withdata = box.schema.create_space('withdata')
+s_withdata:create_index('primary', 'tree')
+s_withdata:insert(1, 2, 3, 4, 5)
+s_withdata:insert(4, 5, 6, 7, 8)
+s_withdata:create_index('secondary', 'hash', {parts = {1, 'num', 2, 'num' }})
+box.errinj.set("ERRINJ_WAL_IO", true)
+test = box.schema.create_space('test')
+s_disabled:create_index('primary', 'hash')
+s_disabled.enabled
+s_disabled:insert(0)
+s_withindex:create_index('secondary', 'tree', { parts = { 1, 'num'} })
+s_withindex.index.secondary
+s_withdata.index.secondary:drop()
+s_withdata.index.secondary.unique
+s_withdata:drop()
+box.space['withdata'].enabled
+s_withdata:create_index('another', 'tree', {parts = { 4, 'num' }, unique = false})
+s_withdata.index.another
+box.errinj.set("ERRINJ_WAL_IO", false)
+test = box.schema.create_space('test')
+s_disabled:create_index('primary', 'hash')
+s_disabled.enabled
+s_disabled:insert(0)
+s_withindex:create_index('secondary', 'tree', { parts = { 1, 'num'} })
+s_withindex.index.secondary.unique
+s_withdata.index.secondary:drop()
+s_withdata.index.secondary
+s_withdata:drop()
+box.space['withdata']
+s_withdata:create_index('another', 'tree', {parts = { 4, 'num' }, unique = false})
+s_withdata.index.another
+test:drop()
+s_disabled:drop()
+s_withindex:drop()
--
GitLab