From 7203b9487038689e2dc820bbf5072d4be1cb47fa Mon Sep 17 00:00:00 2001
From: Alexandr Lyapunov <a.lyapunov@corp.mail.ru>
Date: Thu, 9 Jul 2015 19:01:15 +0300
Subject: [PATCH] fixed gh-727 : RTREE index safety if out of memory by adding
preallocated number of pages
---
src/box/memtx_engine.cc | 44 ++++++++++++++-
src/box/memtx_engine.h | 7 +++
test/box/errinj_index.result | 99 ++++++++++++++--------------------
test/box/errinj_index.test.lua | 19 ++-----
4 files changed, 96 insertions(+), 73 deletions(-)
diff --git a/src/box/memtx_engine.cc b/src/box/memtx_engine.cc
index 8647dadd43..885b663aa4 100644
--- a/src/box/memtx_engine.cc
+++ b/src/box/memtx_engine.cc
@@ -58,6 +58,12 @@ static bool memtx_index_arena_initialized = false;
static struct slab_arena memtx_index_arena;
static struct slab_cache memtx_index_arena_slab_cache;
static struct mempool memtx_index_extent_pool;
+static int memtx_index_num_reserved_extents;
+static void *memtx_index_reserved_extents;
+
+enum {
+ RESERVE_EXTENTS_BEFORE_REPLACE = 16
+};
/**
* A version of space_replace for a space which has
@@ -158,6 +164,7 @@ memtx_replace_all_keys(struct txn *txn, struct space *space,
struct tuple *old_tuple, struct tuple *new_tuple,
enum dup_replace_mode mode)
{
+ memtx_index_extent_reserve(RESERVE_EXTENTS_BEFORE_REPLACE);
uint32_t i = 0;
try {
/* Update the primary key */
@@ -828,6 +835,9 @@ memtx_index_arena_init()
mempool_create(&memtx_index_extent_pool,
&memtx_index_arena_slab_cache,
MEMTX_EXTENT_SIZE);
+ /* Empty reserved list */
+ memtx_index_num_reserved_extents = 0;
+ memtx_index_reserved_extents = 0;
/* Done */
memtx_index_arena_initialized = true;
}
@@ -838,7 +848,19 @@ memtx_index_arena_init()
void *
memtx_index_extent_alloc()
{
- ERROR_INJECT(ERRINJ_INDEX_ALLOC, return 0);
+ if (memtx_index_reserved_extents) {
+ assert(memtx_index_num_reserved_extents > 0);
+ memtx_index_num_reserved_extents--;
+ void *result = memtx_index_reserved_extents;
+ memtx_index_reserved_extents = *(void **)
+ memtx_index_reserved_extents;
+ return result;
+ }
+ ERROR_INJECT(ERRINJ_INDEX_ALLOC,
+ /* same error as in mempool_alloc */
+ tnt_raise(OutOfMemory, MEMTX_EXTENT_SIZE,
+ "mempool", "new slab")
+ );
return mempool_alloc(&memtx_index_extent_pool);
}
@@ -850,3 +872,23 @@ memtx_index_extent_free(void *extent)
{
return mempool_free(&memtx_index_extent_pool, extent);
}
+
+/**
+ * Reserve num extents in pool.
+ * Ensure that next num extent_alloc will succeed w/o an error
+ */
+void
+memtx_index_extent_reserve(int num)
+{
+ ERROR_INJECT(ERRINJ_INDEX_ALLOC,
+ /* same error as in mempool_alloc */
+ tnt_raise(OutOfMemory, MEMTX_EXTENT_SIZE,
+ "mempool", "new slab")
+ );
+ while (memtx_index_num_reserved_extents < num) {
+ void *ext = mempool_alloc(&memtx_index_extent_pool);
+ *(void **)ext = memtx_index_reserved_extents;
+ memtx_index_reserved_extents = ext;
+ memtx_index_num_reserved_extents++;
+ }
+}
\ No newline at end of file
diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
index b8e771687c..ed0561f1c8 100644
--- a/src/box/memtx_engine.h
+++ b/src/box/memtx_engine.h
@@ -95,4 +95,11 @@ memtx_index_extent_alloc();
void
memtx_index_extent_free(void *extent);
+/**
+ * Reserve num extents in pool.
+ * Ensure that next num extent_alloc will succeed w/o an error
+ */
+void
+memtx_index_extent_reserve(int num);
+
#endif /* TARANTOOL_BOX_MEMTX_ENGINE_H_INCLUDED */
diff --git a/test/box/errinj_index.result b/test/box/errinj_index.result
index dc640cc3d8..f3943dccac 100644
--- a/test/box/errinj_index.result
+++ b/test/box/errinj_index.result
@@ -93,11 +93,11 @@ res
...
for i = 501,2500 do s:insert{i, i} end
---
-- error: Failed to allocate 16384 bytes in MemtxTree for replace
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
s:delete{1}
---
-- [1, 1, 'test1']
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
res = {}
---
@@ -107,7 +107,8 @@ for i = 1,10 do table.insert(res, (s:get{i})) end
...
res
---
-- - [2, 2, 'test2']
+- - [1, 1, 'test1']
+ - [2, 2, 'test2']
- [3, 3, 'test3']
- [4, 4, 'test4']
- [5, 5, 'test5']
@@ -125,29 +126,10 @@ for i = 501,510 do table.insert(res, (s:get{i})) end
...
res
---
-- - [501, 501]
- - [502, 502]
- - [503, 503]
- - [504, 504]
- - [505, 505]
- - [506, 506]
- - [507, 507]
- - [508, 508]
- - [509, 509]
- - [510, 510]
-...
-res = {}
----
-...
-for i = 2001,2010 do table.insert(res, (s:get{i})) end
----
-...
-res
----
- []
...
---count must be greater that 1000 but less than 2000
-function check_iter_and_size() local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end return count <= 1000 and "fail 1" or count >= 2000 and "fail 2" or "ok" end
+--count must be exactly 10
+function check_iter_and_size() local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end return count == 10 and "ok" or "fail" end
---
...
check_iter_and_size()
@@ -156,11 +138,11 @@ check_iter_and_size()
...
for i = 2501,3500 do s:insert{i, i} end
---
-- error: Failed to allocate 16384 bytes in MemtxTree for replace
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
s:delete{2}
---
-- [2, 2, 'test2']
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
check_iter_and_size()
---
@@ -174,7 +156,9 @@ for i = 1,10 do table.insert(res, (s:get{i})) end
...
res
---
-- - [3, 3, 'test3']
+- - [1, 1, 'test1']
+ - [2, 2, 'test2']
+ - [3, 3, 'test3']
- [4, 4, 'test4']
- [5, 5, 'test5']
- [6, 6, 'test6']
@@ -183,18 +167,6 @@ res
- [9, 9, 'test9']
- [10, 10, 'test10']
...
-for i = 3501,4500 do s:insert{i, i} end
----
-- error: Failed to allocate 16384 bytes in MemtxTree for replace
-...
-s:delete{3}
----
-- [3, 3, 'test3']
-...
-check_iter_and_size()
----
-- ok
-...
errinj.set("ERRINJ_INDEX_ALLOC", false)
---
- ok
@@ -210,7 +182,10 @@ for i = 1,10 do table.insert(res, (s:get{i})) end
...
res
---
-- - [4, 4, 'test4']
+- - [1, 1, 'test1']
+ - [2, 2, 'test2']
+ - [3, 3, 'test3']
+ - [4, 4, 'test4']
- [5, 5, 'test5']
- [6, 6, 'test6']
- [7, 7, 'test7']
@@ -230,7 +205,10 @@ for i = 1,10 do table.insert(res, (s:get{i})) end
...
res
---
-- - [4, 4, 'test4']
+- - [1, 1, 'test1']
+ - [2, 2, 'test2']
+ - [3, 3, 'test3']
+ - [4, 4, 'test4']
- [5, 5, 'test5']
- [6, 6, 'test6']
- [7, 7, 'test7']
@@ -352,11 +330,11 @@ res
...
for i = 501,2500 do s:insert{i, i} end
---
-- error: Failed to allocate 10 bytes in hash_table for key
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
s:delete{1}
---
-- [1, 1, 'test1']
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
res = {}
---
@@ -366,7 +344,8 @@ for i = 1,10 do table.insert(res, (s:get{i})) end
...
res
---
-- - [2, 2, 'test2']
+- - [1, 1, 'test1']
+ - [2, 2, 'test2']
- [3, 3, 'test3']
- [4, 4, 'test4']
- [5, 5, 'test5']
@@ -396,23 +375,19 @@ res
---
- []
...
---since every insertion is rejected, count must be (10 - number of deletions)
-function check_iter_and_size(size_must_be) local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end print (count) return count ~= size_must_be and "fail 1" or "ok" end
----
-...
-check_iter_and_size(9)
+check_iter_and_size()
---
- ok
...
for i = 2501,3500 do s:insert{i, i} end
---
-- error: Failed to allocate 9 bytes in hash_table for key
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
s:delete{2}
---
-- [2, 2, 'test2']
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
-check_iter_and_size(8)
+check_iter_and_size()
---
- ok
...
@@ -424,7 +399,9 @@ for i = 1,10 do table.insert(res, (s:get{i})) end
...
res
---
-- - [3, 3, 'test3']
+- - [1, 1, 'test1']
+ - [2, 2, 'test2']
+ - [3, 3, 'test3']
- [4, 4, 'test4']
- [5, 5, 'test5']
- [6, 6, 'test6']
@@ -435,13 +412,13 @@ res
...
for i = 3501,4500 do s:insert{i, i} end
---
-- error: Failed to allocate 8 bytes in hash_table for key
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
s:delete{3}
---
-- [3, 3, 'test3']
+- error: Failed to allocate 16384 bytes in mempool for new slab
...
-check_iter_and_size(7)
+check_iter_and_size()
---
- ok
...
@@ -460,7 +437,10 @@ for i = 1,10 do table.insert(res, (s:get{i})) end
...
res
---
-- - [4, 4, 'test4']
+- - [1, 1, 'test1']
+ - [2, 2, 'test2']
+ - [3, 3, 'test3']
+ - [4, 4, 'test4']
- [5, 5, 'test5']
- [6, 6, 'test6']
- [7, 7, 'test7']
@@ -480,7 +460,10 @@ for i = 1,10 do table.insert(res, (s:get{i})) end
...
res
---
-- - [4, 4, 'test4']
+- - [1, 1, 'test1']
+ - [2, 2, 'test2']
+ - [3, 3, 'test3']
+ - [4, 4, 'test4']
- [5, 5, 'test5']
- [6, 6, 'test6']
- [7, 7, 'test7']
diff --git a/test/box/errinj_index.test.lua b/test/box/errinj_index.test.lua
index c725b490fe..6068965ee2 100644
--- a/test/box/errinj_index.test.lua
+++ b/test/box/errinj_index.test.lua
@@ -31,12 +31,9 @@ res
res = {}
for i = 501,510 do table.insert(res, (s:get{i})) end
res
-res = {}
-for i = 2001,2010 do table.insert(res, (s:get{i})) end
-res
---count must be greater that 1000 but less than 2000
-function check_iter_and_size() local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end return count <= 1000 and "fail 1" or count >= 2000 and "fail 2" or "ok" end
+--count must be exactly 10
+function check_iter_and_size() local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end return count == 10 and "ok" or "fail" end
check_iter_and_size()
for i = 2501,3500 do s:insert{i, i} end
@@ -46,10 +43,6 @@ res = {}
for i = 1,10 do table.insert(res, (s:get{i})) end
res
-for i = 3501,4500 do s:insert{i, i} end
-s:delete{3}
-check_iter_and_size()
-
errinj.set("ERRINJ_INDEX_ALLOC", false)
for i = 4501,5500 do s:insert{i, i} end
@@ -102,20 +95,18 @@ res = {}
for i = 2001,2010 do table.insert(res, (s:get{i})) end
res
---since every insertion is rejected, count must be (10 - number of deletions)
-function check_iter_and_size(size_must_be) local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end print (count) return count ~= size_must_be and "fail 1" or "ok" end
-check_iter_and_size(9)
+check_iter_and_size()
for i = 2501,3500 do s:insert{i, i} end
s:delete{2}
-check_iter_and_size(8)
+check_iter_and_size()
res = {}
for i = 1,10 do table.insert(res, (s:get{i})) end
res
for i = 3501,4500 do s:insert{i, i} end
s:delete{3}
-check_iter_and_size(7)
+check_iter_and_size()
errinj.set("ERRINJ_INDEX_ALLOC", false)
--
GitLab