From 2ab97c0129faeab5f2eda17a53559c512072e628 Mon Sep 17 00:00:00 2001
From: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Thu, 13 Jul 2017 15:26:44 +0300
Subject: [PATCH] vinyl: introduce vy_stmt_env and vinyl_max_tuple_size option
Closes #2569
---
src/box/lua/load_cfg.lua | 2 ++
src/box/tuple.c | 6 ++++++
src/box/tuple.h | 3 +++
src/box/vinyl.c | 37 +++++++++++++++++----------------
src/box/vy_stmt.c | 17 ++++++++++++++-
src/box/vy_stmt.h | 25 ++++++++++++++++++++++
test/app-tap/init_script.result | 25 +++++++++++-----------
test/box/admin.result | 2 ++
test/box/cfg.result | 4 ++++
test/vinyl/gh.result | 4 ++--
test/vinyl/gh.test.lua | 2 +-
test/vinyl/quota.result | 3 +--
test/vinyl/vinyl.lua | 17 ++++++++-------
13 files changed, 103 insertions(+), 44 deletions(-)
diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index be94726d1f..b7fae09b41 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -20,6 +20,7 @@ local default_cfg = {
vinyl_dir = '.',
vinyl_memory = 128 * 1024 * 1024,
vinyl_cache = 128 * 1024 * 1024,
+ vinyl_max_tuple_size = 1024 * 1024,
vinyl_read_threads = 1,
vinyl_write_threads = 2,
vinyl_timeout = 60,
@@ -66,6 +67,7 @@ local template_cfg = {
vinyl_dir = 'string',
vinyl_memory = 'number',
vinyl_cache = 'number',
+ vinyl_max_tuple_size = 'number',
vinyl_read_threads = 'number',
vinyl_write_threads = 'number',
vinyl_timeout = 'number',
diff --git a/src/box/tuple.c b/src/box/tuple.c
index 801f51dee6..440350dcd8 100644
--- a/src/box/tuple.c
+++ b/src/box/tuple.c
@@ -361,6 +361,12 @@ tuple_arena_create(struct slab_arena *arena, struct quota *quota,
}
}
+void
+tuple_arena_destroy(struct slab_arena *arena)
+{
+ slab_arena_destroy(arena);
+}
+
void
tuple_free(void)
{
diff --git a/src/box/tuple.h b/src/box/tuple.h
index 8ea34362b8..88c2218c95 100644
--- a/src/box/tuple.h
+++ b/src/box/tuple.h
@@ -63,6 +63,9 @@ tuple_arena_create(struct slab_arena *arena, struct quota *quota,
uint64_t arena_max_size, uint32_t tuple_max_size,
const char *arena_name);
+void
+tuple_arena_destroy(struct slab_arena *arena);
+
/** Cleanup tuple library */
void
tuple_free(void);
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 55f3c833d9..748d334575 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -122,8 +122,6 @@ struct vy_env {
struct vy_squash_queue *squash_queue;
/** Mempool for struct vy_cursor */
struct mempool cursor_pool;
- /** Allocator for tuples */
- struct lsregion allocator;
/** Memory quota */
struct vy_quota quota;
/** Timer for updating quota watermark. */
@@ -134,6 +132,8 @@ struct vy_env {
struct vy_cache_env cache_env;
/** Environment for run subsystem */
struct vy_run_env run_env;
+ /** Environment for statements subsystem. */
+ struct vy_stmt_env stmt_env;
/** Local recovery context. */
struct vy_recovery *recovery;
/** Local recovery vclock. */
@@ -1505,7 +1505,7 @@ vy_scheduler_peek_dump(struct vy_scheduler *scheduler, struct vy_task **ptask)
*/
return 0;
}
- if (lsregion_used(&scheduler->env->allocator) == 0) {
+ if (lsregion_used(&scheduler->env->stmt_env.allocator) == 0) {
/*
* Quota must be exceeded by a pending transaction,
* there's nothing we can do about that.
@@ -1898,7 +1898,7 @@ vy_scheduler_complete_dump(struct vy_scheduler *scheduler)
* so the current dump round has been finished.
* Free memory, release quota, and signal dump completion.
*/
- struct lsregion *allocator = &scheduler->env->allocator;
+ struct lsregion *allocator = &scheduler->env->stmt_env.allocator;
struct vy_quota *quota = &scheduler->env->quota;
size_t mem_used_before = lsregion_used(allocator);
lsregion_gc(allocator, min_generation - 1);
@@ -3666,11 +3666,11 @@ vy_prepare(struct vy_env *env, struct vy_tx *tx)
return -1;
}
- size_t mem_used_before = lsregion_used(&env->allocator);
+ size_t mem_used_before = lsregion_used(&env->stmt_env.allocator);
int rc = vy_tx_prepare(tx);
- size_t mem_used_after = lsregion_used(&env->allocator);
+ size_t mem_used_after = lsregion_used(&env->stmt_env.allocator);
assert(mem_used_after >= mem_used_before);
size_t write_size = mem_used_after - mem_used_before;
/*
@@ -3707,11 +3707,11 @@ vy_commit(struct vy_env *env, struct vy_tx *tx, int64_t lsn)
* it silently fails. But if it succeeds, we
* need to account the memory in the quota.
*/
- size_t mem_used_before = lsregion_used(&env->allocator);
+ size_t mem_used_before = lsregion_used(&env->stmt_env.allocator);
vy_tx_commit(tx, lsn);
- size_t mem_used_after = lsregion_used(&env->allocator);
+ size_t mem_used_after = lsregion_used(&env->stmt_env.allocator);
assert(mem_used_after >= mem_used_before);
/* We can't abort the transaction at this point, use force. */
vy_quota_force_use(&env->quota, mem_used_after - mem_used_before);
@@ -3821,14 +3821,16 @@ vy_env_new(void)
if (e->squash_queue == NULL)
goto error_squash_queue;
- struct slab_cache *slab_cache = cord_slab_cache();
- lsregion_create(&e->allocator, slab_cache->arena);
+ vy_stmt_env_create(&e->stmt_env, e->conf->memory,
+ cfg_geti("vinyl_max_tuple_size"));
if (vy_index_env_create(&e->index_env, e->conf->path,
- &e->allocator, &e->scheduler->generation,
- vy_squash_schedule, e) != 0)
+ &e->stmt_env.allocator,
+ &e->scheduler->generation, vy_squash_schedule,
+ e) != 0)
goto error_index_env;
+ struct slab_cache *slab_cache = cord_slab_cache();
mempool_create(&e->cursor_pool, slab_cache,
sizeof(struct vy_cursor));
vy_quota_init(&e->quota, vy_scheduler_quota_exceeded_cb,
@@ -3837,13 +3839,12 @@ vy_env_new(void)
ev_timer_init(&e->quota_timer, vy_env_quota_timer_cb, 0, 1.);
e->quota_timer.data = e;
ev_timer_start(loop(), &e->quota_timer);
- vy_cache_env_create(&e->cache_env, slab_cache,
- e->conf->cache);
+ vy_cache_env_create(&e->cache_env, slab_cache, e->conf->cache);
vy_run_env_create(&e->run_env);
vy_log_init(e->conf->path);
return e;
error_index_env:
- lsregion_destroy(&e->allocator);
+ vy_stmt_env_destroy(&e->stmt_env);
vy_squash_queue_delete(e->squash_queue);
error_squash_queue:
vy_scheduler_delete(e->scheduler);
@@ -3870,7 +3871,7 @@ vy_env_delete(struct vy_env *e)
mempool_destroy(&e->cursor_pool);
vy_run_env_destroy(&e->run_env);
vy_index_env_destroy(&e->index_env);
- lsregion_destroy(&e->allocator);
+ vy_stmt_env_destroy(&e->stmt_env);
vy_cache_env_destroy(&e->cache_env);
if (e->recovery != NULL)
vy_recovery_delete(e->recovery);
@@ -4666,11 +4667,11 @@ vy_squash_process(struct vy_squash *squash)
* Insert the resulting REPLACE statement to the mem
* and adjust the quota.
*/
- size_t mem_used_before = lsregion_used(&env->allocator);
+ size_t mem_used_before = lsregion_used(&env->stmt_env.allocator);
const struct tuple *region_stmt = NULL;
rc = vy_index_set(index, mem, result, ®ion_stmt);
tuple_unref(result);
- size_t mem_used_after = lsregion_used(&env->allocator);
+ size_t mem_used_after = lsregion_used(&env->stmt_env.allocator);
assert(mem_used_after >= mem_used_before);
if (rc == 0) {
/*
diff --git a/src/box/vy_stmt.c b/src/box/vy_stmt.c
index 2e1849b9a7..c39156f338 100644
--- a/src/box/vy_stmt.c
+++ b/src/box/vy_stmt.c
@@ -38,7 +38,6 @@
#include "diag.h"
#include <small/region.h>
-#include "small/lsregion.h"
#include "error.h"
#include "tuple_format.h"
@@ -49,6 +48,22 @@ struct tuple_format_vtab vy_tuple_format_vtab = {
vy_tuple_delete,
};
+void
+vy_stmt_env_create(struct vy_stmt_env *env, uint64_t arena_max_size,
+ uint32_t tuple_max_size)
+{
+ tuple_arena_create(&env->arena, &env->quota, arena_max_size,
+ tuple_max_size, "vinyl");
+ lsregion_create(&env->allocator, &env->arena);
+}
+
+void
+vy_stmt_env_destroy(struct vy_stmt_env *env)
+{
+ lsregion_destroy(&env->allocator);
+ tuple_arena_destroy(&env->arena);
+}
+
void
vy_tuple_delete(struct tuple_format *format, struct tuple *tuple)
{
diff --git a/src/box/vy_stmt.h b/src/box/vy_stmt.h
index 83545b8384..bc335e207f 100644
--- a/src/box/vy_stmt.h
+++ b/src/box/vy_stmt.h
@@ -42,6 +42,9 @@
#include "tuple.h"
#include "tuple_compare.h"
#include "iproto_constants.h"
+#include "small/lsregion.h"
+#include "small/slab_arena.h"
+#include "small/quota.h"
#if defined(__cplusplus)
extern "C" {
@@ -62,8 +65,30 @@ static_assert(VY_UPSERT_THRESHOLD <= UINT8_MAX, "n_upserts max value");
static_assert(VY_UPSERT_INF == VY_UPSERT_THRESHOLD + 1,
"inf must be threshold + 1");
+/** Vinyl statement vtable. */
extern struct tuple_format_vtab vy_tuple_format_vtab;
+/** Vinyl statement environment. */
+struct vy_stmt_env {
+ struct lsregion allocator;
+ struct slab_arena arena;
+ struct quota quota;
+};
+
+/**
+ * Initialize vinyl statement environment.
+ * @param env[out] Vinyl statement environment.
+ * @param arena_max_size Memory limit for vinyl statement arena.
+ * @param tuple_max_size Memory limit for a single vinyl
+ * statement.
+ */
+void
+vy_stmt_env_create(struct vy_stmt_env *env, uint64_t arena_max_size,
+ uint32_t tuple_max_size);
+
+void
+vy_stmt_env_destroy(struct vy_stmt_env *env);
+
/**
* There are two groups of statements:
*
diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result
index 981616036e..435d3a920f 100644
--- a/test/app-tap/init_script.result
+++ b/test/app-tap/init_script.result
@@ -26,18 +26,19 @@ box.cfg
21 vinyl_bloom_fpr:0.05
22 vinyl_cache:134217728
23 vinyl_dir:.
-24 vinyl_memory:134217728
-25 vinyl_page_size:8192
-26 vinyl_range_size:1073741824
-27 vinyl_read_threads:1
-28 vinyl_run_count_per_level:2
-29 vinyl_run_size_ratio:3.5
-30 vinyl_timeout:60
-31 vinyl_write_threads:2
-32 wal_dir:.
-33 wal_dir_rescan_delay:2
-34 wal_max_size:268435456
-35 wal_mode:write
+24 vinyl_max_tuple_size:1048576
+25 vinyl_memory:134217728
+26 vinyl_page_size:8192
+27 vinyl_range_size:1073741824
+28 vinyl_read_threads:1
+29 vinyl_run_count_per_level:2
+30 vinyl_run_size_ratio:3.5
+31 vinyl_timeout:60
+32 vinyl_write_threads:2
+33 wal_dir:.
+34 wal_dir_rescan_delay:2
+35 wal_max_size:268435456
+36 wal_mode:write
--
-- Test insert from detached fiber
--
diff --git a/test/box/admin.result b/test/box/admin.result
index bda03bb63c..7b82b14488 100644
--- a/test/box/admin.result
+++ b/test/box/admin.result
@@ -64,6 +64,8 @@ cfg_filter(box.cfg)
- 134217728
- - vinyl_dir
- <hidden>
+ - - vinyl_max_tuple_size
+ - 1048576
- - vinyl_memory
- 134217728
- - vinyl_page_size
diff --git a/test/box/cfg.result b/test/box/cfg.result
index 9392c32420..e1448a6a2a 100644
--- a/test/box/cfg.result
+++ b/test/box/cfg.result
@@ -60,6 +60,8 @@ cfg_filter(box.cfg)
- 134217728
- - vinyl_dir
- <hidden>
+ - - vinyl_max_tuple_size
+ - 1048576
- - vinyl_memory
- 134217728
- - vinyl_page_size
@@ -137,6 +139,8 @@ cfg_filter(box.cfg)
- 134217728
- - vinyl_dir
- <hidden>
+ - - vinyl_max_tuple_size
+ - 1048576
- - vinyl_memory
- 134217728
- - vinyl_page_size
diff --git a/test/vinyl/gh.result b/test/vinyl/gh.result
index 2ca16889e7..184471300f 100644
--- a/test/vinyl/gh.result
+++ b/test/vinyl/gh.result
@@ -647,9 +647,9 @@ s = box.schema.space.create('vinyl', { engine = 'vinyl' })
i = box.space.vinyl:create_index('primary')
---
...
-s:replace({1, string.rep('x', 10 * 1024 * 1024)})
+_ = s:replace({1, string.rep('x', 35 * 1024 * 1024)})
---
-- error: Failed to allocate 10485799 bytes in lsregion_alloc for mem_stmt
+- error: Failed to allocate 36700199 bytes in lsregion_alloc for mem_stmt
...
s:drop()
---
diff --git a/test/vinyl/gh.test.lua b/test/vinyl/gh.test.lua
index 9756d153b8..eacccc9b8a 100644
--- a/test/vinyl/gh.test.lua
+++ b/test/vinyl/gh.test.lua
@@ -270,5 +270,5 @@ s:drop()
-- https://github.com/tarantool/tarantool/issues/2588
s = box.schema.space.create('vinyl', { engine = 'vinyl' })
i = box.space.vinyl:create_index('primary')
-s:replace({1, string.rep('x', 10 * 1024 * 1024)})
+_ = s:replace({1, string.rep('x', 35 * 1024 * 1024)})
s:drop()
diff --git a/test/vinyl/quota.result b/test/vinyl/quota.result
index 261de8c33a..fdadbad192 100644
--- a/test/vinyl/quota.result
+++ b/test/vinyl/quota.result
@@ -86,11 +86,10 @@ box.info.vinyl().memory.used
...
_ = space:replace{1, 1, string.rep('a', 1024 * 1024 * 5)}
---
-- error: Failed to allocate 5242924 bytes in lsregion_alloc for mem_stmt
...
box.info.vinyl().memory.used
---
-- 0
+- 5341228
...
space:drop()
---
diff --git a/test/vinyl/vinyl.lua b/test/vinyl/vinyl.lua
index 41510f1d03..34bd948ffc 100644
--- a/test/vinyl/vinyl.lua
+++ b/test/vinyl/vinyl.lua
@@ -5,14 +5,15 @@ box.cfg {
memtx_memory = 512 * 1024 * 1024,
memtx_max_tuple_size = 4 * 1024 * 1024,
rows_per_wal = 1000000,
- vinyl_read_threads = 2;
- vinyl_write_threads = 3;
- vinyl_memory = 512 * 1024 * 1024;
- vinyl_range_size = 1024 * 64;
- vinyl_page_size = 1024;
- vinyl_run_count_per_level = 1;
- vinyl_run_size_ratio = 2;
- vinyl_cache = 10240; -- 10kB
+ vinyl_read_threads = 2,
+ vinyl_write_threads = 3,
+ vinyl_memory = 512 * 1024 * 1024,
+ vinyl_range_size = 1024 * 64,
+ vinyl_page_size = 1024,
+ vinyl_run_count_per_level = 1,
+ vinyl_run_size_ratio = 2,
+ vinyl_cache = 10240, -- 10kB
+ vinyl_max_tuple_size = 1024 * 1024 * 6,
}
function box_info_sort(data)
--
GitLab