From 6606d587596e13dec4db1812ffb702f3c345c081 Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov@tarantool.org>
Date: Tue, 29 Aug 2023 11:59:37 +0300
Subject: [PATCH] vinyl: add statistic for total size of memory occupied by
tuples
Vinyl tuples returned to the user are allocated with malloc. They may be
pinned by Lua indefinitely. Currently, there's no way to figure out how
much memory is occupied by these tuples. This commit adds a statistic to
box.stat.vinyl() that accounts them.
Closes #8485
@TarantoolBot document
Title: Document `memory.tuple` statistic of `box.stat.vinyl()`
The new statistic shows the total size of memory in bytes occupied by
Vinyl tuples. It includes cached tuples and tuples pinned by the Lua
world.
---
.../unreleased/gh-8485-vy-tuple-stat.md | 5 ++
src/box/vinyl.c | 1 +
src/box/vy_stmt.c | 17 +++++-
src/box/vy_stmt.h | 7 +++
.../vinyl-luatest/gh_8485_tuple_stat_test.lua | 61 +++++++++++++++++++
test/vinyl/stat.result | 13 +++-
test/vinyl/stat.test.lua | 4 ++
7 files changed, 103 insertions(+), 5 deletions(-)
create mode 100644 changelogs/unreleased/gh-8485-vy-tuple-stat.md
create mode 100644 test/vinyl-luatest/gh_8485_tuple_stat_test.lua
diff --git a/changelogs/unreleased/gh-8485-vy-tuple-stat.md b/changelogs/unreleased/gh-8485-vy-tuple-stat.md
new file mode 100644
index 0000000000..eeaff6c89a
--- /dev/null
+++ b/changelogs/unreleased/gh-8485-vy-tuple-stat.md
@@ -0,0 +1,5 @@
+## feature/vinyl
+
+* Introduced the `memory.tuple` statistic for `box.stat.vinyl()` that shows
+ the total size of memory occupied by all tuples allocated by the Vinyl engine
+ (gh-8485).
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 2cb41a222d..fc3201042d 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -296,6 +296,7 @@ vy_info_append_memory(struct vy_env *env, struct info_handler *h)
info_table_begin(h, "memory");
info_append_int(h, "tx", vy_tx_manager_mem_used(env->xm));
info_append_int(h, "level0", lsregion_used(&env->mem_env.allocator));
+ info_append_int(h, "tuple", env->stmt_env.sum_tuple_size);
info_append_int(h, "tuple_cache", env->cache_env.mem_used);
info_append_int(h, "page_index", env->lsm_env.page_index_size);
info_append_int(h, "bloom_filter", env->lsm_env.bloom_size);
diff --git a/src/box/vy_stmt.c b/src/box/vy_stmt.c
index 4105b82d03..a6659082b4 100644
--- a/src/box/vy_stmt.c
+++ b/src/box/vy_stmt.c
@@ -98,6 +98,8 @@ vy_tuple_new(struct tuple_format *format, const char *data, const char *end)
static void
vy_tuple_delete(struct tuple_format *format, struct tuple *tuple)
{
+ struct vy_stmt_env *env = format->engine;
+ size_t size = tuple_size(tuple);
say_debug("%s(%p)", __func__, tuple);
assert(tuple_is_unreferenced(tuple));
/*
@@ -105,10 +107,15 @@ vy_tuple_delete(struct tuple_format *format, struct tuple *tuple)
* multithread unsafe modifications of a reference
* counter.
*/
- if (cord_is_main())
+ if (cord_is_main()) {
+ if (format != env->key_format) {
+ assert(env->sum_tuple_size >= size);
+ env->sum_tuple_size -= size;
+ }
tuple_format_unref(format);
+ }
#ifndef NDEBUG
- memset(tuple, '#', tuple_size(tuple)); /* fail early */
+ memset(tuple, '#', size); /* fail early */
#endif
free(tuple);
}
@@ -119,6 +126,7 @@ vy_stmt_env_create(struct vy_stmt_env *env)
env->tuple_format_vtab.tuple_new = vy_tuple_new;
env->tuple_format_vtab.tuple_delete = vy_tuple_delete;
env->max_tuple_size = 1024 * 1024;
+ env->sum_tuple_size = 0;
env->key_format = vy_simple_stmt_format_new(env, NULL, 0);
if (env->key_format == NULL)
panic("failed to create vinyl key format");
@@ -185,8 +193,11 @@ vy_stmt_alloc(struct tuple_format *format, uint32_t data_offset, uint32_t bsize)
format->id, data_offset, bsize, tuple);
tuple_create(tuple, 1, tuple_format_id(format),
data_offset, bsize, false);
- if (cord_is_main())
+ if (cord_is_main()) {
+ if (format != env->key_format)
+ env->sum_tuple_size += total_size;
tuple_format_ref(format);
+ }
vy_stmt_set_lsn(tuple, 0);
vy_stmt_set_type(tuple, 0);
vy_stmt_set_flags(tuple, 0);
diff --git a/src/box/vy_stmt.h b/src/box/vy_stmt.h
index 3938c5a2af..771f26bbce 100644
--- a/src/box/vy_stmt.h
+++ b/src/box/vy_stmt.h
@@ -74,6 +74,13 @@ struct vy_stmt_env {
* @see box.cfg.vinyl_max_tuple_size
*/
size_t max_tuple_size;
+ /**
+ * Size of memory occupied by all vinyl tuples allocated
+ * in the main thread. Note, this doesn't include keys,
+ * which should be fine because keys shouldn't stay in
+ * memory for long.
+ */
+ size_t sum_tuple_size;
/**
* Tuple format used for creating key statements (e.g.
* statements read from secondary index runs). It doesn't
diff --git a/test/vinyl-luatest/gh_8485_tuple_stat_test.lua b/test/vinyl-luatest/gh_8485_tuple_stat_test.lua
new file mode 100644
index 0000000000..4df1b7a22c
--- /dev/null
+++ b/test/vinyl-luatest/gh_8485_tuple_stat_test.lua
@@ -0,0 +1,61 @@
+local server = require('luatest.server')
+local t = require('luatest')
+
+local g = t.group()
+
+g.before_all(function(cg)
+ cg.server = server:new()
+ cg.server:start()
+ cg.server:exec(function()
+ box.schema.create_space('test', {engine = 'vinyl'})
+ box.space.test:create_index('primary', {
+ parts = {1, 'unsigned'},
+ })
+ box.space.test:create_index('secondary', {
+ parts = {2, 'string'}, unique = false,
+ })
+ for i = 1, 10 do
+ box.space.test:insert({i, string.rep('x', 1000)})
+ if i == 5 then
+ box.snapshot()
+ end
+ end
+ collectgarbage()
+ end)
+end)
+
+g.after_all(function(cg)
+ cg.server:drop()
+end)
+
+g.test_tuple_stat = function(cg)
+ cg.server:exec(function()
+ local function gc()
+ box.tuple.new() -- drop blessed tuple ref
+ collectgarbage()
+ end
+ -- Initial value is 0.
+ gc()
+ t.assert_equals(box.stat.vinyl().memory.tuple, 0)
+
+ -- Tuples pinned by Lua.
+ box.cfg({vinyl_cache = 0})
+ local ret = box.space.test:select()
+ t.assert_equals(#ret, 10)
+ t.assert_almost_equals(box.stat.vinyl().memory.tuple, 10 * 1000, 1000)
+ ret = nil -- luacheck: ignore
+ gc()
+ t.assert_equals(box.stat.vinyl().memory.tuple, 0)
+
+ -- Tuples pinned by cache.
+ box.cfg({vinyl_cache = 100 * 1000})
+ ret = box.space.test:select()
+ t.assert_equals(#ret, 10)
+ t.assert_almost_equals(box.stat.vinyl().memory.tuple, 10 * 1000, 1000)
+ ret = nil -- luacheck: ignore
+ gc()
+ t.assert_almost_equals(box.stat.vinyl().memory.tuple, 10 * 1000, 1000)
+ box.cfg({vinyl_cache = 0})
+ t.assert_equals(box.stat.vinyl().memory.tuple, 0)
+ end)
+end
diff --git a/test/vinyl/stat.result b/test/vinyl/stat.result
index c4dc9af394..909b6039c0 100644
--- a/test/vinyl/stat.result
+++ b/test/vinyl/stat.result
@@ -256,8 +256,9 @@ gstat()
tuple_cache: 0
tx: 0
level0: 0
- page_index: 0
bloom_filter: 0
+ page_index: 0
+ tuple: 0
disk:
data_compacted: 0
data: 0
@@ -1007,6 +1008,13 @@ stat_diff(gstat(), st, 'tx')
---
- commit: 1
...
+-- free tuples pinned by Lua
+_ = nil
+---
+...
+_ = collectgarbage()
+---
+...
-- box.stat.reset
box.stat.reset()
---
@@ -1140,8 +1148,9 @@ gstat()
tuple_cache: 14313
tx: 0
level0: 261562
- page_index: 1250
bloom_filter: 140
+ page_index: 1250
+ tuple: 13689
disk:
data_compacted: 104299
data: 104299
diff --git a/test/vinyl/stat.test.lua b/test/vinyl/stat.test.lua
index a8657ccf49..83c595509c 100644
--- a/test/vinyl/stat.test.lua
+++ b/test/vinyl/stat.test.lua
@@ -323,6 +323,10 @@ stat_diff(gstat(), st, 'tx')
box.commit()
stat_diff(gstat(), st, 'tx')
+-- free tuples pinned by Lua
+_ = nil
+_ = collectgarbage()
+
-- box.stat.reset
box.stat.reset()
istat()
--
GitLab