From 3ba2779e47be4404da0d7689f571db0b2f89ecbb Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov.dev@gmail.com>
Date: Tue, 27 Jun 2017 13:21:41 +0300
Subject: [PATCH] vinyl: account squashed upserts per index
Replace box.info.vinyl().performance.upsert_{squashed,applied} with per
index index.info().upsert.{squashed,applied}.
Needed for #1662
---
src/box/vinyl.c | 38 ++++++++++------------
src/box/vy_stat.h | 7 +++++
test/vinyl/upsert.result | 64 ++++++++++++++------------------------
test/vinyl/upsert.test.lua | 48 +++++++++++++---------------
4 files changed, 67 insertions(+), 90 deletions(-)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 47ac2565bb..899570258d 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -183,10 +183,6 @@ enum vy_stat_name {
VY_STAT_TX_WRITE,
VY_STAT_CURSOR,
VY_STAT_CURSOR_OPS,
- /* How many upsert chains was squashed */
- VY_STAT_UPSERT_SQUASHED,
- /* How many upserts was applied on read */
- VY_STAT_UPSERT_APPLIED,
VY_STAT_LAST,
};
@@ -197,8 +193,6 @@ static const char *vy_stat_strings[] = {
"tx_write",
"cursor",
"cursor_ops",
- "upsert_squashed",
- "upsert_applied"
};
struct vy_stat {
@@ -2683,7 +2677,6 @@ vy_index_commit_upsert(struct vy_index *index, struct vy_mem *mem,
*/
assert(index->id == 0);
- struct vy_stat *stat = index->env->stat;
const struct tuple *older;
int64_t lsn = vy_stmt_lsn(stmt);
uint8_t n_upserts = vy_stmt_n_upserts(stmt);
@@ -2740,7 +2733,7 @@ vy_index_commit_upsert(struct vy_index *index, struct vy_mem *mem,
vy_apply_upsert(stmt, older, index->key_def,
index->space_format,
index->upsert_format, false);
- rmean_collect(stat->rmean, VY_STAT_UPSERT_APPLIED, 1);
+ index->stat.upsert.applied++;
if (upserted == NULL) {
/* OOM */
@@ -2789,7 +2782,7 @@ vy_index_commit_upsert(struct vy_index *index, struct vy_mem *mem,
assert(rc == 0); (void)rc;
tuple_unref(upserted);
vy_mem_commit_stmt(mem, region_stmt);
- rmean_collect(stat->rmean, VY_STAT_UPSERT_SQUASHED, 1);
+ index->stat.upsert.squashed++;
}
}
@@ -4544,6 +4537,11 @@ vy_index_info(struct vy_index *index, struct info_handler *h)
vy_info_append_stmt_counter(h, "get", &stat->get);
vy_info_append_stmt_counter(h, "put", &stat->put);
+ info_table_begin(h, "upsert");
+ info_append_int(h, "squashed", stat->upsert.squashed);
+ info_append_int(h, "applied", stat->upsert.applied);
+ info_table_end(h);
+
info_table_begin(h, "memory");
vy_info_append_stmt_counter(h, NULL, &stat->memory.count);
info_table_begin(h, "iterator");
@@ -5256,7 +5254,6 @@ static int
vy_tx_set(struct vy_tx *tx, struct vy_index *index, struct tuple *stmt)
{
assert(vy_stmt_type(stmt) != 0);
- struct vy_stat *stat = index->env->stat;
/**
* A statement in write set must have and unique lsn
* in order to differ it from cachable statements in mem and run.
@@ -5278,11 +5275,11 @@ vy_tx_set(struct vy_tx *tx, struct vy_index *index, struct tuple *stmt)
stmt = vy_apply_upsert(stmt, old->stmt, index->key_def,
index->space_format,
index->upsert_format, true);
- rmean_collect(stat->rmean, VY_STAT_UPSERT_APPLIED, 1);
+ index->stat.upsert.applied++;
if (stmt == NULL)
return -1;
assert(vy_stmt_type(stmt) != 0);
- rmean_collect(stat->rmean, VY_STAT_UPSERT_SQUASHED, 1);
+ index->stat.upsert.squashed++;
}
assert(tx->write_size >= tuple_size(old->stmt));
tx->write_size -= tuple_size(old->stmt);
@@ -7605,7 +7602,7 @@ vy_merge_iterator_next_lsn(struct vy_merge_iterator *itr, struct tuple **ret)
static NODISCARD int
vy_merge_iterator_squash_upsert(struct vy_merge_iterator *itr,
struct tuple **ret, bool suppress_error,
- struct vy_stat *stat)
+ int64_t *upserts_applied)
{
*ret = NULL;
struct tuple *t = itr->curr_stmt;
@@ -7628,8 +7625,7 @@ vy_merge_iterator_squash_upsert(struct vy_merge_iterator *itr,
assert(itr->is_primary);
applied = vy_apply_upsert(t, next, itr->key_def, itr->format,
itr->upsert_format, suppress_error);
- if (stat != NULL)
- rmean_collect(stat->rmean, VY_STAT_UPSERT_APPLIED, 1);
+ ++*upserts_applied;
tuple_unref(t);
if (applied == NULL)
return -1;
@@ -7946,7 +7942,6 @@ vy_read_iterator_next(struct vy_read_iterator *itr, struct tuple **result)
struct tuple *t = NULL;
struct vy_merge_iterator *mi = &itr->merge_iterator;
struct vy_index *index = itr->index;
- struct vy_stat *stat = index->env->stat;
int rc = 0;
while (true) {
if (vy_read_iterator_merge_next_key(itr, &t)) {
@@ -7966,7 +7961,8 @@ vy_read_iterator_next(struct vy_read_iterator *itr, struct tuple **result)
rc = 0; /* No more data. */
break;
}
- rc = vy_merge_iterator_squash_upsert(mi, &t, true, stat);
+ rc = vy_merge_iterator_squash_upsert(mi, &t, true,
+ &index->stat.upsert.applied);
if (rc != 0) {
if (rc == -1)
goto clear;
@@ -7991,8 +7987,7 @@ vy_read_iterator_next(struct vy_read_iterator *itr, struct tuple **result)
mi->format,
mi->upsert_format,
true);
- rmean_collect(stat->rmean,
- VY_STAT_UPSERT_APPLIED, 1);
+ index->stat.upsert.applied++;
tuple_unref(t);
t = applied;
assert(vy_stmt_type(t) == IPROTO_REPLACE);
@@ -8601,7 +8596,6 @@ vy_squash_process(struct vy_squash *squash)
struct vy_index *index = squash->index;
struct vy_env *env = index->env;
- struct vy_stat *stat = env->stat;
struct key_def *def = index->key_def;
/* Upserts enabled only in the primary index. */
@@ -8710,7 +8704,7 @@ vy_squash_process(struct vy_squash *squash)
struct tuple *applied =
vy_apply_upsert(mem_stmt, result, def, mem->format,
mem->upsert_format, true);
- rmean_collect(stat->rmean, VY_STAT_UPSERT_APPLIED, 1);
+ index->stat.upsert.applied++;
tuple_unref(result);
if (applied == NULL)
return -1;
@@ -8748,7 +8742,7 @@ vy_squash_process(struct vy_squash *squash)
}
}
- rmean_collect(stat->rmean, VY_STAT_UPSERT_SQUASHED, 1);
+ index->stat.upsert.squashed++;
/*
* Insert the resulting REPLACE statement to the mem
diff --git a/src/box/vy_stat.h b/src/box/vy_stat.h
index b1fe1dde4c..d844c4f197 100644
--- a/src/box/vy_stat.h
+++ b/src/box/vy_stat.h
@@ -117,6 +117,13 @@ struct vy_index_stat {
struct vy_stmt_counter get;
/** Number of statements written to this index. */
struct vy_stmt_counter put;
+ /** Upsert statistics. */
+ struct {
+ /** How many upsert chains have been squashed. */
+ int64_t squashed;
+ /** How many upserts have been applied on read. */
+ int64_t applied;
+ } upsert;
/** Memory related statistics. */
struct {
/** Number of statements stored in memory. */
diff --git a/test/vinyl/upsert.result b/test/vinyl/upsert.result
index 27f6725164..4bbd183a6b 100644
--- a/test/vinyl/upsert.result
+++ b/test/vinyl/upsert.result
@@ -491,8 +491,8 @@ test_run:cmd("setopt delimiter ';'")
...
function upsert_stat_diff(stat2, stat1)
return {
- squashed = stat2.upsert_squashed.total - stat1.upsert_squashed.total,
- applied = stat2.upsert_applied.total - stat1.upsert_applied.total
+ squashed = stat2.upsert.squashed - stat1.upsert.squashed,
+ applied = stat2.upsert.applied - stat1.upsert.applied
}
end;
---
@@ -501,15 +501,15 @@ test_run:cmd("setopt delimiter ''");
---
- true
...
-stat1 = box.info.vinyl().performance
----
-...
space = box.schema.space.create('test', { engine = 'vinyl' })
---
...
index = space:create_index('primary')
---
...
+stat1 = index:info()
+---
+...
-- separate upserts w/o on disk data
space:upsert({1, 1, 1}, {{'+', 2, 10}})
---
@@ -520,7 +520,7 @@ space:upsert({1, 1, 1}, {{'-', 2, 20}})
space:upsert({1, 1, 1}, {{'=', 2, 20}})
---
...
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
---
...
upsert_stat_diff(stat2, stat1)
@@ -531,7 +531,7 @@ upsert_stat_diff(stat2, stat1)
stat1 = stat2
---
...
-space.index.primary:info().rows
+stat1.rows
---
- 3
...
@@ -551,7 +551,7 @@ space:upsert({2, 1, 1}, {{'=', 2, 20}})
box.commit()
---
...
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
---
...
upsert_stat_diff(stat2, stat1)
@@ -562,7 +562,7 @@ upsert_stat_diff(stat2, stat1)
stat1 = stat2
---
...
-space.index.primary:info().rows
+stat1.rows
---
- 4
...
@@ -570,7 +570,7 @@ box.snapshot()
---
- ok
...
-space.index.primary:info().rows
+index:info().rows
---
- 2
...
@@ -581,7 +581,7 @@ space:upsert({1, 1, 1}, {{'+', 2, 10}})
space:upsert({1, 1, 1}, {{'-', 2, 20}})
---
...
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
---
...
upsert_stat_diff(stat2, stat1)
@@ -592,7 +592,7 @@ upsert_stat_diff(stat2, stat1)
stat1 = stat2
---
...
-space.index.primary:info().rows
+stat1.rows
---
- 4
...
@@ -601,7 +601,7 @@ space:get({1})
---
- [1, 10, 1]
...
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
---
...
upsert_stat_diff(stat2, stat1)
@@ -616,7 +616,7 @@ space:get({2})
---
- [2, 20, 1]
...
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
---
...
upsert_stat_diff(stat2, stat1)
@@ -632,7 +632,7 @@ space:select({})
- - [1, 10, 1]
- [2, 20, 1]
...
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
---
...
upsert_stat_diff(stat2, stat1)
@@ -647,7 +647,7 @@ stat1 = stat2
for i = 0, 999 do space:upsert({3, 0, 0}, {{'+', 2, 1}}) end
---
...
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
---
...
upsert_stat_diff(stat2, stat1)
@@ -662,7 +662,7 @@ space:get{3}
---
- [3, 999, 0]
...
-space.index.primary:info().rows
+stat1.rows
---
- 1004
...
@@ -697,10 +697,7 @@ s:select() --both upserts are ignored due to primary key change
--
-- gh-2520 use cache as a hint when applying upserts.
--
-old_stat = box.info.vinyl().performance
----
-...
-old_disk_lookup_count = s.index.test:info().disk.iterator.lookup
+old_stat = s.index.test:info()
---
...
-- insert the first upsert
@@ -718,10 +715,7 @@ s:get{100}
- [100]
...
-- a lookup in a run was done to populate the cache
-new_stat = box.info.vinyl().performance
----
-...
-new_disk_lookup_count = s.index.test:info().disk.iterator.lookup
+new_stat = s.index.test:info()
---
...
upsert_stat_diff(new_stat, old_stat)
@@ -729,14 +723,11 @@ upsert_stat_diff(new_stat, old_stat)
- squashed: 0
applied: 1
...
-old_stat = new_stat
----
-...
-new_disk_lookup_count - old_disk_lookup_count
+new_stat.disk.iterator.lookup - old_stat.disk.iterator.lookup
---
- 1
...
-old_disk_lookup_count = new_disk_lookup_count
+old_stat = new_stat
---
...
-- Add another upsert: the cached REPLACE will be used and the upsert will
@@ -759,10 +750,7 @@ s:get{100}
-- go no further than the latest dump to locate the latest
-- value of the key
--
-new_stat = box.info.vinyl().performance
----
-...
-new_disk_lookup_count = s.index.test:info().disk.iterator.lookup
+new_stat = s.index.test:info()
---
...
upsert_stat_diff(new_stat, old_stat)
@@ -770,16 +758,10 @@ upsert_stat_diff(new_stat, old_stat)
- squashed: 0
applied: 0
...
-old_stat = new_stat
----
-...
-new_disk_lookup_count - old_disk_lookup_count
+new_stat.disk.iterator.lookup - old_stat.disk.iterator.lookup
---
- 1
...
-old_disk_lookup_count = new_disk_lookup_count
----
-...
s:drop()
---
...
diff --git a/test/vinyl/upsert.test.lua b/test/vinyl/upsert.test.lua
index 5200402c7f..7f1acc8cd3 100644
--- a/test/vinyl/upsert.test.lua
+++ b/test/vinyl/upsert.test.lua
@@ -194,27 +194,27 @@ space:drop()
test_run:cmd("setopt delimiter ';'")
function upsert_stat_diff(stat2, stat1)
return {
- squashed = stat2.upsert_squashed.total - stat1.upsert_squashed.total,
- applied = stat2.upsert_applied.total - stat1.upsert_applied.total
+ squashed = stat2.upsert.squashed - stat1.upsert.squashed,
+ applied = stat2.upsert.applied - stat1.upsert.applied
}
end;
test_run:cmd("setopt delimiter ''");
-stat1 = box.info.vinyl().performance
-
space = box.schema.space.create('test', { engine = 'vinyl' })
index = space:create_index('primary')
+stat1 = index:info()
+
-- separate upserts w/o on disk data
space:upsert({1, 1, 1}, {{'+', 2, 10}})
space:upsert({1, 1, 1}, {{'-', 2, 20}})
space:upsert({1, 1, 1}, {{'=', 2, 20}})
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
upsert_stat_diff(stat2, stat1)
stat1 = stat2
-space.index.primary:info().rows
+stat1.rows
-- in-tx upserts
box.begin()
@@ -223,50 +223,50 @@ space:upsert({2, 1, 1}, {{'-', 2, 20}})
space:upsert({2, 1, 1}, {{'=', 2, 20}})
box.commit()
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
upsert_stat_diff(stat2, stat1)
stat1 = stat2
-space.index.primary:info().rows
+stat1.rows
box.snapshot()
-space.index.primary:info().rows
+index:info().rows
-- upsert with on disk data
space:upsert({1, 1, 1}, {{'+', 2, 10}})
space:upsert({1, 1, 1}, {{'-', 2, 20}})
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
upsert_stat_diff(stat2, stat1)
stat1 = stat2
-space.index.primary:info().rows
+stat1.rows
-- count of applied apserts
space:get({1})
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
upsert_stat_diff(stat2, stat1)
stat1 = stat2
space:get({2})
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
upsert_stat_diff(stat2, stat1)
stat1 = stat2
space:select({})
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
upsert_stat_diff(stat2, stat1)
stat1 = stat2
-- start upsert optimizer
for i = 0, 999 do space:upsert({3, 0, 0}, {{'+', 2, 1}}) end
-stat2 = box.info.vinyl().performance
+stat2 = index:info()
upsert_stat_diff(stat2, stat1)
stat1 = stat2
space:get{3}
-space.index.primary:info().rows
+stat1.rows
space:drop()
@@ -283,8 +283,7 @@ s:select() --both upserts are ignored due to primary key change
--
-- gh-2520 use cache as a hint when applying upserts.
--
-old_stat = box.info.vinyl().performance
-old_disk_lookup_count = s.index.test:info().disk.iterator.lookup
+old_stat = s.index.test:info()
-- insert the first upsert
s:upsert({100}, {{'=', 2, 200}})
-- force a dump, the inserted upsert is now on disk
@@ -292,12 +291,10 @@ box.snapshot()
-- populate the cache
s:get{100}
-- a lookup in a run was done to populate the cache
-new_stat = box.info.vinyl().performance
-new_disk_lookup_count = s.index.test:info().disk.iterator.lookup
+new_stat = s.index.test:info()
upsert_stat_diff(new_stat, old_stat)
+new_stat.disk.iterator.lookup - old_stat.disk.iterator.lookup
old_stat = new_stat
-new_disk_lookup_count - old_disk_lookup_count
-old_disk_lookup_count = new_disk_lookup_count
-- Add another upsert: the cached REPLACE will be used and the upsert will
-- be applied immediately
s:upsert({100}, {{'=', 2, 300}})
@@ -310,11 +307,8 @@ s:get{100}
-- go no further than the latest dump to locate the latest
-- value of the key
--
-new_stat = box.info.vinyl().performance
-new_disk_lookup_count = s.index.test:info().disk.iterator.lookup
+new_stat = s.index.test:info()
upsert_stat_diff(new_stat, old_stat)
-old_stat = new_stat
-new_disk_lookup_count - old_disk_lookup_count
-old_disk_lookup_count = new_disk_lookup_count
+new_stat.disk.iterator.lookup - old_stat.disk.iterator.lookup
s:drop()
--
GitLab