From 47c984180786529fcdecefbb2b45d4b16e27d69e Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov.dev@gmail.com>
Date: Wed, 14 Jun 2017 18:45:57 +0300
Subject: [PATCH] vinyl: report memory and disk totals in index.info
Replace the following index.info fields
memory_used # size of statements in memory
size # size of statements on disk
count # number of statements on disk and in memory
page_count # number of pages on disk
with
memory
rows # number of statements in memory
bytes # size of statements in memory
disk
rows # number of statements on disk
bytes # size of statements on disk (unpacked)
bytes_compressed # size of statements on disk (packed)
pages # number of pages on disk
rows # total number of all statements
bytes # total size of all statements
To achieve that, this patch introduces new classes that can be used for
accounting statements on disk and in memory, vy_stmt_disk_counter and
vy_stmt_counter, and makes vy_slice, vy_run, vy_range, and vy_index use
them instead of counting rows, bytes, and pages directly. The difference
between the two classes is that vy_stmt_counter only accounts rows and
bytes, while vy_stmt_disk_counter also includes bytes_compressed and
pages. The classes will be reused for accounting reads and writes later.
Needed for #1662
---
src/box/vinyl.c | 122 +++++++++++---------
src/box/vy_mem.c | 26 ++---
src/box/vy_mem.h | 5 +-
src/box/vy_run.c | 36 ++++--
src/box/vy_run.h | 11 +-
src/box/vy_stat.h | 114 ++++++++++++++++++
test/vinyl/ddl.result | 10 +-
test/vinyl/ddl.test.lua | 10 +-
test/vinyl/info.result | 224 +++++++++++++++++++++++++-----------
test/vinyl/recover.result | 28 ++++-
test/vinyl/recover.test.lua | 15 +--
test/vinyl/upsert.result | 10 +-
test/vinyl/upsert.test.lua | 10 +-
13 files changed, 434 insertions(+), 187 deletions(-)
create mode 100644 src/box/vy_stat.h
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 7f3b2acf9a..d0a2be9266 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -36,6 +36,7 @@
#include "vy_log.h"
#include "vy_upsert.h"
#include "vy_write_iterator.h"
+#include "vy_stat.h"
#define RB_COMPACT 1
#include <small/rb.h>
@@ -350,11 +351,8 @@ struct vy_range {
struct tuple *end;
/** The index this range belongs to. */
struct vy_index *index;
- /*
- * An estimate of the size of data stored on disk for this range
- * (sum of slice->size).
- */
- uint64_t size;
+ /** An estimate of the number of statements in this range. */
+ struct vy_disk_stmt_counter count;
/**
* List of run slices in this range, linked by vy_slice->in_range.
* The newer a slice, the closer it to the list head.
@@ -486,11 +484,6 @@ struct vy_index {
* The newer an index, the closer it to the list head.
*/
struct rlist sealed;
- /**
- * Total amount of memory used by this index
- * (sum of mem->used).
- */
- size_t mem_used;
/**
* Generation of in-memory data stored in this index
* (min over mem->generation).
@@ -515,13 +508,8 @@ struct vy_index {
int range_count;
/** Number of runs in all ranges. */
int run_count;
- /** Number of pages in all runs. */
- int page_count;
- /**
- * Total number of statements in this index,
- * stored both in memory and on disk.
- */
- uint64_t stmt_count;
+ /** Index statistics. */
+ struct vy_index_stat stat;
/** Size of data stored on disk. */
uint64_t size;
/** Histogram of number of runs in range. */
@@ -1217,9 +1205,7 @@ vy_index_add_run(struct vy_index *index, struct vy_run *run)
assert(rlist_empty(&run->in_index));
rlist_add_entry(&index->runs, run, in_index);
index->run_count++;
- index->page_count += run->info.page_count;
- index->stmt_count += run->row_count;
- index->size += run->size;
+ vy_disk_stmt_counter_add(&index->stat.disk.count, &run->count);
}
static void
@@ -1229,9 +1215,7 @@ vy_index_remove_run(struct vy_index *index, struct vy_run *run)
assert(!rlist_empty(&run->in_index));
rlist_del_entry(run, in_index);
index->run_count--;
- index->page_count -= run->info.page_count;
- index->stmt_count -= run->row_count;
- index->size -= run->size;
+ vy_disk_stmt_counter_sub(&index->stat.disk.count, &run->count);
}
static void
@@ -1285,7 +1269,7 @@ vy_range_add_slice(struct vy_range *range, struct vy_slice *slice)
{
rlist_add_entry(&range->slices, slice, in_range);
range->slice_count++;
- range->size += slice->size;
+ vy_disk_stmt_counter_add(&range->count, &slice->count);
}
/** Add a run slice to a range's list before @next_slice. */
@@ -1295,7 +1279,7 @@ vy_range_add_slice_before(struct vy_range *range, struct vy_slice *slice,
{
rlist_add_tail(&next_slice->in_range, &slice->in_range);
range->slice_count++;
- range->size += slice->size;
+ vy_disk_stmt_counter_add(&range->count, &slice->count);
}
/** Remove a run slice from a range's list. */
@@ -1303,11 +1287,10 @@ static void
vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice)
{
assert(range->slice_count > 0);
- assert(range->size >= slice->size);
assert(!rlist_empty(&range->slices));
rlist_del_entry(slice, in_range);
range->slice_count--;
- range->size -= slice->size;
+ vy_disk_stmt_counter_sub(&range->count, &slice->count);
}
/**
@@ -1875,7 +1858,7 @@ vy_range_needs_split(struct vy_range *range, const char **p_split_key)
slice = rlist_last_entry(&range->slices, struct vy_slice, in_range);
/* The range is too small to be split. */
- if (slice->size < (uint64_t)index->opts.range_size * 4 / 3)
+ if (slice->count.bytes_compressed < index->opts.range_size * 4 / 3)
return false;
/* Find the median key in the oldest run (approximately). */
@@ -2091,16 +2074,17 @@ vy_range_update_compact_priority(struct vy_range *range)
struct vy_slice *slice;
rlist_foreach_entry(slice, &range->slices, in_range) {
+ uint64_t size = slice->count.bytes_compressed;
/*
* The size of the first level is defined by
* the size of the most recent run.
*/
if (target_run_size == 0)
- target_run_size = slice->size;
- total_size += slice->size;
+ target_run_size = size;
+ total_size += size;
level_run_count++;
total_run_count++;
- while (slice->size > target_run_size) {
+ while (size > target_run_size) {
/*
* The run size exceeds the threshold
* set for the current level. Move this
@@ -2158,7 +2142,7 @@ vy_range_needs_coalesce(struct vy_range *range,
struct vy_range *it;
/* Size of the coalesced range. */
- uint64_t total_size = range->size;
+ uint64_t total_size = range->count.bytes_compressed;
/* Coalesce ranges until total_size > max_size. */
uint64_t max_size = index->opts.range_size / 2;
@@ -2173,17 +2157,19 @@ vy_range_needs_coalesce(struct vy_range *range,
for (it = vy_range_tree_next(index->tree, range);
it != NULL && !vy_range_is_scheduled(it);
it = vy_range_tree_next(index->tree, it)) {
- if (total_size + it->size > max_size)
+ uint64_t size = it->count.bytes_compressed;
+ if (total_size + size > max_size)
break;
- total_size += it->size;
+ total_size += size;
*p_last = it;
}
for (it = vy_range_tree_prev(index->tree, range);
it != NULL && !vy_range_is_scheduled(it);
it = vy_range_tree_prev(index->tree, it)) {
- if (total_size + it->size > max_size)
+ uint64_t size = it->count.bytes_compressed;
+ if (total_size + size > max_size)
break;
- total_size += it->size;
+ total_size += size;
*p_first = it;
}
return *p_first != *p_last;
@@ -2251,7 +2237,7 @@ vy_range_maybe_coalesce(struct vy_range *range)
vy_index_remove_range(index, it);
rlist_splice(&result->slices, &it->slices);
result->slice_count += it->slice_count;
- result->size += it->size;
+ vy_disk_stmt_counter_add(&result->count, &it->count);
vy_range_delete(it);
it = next;
}
@@ -2625,7 +2611,7 @@ vy_index_set(struct vy_index *index, struct vy_mem *mem,
}
/* We can't free region_stmt below, so let's add it to the stats */
- index->mem_used += tuple_size(stmt);
+ index->stat.memory.count.bytes += tuple_size(stmt);
if (vy_stmt_type(*region_stmt) != IPROTO_UPSERT)
return vy_mem_insert(mem, *region_stmt);
@@ -2652,7 +2638,7 @@ vy_index_commit_stmt(struct vy_index *index, struct vy_mem *mem,
{
vy_mem_commit_stmt(mem, stmt);
- index->stmt_count++;
+ index->stat.memory.count.rows++;
if (vy_stmt_type(stmt) == IPROTO_UPSERT)
vy_index_commit_upsert(index, mem, stmt);
@@ -2747,7 +2733,8 @@ vy_index_commit_upsert(struct vy_index *index, struct vy_mem *mem,
* If there are no other mems and runs and n_upserts == 0,
* then we can turn the UPSERT into the REPLACE.
*/
- if (n_upserts == 0 && index->mem_used == index->mem->used &&
+ if (n_upserts == 0 &&
+ index->stat.memory.count.rows == index->mem->count.rows &&
index->run_count == 0) {
older = vy_mem_older_lsn(mem, stmt);
assert(older == NULL || vy_stmt_type(older) != IPROTO_UPSERT);
@@ -3136,8 +3123,7 @@ vy_task_dump_complete(struct vy_task *task)
if (mem->generation >= task->generation)
continue;
rlist_del_entry(mem, in_sealed);
- index->mem_used -= mem->used;
- index->stmt_count -= mem->tree.size;
+ vy_stmt_counter_sub(&index->stat.memory.count, &mem->count);
vy_scheduler_remove_mem(scheduler, mem);
vy_mem_delete(mem);
}
@@ -3243,7 +3229,8 @@ vy_task_dump_new(struct vy_index *index, struct vy_task **p_task)
* The tree is empty so we can delete it
* right away, without involving a worker.
*/
- index->mem_used -= mem->used;
+ vy_stmt_counter_sub(&index->stat.memory.count,
+ &mem->count);
rlist_del_entry(mem, in_sealed);
vy_scheduler_remove_mem(scheduler, mem);
vy_mem_delete(mem);
@@ -3534,7 +3521,7 @@ vy_task_compact_new(struct vy_range *range, struct vy_task **p_task)
&index->env->run_env) != 0)
goto err_wi_sub;
- task->max_output_count += slice->row_count;
+ task->max_output_count += slice->count.rows;
new_run->dump_lsn = MAX(new_run->dump_lsn,
slice->run->dump_lsn);
@@ -4488,17 +4475,43 @@ vy_info(struct vy_env *env, struct info_handler *h)
info_end(h);
}
+static void
+vy_info_append_stmt_counter(struct info_handler *h, const char *name,
+ const struct vy_stmt_counter *count)
+{
+ info_table_begin(h, name);
+ info_append_int(h, "rows", count->rows);
+ info_append_int(h, "bytes", count->bytes);
+ info_table_end(h);
+}
+
+static void
+vy_info_append_disk_stmt_counter(struct info_handler *h, const char *name,
+ const struct vy_disk_stmt_counter *count)
+{
+ info_table_begin(h, name);
+ info_append_int(h, "rows", count->rows);
+ info_append_int(h, "bytes", count->bytes);
+ info_append_int(h, "bytes_compressed", count->bytes_compressed);
+ info_append_int(h, "pages", count->pages);
+ info_table_end(h);
+}
+
void
vy_index_info(struct vy_index *index, struct info_handler *h)
{
char buf[1024];
+ struct vy_index_stat *stat = &index->stat;
+
info_begin(h);
+ info_append_int(h, "rows", stat->disk.count.rows +
+ stat->memory.count.rows);
+ info_append_int(h, "bytes", stat->disk.count.bytes +
+ stat->memory.count.bytes);
+ vy_info_append_stmt_counter(h, "memory", &stat->memory.count);
+ vy_info_append_disk_stmt_counter(h, "disk", &stat->disk.count);
info_append_int(h, "range_size", index->opts.range_size);
info_append_int(h, "page_size", index->opts.page_size);
- info_append_int(h, "memory_used", index->mem_used);
- info_append_int(h, "size", index->size);
- info_append_int(h, "count", index->stmt_count);
- info_append_int(h, "page_count", index->page_count);
info_append_int(h, "range_count", index->range_count);
info_append_int(h, "run_count", index->run_count);
info_append_int(h, "run_avg", index->run_count / index->range_count);
@@ -4672,15 +4685,13 @@ vy_index_commit_drop(struct vy_index *index)
static void
vy_index_swap(struct vy_index *old_index, struct vy_index *new_index)
{
- assert(old_index->mem_used == 0);
- assert(new_index->mem_used == 0);
+ assert(old_index->stat.memory.count.rows == 0);
+ assert(new_index->stat.memory.count.rows == 0);
SWAP(old_index->dump_lsn, new_index->dump_lsn);
SWAP(old_index->range_count, new_index->range_count);
SWAP(old_index->run_count, new_index->run_count);
- SWAP(old_index->page_count, new_index->page_count);
- SWAP(old_index->stmt_count, new_index->stmt_count);
- SWAP(old_index->size, new_index->size);
+ SWAP(old_index->stat, new_index->stat);
SWAP(old_index->run_hist, new_index->run_hist);
SWAP(old_index->tree, new_index->tree);
rlist_swap(&old_index->runs, &new_index->runs);
@@ -4956,7 +4967,8 @@ vy_prepare_alter_space(struct space *old_space, struct space *new_space)
if (pk->env->status != VINYL_ONLINE)
return 0;
/* The space is empty. Allow alter. */
- if (pk->stmt_count == 0)
+ if (pk->stat.disk.count.rows == 0 &&
+ pk->stat.memory.count.rows == 0)
return 0;
if (old_space->index_count < new_space->index_count) {
diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
@@ -5148,7 +5160,7 @@ vy_index_delete(struct vy_index *index)
size_t
vy_index_bsize(struct vy_index *index)
{
- return index->mem_used;
+ return index->stat.memory.count.bytes;
}
/** True if the transaction is in a read view. */
diff --git a/src/box/vy_mem.c b/src/box/vy_mem.c
index ea55fe7b84..fa37a597e8 100644
--- a/src/box/vy_mem.c
+++ b/src/box/vy_mem.c
@@ -64,7 +64,7 @@ vy_mem_new(struct lsregion *allocator, int64_t generation,
struct tuple_format *format_with_colmask,
struct tuple_format *upsert_format, uint32_t schema_version)
{
- struct vy_mem *index = malloc(sizeof(*index));
+ struct vy_mem *index = calloc(1, sizeof(*index));
if (!index) {
diag_set(OutOfMemory, sizeof(*index),
"malloc", "struct vy_mem");
@@ -72,9 +72,7 @@ vy_mem_new(struct lsregion *allocator, int64_t generation,
}
index->min_lsn = INT64_MAX;
index->max_lsn = -1;
- index->used = 0;
index->key_def = key_def;
- index->version = 0;
index->generation = generation;
index->schema_version = schema_version;
index->allocator = allocator;
@@ -89,7 +87,6 @@ vy_mem_new(struct lsregion *allocator, int64_t generation,
vy_mem_tree_extent_free, index);
rlist_create(&index->in_sealed);
rlist_create(&index->in_dump_fifo);
- index->pin_count = 0;
ipc_cond_create(&index->pin_cond);
return index;
}
@@ -99,7 +96,7 @@ vy_mem_update_formats(struct vy_mem *mem, struct tuple_format *new_format,
struct tuple_format *new_format_with_colmask,
struct tuple_format *new_upsert_format)
{
- assert(mem->used == 0);
+ assert(mem->count.rows == 0);
tuple_format_ref(mem->format, -1);
tuple_format_ref(mem->format_with_colmask, -1);
tuple_format_ref(mem->upsert_format, -1);
@@ -160,16 +157,14 @@ vy_mem_insert_upsert(struct vy_mem *mem, const struct tuple *stmt)
return -1;
assert(! vy_mem_tree_iterator_is_invalid(&inserted));
assert(*vy_mem_tree_iterator_get_elem(&mem->tree, &inserted) == stmt);
+ if (replaced_stmt == NULL)
+ mem->count.rows++;
+ mem->count.bytes += size;
/*
* All iterators begin to see the new statement, and
* will be aborted in case of rollback.
*/
mem->version++;
- /*
- * We will not be able to free memory even in case of
- * rollback.
- */
- mem->used += size;
/*
* Update n_upserts if needed. Get the previous statement
* from the inserted one and if it has the same key, then
@@ -219,16 +214,14 @@ vy_mem_insert(struct vy_mem *mem, const struct tuple *stmt)
const struct tuple *replaced_stmt = NULL;
if (vy_mem_tree_insert(&mem->tree, stmt, &replaced_stmt))
return -1;
+ if (replaced_stmt == NULL)
+ mem->count.rows++;
+ mem->count.bytes += size;
/*
* All iterators begin to see the new statement, and
* will be aborted in case of rollback.
*/
mem->version++;
- /*
- * We will not be able to free memory even in case of
- * rollback.
- */
- mem->used += size;
return 0;
}
@@ -253,7 +246,8 @@ vy_mem_rollback_stmt(struct vy_mem *mem, const struct tuple *stmt)
int rc = vy_mem_tree_delete(&mem->tree, stmt);
assert(rc == 0);
(void) rc;
-
+ /* We can't free memory in case of rollback. */
+ mem->count.rows--;
mem->version++;
}
diff --git a/src/box/vy_mem.h b/src/box/vy_mem.h
index efad0dc397..ff9ffbc6a1 100644
--- a/src/box/vy_mem.h
+++ b/src/box/vy_mem.h
@@ -40,6 +40,7 @@
#include "index.h" /* enum iterator_type */
#include "vy_stmt.h" /* for comparators */
#include "vy_stmt_iterator.h" /* struct vy_stmt_iterator */
+#include "vy_stat.h"
#if defined(__cplusplus)
extern "C" {
@@ -139,8 +140,8 @@ struct vy_mem {
struct rlist in_dump_fifo;
/** BPS tree */
struct vy_mem_tree tree;
- /** The total size of all tuples in this tree in bytes */
- size_t used;
+ /** Number of statements. */
+ struct vy_stmt_counter count;
/** The min and max values of stmt->lsn in this tree. */
int64_t min_lsn;
int64_t max_lsn;
diff --git a/src/box/vy_run.c b/src/box/vy_run.c
index cb4602baec..cf287f5d4a 100644
--- a/src/box/vy_run.c
+++ b/src/box/vy_run.c
@@ -364,6 +364,7 @@ vy_slice_new(int64_t id, struct vy_run *run,
slice->end = end;
rlist_create(&slice->in_range);
ipc_cond_create(&slice->pin_cond);
+ /** Lookup the first and the last pages spanned by the slice. */
bool unused;
if (slice->begin == NULL) {
slice->first_page_no = 0;
@@ -386,11 +387,16 @@ vy_slice_new(int64_t id, struct vy_run *run,
}
}
assert(slice->last_page_no >= slice->first_page_no);
- uint64_t count = slice->last_page_no - slice->first_page_no + 1;
- slice->size = DIV_ROUND_UP(run->size * count,
- run->info.page_count);
- slice->row_count = DIV_ROUND_UP(run->row_count * count,
- run->info.page_count);
+ /** Estimate the number of statements in the slice. */
+ uint32_t run_pages = run->info.page_count;
+ uint32_t slice_pages = slice->last_page_no - slice->first_page_no + 1;
+ slice->count.pages = slice_pages;
+ slice->count.rows = DIV_ROUND_UP(run->count.rows *
+ slice_pages, run_pages);
+ slice->count.bytes = DIV_ROUND_UP(run->count.bytes *
+ slice_pages, run_pages);
+ slice->count.bytes_compressed = DIV_ROUND_UP(
+ run->count.bytes_compressed * slice_pages, run_pages);
return slice;
}
@@ -1822,6 +1828,16 @@ static struct vy_stmt_iterator_iface vy_run_iterator_iface = {
/* }}} vy_run_iterator API implementation */
+/** Account a page to run statistics. */
+static void
+vy_run_acct_page(struct vy_run *run, struct vy_page_info *page)
+{
+ run->count.rows += page->row_count;
+ run->count.bytes += page->unpacked_size;
+ run->count.bytes_compressed += page->size;
+ run->count.pages++;
+}
+
int
vy_run_recover(struct vy_run *run, const char *dir,
uint32_t space_id, uint32_t iid)
@@ -1910,8 +1926,7 @@ vy_run_recover(struct vy_run *run, const char *dir,
run->info.page_count = page_no;
goto fail_close;
}
- run->size += page->size;
- run->row_count += page->row_count;
+ vy_run_acct_page(run, page);
}
/* We don't need to keep metadata file open any longer. */
@@ -2136,8 +2151,7 @@ vy_run_write_page(struct vy_run *run, struct xlog *data_xlog,
assert(page->row_count > 0);
run->info.page_count++;
- run->size += page->size;
- run->row_count += page->row_count;
+ vy_run_acct_page(run, page);
ibuf_destroy(&page_index_buf);
return !end_of_run ? 0: 1;
@@ -2494,8 +2508,8 @@ vy_run_write(struct vy_run *run, const char *dirpath,
if (vy_run_write_index(run, dirpath, space_id, iid) != 0)
return -1;
- *written += run->size;
- *dumped_statements += run->row_count;
+ *written += run->count.bytes_compressed;
+ *dumped_statements += run->count.rows;
return 0;
}
diff --git a/src/box/vy_run.h b/src/box/vy_run.h
index db7066e384..436ed33c29 100644
--- a/src/box/vy_run.h
+++ b/src/box/vy_run.h
@@ -38,6 +38,7 @@
#include "index.h" /* enum iterator_type */
#include "vy_stmt.h" /* for comparators */
#include "vy_stmt_iterator.h" /* struct vy_stmt_iterator */
+#include "vy_stat.h"
#include "small/mempool.h"
#include "salad/bloom.h"
@@ -115,10 +116,8 @@ struct vy_run {
int fd;
/** Unique ID of this run. */
int64_t id;
- /** Size of the run on disk. */
- uint64_t size;
- /** Number of statements in the run. */
- uint64_t row_count;
+ /** Number of statements in this run. */
+ struct vy_disk_stmt_counter count;
/** Max LSN stored on disk. */
int64_t dump_lsn;
/**
@@ -182,10 +181,8 @@ struct vy_slice {
*/
uint32_t first_page_no;
uint32_t last_page_no;
- /** An estimate of the size of this slice on disk. */
- uint64_t size;
/** An estimate of the number of statements in this slice. */
- uint32_t row_count;
+ struct vy_disk_stmt_counter count;
};
/** Position of a particular stmt in vy_run. */
diff --git a/src/box/vy_stat.h b/src/box/vy_stat.h
new file mode 100644
index 0000000000..8abc59ee4e
--- /dev/null
+++ b/src/box/vy_stat.h
@@ -0,0 +1,114 @@
+#ifndef INCLUDES_TARANTOOL_BOX_VY_STAT_H
+#define INCLUDES_TARANTOOL_BOX_VY_STAT_H
+/*
+ * Copyright 2010-2017, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif /* defined(__cplusplus) */
+
+/** Used for accounting statements stored in memory. */
+struct vy_stmt_counter {
+ /** Number of statements. */
+ int64_t rows;
+ /** Size, in bytes. */
+ int64_t bytes;
+};
+
+/** Used for accounting statements stored on disk. */
+struct vy_disk_stmt_counter {
+ /** Number of statements. */
+ int64_t rows;
+ /** Size when uncompressed, in bytes. */
+ int64_t bytes;
+ /** Size when compressed, in bytes */
+ int64_t bytes_compressed;
+ /** Number of pages. */
+ int64_t pages;
+};
+
+/** Vinyl index statistics. */
+struct vy_index_stat {
+ /** Memory related statistics. */
+ struct {
+ /** Number of statements stored in memory. */
+ struct vy_stmt_counter count;
+ } memory;
+ /** Disk related statistics. */
+ struct {
+ /** Number of statements stored on disk. */
+ struct vy_disk_stmt_counter count;
+ } disk;
+};
+
+static inline void
+vy_stmt_counter_add(struct vy_stmt_counter *c1,
+ const struct vy_stmt_counter *c2)
+{
+ c1->rows += c2->rows;
+ c1->bytes += c2->bytes;
+}
+
+static inline void
+vy_stmt_counter_sub(struct vy_stmt_counter *c1,
+ const struct vy_stmt_counter *c2)
+{
+ c1->rows -= c2->rows;
+ c1->bytes -= c2->bytes;
+}
+
+static inline void
+vy_disk_stmt_counter_add(struct vy_disk_stmt_counter *c1,
+ const struct vy_disk_stmt_counter *c2)
+{
+ c1->rows += c2->rows;
+ c1->bytes += c2->bytes;
+ c1->bytes_compressed += c2->bytes_compressed;
+ c1->pages += c2->pages;
+}
+
+static inline void
+vy_disk_stmt_counter_sub(struct vy_disk_stmt_counter *c1,
+ const struct vy_disk_stmt_counter *c2)
+{
+ c1->rows -= c2->rows;
+ c1->bytes -= c2->bytes;
+ c1->bytes_compressed -= c2->bytes_compressed;
+ c1->pages -= c2->pages;
+}
+
+#if defined(__cplusplus)
+} /* extern "C" */
+#endif /* defined(__cplusplus) */
+
+#endif /* INCLUDES_TARANTOOL_BOX_VY_STAT_H */
diff --git a/test/vinyl/ddl.result b/test/vinyl/ddl.result
index c11346498b..092a61c5a2 100644
--- a/test/vinyl/ddl.result
+++ b/test/vinyl/ddl.result
@@ -99,7 +99,7 @@ box.snapshot()
---
- ok
...
-while space.index.primary:info().count ~= 0 do fiber.sleep(0.01) end
+while space.index.primary:info().rows ~= 0 do fiber.sleep(0.01) end
---
...
-- after a dump REPLACE + DELETE = nothing, so the space is empty now and
@@ -173,7 +173,7 @@ box.snapshot()
- ok
...
-- Wait until the dump is finished.
-while space.index.primary:info().count ~= 0 do fiber.sleep(0.01) end
+while space.index.primary:info().rows ~= 0 do fiber.sleep(0.01) end
---
...
index2 = space:create_index('secondary', { parts = {2, 'unsigned'} })
@@ -314,7 +314,7 @@ box.snapshot()
---
- ok
...
-pk:info().page_count
+pk:info().disk.pages
---
- 4
...
@@ -361,7 +361,7 @@ box.snapshot()
while pk:info().run_count ~= 1 do fiber.sleep(0.01) end
---
...
-pk:info().page_count
+pk:info().disk.pages
---
- 6
...
@@ -377,7 +377,7 @@ pk:info().bloom_fpr
---
- 0.2
...
-est_bsize / page_size == pk:info().page_count
+est_bsize / page_size == pk:info().disk.pages
---
- true
...
diff --git a/test/vinyl/ddl.test.lua b/test/vinyl/ddl.test.lua
index 0887652450..752b22eff6 100644
--- a/test/vinyl/ddl.test.lua
+++ b/test/vinyl/ddl.test.lua
@@ -37,7 +37,7 @@ space:delete({1})
-- must fail because vy_mems have data
index2 = space:create_index('secondary', { parts = {2, 'unsigned'} })
box.snapshot()
-while space.index.primary:info().count ~= 0 do fiber.sleep(0.01) end
+while space.index.primary:info().rows ~= 0 do fiber.sleep(0.01) end
-- after a dump REPLACE + DELETE = nothing, so the space is empty now and
-- can be altered.
@@ -66,7 +66,7 @@ space:insert({2, 3})
space:delete({2})
box.snapshot()
-- Wait until the dump is finished.
-while space.index.primary:info().count ~= 0 do fiber.sleep(0.01) end
+while space.index.primary:info().rows ~= 0 do fiber.sleep(0.01) end
index2 = space:create_index('secondary', { parts = {2, 'unsigned'} })
space:drop()
@@ -127,7 +127,7 @@ pad = string.rep('I', pad_size)
for i = 1, 20 do space:replace{i, pad} end
est_bsize = pad_size * 20
box.snapshot()
-pk:info().page_count
+pk:info().disk.pages
pk:info().page_size
pk:info().run_count
pk:info().bloom_fpr
@@ -144,11 +144,11 @@ est_bsize = est_bsize + pad_size * 20
box.snapshot()
-- Wait for compaction
while pk:info().run_count ~= 1 do fiber.sleep(0.01) end
-pk:info().page_count
+pk:info().disk.pages
pk:info().page_size
pk:info().run_count
pk:info().bloom_fpr
-est_bsize / page_size == pk:info().page_count
+est_bsize / page_size == pk:info().disk.pages
space:drop()
--
diff --git a/test/vinyl/info.result b/test/vinyl/info.result
index 36e0c5ce14..4d5206db80 100644
--- a/test/vinyl/info.result
+++ b/test/vinyl/info.result
@@ -156,181 +156,277 @@ end;
info;
---
- - - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
- - bloom_fpr: 0.05
- - count: 0
- - memory_used: 0
- - page_count: 0
+ - bytes: 0
+ - disk:
+ - bytes: 0
+ - bytes_compressed: 0
+ - pages: 0
+ - rows: 0
+ - memory:
+ - bytes: 0
+ - rows: 0
- page_size: 1024
- range_count: 1
- range_size: 65536
+ - rows: 0
- run_avg: 0
- run_count: 0
- run_histogram: '[0]:1'
- - size: 0
...
for i = 1, 16 do
box.space['i'..i]:drop()
diff --git a/test/vinyl/recover.result b/test/vinyl/recover.result
index 89413ffedd..c6142ef80a 100644
--- a/test/vinyl/recover.result
+++ b/test/vinyl/recover.result
@@ -128,17 +128,35 @@ vyinfo1 = tmp:get('vyinfo')[2]
vyinfo2 = s.index.primary:info()
---
...
--- Clear metatable for the sake of output format.
-_ = setmetatable(vyinfo1, nil)
+vyinfo1.memory.rows == vyinfo2.memory.rows
---
+- true
+...
+vyinfo1.memory.bytes == vyinfo2.memory.bytes
+---
+- true
...
-success = true
+vyinfo1.disk.rows == vyinfo2.disk.rows
---
+- true
+...
+vyinfo1.disk.bytes == vyinfo2.disk.bytes
+---
+- true
...
-for k, v in pairs(vyinfo1) do if v ~= vyinfo2[k] then success = false end end
+vyinfo1.disk.bytes_compressed == vyinfo2.disk.bytes_compressed
---
+- true
+...
+vyinfo1.disk.pages == vyinfo2.disk.pages
+---
+- true
+...
+vyinfo1.run_count == vyinfo2.run_count
+---
+- true
...
-success or {vyinfo1, vyinfo2}
+vyinfo1.range_count == vyinfo2.range_count
---
- true
...
diff --git a/test/vinyl/recover.test.lua b/test/vinyl/recover.test.lua
index aa3f45c7d8..046b016c20 100644
--- a/test/vinyl/recover.test.lua
+++ b/test/vinyl/recover.test.lua
@@ -78,13 +78,14 @@ tmp = box.space.tmp
vyinfo1 = tmp:get('vyinfo')[2]
vyinfo2 = s.index.primary:info()
--- Clear metatable for the sake of output format.
-_ = setmetatable(vyinfo1, nil)
-
-success = true
-for k, v in pairs(vyinfo1) do if v ~= vyinfo2[k] then success = false end end
-
-success or {vyinfo1, vyinfo2}
+vyinfo1.memory.rows == vyinfo2.memory.rows
+vyinfo1.memory.bytes == vyinfo2.memory.bytes
+vyinfo1.disk.rows == vyinfo2.disk.rows
+vyinfo1.disk.bytes == vyinfo2.disk.bytes
+vyinfo1.disk.bytes_compressed == vyinfo2.disk.bytes_compressed
+vyinfo1.disk.pages == vyinfo2.disk.pages
+vyinfo1.run_count == vyinfo2.run_count
+vyinfo1.range_count == vyinfo2.range_count
s:drop()
diff --git a/test/vinyl/upsert.result b/test/vinyl/upsert.result
index e81e324689..32f5105c3e 100644
--- a/test/vinyl/upsert.result
+++ b/test/vinyl/upsert.result
@@ -531,7 +531,7 @@ upsert_stat_diff(stat2, stat1)
stat1 = stat2
---
...
-space.index.primary:info().count
+space.index.primary:info().rows
---
- 3
...
@@ -562,7 +562,7 @@ upsert_stat_diff(stat2, stat1)
stat1 = stat2
---
...
-space.index.primary:info().count
+space.index.primary:info().rows
---
- 4
...
@@ -570,7 +570,7 @@ box.snapshot()
---
- ok
...
-space.index.primary:info().count
+space.index.primary:info().rows
---
- 2
...
@@ -592,7 +592,7 @@ upsert_stat_diff(stat2, stat1)
stat1 = stat2
---
...
-space.index.primary:info().count
+space.index.primary:info().rows
---
- 4
...
@@ -662,7 +662,7 @@ space:get{3}
---
- [3, 999, 0]
...
-space.index.primary:info().count
+space.index.primary:info().rows
---
- 1004
...
diff --git a/test/vinyl/upsert.test.lua b/test/vinyl/upsert.test.lua
index de4547c7ea..82af24a698 100644
--- a/test/vinyl/upsert.test.lua
+++ b/test/vinyl/upsert.test.lua
@@ -214,7 +214,7 @@ stat2 = box.info.vinyl().performance
upsert_stat_diff(stat2, stat1)
stat1 = stat2
-space.index.primary:info().count
+space.index.primary:info().rows
-- in-tx upserts
box.begin()
@@ -227,11 +227,11 @@ stat2 = box.info.vinyl().performance
upsert_stat_diff(stat2, stat1)
stat1 = stat2
-space.index.primary:info().count
+space.index.primary:info().rows
box.snapshot()
-space.index.primary:info().count
+space.index.primary:info().rows
-- upsert with on disk data
space:upsert({1, 1, 1}, {{'+', 2, 10}})
@@ -241,7 +241,7 @@ stat2 = box.info.vinyl().performance
upsert_stat_diff(stat2, stat1)
stat1 = stat2
-space.index.primary:info().count
+space.index.primary:info().rows
-- count of applied apserts
space:get({1})
@@ -266,7 +266,7 @@ upsert_stat_diff(stat2, stat1)
stat1 = stat2
space:get{3}
-space.index.primary:info().count
+space.index.primary:info().rows
space:drop()
--
GitLab