diff --git a/include/salloc.h b/include/salloc.h
index fe43308f9d843b36d7be71ca1b001a9058e12b59..773bdd08719d5938ff002774a650a4d050e33efd 100644
--- a/include/salloc.h
+++ b/include/salloc.h
@@ -38,10 +38,9 @@ bool salloc_init(size_t size, size_t minimal, double factor);
void salloc_free(void);
void *salloc(size_t size, const char *what);
void sfree(void *ptr);
+void sfree_delayed(void *ptr);
void slab_validate();
-
void salloc_protect(void);
-void salloc_batch_mode(bool mode);
/** Statistics on utilization of a single slab class. */
struct slab_cache_stats {
diff --git a/src/box/space.cc b/src/box/space.cc
index d9c255010411abbeb9b31634804c152533b49948..3ee39e0ca830d689371baf277ce7dc96b1b48d9a 100644
--- a/src/box/space.cc
+++ b/src/box/space.cc
@@ -216,7 +216,7 @@ space_free(void)
mh_i32ptr_node(spaces, i)->val;
space_delete(space);
}
- tuple_free();
+ tuple_format_free();
}
@@ -265,7 +265,7 @@ void
space_init(void)
{
spaces = mh_i32ptr_new();
- tuple_init();
+ tuple_format_init();
/* configure regular spaces */
space_config();
diff --git a/src/box/tuple.cc b/src/box/tuple.cc
index 0daec173676555e64ebe8618fb1fc20d6ed4bd7b..94e3472df7ba4eee2223075fe42ea0bccd36f974 100644
--- a/src/box/tuple.cc
+++ b/src/box/tuple.cc
@@ -203,6 +203,8 @@ tuple_init_field_map(struct tuple *tuple, struct tuple_format *format)
}
}
+extern uint32_t snapshot_version;
+
/** Allocate a tuple */
struct tuple *
tuple_alloc(struct tuple_format *format, size_t size)
@@ -212,6 +214,7 @@ tuple_alloc(struct tuple_format *format, size_t size)
struct tuple *tuple = (struct tuple *)(ptr + format->field_map_size);
tuple->refs = 0;
+ tuple->version = snapshot_version;
tuple->bsize = size;
tuple->format_id = tuple_format_id(format);
@@ -229,7 +232,10 @@ tuple_free(struct tuple *tuple)
say_debug("tuple_free(%p)", tuple);
assert(tuple->refs == 0);
char *ptr = (char *) tuple - tuple_format(tuple)->field_map_size;
- sfree(ptr);
+ if (tuple->version == snapshot_version)
+ sfree(ptr);
+ else
+ sfree_delayed(ptr);
}
/**
@@ -510,13 +516,13 @@ tuple_compare_with_key(const struct tuple *tuple, const char *key,
}
void
-tuple_init()
+tuple_format_init()
{
tuple_format_ber = tuple_format_new(NULL, 0);
}
void
-tuple_free()
+tuple_format_free()
{
for (struct tuple_format **format = tuple_formats;
format < tuple_formats + formats_size;
diff --git a/src/box/tuple.h b/src/box/tuple.h
index bd893244c2040773087d8a61e09f1f2f5e2a796e..9b663cbb01a2aa17ba3fc46fe9428bf720658dff 100644
--- a/src/box/tuple.h
+++ b/src/box/tuple.h
@@ -115,6 +115,8 @@ tuple_format_new(struct key_def *key_def, uint32_t key_count);
*/
struct tuple
{
+ /** snapshot generation version */
+ uint32_t version;
/** reference counter */
uint16_t refs;
/** format identifier */
@@ -370,10 +372,10 @@ tuple_to_luabuf(struct tuple *tuple, struct luaL_Buffer *b);
/** Initialize tuple library */
void
-tuple_init();
+tuple_format_init();
/** Cleanup tuple library */
void
-tuple_free();
+tuple_format_free();
#endif /* TARANTOOL_BOX_TUPLE_H_INCLUDED */
diff --git a/src/salloc.cc b/src/salloc.cc
index 0a66d1d85faf7cc5c1cd10c707e07ac021cfe65d..da14adb48a4a3f91caa01a9be6f461775b7717df 100644
--- a/src/salloc.cc
+++ b/src/salloc.cc
@@ -40,7 +40,6 @@
#include <third_party/queue.h>
#include "tarantool/util.h"
#include <tbuf.h>
-#include <qbuf.h>
#include <say.h>
#include "exception.h"
@@ -62,14 +61,16 @@ static const size_t MAX_SLAB_ITEM = 1 << 20;
size_t MAX_SLAB_ITEM_COUNT;
struct slab_item {
- struct slab_item *next;
+ SLIST_ENTRY(slab_item) next;
};
+SLIST_HEAD(item_slist_head, slab_item);
+
struct slab {
uint32_t magic;
size_t used;
size_t items;
- struct slab_item *free;
+ struct item_slist_head free;
struct slab_cache *cache;
void *brk;
SLIST_ENTRY(slab) link;
@@ -89,9 +90,8 @@ struct slab_cache {
struct arena {
void *mmap_base;
size_t mmap_size;
- bool delayed_free_mode;
+ int64_t delayed_free_count;
size_t delayed_free_batch;
- struct qbuf delayed_q;
void *base;
size_t size;
size_t used;
@@ -99,6 +99,7 @@ struct arena {
};
static uint32_t slab_active_caches;
+static struct item_slist_head free_delayed;
static struct slab_cache slab_caches[256];
static struct arena arena;
@@ -129,17 +130,15 @@ slab_caches_init(size_t minimal, double factor)
MAX_SLAB_ITEM_COUNT = (size_t) (SLAB_SIZE - sizeof(struct slab)) /
slab_caches[0].item_size;
+
+ SLIST_INIT(&free_delayed);
}
static bool
arena_init(struct arena *arena, size_t size)
{
- arena->delayed_free_mode = false;
arena->delayed_free_batch = 100;
-
- int rc = qbuf_init(&arena->delayed_q, 4096);
- if (rc == -1)
- return false;
+ arena->delayed_free_count = 0;
arena->used = 0;
arena->size = size - size % SLAB_SIZE;
@@ -149,7 +148,6 @@ arena_init(struct arena *arena, size_t size)
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (arena->mmap_base == MAP_FAILED) {
say_syserror("mmap");
- qbuf_free(&arena->delayed_q);
return false;
}
@@ -164,10 +162,6 @@ void salloc_protect(void) {
mprotect(arena.mmap_base, arena.mmap_size, PROT_READ);
}
-void salloc_batch_mode(bool mode) {
- arena.delayed_free_mode = mode;
-}
-
static void *
arena_alloc(struct arena *arena)
{
@@ -201,7 +195,6 @@ salloc_free(void)
{
if (arena.mmap_base != NULL)
munmap(arena.mmap_base, arena.mmap_size);
- qbuf_free(&arena.delayed_q);
memset(&arena, 0, sizeof(struct arena));
}
@@ -211,7 +204,7 @@ format_slab(struct slab_cache *cache, struct slab *slab)
assert(cache->item_size <= MAX_SLAB_ITEM);
slab->magic = SLAB_MAGIC;
- slab->free = NULL;
+ SLIST_INIT(&slab->free);
slab->cache = cache;
slab->items = 0;
slab->used = 0;
@@ -288,21 +281,20 @@ valid_item(struct slab *slab, void *item)
}
#endif
-static void
-sfree_do(void *ptr)
+void
+sfree(void *ptr)
{
struct slab *slab = slab_header(ptr);
struct slab_cache *cache = slab->cache;
struct slab_item *item = (struct slab_item *) ptr;
- if (fully_formatted(slab) && slab->free == NULL)
+ if (fully_formatted(slab) && SLIST_EMPTY(&slab->free))
TAILQ_INSERT_TAIL(&cache->free_slabs, slab, cache_free_link);
assert(valid_item(slab, item));
- assert(slab->free == NULL || valid_item(slab, slab->free));
+ assert(SLIST_EMPTY(&slab->free) || valid_item(slab, SLIST_FIRST(&slab->free)));
- item->next = slab->free;
- slab->free = item;
+ SLIST_INSERT_HEAD(&slab->free, item, next);
slab->used -= cache->item_size + sizeof(red_zone);
slab->items -= 1;
@@ -319,26 +311,26 @@ static void
sfree_batch(void)
{
ssize_t batch = arena.delayed_free_batch;
- size_t n = qbuf_n(&arena.delayed_q);
- while (batch-- > 0 && n-- > 0) {
- void *ptr = qbuf_pop(&arena.delayed_q);
- assert(ptr != NULL);
- sfree_do(ptr);
+
+ while (--batch >= 0 && !SLIST_EMPTY(&free_delayed)) {
+ assert(arena.delayed_free_count > 0);
+ struct slab_item *item = SLIST_FIRST(&free_delayed);
+ SLIST_REMOVE_HEAD(&free_delayed, next);
+ arena.delayed_free_count--;
+ sfree(item);
}
}
void
-sfree(void *ptr)
+sfree_delayed(void *ptr)
{
if (ptr == NULL)
return;
- if (arena.delayed_free_mode) {
- if (qbuf_push(&arena.delayed_q, ptr) == -1)
- panic("failed to add tuple to the delayed queue");
- return;
- }
- sfree_batch();
- return sfree_do(ptr);
+ struct slab_item *item = (struct slab_item *)ptr;
+ struct slab *slab = slab_header(item);
+ assert(valid_item(slab, item));
+ SLIST_INSERT_HEAD(&free_delayed, item, next);
+ arena.delayed_free_count++;
}
void *
@@ -348,8 +340,7 @@ salloc(size_t size, const char *what)
struct slab *slab;
struct slab_item *item;
- if (! arena.delayed_free_mode)
- sfree_batch();
+ sfree_batch();
if ((cache = cache_for(size)) == NULL ||
(slab = slab_of(cache)) == NULL) {
@@ -358,21 +349,20 @@ salloc(size_t size, const char *what)
"slab allocator", what);
}
- if (slab->free == NULL) {
+ if (SLIST_EMPTY(&slab->free)) {
assert(valid_item(slab, slab->brk));
item = (struct slab_item *) slab->brk;
memcpy((char *)item + cache->item_size, red_zone, sizeof(red_zone));
slab->brk = (char *) slab->brk + cache->item_size + sizeof(red_zone);
} else {
- assert(valid_item(slab, slab->free));
- item = slab->free;
-
+ item = SLIST_FIRST(&slab->free);
+ assert(valid_item(slab, item));
(void) VALGRIND_MAKE_MEM_DEFINED(item, sizeof(void *));
- slab->free = item->next;
+ SLIST_REMOVE_HEAD(&slab->free, next);
(void) VALGRIND_MAKE_MEM_UNDEFINED(item, sizeof(void *));
}
- if (fully_formatted(slab) && slab->free == NULL)
+ if (fully_formatted(slab) && SLIST_EMPTY(&slab->free))
TAILQ_REMOVE(&cache->free_slabs, slab, cache_free_link);
slab->used += cache->item_size + sizeof(red_zone);
diff --git a/src/tarantool.cc b/src/tarantool.cc
index 0954edb2c81e9a1566631d654dd6cbfa33a1bec5..bc68749f49f5940ff8320ab3e7269a5b7d76db43 100644
--- a/src/tarantool.cc
+++ b/src/tarantool.cc
@@ -84,6 +84,8 @@ struct tarantool_cfg cfg;
static ev_signal *sigs = NULL;
int snapshot_pid = 0; /* snapshot processes pid */
+uint32_t snapshot_version = 0;
+
extern const void *opt_def;
static int
@@ -328,7 +330,8 @@ snapshot(void)
if (snapshot_pid)
return EINPROGRESS;
- salloc_batch_mode(true);
+ /* increment snapshot version */
+ snapshot_version++;
pid_t p = fork();
if (p < 0) {
@@ -340,7 +343,6 @@ snapshot(void)
say_warn("waiting for dumper %d", p);
int status = wait_for_child(p);
say_warn("dumper finished %d", p);
- salloc_batch_mode(true);
snapshot_pid = 0;
return (WIFSIGNALED(status) ? EINTR : WEXITSTATUS(status));
}