From 706776a0723d50698c73b1748ee9ccca076a186b Mon Sep 17 00:00:00 2001 From: Konstantin Osipov <kostja@tarantool.org> Date: Tue, 17 Dec 2013 22:36:45 +0400 Subject: [PATCH] Remove salloc and replace it with small object allocator. --- src/CMakeLists.txt | 1 - src/box/bitset_index.cc | 5 +- src/box/box.cc | 18 +- src/box/lua/slab.cc | 31 +-- src/box/tuple.cc | 56 ++++- src/box/tuple.h | 15 +- src/lib/small/small.c | 11 +- src/lib/small/small.h | 23 +- src/salloc.cc | 495 ---------------------------------------- src/salloc.h | 96 -------- src/tarantool.cc | 5 - test/wal/oom.result | 28 +-- 12 files changed, 118 insertions(+), 666 deletions(-) delete mode 100644 src/salloc.cc delete mode 100644 src/salloc.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a61453ed06..77813b24f8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,7 +48,6 @@ set (common_sources coeio.cc iobuf.cc coio_buf.cc - salloc.cc pickle.cc coro.cc stat.cc diff --git a/src/box/bitset_index.cc b/src/box/bitset_index.cc index 971533ba42..6f085fcd19 100644 --- a/src/box/bitset_index.cc +++ b/src/box/bitset_index.cc @@ -31,7 +31,6 @@ #include <string.h> -#include "salloc.h" #include "tuple.h" #include "exception.h" #include "pickle.h" @@ -44,10 +43,10 @@ static inline size_t tuple_to_value(struct tuple *tuple) { /* - * @todo salloc_index_to_ptr is broken + * @todo small_ptr_compress() is broken * https://github.com/tarantool/tarantool/issues/49 */ - /* size_t value = salloc_ptr_to_index(tuple); */ + /* size_t value = small_ptr_compress(tuple); */ size_t value = (intptr_t) tuple >> 2; assert(value_to_tuple(value) == tuple); return value; diff --git a/src/box/box.cc b/src/box/box.cc index 0d2fc8b25a..0be2b85f41 100644 --- a/src/box/box.cc +++ b/src/box/box.cc @@ -49,7 +49,6 @@ extern "C" { #include "request.h" #include "txn.h" #include "fiber.h" -#include "salloc.h" static void process_replica(struct port *port, struct request *request); static void process_ro(struct port *port, struct request *request); @@ -59,7 +58,6 @@ box_process_func box_process_ro = process_ro; static int stat_base; int snapshot_pid = 0; /* snapshot processes pid */ -uint32_t snapshot_version = 0; /** The snapshot row metadata repeats the structure of REPLACE request. */ @@ -300,7 +298,7 @@ void box_free(void) { schema_free(); - tuple_format_free(); + tuple_free(); recovery_free(); } @@ -309,7 +307,8 @@ box_init() { title("loading", NULL); - tuple_format_init(); + tuple_init(cfg.slab_alloc_arena, cfg.slab_alloc_minimal, + cfg.slab_alloc_factor); schema_init(); /* recovery initialization */ @@ -391,13 +390,14 @@ box_snapshot(void) if (p > 0) { snapshot_pid = p; /* increment snapshot version */ - snapshot_version++; + tuple_begin_snapshot(); int status = wait_for_child(p); + tuple_end_snapshot(); snapshot_pid = 0; return (WIFSIGNALED(status) ? EINTR : WEXITSTATUS(status)); } - salloc_protect(); + slab_arena_mprotect(&tuple_arena); title("dumper", "%" PRIu32, getppid()); fiber_set_name(fiber, status); @@ -415,9 +415,9 @@ box_snapshot(void) void box_init_storage(const char *dirname) { - struct log_dir dir = snap_dir; - dir.dirname = (char *) dirname; - init_storage(&dir, NULL); + struct log_dir dir = snap_dir; + dir.dirname = (char *) dirname; + init_storage(&dir, NULL); } void diff --git a/src/box/lua/slab.cc b/src/box/lua/slab.cc index e9509d6ddc..0a88bd0c9c 100644 --- a/src/box/lua/slab.cc +++ b/src/box/lua/slab.cc @@ -35,11 +35,14 @@ extern "C" { #include <lualib.h> } /* extern "C" */ -#include <salloc.h> +#include "box/tuple.h" +#include "small/small.h" /** A callback passed into salloc_stat() and invoked for every slab class. */ +extern "C" { + static int -salloc_stat_lua_cb(const struct slab_cache_stats *cstat, void *cb_ctx) +small_stats_lua_cb(const struct mempool_stats *stats, void *cb_ctx) { struct lua_State *L = (struct lua_State *) cb_ctx; @@ -47,50 +50,52 @@ salloc_stat_lua_cb(const struct slab_cache_stats *cstat, void *cb_ctx) * Create a Lua table for every slab class. A class is * defined by its item size. */ - lua_pushnumber(L, cstat->item_size); + lua_pushnumber(L, stats->objsize); lua_newtable(L); lua_pushstring(L, "slabs"); - luaL_pushnumber64(L, cstat->slabs); + luaL_pushnumber64(L, stats->slabcount); lua_settable(L, -3); lua_pushstring(L, "items"); - luaL_pushnumber64(L, cstat->items); + luaL_pushnumber64(L, stats->objcount); lua_settable(L, -3); lua_pushstring(L, "bytes_used"); - luaL_pushnumber64(L, cstat->bytes_used); + luaL_pushnumber64(L, stats->totals.used); lua_settable(L, -3); lua_pushstring(L, "bytes_free"); - luaL_pushnumber64(L, cstat->bytes_free); + luaL_pushnumber64(L, stats->totals.total - stats->totals.used); lua_settable(L, -3); lua_pushstring(L, "item_size"); - luaL_pushnumber64(L, cstat->item_size); + luaL_pushnumber64(L, stats->objsize); lua_settable(L, -3); lua_settable(L, -3); return 0; } +} /* extern "C" */ + static int lbox_slab_info(struct lua_State *L) { - struct slab_arena_stats astat; + struct small_stats totals; lua_newtable(L); lua_pushstring(L, "slabs"); lua_newtable(L); - salloc_stat(salloc_stat_lua_cb, &astat, L); + small_stats(&talloc, &totals, small_stats_lua_cb, L); lua_settable(L, -3); lua_pushstring(L, "arena_used"); - luaL_pushnumber64(L, astat.used); + luaL_pushnumber64(L, totals.used); lua_settable(L, -3); lua_pushstring(L, "arena_size"); - luaL_pushnumber64(L, astat.size); + luaL_pushnumber64(L, totals.total); lua_settable(L, -3); return 1; } @@ -98,7 +103,7 @@ lbox_slab_info(struct lua_State *L) static int lbox_slab_check(struct lua_State *L __attribute__((unused))) { - slab_validate(); + slab_cache_check(talloc.cache); return 0; } diff --git a/src/box/tuple.cc b/src/box/tuple.cc index 4ffa22d333..e0fd30e325 100644 --- a/src/box/tuple.cc +++ b/src/box/tuple.cc @@ -28,7 +28,7 @@ */ #include "tuple.h" -#include <salloc.h> +#include "small/small.h" #include "tbuf.h" #include "key_def.h" @@ -42,7 +42,12 @@ struct tuple_format *tuple_format_ber; static intptr_t recycled_format_ids = FORMAT_ID_NIL; static uint32_t formats_size, formats_capacity; -extern int snapshot_pid; + +uint32_t snapshot_version; + +struct slab_arena tuple_arena; +static struct slab_cache tuple_slab_cache; +struct small_alloc talloc; /** Extract all available type info from keys. */ void @@ -241,14 +246,13 @@ tuple_init_field_map(struct tuple_format *format, struct tuple *tuple, uint32_t * after the snapshot has finished, otherwise it'll write bad data * to the snapshot file). */ -extern uint32_t snapshot_version; /** Allocate a tuple */ struct tuple * tuple_alloc(struct tuple_format *format, size_t size) { size_t total = sizeof(struct tuple) + size + format->field_map_size; - char *ptr = (char *) salloc(total, "tuple"); + char *ptr = (char *) smalloc(&talloc, total, "tuple"); struct tuple *tuple = (struct tuple *)(ptr + format->field_map_size); tuple->refs = 0; @@ -273,10 +277,10 @@ tuple_delete(struct tuple *tuple) struct tuple_format *format = tuple_format(tuple); char *ptr = (char *) tuple - format->field_map_size; tuple_format_ref(format, -1); - if (snapshot_pid == 0 || tuple->version == snapshot_version) - sfree(ptr); + if (tuple->version == snapshot_version) + smfree(&talloc, ptr); else - sfree_delayed(ptr); + smfree_delayed(&talloc, ptr); } /** @@ -520,15 +524,36 @@ tuple_compare_with_key(const struct tuple *tuple, const char *key, } void -tuple_format_init() +tuple_init(float arena_prealloc, uint32_t objsize_min, + float alloc_factor) { tuple_format_ber = tuple_format_new(&rlist_nil); /* Make sure this one stays around. */ tuple_format_ref(tuple_format_ber, 1); + + int flags; + if (access("/proc/user_beancounters", F_OK) == 0) { + say_warn("Disable shared arena since running under OpenVZ " + "(https://bugzilla.openvz.org/show_bug.cgi?id=2805)"); + flags = MAP_PRIVATE; + } else { + flags = MAP_SHARED; + } + size_t slab_size = 4*1024*1024; + size_t prealloc = arena_prealloc * 1024 * 1024 * 1024; + + slab_arena_create(&tuple_arena, slab_size, + prealloc, prealloc, flags); + slab_cache_create(&tuple_slab_cache, &tuple_arena, + slab_size); + small_alloc_create(&talloc, &tuple_slab_cache, + objsize_min, + (slab_size - mslab_sizeof())/4, + alloc_factor); } void -tuple_format_free() +tuple_free() { /* Clear recycled ids. */ while (recycled_format_ids != FORMAT_ID_NIL) { @@ -543,3 +568,16 @@ tuple_format_free() free(*format); /* ignore the reference count. */ free(tuple_formats); } + +void +tuple_begin_snapshot() +{ + snapshot_version++; + small_alloc_setopt(&talloc, SMALL_DELAYED_FREE_MODE, true); +} + +void +tuple_end_snapshot() +{ + small_alloc_setopt(&talloc, SMALL_DELAYED_FREE_MODE, false); +} diff --git a/src/box/tuple.h b/src/box/tuple.h index 432d187516..45eef4cd30 100644 --- a/src/box/tuple.h +++ b/src/box/tuple.h @@ -36,6 +36,9 @@ enum { FORMAT_ID_MAX = UINT16_MAX - 1, FORMAT_ID_NIL = UINT16_MAX }; struct tbuf; +extern struct small_alloc talloc; +extern struct slab_arena tuple_arena; + /** * @brief In-memory tuple format */ @@ -95,7 +98,6 @@ extern struct tuple_format **tuple_formats; */ extern struct tuple_format *tuple_format_ber; - static inline uint32_t tuple_format_id(struct tuple_format *format) { @@ -448,10 +450,17 @@ tuple_to_tbuf(struct tuple *tuple, struct tbuf *buf); /** Initialize tuple library */ void -tuple_format_init(); +tuple_init(float slab_alloc_arena, uint32_t slab_alloc_minimal, + float alloc_factor); /** Cleanup tuple library */ void -tuple_format_free(); +tuple_free(); + +void +tuple_begin_snapshot(); + +void +tuple_end_snapshot(); #endif /* TARANTOOL_BOX_TUPLE_H_INCLUDED */ diff --git a/src/lib/small/small.c b/src/lib/small/small.c index 4fbb6124de..b9aa2c30a6 100644 --- a/src/lib/small/small.c +++ b/src/lib/small/small.c @@ -262,14 +262,6 @@ smfree(struct small_alloc *alloc, void *ptr) } } -void -smfree_delayed(struct small_alloc *alloc, void *ptr) -{ - assert(alloc->is_delayed_free_mode); - if (ptr) - lifo_push(&alloc->delayed, ptr); -} - /** Simplify iteration over small allocator mempools. */ struct mempool_iterator { @@ -331,6 +323,7 @@ small_stats(struct small_alloc *alloc, mempool_stats(pool, &stats); totals->used += stats.totals.used; totals->total += stats.totals.total; - cb(cb_ctx, &stats); + if (cb(&stats, cb_ctx)) + break; } } diff --git a/src/lib/small/small.h b/src/lib/small/small.h index 839857cd29..e1f1a8390b 100644 --- a/src/lib/small/small.h +++ b/src/lib/small/small.h @@ -190,14 +190,19 @@ smalloc_nothrow(struct small_alloc *alloc, size_t size); void smfree(struct small_alloc *alloc, void *ptr); - /** * Free memory chunk allocated by the small allocator * if not in snapshot mode, otherwise put to the delayed * free list. */ -void -smfree_delayed(struct small_alloc *alloc, void *ptr); +static inline void +smfree_delayed(struct small_alloc *alloc, void *ptr) +{ + if (alloc->is_delayed_free_mode && ptr) + lifo_push(&alloc->delayed, ptr); + else + smfree(alloc, ptr); +} /** * @brief Return an unique index associated with a chunk allocated @@ -229,8 +234,8 @@ small_ptr_compress(struct small_alloc *alloc, void *ptr); void * small_ptr_decompress(struct small_alloc *alloc, size_t val); -typedef void (*mempool_stats_cb)(void *cb_ctx, - struct mempool_stats *stats); +typedef int (*mempool_stats_cb)(const struct mempool_stats *stats, + void *cb_ctx); void small_stats(struct small_alloc *alloc, @@ -242,19 +247,19 @@ small_stats(struct small_alloc *alloc, #include "exception.h" static inline void * -smalloc(struct small_alloc *alloc, size_t size) +smalloc(struct small_alloc *alloc, size_t size, const char *where) { void *ptr = smalloc_nothrow(alloc, size); if (ptr == NULL) tnt_raise(LoggedError, ER_MEMORY_ISSUE, - size, "small object allocator", "new slab"); + size, "slab allocator", where); return ptr; } static inline void * -smalloc0(struct small_alloc *alloc, size_t size) +smalloc0(struct small_alloc *alloc, size_t size, const char *where) { - return memset(smalloc(alloc, size), 0, size); + return memset(smalloc(alloc, size, where), 0, size); } #endif /* defined(__cplusplus) */ diff --git a/src/salloc.cc b/src/salloc.cc deleted file mode 100644 index fe54e6d1fa..0000000000 --- a/src/salloc.cc +++ /dev/null @@ -1,495 +0,0 @@ -/* - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * 1. Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF - * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#include "salloc.h" - -#include "trivia/config.h" -#include <stdbool.h> -#include <stddef.h> -#include <stdint.h> -#include <stdio.h> -#include <string.h> -#include <sys/mman.h> - -#include "third_party/valgrind/memcheck.h" -#include <third_party/queue.h> -#include "trivia/util.h" -#include <tbuf.h> -#include <say.h> -#include "exception.h" - -#define SLAB_ALIGN_PTR(ptr) (void *)((uintptr_t)(ptr) & ~(SLAB_SIZE - 1)) - -extern int snapshot_pid; - - -static bool private_arena = false; - -#ifdef SLAB_DEBUG -#undef NDEBUG -uint8_t red_zone[4] = { 0xfa, 0xfa, 0xfa, 0xfa }; -#else -uint8_t red_zone[0] = { }; -#endif - -static const uint32_t SLAB_MAGIC = 0x51abface; -static const size_t SLAB_SIZE = 1 << 22; -static const size_t MAX_SLAB_ITEM = 1 << 20; - -/* maximum number of items in one slab */ -/* updated in slab_classes_init, depends on salloc_init params */ -size_t MAX_SLAB_ITEM_COUNT; - -struct slab_item { - SLIST_ENTRY(slab_item) next; -}; - -SLIST_HEAD(item_slist_head, slab_item); - -struct slab { - uint32_t magic; - size_t used; - size_t items; - size_t used_real; - size_t alloc_real; - struct item_slist_head free; - struct slab_cache *cache; - void *brk; - SLIST_ENTRY(slab) link; - SLIST_ENTRY(slab) free_link; - TAILQ_ENTRY(slab) cache_free_link; - TAILQ_ENTRY(slab) cache_link; -}; - -SLIST_HEAD(slab_slist_head, slab); -TAILQ_HEAD(slab_tailq_head, slab); - -struct slab_cache { - size_t item_size; - struct slab_tailq_head slabs, free_slabs; -}; - -struct arena { - void *mmap_base; - size_t mmap_size; - /** How items tuples do we have stacked for delayed free. */ - int64_t delayed_free_count; - /** How many items in the delayed free list to free at once. */ - size_t delayed_free_batch; - void *base; - size_t size; - size_t used; - struct slab_slist_head slabs, free_slabs; -}; - -static uint32_t slab_active_caches; -/** - * Delayed garbage collection for items which are used - * in a forked process. - */ -static struct item_slist_head free_delayed; -static struct slab_cache slab_caches[256]; -static struct arena arena; - -static struct slab * -slab_header(void *ptr) -{ - struct slab *slab = (struct slab *) SLAB_ALIGN_PTR(ptr); - assert(slab->magic == SLAB_MAGIC); - return slab; -} - -static void -slab_caches_init(size_t minimal, double factor) -{ - uint32_t i; - size_t size; - const size_t ptr_size = sizeof(void *); - - for (i = 0, size = minimal; i < nelem(slab_caches) && size <= MAX_SLAB_ITEM; i++) { - slab_caches[i].item_size = size - sizeof(red_zone); - TAILQ_INIT(&slab_caches[i].free_slabs); - - size = MAX((size_t)(size * factor) & ~(ptr_size - 1), - (size + ptr_size) & ~(ptr_size - 1)); - } - - slab_active_caches = i; - - MAX_SLAB_ITEM_COUNT = (size_t) (SLAB_SIZE - sizeof(struct slab)) / - slab_caches[0].item_size; - - SLIST_INIT(&free_delayed); -} - -static bool -arena_init(struct arena *arena, size_t size) -{ - arena->delayed_free_batch = 100; - arena->delayed_free_count = 0; - - arena->used = 0; - arena->size = size - size % SLAB_SIZE; - arena->mmap_size = size - size % SLAB_SIZE + SLAB_SIZE; /* spend SLAB_SIZE bytes on align :-( */ - - int flags = MAP_SHARED | MAP_ANONYMOUS; - if (access("/proc/user_beancounters", F_OK) == 0) { - say_warn("Disable shared arena since running under OpenVZ " - "(https://bugzilla.openvz.org/show_bug.cgi?id=2805)"); - flags = MAP_PRIVATE | MAP_ANONYMOUS; - private_arena = true; - } - - arena->mmap_base = mmap(NULL, arena->mmap_size, - PROT_READ | PROT_WRITE, flags, -1, 0); - if (arena->mmap_base == MAP_FAILED) { - say_syserror("mmap"); - return false; - } - - arena->base = (char *)SLAB_ALIGN_PTR(arena->mmap_base) + SLAB_SIZE; - SLIST_INIT(&arena->slabs); - SLIST_INIT(&arena->free_slabs); - - return true; -} - -/** - * Protect slab arena from changes. A safeguard used in a forked - * process to prevent changes to the master process arena. - */ -void -salloc_protect(void) -{ - mprotect(arena.mmap_base, arena.mmap_size, PROT_READ); -} - -static void * -arena_alloc(struct arena *arena) -{ - void *ptr; - const size_t size = SLAB_SIZE; - - if (arena->size - arena->used < size) - return NULL; - - ptr = (char *)arena->base + arena->used; - arena->used += size; - - return ptr; -} - -bool -salloc_init(size_t size, size_t minimal, double factor) -{ - if (size < SLAB_SIZE * 2) - return false; - - if (!arena_init(&arena, size)) - return false; - - slab_caches_init(MAX(sizeof(void *), minimal), factor); - return true; -} - -void -salloc_free(void) -{ - if (arena.mmap_base != NULL) - munmap(arena.mmap_base, arena.mmap_size); - memset(&arena, 0, sizeof(struct arena)); -} - -static void -format_slab(struct slab_cache *cache, struct slab *slab) -{ - assert(cache->item_size <= MAX_SLAB_ITEM); - - slab->magic = SLAB_MAGIC; - SLIST_INIT(&slab->free); - slab->cache = cache; - slab->items = 0; - slab->used = 0; - slab->brk = (char *)CACHEALIGN((char *)slab + sizeof(struct slab)); - - TAILQ_INSERT_HEAD(&cache->slabs, slab, cache_link); - TAILQ_INSERT_HEAD(&cache->free_slabs, slab, cache_free_link); -} - -static bool -fully_formatted(struct slab *slab) -{ - return (char *) slab->brk + slab->cache->item_size >= (char *)slab + SLAB_SIZE; -} - -void -slab_validate(void) -{ - struct slab *slab; - - SLIST_FOREACH(slab, &arena.slabs, link) { - for (char *p = (char *)slab + sizeof(struct slab); - p + slab->cache->item_size < (char *)slab + SLAB_SIZE; - p += slab->cache->item_size + sizeof(red_zone)) { - assert(memcmp(p + slab->cache->item_size, red_zone, sizeof(red_zone)) == 0); - } - } -} - -static struct slab_cache * -cache_for(size_t size) -{ - for (uint32_t i = 0; i < slab_active_caches; i++) - if (slab_caches[i].item_size >= size) - return &slab_caches[i]; - - return NULL; -} - -static struct slab * -slab_of(struct slab_cache *cache) -{ - struct slab *slab; - - if (!TAILQ_EMPTY(&cache->free_slabs)) { - slab = TAILQ_FIRST(&cache->free_slabs); - assert(slab->magic == SLAB_MAGIC); - return slab; - } - - if (!SLIST_EMPTY(&arena.free_slabs)) { - slab = SLIST_FIRST(&arena.free_slabs); - assert(slab->magic == SLAB_MAGIC); - SLIST_REMOVE_HEAD(&arena.free_slabs, free_link); - format_slab(cache, slab); - return slab; - } - - if ((slab = (struct slab *) arena_alloc(&arena)) != NULL) { - format_slab(cache, slab); - SLIST_INSERT_HEAD(&arena.slabs, slab, link); - return slab; - } - - return NULL; -} - -#ifndef NDEBUG -static bool -valid_item(struct slab *slab, void *item) -{ - return (char *)item >= (char *)(slab) + sizeof(struct slab) && - (char *)item < (char *)(slab) + sizeof(struct slab) + SLAB_SIZE; -} -#endif - -void -sfree(void *ptr) -{ - struct slab *slab = slab_header(ptr); - struct slab_cache *cache = slab->cache; - struct slab_item *item = (struct slab_item *) ptr; - - if (fully_formatted(slab) && SLIST_EMPTY(&slab->free)) - TAILQ_INSERT_TAIL(&cache->free_slabs, slab, cache_free_link); - - assert(valid_item(slab, item)); - assert(SLIST_EMPTY(&slab->free) || valid_item(slab, SLIST_FIRST(&slab->free))); - - SLIST_INSERT_HEAD(&slab->free, item, next); - slab->used -= cache->item_size + sizeof(red_zone); - slab->items -= 1; - - if (slab->items == 0) { - TAILQ_REMOVE(&cache->free_slabs, slab, cache_free_link); - TAILQ_REMOVE(&cache->slabs, slab, cache_link); - SLIST_INSERT_HEAD(&arena.free_slabs, slab, free_link); - } - - VALGRIND_FREELIKE_BLOCK(item, sizeof(red_zone)); -} - -static void -sfree_batch(void) -{ - if (snapshot_pid) - return; - - ssize_t batch = arena.delayed_free_batch; - - while (--batch >= 0 && !SLIST_EMPTY(&free_delayed)) { - assert(arena.delayed_free_count > 0); - struct slab_item *item = SLIST_FIRST(&free_delayed); - SLIST_REMOVE_HEAD(&free_delayed, next); - arena.delayed_free_count--; - sfree(item); - } -} - -void -sfree_delayed(void *ptr) -{ - if (ptr == NULL) - return; - if (private_arena) - return sfree(ptr); - struct slab_item *item = (struct slab_item *)ptr; - struct slab *slab = slab_header(item); - (void) slab; - assert(valid_item(slab, item)); - SLIST_INSERT_HEAD(&free_delayed, item, next); - arena.delayed_free_count++; -} - -void * -salloc(size_t size, const char *what) -{ - struct slab_cache *cache; - struct slab *slab; - struct slab_item *item; - - sfree_batch(); - - if ((cache = cache_for(size)) == NULL || - (slab = slab_of(cache)) == NULL) { - - tnt_raise(LoggedError, ER_MEMORY_ISSUE, size, - "slab allocator", what); - } - - if (SLIST_EMPTY(&slab->free)) { - assert(valid_item(slab, slab->brk)); - item = (struct slab_item *) slab->brk; - memcpy((char *)item + cache->item_size, red_zone, sizeof(red_zone)); - slab->brk = (char *) slab->brk + cache->item_size + sizeof(red_zone); - } else { - item = SLIST_FIRST(&slab->free); - assert(valid_item(slab, item)); - (void) VALGRIND_MAKE_MEM_DEFINED(item, sizeof(void *)); - SLIST_REMOVE_HEAD(&slab->free, next); - (void) VALGRIND_MAKE_MEM_UNDEFINED(item, sizeof(void *)); - } - - if (fully_formatted(slab) && SLIST_EMPTY(&slab->free)) - TAILQ_REMOVE(&cache->free_slabs, slab, cache_free_link); - - slab->used_real += size + sizeof(red_zone); - slab->alloc_real += cache->item_size + sizeof(red_zone); - slab->used += cache->item_size + sizeof(red_zone); - slab->items += 1; - - VALGRIND_MALLOCLIKE_BLOCK(item, cache->item_size, sizeof(red_zone), 0); - return (void *)item; -} - -size_t -salloc_ptr_to_index(void *ptr) -{ - struct slab *slab = slab_header(ptr); - struct slab_item *item = (struct slab_item *) ptr; - struct slab_cache *clazz = slab->cache; - - (void) item; - assert(valid_item(slab, item)); - - void *brk_start = (char *)CACHEALIGN((char *)slab+sizeof(struct slab)); - ptrdiff_t item_no = ((const char *) ptr - (const char *) brk_start) / clazz->item_size; - assert(item_no >= 0); - - ptrdiff_t slab_no = ((const char *) slab - (const char *) arena.base) / SLAB_SIZE; - assert(slab_no >= 0); - - size_t index = (size_t)slab_no * MAX_SLAB_ITEM_COUNT + (size_t) item_no; - - assert(salloc_ptr_from_index(index) == ptr); - - return index; -} - -void * -salloc_ptr_from_index(size_t index) -{ - size_t slab_no = index / MAX_SLAB_ITEM_COUNT; - size_t item_no = index % MAX_SLAB_ITEM_COUNT; - - struct slab *slab = slab_header( - (void *) ((size_t) arena.base + SLAB_SIZE * slab_no)); - struct slab_cache *clazz = slab->cache; - - void *brk_start = (char *)CACHEALIGN((char *)slab+sizeof(struct slab)); - struct slab_item *item = (struct slab_item *)((char *) brk_start + item_no * clazz->item_size); - assert(valid_item(slab, item)); - - return (void *) item; -} - -/** - * Collect slab allocator statistics. - * - * @param cb - a callback to receive statistic item - * @param astat - a structure to fill with of arena - * @user_data - user's data that will be sent to cb - * - */ -int -salloc_stat(salloc_stat_cb cb, struct slab_arena_stats *astat, void *cb_ctx) -{ - if (astat) { - astat->used = arena.used; - astat->size = arena.size; - } - - if (cb) { - struct slab *slab; - struct slab_cache_stats st; - - for (int i = 0; i < slab_active_caches; i++) { - memset(&st, 0, sizeof(st)); - TAILQ_FOREACH(slab, &slab_caches[i].slabs, cache_link) - { - st.slabs++; - st.items += slab->items; - st.bytes_free += SLAB_SIZE; - st.bytes_free -= slab->used; - st.bytes_free -= sizeof(struct slab); - st.bytes_used += sizeof(struct slab); - st.bytes_used += slab->used; - st.bytes_alloc_real += slab->alloc_real + sizeof(struct slab); - st.bytes_used_real += slab->used_real + sizeof(struct slab); - } - st.item_size = slab_caches[i].item_size; - - if (st.slabs == 0) - continue; - int res = cb(&st, cb_ctx); - if (res != 0) - return res; - } - } - return 0; -} diff --git a/src/salloc.h b/src/salloc.h deleted file mode 100644 index b592594670..0000000000 --- a/src/salloc.h +++ /dev/null @@ -1,96 +0,0 @@ -#ifndef TARANTOOL_SALLOC_H_INCLUDED -#define TARANTOOL_SALLOC_H_INCLUDED -/* - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * 1. Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF - * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#include <stddef.h> -#include <stdbool.h> -#include "trivia/util.h" /* for uint64_t */ - -struct tbuf; - -bool salloc_init(size_t size, size_t minimal, double factor); -void salloc_free(void); -void *salloc(size_t size, const char *what); -void sfree(void *ptr); -void sfree_delayed(void *ptr); -void slab_validate(); -void salloc_protect(void); - -/** Statistics on utilization of a single slab class. */ -struct slab_cache_stats { - int64_t item_size; - int64_t slabs; - int64_t items; - int64_t bytes_used; - int64_t bytes_free; - int64_t bytes_used_real; - int64_t bytes_alloc_real; -}; - -/** Statistics on utilization of the slab allocator. */ -struct slab_arena_stats { - size_t size; - size_t used; -}; - -typedef int (*salloc_stat_cb)(const struct slab_cache_stats *st, void *ctx); - -int -salloc_stat(salloc_stat_cb cb, struct slab_arena_stats *astat, void *cb_ctx); - -/** - * @brief Return an unique index associated with a chunk allocated by salloc. - * This index space is more dense than pointers space, especially in the less - * significant bits. This number is needed because some types of box's indexes - * (e.g. BITSET) have better performance then they operate on sequencial - * offsets (i.e. dense space) instead of memory pointers (sparse space). - * - * The calculation is based on SLAB number and the position of an item within - * it. Current implementation only guarantees that adjacent chunks from one - * SLAB will have consecutive indexes. That is, if two chunks were sequencially - * allocated from one chunk they will have sequencial ids. If a second chunk was - * allocated from another SLAB thеn the difference between indexes may be more - * then one. - * - * @param ptr pointer to memory allocated by salloc - * @return unique index - */ -size_t -salloc_ptr_to_index(void *ptr); - -/** - * @brief Perform the opposite action of a salloc_ptr_to_index. - * @param index unique index - * @see salloc_ptr_to_index - * @return point to memory area associated with \a index. - */ -void * -salloc_ptr_from_index(size_t index); - -#endif /* TARANTOOL_SALLOC_H_INCLUDED */ diff --git a/src/tarantool.cc b/src/tarantool.cc index cae95ccbe0..546a12e39e 100644 --- a/src/tarantool.cc +++ b/src/tarantool.cc @@ -54,7 +54,6 @@ #include "mutex.h" #include <crc32.h> #include "memory.h" -#include <salloc.h> #include <say.h> #include <stat.h> #include <limits.h> @@ -780,10 +779,6 @@ main(int argc, char **argv) replication_prefork(); iobuf_init_readahead(cfg.readahead); coeio_init(); - if (!salloc_init(cfg.slab_alloc_arena * (1 << 30) /* GB */, - cfg.slab_alloc_minimal, cfg.slab_alloc_factor)) - panic("can't initialize slab allocator"); - signal_init(); try { diff --git a/test/wal/oom.result b/test/wal/oom.result index 3aa6c08e13..352277546a 100644 --- a/test/wal/oom.result +++ b/test/wal/oom.result @@ -15,11 +15,11 @@ while true do i = i + 1 end; --- -- error: Failed to allocate 19435 bytes in slab allocator for tuple +- error: Failed to allocate 188 bytes in slab allocator for tuple ... space:len(); --- -- 4853 +- 42 ... i = 1; --- @@ -29,11 +29,11 @@ while true do i = i + 1 end; --- -- error: Failed to allocate 4095 bytes in slab allocator for tuple +- error: Failed to allocate 188 bytes in slab allocator for tuple ... space:len(); --- -- 5871 +- 84 ... i = 1; --- @@ -43,12 +43,12 @@ while true do i = i + 1 end; --- -- error: Failed to allocate 2051 bytes in slab allocator for tuple +- error: Failed to allocate 188 bytes in slab allocator for tuple ... --# setopt delimiter '' space:len() --- -- 6378 +- 126 ... space.index['primary']:select(0) --- @@ -132,14 +132,14 @@ t - [39, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - [40, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - [41, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - - [42, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - - [43, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - - [44, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - - [45, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - - [46, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - - [47, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - - [48, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] - - [49, 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest'] + - [42, 'test'] + - [43, 'testtest'] + - [44, 'testtesttest'] + - [45, 'testtesttesttest'] + - [46, 'testtesttesttesttest'] + - [47, 'testtesttesttesttesttest'] + - [48, 'testtesttesttesttesttesttest'] + - [49, 'testtesttesttesttesttesttesttest'] ... space:truncate() --- -- GitLab