diff --git a/src/lib/small/mempool.c b/src/lib/small/mempool.c index 905e05d41180416811b26e4ee5948149f68de1b8..91c990c1461172fa92d7f23e28361547e8b8000e 100644 --- a/src/lib/small/mempool.c +++ b/src/lib/small/mempool.c @@ -46,9 +46,10 @@ rb_gen(, mslab_tree_, mslab_tree_t, struct mslab, node, mslab_cmp) static inline void mslab_create(struct mslab *slab, struct mempool *pool) { - slab->ffi = 0; slab->nfree = pool->objcount; slab->pool = pool; + slab->untouched_offset = 0; + slab->freed_list = 0; /* A bit is set if a slot is free. */ memset(slab->map, 0xFF, sizeof(slab->map[0]) * pool->mapsize); } @@ -84,43 +85,47 @@ void * mslab_alloc(struct mslab *slab) { assert(slab->nfree); - uint32_t idx = __builtin_ffsl(slab->map[slab->ffi]); - while (idx == 0) { - if (slab->ffi == slab->pool->mapsize - 1) { - /* - * mslab_alloc() shouldn't be called - * on a full slab. - */ - assert(false); - return NULL; - } - slab->ffi++; - idx = __builtin_ffsl(slab->map[slab->ffi]); + void *result; + uint32_t idx; + if (slab->freed_list) { + /* return pointer from garbage */ + idx = mslab_idx(slab, slab->freed_list); + result = slab->freed_list; + slab->freed_list = *(void **)slab->freed_list; + } else { + /* return pointer to new object */ + idx = slab->untouched_offset++; + result = mslab_obj(slab, idx); } - /* - * find-first-set returns bit index starting from 1, - * or 0 if no bit is set. Rebase the index to offset 0. - */ - idx--; + /* Mark the position as occupied. */ - slab->map[slab->ffi] ^= ((mbitmap_t) 1) << idx; + const uint32_t slot = idx / MEMPOOL_MAP_BIT; + const uint32_t bit_no = idx & (MEMPOOL_MAP_BIT-1); + slab->map[slot] ^= ((mbitmap_t) 1) << (mbitmap_t) bit_no; + /* If the slab is full, remove it from the rb tree. */ if (--slab->nfree == 0) mslab_tree_remove(&slab->pool->free_slabs, slab); - /* Return the pointer at the free slot */ - return mslab_obj(slab, idx + slab->ffi * MEMPOOL_MAP_BIT); + + return result; } void mslab_free(struct mempool *pool, struct mslab *slab, void *ptr) { + /* put object to garbage list */ + *(void **)ptr = slab->freed_list; + slab->freed_list = ptr; + uint32_t idx = mslab_idx(slab, ptr); - uint32_t bit_no = idx & (MEMPOOL_MAP_BIT-1); - idx /= MEMPOOL_MAP_BIT; - slab->map[idx] |= ((mbitmap_t) 1) << bit_no; + + /* Mark the position as free. */ + const uint32_t slot = idx / MEMPOOL_MAP_BIT; + const uint32_t bit_no = idx & (MEMPOOL_MAP_BIT-1); + slab->map[slot] |= ((mbitmap_t) 1) << bit_no; + slab->nfree++; - if (idx < slab->ffi) - slab->ffi = idx; + if (slab->nfree == 1) { /** * Add this slab to the rbtree which contains partially diff --git a/src/lib/small/mempool.h b/src/lib/small/mempool.h index 55fa00a0c319e57df3044c9252518a0df521dbbe..d8bb0125016e1d4003bf8d7cb818efd33ec1cae5 100644 --- a/src/lib/small/mempool.h +++ b/src/lib/small/mempool.h @@ -93,8 +93,10 @@ enum { /** mslab - a standard slab formatted to store objects of equal size. */ struct mslab { struct slab slab; - /** Index of the first bitmap element which has a free slot. */ - uint32_t ffi; + /* Pointer to earlier freed list head */ + void *freed_list; + /** Offset of an object that was never allocated in mslab */ + uint32_t untouched_offset; /** Number of available slots in the slab. */ uint32_t nfree; /** Used if this slab is a member of free_slabs tree. */ diff --git a/test/unit/mempool.c b/test/unit/mempool.c index eb84b15404fc4fb219dc90c78051554494fb97f7..aef5fcb5a14439032cd6da5692a12ee66aae2562 100644 --- a/test/unit/mempool.c +++ b/test/unit/mempool.c @@ -45,6 +45,7 @@ alloc_checked() if (ptrs[pos]) { assert(ptrs[pos][0] == pos); free_checked(ptrs[pos]); + ptrs[pos] = 0; } if (! allocating) return NULL;