diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
index 2c4f339d58d6e17da6af52126ca0ee9ff28e104d..24c0eabc813a3b74d8e9e60be4700aae5fc9da73 100644
--- a/src/box/memtx_engine.h
+++ b/src/box/memtx_engine.h
@@ -66,7 +66,7 @@ struct MemtxEngine: public Engine {
};
enum {
- MEMTX_EXTENT_SIZE = 16 * 1024,
+ MEMTX_EXTENT_SIZE = 32 * 1024,
MEMTX_SLAB_SIZE = 4 * 1024 * 1024
};
diff --git a/src/lib/salad/bps_tree.h b/src/lib/salad/bps_tree.h
index 887de62bd13928c37e8a3ccfebb49c262c8f04be..762750785cf2206eaa883a29166814c537860b37 100644
--- a/src/lib/salad/bps_tree.h
+++ b/src/lib/salad/bps_tree.h
@@ -980,7 +980,7 @@ bps_tree_build(struct bps_tree *tree, bps_tree_elem_t *sorted_array,
assert(tree->size == 0);
assert(tree->root == 0);
assert(tree->garbage_head == 0);
- assert(tree->matras.block_count == 0);
+ assert(tree->matras.block_counts[0] == 0);
if (array_size == 0)
return true;
bps_tree_block_id_t leaf_count = (array_size +
diff --git a/src/lib/salad/light.h b/src/lib/salad/light.h
index 074f15cafe333e45587b9ab1ecbe20ca6eb00ce3..ca3c22a01e338cafa0773b2a9a8f61466e19c2a7 100644
--- a/src/lib/salad/light.h
+++ b/src/lib/salad/light.h
@@ -439,7 +439,7 @@ LIGHT(prepare_first_insert)(struct LIGHT(core) *ht)
{
assert(ht->count == 0);
assert(ht->table_size == 0);
- assert(ht->mtable.block_count == 0);
+ assert(ht->mtable.block_counts[0] == 0);
uint32_t slot;
struct LIGHT(record) *record = (struct LIGHT(record) *)
@@ -695,7 +695,7 @@ inline int
LIGHT(selfcheck)(const struct LIGHT(core) *ht)
{
int res = 0;
- if (ht->table_size != ht->mtable.block_count)
+ if (ht->table_size != ht->mtable.block_counts[0])
res |= 64;
uint32_t empty_slot = ht->empty_slot;
if (empty_slot == LIGHT(end)) {
diff --git a/src/lib/small/matras.c b/src/lib/small/matras.c
index 09929583788745bbff44b1819878ab25958de91b..ad814205b0c2b05ab37159db941a5973d4009ddc 100644
--- a/src/lib/small/matras.c
+++ b/src/lib/small/matras.c
@@ -4,6 +4,7 @@
#include "matras.h"
#include <limits.h>
+#include <string.h>
#ifdef WIN32
#include <intrin.h>
#pragma intrinsic (_BitScanReverse)
@@ -17,12 +18,12 @@
* approximate (floored) otherwise)
*/
static matras_id_t
-pt_log2(matras_id_t val)
+matras_log2(matras_id_t val)
{
assert(val > 0);
#ifdef WIN32
- unsigned long res = 0;
- unsigned char nonzero = _BitScanReverse(&res, val);
+ unsigned long res = 0;
+ unsigned char nonzero = _BitScanReverse(&res, val);
assert(nonzero); (void)nonzero;
return (matras_id_t)res;
#else
@@ -37,21 +38,28 @@ pt_log2(matras_id_t val)
*/
void
matras_create(struct matras *m, matras_id_t extent_size, matras_id_t block_size,
- prov_alloc_func alloc_func, prov_free_func free_func)
+ matras_alloc_func alloc_func, matras_free_func free_func)
{
/*extent_size must be power of 2 */
assert((extent_size & (extent_size - 1)) == 0);
/*block_size must be power of 2 */
assert((block_size & (block_size - 1)) == 0);
+ /*block must be not greater than the extent*/
+ assert(block_size <= extent_size);
+ /*extent must be able to store at least two records*/
+ assert(extent_size > sizeof(struct matras_record));
- m->extent = 0;
- m->block_count = 0;
+ m->block_counts[0] = 0;
m->extent_size = extent_size;
m->block_size = block_size;
- matras_id_t log1 = pt_log2(extent_size);
- matras_id_t log2 = pt_log2(block_size);
- matras_id_t log3 = pt_log2(sizeof(void *));
+ m->ver_occ_mask = 1;
+
+ matras_id_t log1 = matras_log2(extent_size);
+ matras_id_t log2 = matras_log2(block_size);
+ assert((sizeof(struct matras_record) &
+ (sizeof(struct matras_record) - 1)) == 0);
+ matras_id_t log3 = matras_log2(sizeof(struct matras_record));
m->log2_capacity = log1 * 3 - log2 - log3 * 2;
m->shift1 = log1 * 2 - log2 - log3;
m->shift2 = log1 - log2;
@@ -69,9 +77,14 @@ matras_create(struct matras *m, matras_id_t extent_size, matras_id_t block_size,
void
matras_destroy(struct matras *m)
{
- if (m->block_count) {
- void* extent1 = m->extent;
- matras_id_t id = m->block_count;
+ while(m->ver_occ_mask != 1) {
+ matras_id_t ver = __builtin_ctzl(m->ver_occ_mask ^ 1);
+ matras_delete_version(m, ver);
+ }
+ if (m->block_counts[0]) {
+ struct matras_record *extent1 = m->roots[0].ptr;
+ matras_id_t id = m->block_counts[0];
+ matras_id_t i, j;
matras_id_t n1 = id >> m->shift1;
id &= m->mask1;
@@ -83,30 +96,30 @@ matras_destroy(struct matras *m)
if (id)
n2++;
- void *extent2 = ((void **)extent1)[n1];
- for (matras_id_t j = 0; j < n2; j++) {
- void *extent3 = ((void **)extent2)[j];
+ struct matras_record *extent2 = extent1[n1].ptr;
+ for (j = 0; j < n2; j++) {
+ struct matras_record *extent3 = extent2[j].ptr;
m->free_func(extent3);
}
m->free_func(extent2);
}
/* free fully loaded extents */
- matras_id_t n2 = m->extent_size / sizeof(void *);
- for (matras_id_t i = 0; i < n1; i++) {
- void *extent2 = ((void **)extent1)[i];
- for (matras_id_t j = 0; j < n2; j++) {
- void *extent3 = ((void **)extent2)[j];
+ matras_id_t n2 = m->extent_size / sizeof(struct matras_record);
+ for ( i = 0; i < n1; i++) {
+ struct matras_record *extent2 = extent1[i].ptr;
+ for (j = 0; j < n2; j++) {
+ struct matras_record *extent3 = extent2[j].ptr;
m->free_func(extent3);
}
m->free_func(extent2);
}
m->free_func(extent1);
- m->block_count = 0;
+ m->block_counts[0] = 0;
}
#ifndef __OPTIMIZE__
- m->extent = (void *)(long long)0xDEADBEEF;
+ m->roots[0].ptr = (struct matras_record *)(void *)(long long)0xDEADBEEF;
#endif
}
@@ -119,8 +132,7 @@ void
matras_reset(struct matras *m)
{
matras_destroy(m);
- m->extent = 0;
- m->block_count = 0;
+ m->block_counts[0] = 0;
}
@@ -133,8 +145,8 @@ matras_reset(struct matras *m)
void *
matras_alloc(struct matras *m, matras_id_t *result_id)
{
- if (m->block_count)
- assert(pt_log2(m->block_count) < m->log2_capacity);
+ if (m->block_counts[0])
+ assert(matras_log2(m->block_counts[0]) < m->log2_capacity);
/* Current block_count is the ID of new block */
/* See "Shifts and masks explanation" for details */
@@ -147,7 +159,7 @@ matras_alloc(struct matras *m, matras_id_t *result_id)
* (n1 == 0 && n2 == 0 && n3 == 0) is identical to (id == 0)
* (n2 == 0 && n3 == 0) is identical to (id ^ mask1 == 0)
*/
- matras_id_t id = m->block_count;
+ matras_id_t id = m->block_counts[0];
matras_id_t extent1_not_empty = id;
matras_id_t n1 = id >> m->shift1;
id &= m->mask1;
@@ -157,33 +169,35 @@ matras_alloc(struct matras *m, matras_id_t *result_id)
matras_id_t extent3_not_empty = id;
matras_id_t n3 = id;
- void *extent1, *extent2, *extent3;
+ struct matras_record *extent1, *extent2, *extent3;
if (extent1_not_empty) {
- extent1 = m->extent;
+ extent1 = m->roots[0].ptr;
} else {
- extent1 = m->alloc_func();
+ extent1 = (struct matras_record *)m->alloc_func();
if (!extent1)
return 0;
- m->extent = extent1;
+ m->roots[0].ptr = extent1;
+ m->roots[0].tag = ~(m->ver_occ_mask ^ 1);
}
if (extent2_not_empty) {
- extent2 = ((void **)extent1)[n1];
+ extent2 = extent1[n1].ptr;
} else {
- extent2 = m->alloc_func();
+ extent2 = (struct matras_record *)m->alloc_func();
if (!extent2) {
if (!extent1_not_empty) /* means - was empty */
m->free_func(extent1);
return 0;
}
- ((void **)extent1)[n1] = extent2;
+ extent1[n1].ptr = extent2;
+ extent1[n1].tag = ~(m->ver_occ_mask ^ 1);
}
if (extent3_not_empty) {
- extent3 = ((void **)extent2)[n2];
+ extent3 = extent2[n2].ptr;
} else {
- extent3 = m->alloc_func();
+ extent3 = (struct matras_record *)m->alloc_func();
if (!extent3) {
if (!extent1_not_empty) /* means - was empty */
m->free_func(extent1);
@@ -191,11 +205,12 @@ matras_alloc(struct matras *m, matras_id_t *result_id)
m->free_func(extent2);
return 0;
}
- ((void **)extent2)[n2] = extent3;
+ extent2[n2].ptr = extent3;
+ extent2[n2].tag = ~(m->ver_occ_mask ^ 1);
}
- *result_id = m->block_count++;
- return (void*)((char*)extent3 + n3 * m->block_size);
+ *result_id = m->block_counts[0]++;
+ return (void *)((char*)extent3 + n3 * m->block_size);
}
/*
@@ -204,14 +219,16 @@ matras_alloc(struct matras *m, matras_id_t *result_id)
void
matras_dealloc(struct matras *m)
{
- assert(m->block_count);
- m->block_count--;
+ assert(m->block_counts[0]);
+ matras_id_t last = m->block_counts[0] - 1;
+ matras_before_change(m, last);
+ m->block_counts[0] = last;
/* Current block_count is the ID of deleting block */
/* See "Shifts and masks explanation" for details */
/* Deleting extents in same way (but reverse order) like in matras_alloc
* See matras_alloc for details. */
- matras_id_t id = m->block_count;
+ matras_id_t id = m->block_counts[0];
matras_id_t extent1_free = !id;
matras_id_t n1 = id >> m->shift1;
id &= m->mask1;
@@ -221,10 +238,10 @@ matras_dealloc(struct matras *m)
matras_id_t extent3_free = !id;
if (extent1_free || extent2_free || extent3_free) {
- void *extent1, *extent2, *extent3;
- extent1 = m->extent;
- extent2 = ((void **)extent1)[n1];
- extent3 = ((void **)extent2)[n2];
+ struct matras_record *extent1, *extent2, *extent3;
+ extent1 = m->roots[0].ptr;
+ extent2 = extent1[n1].ptr;
+ extent3 = extent2[n2].ptr;
if (extent3_free)
m->free_func(extent3);
if (extent2_free)
@@ -246,11 +263,11 @@ matras_dealloc(struct matras *m)
void *
matras_alloc_range(struct matras *m, matras_id_t *id, matras_id_t range_count)
{
- assert(m->block_count % range_count == 0);
+ assert(m->block_counts[0] % range_count == 0);
assert(m->extent_size / m->block_size % range_count == 0);
void *res = matras_alloc(m, id);
if (res)
- m->block_count += (range_count - 1);
+ m->block_counts[0] += (range_count - 1);
return res;
}
@@ -263,9 +280,9 @@ matras_alloc_range(struct matras *m, matras_id_t *id, matras_id_t range_count)
void
matras_dealloc_range(struct matras *m, matras_id_t range_count)
{
- assert(m->block_count % range_count == 0);
+ assert(m->block_counts[0] % range_count == 0);
assert(m->extent_size / m->block_size % range_count == 0);
- m->block_count -= (range_count - 1);
+ m->block_counts[0] -= (range_count - 1);
matras_dealloc(m);
}
@@ -279,17 +296,369 @@ matras_extents_count(const struct matras *m)
* Let's calculate extents count level by level, starting from leafs
* Last level of the tree consists of extents that stores blocks,
* so we can calculate number of extents by block count: */
- matras_id_t c = (m->block_count + (m->extent_size / m->block_size - 1))
+ matras_id_t c = (m->block_counts[0] + (m->extent_size / m->block_size - 1))
/ (m->extent_size / m->block_size);
matras_id_t res = c;
/* two upper levels consist of extents that stores pointers to extents,
* so we can calculate number of extents by lower level extent count:*/
- for (matras_id_t i = 0; i < 2; i++) {
- c = (c + (m->extent_size / sizeof(void *) - 1))
- / (m->extent_size / sizeof(void *));
+ matras_id_t i;
+ for (i = 0; i < 2; i++) {
+ c = (c + (m->extent_size / sizeof(struct matras_record) - 1))
+ / (m->extent_size / sizeof(struct matras_record));
res += c;
}
return res;
}
+
+/*
+ * Create new version of matras memory.
+ * Return 0 if all version IDs are occupied.
+ */
+matras_id_t
+matras_new_version(struct matras *m)
+{
+ matras_id_t ver_id;
+#ifdef WIN32
+ unsigned long res = 0;
+ unsigned char nonzero = _BitScanForward(&res, m->ver_occ_mask);
+ assert(nonzero); (void)nonzero;
+ ver_id = (matras_id_t) res;
+#else
+ ver_id = (matras_id_t)
+ __builtin_ctzl(~m->ver_occ_mask);
+#endif
+ assert(ver_id > 0);
+ if (ver_id >= MATRAS_VERSION_COUNT)
+ return 0;
+ m->ver_occ_mask |= ((matras_version_tag_t)1) << ver_id;
+ m->roots[ver_id] = m->roots[0];
+ m->block_counts[ver_id] = m->block_counts[0];
+ return ver_id;
+}
+
+/*
+ * Delete memory version by specified ID.
+ */
+void
+matras_delete_version(struct matras *m, matras_id_t ver_id)
+{
+ matras_version_tag_t me = ((matras_version_tag_t)1) << ver_id;
+ if (m->block_counts[ver_id]) {
+ matras_id_t step = m->mask2 + 1, j;
+ for (j = 0; j < m->block_counts[ver_id]; j += step) {
+ matras_id_t n1 = j >> m->shift1;
+ matras_id_t n2 = (j & m->mask1) >> m->shift2;
+ matras_version_tag_t owners =
+ m->roots[ver_id].ptr[n1].ptr[n2].tag &
+ m->ver_occ_mask;
+ assert(owners & me);
+ if (owners == me) {
+ m->free_func(m->roots[ver_id].ptr[n1].ptr[n2].ptr);
+ m->roots[ver_id].ptr[n1].ptr[n2].tag ^= me;
+#ifndef __OPTIMIZE__
+ m->roots[ver_id].ptr[n1].ptr[n2].ptr =
+ (struct matras_record *)(void *)
+ (long long)0xDEADBEEF;
+#endif
+ } else {
+ matras_version_tag_t run = owners;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << oth_ver;
+ assert((m->roots[oth_ver].ptr[n1].ptr[n2].tag &
+ m->ver_occ_mask & ~me) ==
+ (owners & ~me));
+ assert(m->roots[oth_ver].ptr[n1].ptr[n2].ptr ==
+ m->roots[ver_id].ptr[n1].ptr[n2].ptr);
+ m->roots[oth_ver].ptr[n1].ptr[n2].tag &= ~me;
+ } while (run);
+ }
+ }
+ step = m->mask1 + 1;
+ for (j = 0; j < m->block_counts[ver_id]; j += step) {
+ matras_id_t n1 = j >> m->shift1;
+ matras_version_tag_t owners =
+ m->roots[ver_id].ptr[n1].tag & m->ver_occ_mask;
+ assert(owners & me);
+ if (owners == me) {
+ m->free_func(m->roots[ver_id].ptr[n1].ptr);
+ m->roots[ver_id].ptr[n1].tag ^= me;
+#ifndef __OPTIMIZE__
+ m->roots[ver_id].ptr[n1].ptr =
+ (struct matras_record *)(void *)
+ (long long)0xDEADBEEF;
+#endif
+ } else {
+ matras_version_tag_t run = owners;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << oth_ver;
+ assert((m->roots[oth_ver].ptr[n1].tag &
+ m->ver_occ_mask & ~me) ==
+ (owners & ~me));
+ assert(m->roots[oth_ver].ptr[n1].ptr ==
+ m->roots[ver_id].ptr[n1].ptr);
+ m->roots[oth_ver].ptr[n1].tag &= ~me;
+ } while (run);
+ }
+ }
+ matras_version_tag_t owners
+ = m->roots[ver_id].tag & m->ver_occ_mask;
+ assert(owners & me);
+ if (owners == me) {
+ m->free_func(m->roots[ver_id].ptr);
+ m->roots[ver_id].tag ^= me;
+#ifndef __OPTIMIZE__
+ m->roots[ver_id].ptr =
+ (struct matras_record *)(void *)
+ (long long)0xDEADBEEF;
+#endif
+ } else {
+ matras_version_tag_t run = owners;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << oth_ver;
+ assert((m->roots[oth_ver].tag &
+ m->ver_occ_mask & ~me) ==
+ (owners & ~me));
+ assert(m->roots[oth_ver].ptr ==
+ m->roots[ver_id].ptr);
+ m->roots[oth_ver].tag &= ~me;
+ } while (run);
+ }
+ m->block_counts[ver_id] = 0;
+ }
+ if (m->block_counts[0]) {
+ matras_id_t step = m->mask2 + 1, j;
+ for (j = 0; j < m->block_counts[0]; j += step) {
+ matras_id_t n1 = j >> m->shift1;
+ matras_id_t n2 = (j & m->mask1) >> m->shift2;
+ matras_version_tag_t run =
+ m->roots[0].ptr[n1].ptr[n2].tag &
+ m->ver_occ_mask;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << oth_ver;
+ m->roots[oth_ver].ptr[n1].ptr[n2].tag |= me;
+ } while (run);
+ }
+ step = m->mask1 + 1;
+ for (j = 0; j < m->block_counts[0]; j += step) {
+ matras_id_t n1 = j >> m->shift1;
+ matras_version_tag_t run =
+ m->roots[0].ptr[n1].tag & m->ver_occ_mask;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << oth_ver;
+ m->roots[oth_ver].ptr[n1].tag |= me;
+ } while (run);
+ }
+ matras_version_tag_t run =
+ m->roots[0].tag & m->ver_occ_mask;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << oth_ver;
+ m->roots[oth_ver].tag |= me;
+ } while (run);
+ }
+ m->ver_occ_mask ^= me;
+}
+
+/*
+ * Notify matras that memory at given ID will be changed.
+ * Returns true if ok, and false if failed to allocate memory.
+ */
+void *
+matras_before_change(struct matras *m, matras_id_t id)
+{
+ assert(id < m->block_counts[0]);
+
+ /* see "Shifts and masks explanation" for details */
+ matras_id_t n1 = id >> m->shift1;
+ matras_id_t n2 = (id & m->mask1) >> m->shift2;
+ matras_id_t n3 = id & m->mask2;
+
+ struct matras_record *l1 = &m->roots[0];
+ struct matras_record *l2 = &l1->ptr[n1];
+ struct matras_record *l3 = &l2->ptr[n2];
+ matras_version_tag_t owner_mask3 = l3->tag & m->ver_occ_mask;
+ assert(owner_mask3 & 1);
+ if (owner_mask3 == 1)
+ return &((char *)(l3->ptr))[n3 * m->block_size]; /* private page */
+
+ struct matras_record *new_extent3 =
+ (struct matras_record *)m->alloc_func();
+ if (!new_extent3)
+ return 0;
+
+ matras_version_tag_t owner_mask1 = l1->tag & m->ver_occ_mask;
+ assert(owner_mask1 & 1);
+ matras_version_tag_t owner_mask2 = l2->tag & m->ver_occ_mask;
+ assert(owner_mask2 & 1);
+ struct matras_record *new_extent1 = 0, *new_extent2 = 0;
+ if (owner_mask1 != 1) {
+ new_extent1 = (struct matras_record *)m->alloc_func();
+ if (!new_extent1) {
+ m->free_func(new_extent3);
+ return 0;
+ }
+ }
+ if (owner_mask2 != 1) {
+ new_extent2 = (struct matras_record *)m->alloc_func();
+ if (!new_extent2) {
+ m->free_func(new_extent3);
+ if (owner_mask1 != 1)
+ m->free_func(new_extent1);
+ return 0;
+ }
+ }
+ matras_version_tag_t new_tag = ~(m->ver_occ_mask ^ 1);
+
+ if (owner_mask1 != 1) {
+ memcpy(new_extent1, l1->ptr, m->extent_size);
+ l1->tag = new_tag;
+ l1->ptr = new_extent1;
+ matras_version_tag_t run = owner_mask1 ^ 1;
+ matras_version_tag_t oth_tag = run & m->ver_occ_mask;
+ do {
+ uint32_t ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << ver;
+ m->roots[ver].tag = oth_tag;
+ } while (run);
+ l2 = &l1->ptr[n1];
+ }
+
+ if (owner_mask2 != 1) {
+ memcpy(new_extent2, l2->ptr, m->extent_size);
+ l2->tag = new_tag;
+ l2->ptr = new_extent2;
+ matras_version_tag_t run = owner_mask2 ^ 1;
+ matras_version_tag_t oth_tag = run & m->ver_occ_mask;
+ do {
+ uint32_t ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << ver;
+ m->roots[ver].ptr[n1].tag = oth_tag;
+ } while (run);
+ l3 = &l2->ptr[n2];
+ }
+
+ memcpy(new_extent3, l3->ptr, m->extent_size);
+ l3->tag = new_tag;
+ l3->ptr = new_extent3;
+ matras_version_tag_t run = owner_mask3 ^ 1;
+ matras_version_tag_t oth_tag = run & m->ver_occ_mask;
+ do {
+ uint32_t ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t)1) << ver;
+ m->roots[ver].ptr[n1].ptr[n2].tag = oth_tag;
+ } while (run);
+
+ return &((char *)new_extent3)[n3 * m->block_size]; ;
+}
+
+/*
+ * Debug check that ensures internal consistency.
+ * Must return 0. If i returns not 0, smth is terribly wrong.
+ */
+matras_version_tag_t
+matras_debug_selfcheck(const struct matras *m)
+{
+ matras_version_tag_t res = 0, i;
+ for (i = 0; i < MATRAS_VERSION_COUNT; i++) {
+ matras_version_tag_t me = ((matras_version_tag_t) 1) << i;
+ if (!(m->ver_occ_mask & me))
+ continue;
+ matras_id_t step = m->mask2 + 1, j;
+ for (j = 0; j < m->block_counts[i]; j += step) {
+ matras_id_t n1 = j >> m->shift1;
+ matras_id_t n2 = (j & m->mask1) >> m->shift2;
+ if (!(m->roots[i].tag & me))
+ res |= (1 | (me << 12));
+ if ((m->roots[i].tag & m->ver_occ_mask) != me) {
+ matras_version_tag_t run = m->roots[i].tag
+ & m->ver_occ_mask;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t) 1)
+ << oth_ver;
+ if (m->roots[i].tag
+ != m->roots[oth_ver].tag)
+ res |= (8 | (me << 12));
+ } while (run);
+ }
+ if (!(m->roots[i].ptr[n1].tag & me))
+ res |= (2 | (me << 12));
+ if ((m->roots[i].ptr[n1].tag & m->ver_occ_mask) != me) {
+ matras_version_tag_t run =
+ m->roots[i].ptr[n1].tag
+ & m->ver_occ_mask;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t) 1)
+ << oth_ver;
+ if (m->roots[i].ptr[n1].tag
+ != m->roots[oth_ver].ptr[n1].tag)
+ res |= (0x80 | (me << 12));
+ } while (run);
+ }
+ if (!(m->roots[i].ptr[n1].ptr[n2].tag & me))
+ res |= (4 | (me << 12));
+ if ((m->roots[i].ptr[n1].ptr[n2].tag & m->ver_occ_mask)
+ != me) {
+ matras_version_tag_t run =
+ m->roots[i].ptr[n1].ptr[n2].tag
+ & m->ver_occ_mask;
+ do {
+ uint32_t oth_ver = __builtin_ctzl(run);
+ run ^= ((matras_version_tag_t) 1)
+ << oth_ver;
+ if (m->roots[i].ptr[n1].ptr[n2].tag
+ != m->roots[oth_ver].ptr[n1].ptr[n2].tag)
+ res |= (0x800 | (me << 12));
+ } while (run);
+ }
+ }
+ }
+ {
+ i = 0;
+ matras_version_tag_t me = ((matras_version_tag_t) 1) << i;
+ matras_id_t step = m->mask2 + 1, j;
+ for (j = 0; j < m->block_counts[i]; j += step) {
+ matras_id_t n1 = j >> m->shift1;
+ matras_id_t n2 = (j & m->mask1) >> m->shift2;
+ if ((m->roots[i].tag & ~m->ver_occ_mask)
+ != ~m->ver_occ_mask)
+ res |= (0x10 | (me << 12));
+ if ((m->roots[i].ptr[n1].tag & ~m->ver_occ_mask)
+ != ~m->ver_occ_mask)
+ res |= (0x20 | (me << 12));
+ if ((m->roots[i].ptr[n1].ptr[n2].tag & ~m->ver_occ_mask)
+ != ~m->ver_occ_mask)
+ res |= (0x40 | (me << 12));
+ }
+ }
+ for (i = 1; i < MATRAS_VERSION_COUNT; i++) {
+ matras_version_tag_t me = ((matras_version_tag_t) 1) << i;
+ if (!(m->ver_occ_mask & me))
+ continue;
+ matras_id_t step = m->mask2 + 1, j;
+ for (j = 0; j < m->block_counts[i]; j += step) {
+ matras_id_t n1 = j >> m->shift1;
+ matras_id_t n2 = (j & m->mask1) >> m->shift2;
+ if (!(m->roots[i].tag & 1))
+ if ((m->roots[i].tag & ~m->ver_occ_mask) != 0)
+ res |= (0x100 | (me << 12));
+ if (!(m->roots[i].ptr[n1].tag & 1))
+ if ((m->roots[i].ptr[n1].tag & ~m->ver_occ_mask)
+ != 0)
+ res |= (0x200 | (me << 12));
+ if (!(m->roots[i].ptr[n1].ptr[n2].tag & 1))
+ if ((m->roots[i].ptr[n1].ptr[n2].tag
+ & ~m->ver_occ_mask) != 0)
+ res |= (0x400 | (me << 12));
+ }
+ }
+ return res;
+}
diff --git a/src/lib/small/matras.h b/src/lib/small/matras.h
index 24595995a65ded6134ead750b8c537477337ee74..40c24f01aeda2623faba816acab22203549844b7 100644
--- a/src/lib/small/matras.h
+++ b/src/lib/small/matras.h
@@ -59,7 +59,7 @@
* level 1 extent
* - second N2 bits - level 1 extent id - stores the address of
* the extent which contains actual blocks
- * - remaining N3 bits - block number in level 1 extent
+ * - remaining N3 bits - block number in level 2 extent
* (Actual values of N1 and N2 are a function of block size B).
* Calculation of N1, N2 and N3 depends on sizes of blocks,
* extents and sizeof(void *).
@@ -70,7 +70,7 @@
* instance:
*
* 1) can provide not more than
- * pow(M / sizeof(void*), L - 1) * (M / N)
+ * pow(M / sizeof(void*) / 2, L - 1) * (M / N)
* blocks
* 2) costs (L - 1) random memory accesses to provide a new block
* or restore a block pointer from block id
@@ -79,6 +79,19 @@
* Of course, the integer type used for block id (matras_id_t,
* usually is a typedef to uint32) also limits the maximum number
* of objects that can be block_count by an instance of matras.
+ *
+ * Additionally matras provides freezing all allocated data. At
+ * any moment user can call matras_new_version method for achieving
+ * unique ID of current data state. Then the user could change,
+ * allocate and dealocate main matras data, meanwhile getting data
+ * from previusly freezed version. This works through copy-on-write
+ * mechanism, thus is cheap enough, and of course this works only
+ * if user correctly notifies mastras about chainging data.
+ * An impotant property of implemented copy-on-write mechnism - is
+ * that main data does not moved, i.e. before data modification the
+ * extent is copied to another location to become the extent
+ * for freezed version. Thus main block address with some particular
+ * block ID is unchanged.
*/
/* }}} */
@@ -90,6 +103,7 @@ typedef uint32_t matras_id_t;
#endif
#include <assert.h>
+#include <stdbool.h>
#if defined(__cplusplus)
extern "C" {
@@ -104,8 +118,27 @@ extern "C" {
* of size M). Is allowed to return NULL, but is not allowed
* to throw an exception
*/
-typedef void *(*prov_alloc_func)();
-typedef void (*prov_free_func)(void *);
+typedef void *(*matras_alloc_func)();
+typedef void (*matras_free_func)(void *);
+
+typedef uint32_t matras_version_tag_t;
+
+struct matras_record {
+ /* pointer to next level of a tree */
+ union {
+ struct matras_record *ptr;
+ matras_version_tag_t ptr_padded;
+ };
+ /* version tag - bitmask of all version referencing ptr above */
+ union {
+ matras_version_tag_t tag;
+ void *tag_padded;
+ };
+};
+
+enum {
+ MATRAS_VERSION_COUNT = 8
+};
/**
* matras - memory allocator of blocks of equal
@@ -113,13 +146,15 @@ typedef void (*prov_free_func)(void *);
*/
struct matras {
/* Pointer to the root extent of matras */
- void *extent;
+ struct matras_record roots[MATRAS_VERSION_COUNT];
+ /* A number of already allocated blocks */
+ matras_id_t block_counts[MATRAS_VERSION_COUNT];
+ /* Bit mask of used versions */
+ matras_version_tag_t ver_occ_mask;
/* Block size (N) */
matras_id_t block_size;
/* Extent size (M) */
matras_id_t extent_size;
- /* A number of already allocated blocks */
- matras_id_t block_count;
/* binary logarithm of maximum possible created blocks count */
matras_id_t log2_capacity;
/* See "Shifts and masks explanation" below */
@@ -127,9 +162,9 @@ struct matras {
/* See "Shifts and masks explanation" below */
matras_id_t mask1, mask2;
/* External extent allocator */
- prov_alloc_func alloc_func;
+ matras_alloc_func alloc_func;
/* External extent deallocator */
- prov_free_func free_func;
+ matras_free_func free_func;
};
/*
@@ -160,7 +195,7 @@ struct matras {
*/
void
matras_create(struct matras *m, matras_id_t extent_size, matras_id_t block_size,
- prov_alloc_func alloc_func, prov_free_func free_func);
+ matras_alloc_func alloc_func, matras_free_func free_func);
/**
* Free all memory used by an instance of matras and
@@ -217,12 +252,45 @@ matras_dealloc_range(struct matras *m, matras_id_t range_count);
static void *
matras_get(const struct matras *m, matras_id_t id);
+/**
+ * Convert block id of a specified version into block address.
+ */
+static void *
+matras_getv(const struct matras *m, matras_id_t id, matras_id_t version);
+
/*
* Getting number of allocated extents (of size extent_size each)
*/
matras_id_t
matras_extents_count(const struct matras *m);
+/*
+ * Create new version of matras memory.
+ * Return 0 if all version IDs are occupied.
+ */
+matras_id_t
+matras_new_version(struct matras *m);
+
+/*
+ * Delete memory version by specified ID.
+ */
+void
+matras_delete_version(struct matras *m, matras_id_t ver_id);
+
+/*
+ * Notify matras that memory at given ID will be changed.
+ * Returns true if ok, and false if failed to allocate memory.
+ * Only needed (and does any work) if some versions are used.
+ */
+void *
+matras_before_change(struct matras *m, matras_id_t id);
+
+/*
+ * Debug check that ensures internal consistency.
+ * Must return 0. If i returns not 0, smth is terribly wrong.
+ */
+matras_version_tag_t
+matras_debug_selfcheck(const struct matras *m);
/**
* matras_get implementation
@@ -230,14 +298,32 @@ matras_extents_count(const struct matras *m);
static inline void *
matras_get(const struct matras *m, matras_id_t id)
{
- assert(id < m->block_count);
+ assert(id < m->block_counts[0]);
+
+ /* see "Shifts and masks explanation" for details */
+ matras_id_t n1 = id >> m->shift1;
+ matras_id_t n2 = (id & m->mask1) >> m->shift2;
+ matras_id_t n3 = (id & m->mask2);
+
+ char *extent = (char *)m->roots[0].ptr[n1].ptr[n2].ptr;
+ return &extent[n3 * m->block_size];
+}
+
+/**
+ * matras_getv implementation
+ */
+static inline void *
+matras_getv(const struct matras *m, matras_id_t id, matras_id_t version)
+{
+ assert(id < m->block_counts[version]);
/* see "Shifts and masks explanation" for details */
matras_id_t n1 = id >> m->shift1;
matras_id_t n2 = (id & m->mask1) >> m->shift2;
matras_id_t n3 = (id & m->mask2);
- return (((char***)m->extent)[n1][n2] + n3 * m->block_size);
+ char *extent = (char *)m->roots[version].ptr[n1].ptr[n2].ptr;
+ return &extent[n3 * m->block_size];
}
#if defined(__cplusplus)
diff --git a/test/box/errinj_index.result b/test/box/errinj_index.result
index a7536ba883370eb02af855c927d52f2beb417491..028d76a46973214bcdcf0078c0e9fdbd0afa3f35 100644
--- a/test/box/errinj_index.result
+++ b/test/box/errinj_index.result
@@ -91,9 +91,9 @@ res
- [9, 9, 'test9']
- [10, 10, 'test10']
...
-for i = 501,2500 do s:insert{i, i} end
+for i = 501,5000 do s:insert{i, i} end
---
-- error: Failed to allocate 16384 bytes in MemtxTree for replace
+- error: Failed to allocate 32768 bytes in MemtxTree for replace
...
s:delete{1}
---
@@ -139,24 +139,24 @@ res
res = {}
---
...
-for i = 2001,2010 do table.insert(res, (s:get{i})) end
+for i = 4001,4010 do table.insert(res, (s:get{i})) end
---
...
res
---
- []
...
---count must be greater that 1000 but less than 2000
-function check_iter_and_size() local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end return count <= 1000 and "fail 1" or count >= 2000 and "fail 2" or "ok" end
+--count must be greater that 2000 but less than 4000
+function check_iter_and_size() local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end return count <= 2000 and "fail 1" or count >= 4000 and "fail 2" or "ok" end
---
...
check_iter_and_size()
---
- ok
...
-for i = 2501,3500 do s:insert{i, i} end
+for i = 5001,6000 do s:insert{i, i} end
---
-- error: Failed to allocate 16384 bytes in MemtxTree for replace
+- error: Failed to allocate 32768 bytes in MemtxTree for replace
...
s:delete{2}
---
@@ -183,9 +183,9 @@ res
- [9, 9, 'test9']
- [10, 10, 'test10']
...
-for i = 3501,4500 do s:insert{i, i} end
+for i = 6001,7000 do s:insert{i, i} end
---
-- error: Failed to allocate 16384 bytes in MemtxTree for replace
+- error: Failed to allocate 32768 bytes in MemtxTree for replace
...
s:delete{3}
---
@@ -199,7 +199,7 @@ errinj.set("ERRINJ_INDEX_ALLOC", false)
---
- ok
...
-for i = 4501,5500 do s:insert{i, i} end
+for i = 7001,8000 do s:insert{i, i} end
---
...
res = {}
@@ -240,21 +240,21 @@ res
res = {}
---
...
-for i = 5001,5010 do table.insert(res, (s:get{i})) end
+for i = 7501,7510 do table.insert(res, (s:get{i})) end
---
...
res
---
-- - [5001, 5001]
- - [5002, 5002]
- - [5003, 5003]
- - [5004, 5004]
- - [5005, 5005]
- - [5006, 5006]
- - [5007, 5007]
- - [5008, 5008]
- - [5009, 5009]
- - [5010, 5010]
+- - [7501, 7501]
+ - [7502, 7502]
+ - [7503, 7503]
+ - [7504, 7504]
+ - [7505, 7505]
+ - [7506, 7506]
+ - [7507, 7507]
+ - [7508, 7508]
+ - [7509, 7509]
+ - [7510, 7510]
...
s:drop()
---
diff --git a/test/box/errinj_index.test.lua b/test/box/errinj_index.test.lua
index 05155f261e6c16f7831c0e53ba20f9270f89c5f9..845cc2ff6dc9b3d485b95e9fcd665a065c10dcc8 100644
--- a/test/box/errinj_index.test.lua
+++ b/test/box/errinj_index.test.lua
@@ -22,7 +22,7 @@ res = {}
for _, t in s.index[0]:pairs() do table.insert(res, t) end
res
-for i = 501,2500 do s:insert{i, i} end
+for i = 501,5000 do s:insert{i, i} end
s:delete{1}
res = {}
@@ -32,27 +32,27 @@ res = {}
for i = 501,510 do table.insert(res, (s:get{i})) end
res
res = {}
-for i = 2001,2010 do table.insert(res, (s:get{i})) end
+for i = 4001,4010 do table.insert(res, (s:get{i})) end
res
---count must be greater that 1000 but less than 2000
-function check_iter_and_size() local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end return count <= 1000 and "fail 1" or count >= 2000 and "fail 2" or "ok" end
+--count must be greater that 2000 but less than 4000
+function check_iter_and_size() local count = 0 for _, t in s.index[0]:pairs() do count = count + 1 end return count <= 2000 and "fail 1" or count >= 4000 and "fail 2" or "ok" end
check_iter_and_size()
-for i = 2501,3500 do s:insert{i, i} end
+for i = 5001,6000 do s:insert{i, i} end
s:delete{2}
check_iter_and_size()
res = {}
for i = 1,10 do table.insert(res, (s:get{i})) end
res
-for i = 3501,4500 do s:insert{i, i} end
+for i = 6001,7000 do s:insert{i, i} end
s:delete{3}
check_iter_and_size()
errinj.set("ERRINJ_INDEX_ALLOC", false)
-for i = 4501,5500 do s:insert{i, i} end
+for i = 7001,8000 do s:insert{i, i} end
res = {}
for i = 1,10 do table.insert(res, (s:get{i})) end
res
@@ -61,7 +61,7 @@ res = {}
for i = 1,10 do table.insert(res, (s:get{i})) end
res
res = {}
-for i = 5001,5010 do table.insert(res, (s:get{i})) end
+for i = 7501,7510 do table.insert(res, (s:get{i})) end
res
s:drop()
diff --git a/test/unit/matras.cc b/test/unit/matras.cc
index 4970cb7a1de98f39dbbb62efeebca09b08157048..04f26b17f64a3be183be59a0abc39e503c428897 100644
--- a/test/unit/matras.cc
+++ b/test/unit/matras.cc
@@ -64,8 +64,8 @@ void matras_alloc_test()
{
std::cout << "Testing matras_alloc..." << std::endl;
unsigned int maxCapacity = PROV_EXTENT_SIZE / PROV_BLOCK_SIZE;
- maxCapacity *= PROV_EXTENT_SIZE / sizeof(void *);
- maxCapacity *= PROV_EXTENT_SIZE / sizeof(void *);
+ maxCapacity *= PROV_EXTENT_SIZE / sizeof(struct matras_record);
+ maxCapacity *= PROV_EXTENT_SIZE / sizeof(struct matras_record);
struct matras pta;
@@ -136,10 +136,10 @@ void matras_alloc_test()
for (unsigned int j = 0; j < maxCapacity; j++) {
unsigned int res = 0;
- unsigned int prev_block_count = pta.block_count;
+ unsigned int prev_block_count = pta.block_counts[0];
void *data = matras_alloc(&pta, &res);
if (!data) {
- check(prev_block_count == pta.block_count, "Created count changed during memory fail!");
+ check(prev_block_count == pta.block_counts[0], "Created count changed during memory fail!");
break;
}
}
@@ -150,9 +150,102 @@ void matras_alloc_test()
std::cout << "Testing matras_alloc successfully finished" << std::endl;
}
+typedef uint64_t type_t;
+const size_t VER_EXTENT_SIZE = 512;
+int extents_in_use = 0;
+
+void *all()
+{
+ extents_in_use++;
+ return malloc(VER_EXTENT_SIZE);
+}
+
+void dea(void *p)
+{
+ extents_in_use--;
+ free(p);
+}
+void
+matras_vers_test()
+{
+ std::cout << "Testing matras versions..." << std::endl;
+
+ std::vector<type_t> comps[MATRAS_VERSION_COUNT];
+ int use_mask = 1;
+ int cur_num_or_ver = 1;
+ struct matras local;
+ matras_create(&local, VER_EXTENT_SIZE, sizeof(type_t), all, dea);
+ type_t val = 0;
+ for (int s = 10; s < 8000; s = int(s * 1.5)) {
+ for (int k = 0; k < 800; k++) {
+ bool check_me = false;
+ if (rand() % 16 == 0) {
+ bool add_ver;
+ if (cur_num_or_ver == 1)
+ add_ver = true;
+ else if (cur_num_or_ver == MATRAS_VERSION_COUNT)
+ add_ver = false;
+ else
+ add_ver = rand() % 2 == 0;
+ if (add_ver) {
+ cur_num_or_ver++;
+ int new_ver = matras_new_version(&local);
+ use_mask |= (1 << new_ver);
+ comps[new_ver] = comps[0];
+ } else {
+ cur_num_or_ver--;
+ int del_ver;
+ do {
+ del_ver = 1 + rand() % (MATRAS_VERSION_COUNT - 1);
+ } while ((use_mask & (1 << del_ver)) == 0);
+ matras_delete_version(&local, del_ver);
+ comps[del_ver].clear();
+ use_mask &= ~(1 << del_ver);
+ }
+ check_me = true;
+ } else {
+ check_me = rand() % 16 == 0;
+ if (rand() % 8 == 0 && comps[0].size() > 0) {
+ matras_dealloc(&local);
+ comps[0].pop_back();
+ }
+ int p = rand() % s;
+ int mod = 0;
+ while (p >= comps[0].size()) {
+ comps[0].push_back(val * 10000 + mod);
+ matras_id_t tmp;
+ type_t *ptrval = (type_t *)matras_alloc(&local, &tmp);
+ *ptrval = val * 10000 + mod;
+ mod++;
+ }
+ val++;
+ comps[0][p] = val;
+ matras_before_change(&local, p);
+ *(type_t *)matras_get(&local, p) = val;
+ }
+ int checkres = matras_debug_selfcheck(&local);
+ check(checkres == 0, "internal check failed");
+ for (int i = 0; i < MATRAS_VERSION_COUNT; i++) {
+ if ((use_mask & (1 << i)) == 0)
+ continue;
+ check(comps[i].size() == local.block_counts[i], "size mismatch");
+ for (int j = 0; j < comps[i].size(); j++) {
+ type_t val1 = comps[i][j];
+ type_t val2 = *(type_t *)matras_getv(&local, j, i);
+ check(val1 == val2, "data mismatch");
+ }
+ }
+ }
+ }
+ matras_destroy(&local);
+ check(extents_in_use == 0, "memory leak");
+
+ std::cout << "Testing matras_version successfully finished" << std::endl;
+}
int
main(int, const char **)
{
matras_alloc_test();
+ matras_vers_test();
}
diff --git a/test/unit/matras.result b/test/unit/matras.result
index 862dd816368ed437298f25c2c42179a8a8e4d806..cf05e6434aa0ffc246291803110d2c01cf5886b3 100644
--- a/test/unit/matras.result
+++ b/test/unit/matras.result
@@ -1,2 +1,4 @@
Testing matras_alloc...
Testing matras_alloc successfully finished
+Testing matras versions...
+Testing matras_version successfully finished