diff --git a/src/box/hash_index.cc b/src/box/hash_index.cc index 26ba6c75486ff0fda2b66784ebff526b6b9d1a72..adab4e214d9d1904640ed8703585dd11569dd6bd 100644 --- a/src/box/hash_index.cc +++ b/src/box/hash_index.cc @@ -147,7 +147,7 @@ key_hash(const char *key, const struct key_def *key_def) return PMurHash32_Result(h, carry, total_size); } -#define LIGHT_NAME index_ +#define LIGHT_NAME _index #define LIGHT_DATA_TYPE struct tuple * #define LIGHT_KEY_TYPE const char * #define LIGHT_CMP_ARG_TYPE struct key_def * @@ -161,7 +161,7 @@ typedef uint32_t hash_t; struct hash_iterator { struct iterator base; /* Must be the first member. */ - struct index_light *hash_table; + struct light_index_core *hash_table; uint32_t h_pos; }; @@ -178,11 +178,11 @@ hash_iterator_ge(struct iterator *ptr) assert(ptr->free == hash_iterator_free); struct hash_iterator *it = (struct hash_iterator *) ptr; - if (it->h_pos < it->hash_table->table_size * 5) { - struct tuple *res = index_light_get(it->hash_table, it->h_pos); + if (it->h_pos < it->hash_table->table_size) { + struct tuple *res = light_index_get(it->hash_table, it->h_pos); it->h_pos++; - while (it->h_pos < it->hash_table->table_size * 5 - && !index_light_pos_valid(it->hash_table, it->h_pos)) + while (it->h_pos < it->hash_table->table_size + && !light_index_pos_valid(it->hash_table, it->h_pos)) it->h_pos++; return res; } @@ -235,18 +235,18 @@ HashIndex::HashIndex(struct key_def *key_def) HASH_INDEX_EXTENT_SIZE); index_arena_initialized = true; } - hash_table = new struct index_light; + hash_table = new struct light_index_core; if (hash_table == NULL) { tnt_raise(ClientError, ER_MEMORY_ISSUE, sizeof(hash_table), "HashIndex", "hash_table"); } - index_light_create(hash_table, HASH_INDEX_EXTENT_SIZE, + light_index_create(hash_table, HASH_INDEX_EXTENT_SIZE, extent_alloc, extent_free, this->key_def); } HashIndex::~HashIndex() { - index_light_destroy(hash_table); + light_index_destroy(hash_table); delete hash_table; } @@ -273,12 +273,12 @@ HashIndex::random(uint32_t rnd) const { if (hash_table->count == 0) return NULL; - rnd %= (hash_table->table_size * 5); - while (!index_light_pos_valid(hash_table, rnd)) { + rnd %= (hash_table->table_size); + while (!light_index_pos_valid(hash_table, rnd)) { rnd++; - rnd %= (hash_table->table_size * 5); + rnd %= (hash_table->table_size); } - return index_light_get(hash_table, rnd); + return light_index_get(hash_table, rnd); } struct tuple * @@ -289,9 +289,9 @@ HashIndex::findByKey(const char *key, uint32_t part_count) const struct tuple *ret = NULL; uint32_t h = key_hash(key, key_def); - uint32_t k = index_light_find_key(hash_table, h, key); - if (k != index_light_end) - ret = index_light_get(hash_table, k); + uint32_t k = light_index_find_key(hash_table, h, key); + if (k != light_index_end) + ret = light_index_get(hash_table, k); return ret; } @@ -304,27 +304,27 @@ HashIndex::replace(struct tuple *old_tuple, struct tuple *new_tuple, if (new_tuple) { uint32_t h = tuple_hash(new_tuple, key_def); struct tuple *dup_tuple = NULL; - hash_t pos = index_light_replace(hash_table, h, new_tuple, &dup_tuple); - if (pos == index_light_end) - pos = index_light_insert(hash_table, h, new_tuple); + hash_t pos = light_index_replace(hash_table, h, new_tuple, &dup_tuple); + if (pos == light_index_end) + pos = light_index_insert(hash_table, h, new_tuple); ERROR_INJECT(ERRINJ_INDEX_ALLOC, { - index_light_delete(hash_table, pos); - pos = index_light_end; + light_index_delete(hash_table, pos); + pos = light_index_end; }); - if (pos == index_light_end) { + if (pos == light_index_end) { tnt_raise(LoggedError, ER_MEMORY_ISSUE, (ssize_t) hash_table->count, "hash_table", "key"); } errcode = replace_check_dup(old_tuple, dup_tuple, mode); if (errcode) { - index_light_delete(hash_table, pos); + light_index_delete(hash_table, pos); if (dup_tuple) { - uint32_t pos = index_light_insert(hash_table, h, dup_tuple); - if (pos == index_light_end) { + uint32_t pos = light_index_insert(hash_table, h, dup_tuple); + if (pos == light_index_end) { panic("Failed to allocate memory in " "recover of int hash_table"); } @@ -338,9 +338,9 @@ HashIndex::replace(struct tuple *old_tuple, struct tuple *new_tuple, if (old_tuple) { uint32_t h = tuple_hash(old_tuple, key_def); - hash_t slot = index_light_find(hash_table, h, old_tuple); - assert(slot != index_light_end); - index_light_delete(hash_table, slot); + hash_t slot = light_index_find(hash_table, h, old_tuple); + assert(slot != light_index_end); + light_index_delete(hash_table, slot); } return old_tuple; } @@ -375,14 +375,14 @@ HashIndex::initIterator(struct iterator *ptr, enum iterator_type type, switch (type) { case ITER_ALL: it->h_pos = 0; - while (it->h_pos < it->hash_table->table_size * 5 - && !index_light_pos_valid(it->hash_table, it->h_pos)) + while (it->h_pos < it->hash_table->table_size + && !light_index_pos_valid(it->hash_table, it->h_pos)) it->h_pos++; it->base.next = hash_iterator_ge; break; case ITER_EQ: assert(part_count > 0); - it->h_pos = index_light_find_key(hash_table, key_hash(key, key_def), key); + it->h_pos = light_index_find_key(hash_table, key_hash(key, key_def), key); it->base.next = hash_iterator_eq; break; default: diff --git a/src/box/hash_index.h b/src/box/hash_index.h index d3253c45e1d0a0f32635d8d8fc4492194fde5b95..639d08f31715d036b0bb75af3de7e30067ae92a7 100644 --- a/src/box/hash_index.h +++ b/src/box/hash_index.h @@ -31,7 +31,7 @@ #include "index.h" -struct index_light; +struct light_index_core; class HashIndex: public Index { public: @@ -53,7 +53,7 @@ class HashIndex: public Index { virtual size_t memsize() const; protected: - struct index_light *hash_table; + struct light_index_core *hash_table; }; #endif /* TARANTOOL_BOX_HASH_INDEX_H_INCLUDED */ diff --git a/src/lib/salad/light.h b/src/lib/salad/light.h index 08cef2c054ad7a957a4be2f6d14a2e418a39a9fc..bb418ea47d18fdd4397ee8ebb2f0a60c577b338d 100644 --- a/src/lib/salad/light.h +++ b/src/lib/salad/light.h @@ -34,13 +34,15 @@ #include "small/matras.h" /** - * light - Linear probing Incremental Growing Hash Table - */ - - -/** - * Prefix for all names of struct and functions in this header file. - * Mat be empty, but still have to be defined + * Additional user defined name that appended to prefix 'light' + * for all names of structs and functions in this header file. + * All names use pattern: light<LIGHT_NAME>_<name of func/struct> + * May be empty, but still have to be defined (just #define LIGHT_NAME) + * Example: + * #define LIGHT_NAME _test + * ... + * struct light_test_core hash_table; + * light_test_create(&hash_table, ...); */ #ifndef LIGHT_NAME #error "LIGHT_NAME must be defined" @@ -91,41 +93,68 @@ /** * Tools for name substitution: */ -#ifndef CONCAT -#define CONCAT_R(a, b) a##b -#define CONCAT(a, b) CONCAT_R(a, b) +#ifndef CONCAT4 +#define CONCAT4_R(a, b, c, d) a##b##c##d +#define CONCAT4(a, b, c, d) CONCAT4_R(a, b, c, d) #endif #ifdef _ #error '_' must be undefinded! #endif -#define LH(name) CONCAT(LIGHT_NAME, name) +#define LIGHT(name) CONCAT4(light, LIGHT_NAME, _, name) + +/** + * Struct for one record of the hash table + */ +struct LIGHT(record) { + /* hash of a value */ + uint32_t hash; + /* slot of the next record in chain */ + uint32_t next; + /* the value */ + LIGHT_DATA_TYPE value; +}; /** * Main struct for holding hash table */ -struct LH(light) { +struct LIGHT(core) { + /* Number of records added while grow iteration */ + enum { GROW_INCREMENT = 8 }; /* count of values in hash table */ uint32_t count; - /* size of hash table in clusters ( equal to mtable.size ) */ + /* size of hash table ( equal to mtable.size ) */ uint32_t table_size; /* * cover is power of two; - * if table_size is positive, then cover/2 < table_size <= cover + * if count is positive, then cover/2 < count <= cover * cover_mask is cover - 1 */ uint32_t cover_mask; + + /* + * Start of chain of empty slots + */ + uint32_t empty_slot; + struct LIGHT(record) *empty_record; + /* additional parameter for data comparison */ LIGHT_CMP_ARG_TYPE arg; - /* dynamic storage for clusters */ + + /* dynamic storage for records */ struct matras mtable; }; /** * Type of functions for memory allocation and deallocation */ -typedef void *(*LH(light_extent_alloc_t))(); -typedef void (*LH(light_extent_free_t))(void *); +typedef void *(*LIGHT(extent_alloc_t))(); +typedef void (*LIGHT(extent_free_t))(void *); + +/** + * Special result of light_find that means that nothing was found + */ +static const uint32_t LIGHT(end) = 0xFFFFFFFF; /** * @brief Hash table construction. Fills struct light members. @@ -136,18 +165,17 @@ typedef void (*LH(light_extent_free_t))(void *); * @param arg - optional parameter to save for comparing function */ void -LH(light_create)(struct LH(light) *ht, size_t extent_size, - LH(light_extent_alloc_t) extent_alloc_func, - LH(light_extent_free_t) extent_free_func, - LIGHT_CMP_ARG_TYPE arg); - +LIGHT(create)(struct LIGHT(core) *ht, size_t extent_size, + LIGHT(extent_alloc_t) extent_alloc_func, + LIGHT(extent_free_t) extent_free_func, + LIGHT_CMP_ARG_TYPE arg); /** * @brief Hash table destruction. Frees all allocated memory * @param ht - pointer to a hash table struct */ void -LH(light_destroy)(struct LH(light) *ht); +LIGHT(destroy)(struct LIGHT(core) *ht); /** * @brief Find a record with given hash and value @@ -157,7 +185,7 @@ LH(light_destroy)(struct LH(light) *ht); * @return integer ID of found record or light_end if nothing found */ uint32_t -LH(light_find)(const struct LH(light) *ht, uint32_t hash, LIGHT_DATA_TYPE data); +LIGHT(find)(const struct LIGHT(core) *ht, uint32_t hash, LIGHT_DATA_TYPE data); /** * @brief Find a record with given hash and key @@ -167,7 +195,7 @@ LH(light_find)(const struct LH(light) *ht, uint32_t hash, LIGHT_DATA_TYPE data); * @return integer ID of found record or light_end if nothing found */ uint32_t -LH(light_find_key)(const struct LH(light) *ht, uint32_t hash, LIGHT_KEY_TYPE data); +LIGHT(find_key)(const struct LIGHT(core) *ht, uint32_t hash, LIGHT_KEY_TYPE data); /** * @brief Insert a record with given hash and value @@ -177,7 +205,7 @@ LH(light_find_key)(const struct LH(light) *ht, uint32_t hash, LIGHT_KEY_TYPE dat * @return integer ID of inserted record or light_end if failed */ uint32_t -LH(light_insert)(struct LH(light) *ht, uint32_t hash, LIGHT_DATA_TYPE data); +LIGHT(insert)(struct LIGHT(core) *ht, uint32_t hash, LIGHT_DATA_TYPE data); /** * @brief Replace a record with given hash and value @@ -188,15 +216,16 @@ LH(light_insert)(struct LH(light) *ht, uint32_t hash, LIGHT_DATA_TYPE data); * @return integer ID of found record or light_end if nothing found */ uint32_t -LH(light_replace)(struct LH(light) *ht, uint32_t hash, LIGHT_DATA_TYPE data, LIGHT_DATA_TYPE *replaced); +LIGHT(replace)(struct LIGHT(core) *ht, uint32_t hash, + LIGHT_DATA_TYPE data, LIGHT_DATA_TYPE *replaced); /** * @brief Delete a record from a hash table by given record ID * @param ht - pointer to a hash table struct - * @param slotpos - ID of an record. See light_find for details. + * @param slotpos - ID of an record. See LIGHT(find) for details. */ void -LH(light_delete)(struct LH(light) *ht, uint32_t slotpos); +LIGHT(delete)(struct LIGHT(core) *ht, uint32_t slotpos); /** * @brief Get a value from a desired position @@ -204,7 +233,7 @@ LH(light_delete)(struct LH(light) *ht, uint32_t slotpos); * @param slotpos - ID of an record */ LIGHT_DATA_TYPE -LH(light_get)(struct LH(light) *ht, uint32_t slotpos); +LIGHT(get)(struct LIGHT(core) *ht, uint32_t slotpos); /** * @brief Determine if posision holds a value @@ -212,54 +241,39 @@ LH(light_get)(struct LH(light) *ht, uint32_t slotpos); * @param slotpos - ID of an record */ bool -LH(light_pos_valid)(struct LH(light) *ht, uint32_t slotpos); +LIGHT(pos_valid)(struct LIGHT(core) *ht, uint32_t slotpos); -/** - * Internal definitions - */ -enum { LIGHT_CLUSTER_SIZE = 64 }; -static const uint32_t LH(light_end) = 0xFFFFFFFFu; -struct LH(light_cluster) { - uint32_t flags; - uint32_t hash[5]; - LIGHT_DATA_TYPE data[5]; -}; -/** - * Light implementation - */ inline void -LH(light_create)(struct LH(light) *ht, size_t extent_size, - LH(light_extent_alloc_t) extent_alloc_func, - LH(light_extent_free_t) extent_free_func, - LIGHT_CMP_ARG_TYPE arg) +LIGHT(create)(struct LIGHT(core) *ht, size_t extent_size, + LIGHT(extent_alloc_t) extent_alloc_func, + LIGHT(extent_free_t) extent_free_func, + LIGHT_CMP_ARG_TYPE arg) { - /* could be less than LIGHT_CLUSTER_SIZE on 32bit system*/ - assert(sizeof(struct LH(light_cluster)) <= LIGHT_CLUSTER_SIZE); - /* builtins (__builtin_ctz for example) must take uint32_t */ - assert(sizeof(unsigned) == sizeof(uint32_t)); + assert((ht->GROW_INCREMENT & (ht->GROW_INCREMENT - 1)) == 0); + assert(sizeof(LIGHT_DATA_TYPE) >= sizeof(uint32_t)); ht->count = 0; ht->table_size = 0; - ht->cover_mask = 0; + ht->empty_slot = LIGHT(end); + ht->empty_record = 0; ht->arg = arg; matras_create(&ht->mtable, - extent_size, LIGHT_CLUSTER_SIZE, + extent_size, sizeof(struct LIGHT(record)), extent_alloc_func, extent_free_func); } inline void -LH(light_destroy)(struct LH(light) *ht) +LIGHT(destroy)(struct LIGHT(core) *ht) { matras_destroy(&ht->mtable); } inline uint32_t -LH(light_slot)(const struct LH(light) *ht, uint32_t hash) +LIGHT(slot)(const struct LIGHT(core) *ht, uint32_t hash) { - uint32_t high_hash = hash / 5u; uint32_t cover_mask = ht->cover_mask; - uint32_t res = high_hash & cover_mask; + uint32_t res = hash & cover_mask; uint32_t probe = (ht->table_size - res - 1) >> 31; uint32_t shift = __builtin_ctz(~(cover_mask >> 1)); res ^= (probe << shift); @@ -267,261 +281,392 @@ LH(light_slot)(const struct LH(light) *ht, uint32_t hash) } -inline struct LH(light_cluster) * -LH(light_cluster)(const struct LH(light) *ht, uint32_t slot) +inline uint32_t +LIGHT(find)(const struct LIGHT(core) *ht, uint32_t hash, LIGHT_DATA_TYPE value) { - return (struct LH(light_cluster) *) + if (ht->count == 0) + return LIGHT(end); + uint32_t slot = LIGHT(slot)(ht, hash); + struct LIGHT(record) *record = (struct LIGHT(record) *) matras_get(&ht->mtable, slot); + if (record->next == slot) + return LIGHT(end); + while (1) { + if (record->hash == hash && LIGHT_EQUAL((record->value), (value), (ht->arg))) + return slot; + slot = record->next; + if (slot == LIGHT(end)) + return LIGHT(end); + record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); + } + /* unreachable */ + return LIGHT(end); } inline uint32_t -LH(light_find_key)(const struct LH(light) *ht, uint32_t hash, LIGHT_KEY_TYPE data) +LIGHT(replace)(struct LIGHT(core) *ht, uint32_t hash, LIGHT_DATA_TYPE value, LIGHT_DATA_TYPE *replaced) { - if (ht->table_size == 0) - return LH(light_end); - uint32_t slot = LH(light_slot)(ht, hash); + if (ht->count == 0) + return LIGHT(end); + uint32_t slot = LIGHT(slot)(ht, hash); + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); + if (record->next == slot) + return LIGHT(end); while (1) { - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, slot); - __builtin_prefetch(cluster, 0); - uint32_t search_mask = ~(((hash & 017) | 060) * 01010101); - uint32_t mask = cluster->flags & 03737373737; - uint32_t found_mask = ((mask ^ search_mask) + 0101010101) & 04040404040; - while (found_mask) { - uint32_t bit = __builtin_ctz(found_mask); - found_mask ^= (1u << bit); - uint32_t pos = bit / 6; - if (cluster->hash[pos] == hash && - (LIGHT_EQUAL_KEY((cluster->data[pos]), (data), (ht->arg)))) - return slot * 5 + pos; + if (record->hash == hash && LIGHT_EQUAL((record->value), (value), (ht->arg))) { + *replaced = record->value; + record->value = value; + return slot; } - - if (!(cluster->flags >> 31)) - return LH(light_end); - slot++; - if (slot >= ht->table_size) - slot = 0; + slot = record->next; + if (slot == LIGHT(end)) + return LIGHT(end); + record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); } /* unreachable */ - return LH(light_end); + return LIGHT(end); } + + inline uint32_t -LH(light_find)(const struct LH(light) *ht, uint32_t hash, LIGHT_DATA_TYPE data) +LIGHT(find_key)(const struct LIGHT(core) *ht, uint32_t hash, LIGHT_KEY_TYPE key) { - if (ht->table_size == 0) - return LH(light_end); - uint32_t slot = LH(light_slot)(ht, hash); + if (ht->count == 0) + return LIGHT(end); + uint32_t slot = LIGHT(slot)(ht, hash); + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); + if (record->next == slot) + return LIGHT(end); while (1) { - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, slot); - __builtin_prefetch(cluster, 0); - uint32_t search_mask = ~(((hash & 017) | 060) * 01010101); - uint32_t mask = cluster->flags & 03737373737; - uint32_t found_mask = ((mask ^ search_mask) + 0101010101) & 04040404040; - while (found_mask) { - uint32_t bit = __builtin_ctz(found_mask); - found_mask ^= (1u << bit); - uint32_t pos = bit / 6; - if (cluster->hash[pos] == hash && - (LIGHT_EQUAL((cluster->data[pos]), (data), (ht->arg)))) - return slot * 5 + pos; - } - - if (!(cluster->flags >> 31)) - return LH(light_end); - slot++; - if (slot >= ht->table_size) - slot = 0; + if (record->hash == hash && LIGHT_EQUAL_KEY((record->value), (key), (ht->arg))) + return slot; + slot = record->next; + if (slot == LIGHT(end)) + return LIGHT(end); + record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); } /* unreachable */ - return LH(light_end); + return LIGHT(end); } inline uint32_t -LH(light_replace)(struct LH(light) *ht, uint32_t hash, LIGHT_DATA_TYPE data, LIGHT_DATA_TYPE *replaced) +LIGHT(get_empty_prev)(struct LIGHT(record) *record) { - if (ht->table_size == 0) - return LH(light_end); - uint32_t slot = LH(light_slot)(ht, hash); - while (1) { - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, slot); - __builtin_prefetch(cluster, 0); - uint32_t search_mask = ~(((hash & 017) | 060) * 01010101); - uint32_t mask = cluster->flags & 03737373737; - uint32_t found_mask = ((mask ^ search_mask) + 0101010101) & 04040404040; - while (found_mask) { - uint32_t bit = __builtin_ctz(found_mask); - found_mask ^= (1u << bit); - uint32_t pos = bit / 6; - if (cluster->hash[pos] == hash && - (LIGHT_EQUAL((cluster->data[pos]), (data), (ht->arg)))) { - *replaced = cluster->data[pos]; - cluster->data[pos] = data; - return slot * 5 + pos; - } - } + return record->hash; +} - if (!(cluster->flags >> 31)) - return LH(light_end); - slot++; - if (slot >= ht->table_size) - slot = 0; - } - /* unreachable */ - return LH(light_end); +inline void +LIGHT(set_empty_prev)(struct LIGHT(record) *record, uint32_t pos) +{ + record->hash = pos; +} + +inline uint32_t +LIGHT(get_empty_next)(struct LIGHT(record) *record) +{ + return (uint32_t)(uint64_t)record->value; } inline void -LH(light_set_value)(struct LH(light_cluster) *cluster, uint32_t pos, - uint32_t hash_flags, uint32_t hash, LIGHT_DATA_TYPE data) +LIGHT(set_empty_next)(struct LIGHT(record) *record, uint32_t pos) { - uint32_t shift = pos * 6; - cluster->flags |= (((hash & 017) | hash_flags) << shift); - cluster->hash[pos] = hash; - cluster->data[pos] = data; + record->value = (LIGHT_DATA_TYPE)(int64_t)pos; } inline void -LH(light_clr_value)(struct LH(light_cluster) *cluster, uint32_t pos) +LIGHT(enqueue_empty)(struct LIGHT(core) *ht, uint32_t slot, struct LIGHT(record) *record) { - uint32_t shift = pos * 6; - cluster->flags &= ~(077 << shift); + record->next = slot; + if (ht->empty_record) + LIGHT(set_empty_prev)(ht->empty_record, slot); + LIGHT(set_empty_prev)(record, LIGHT(end)); + LIGHT(set_empty_next)(record, ht->empty_slot); + ht->empty_record = record; + ht->empty_slot = slot; } inline void -LH(light_grow)(struct LH(light) *ht) +LIGHT(detach_first_empty)(struct LIGHT(core) *ht) { - uint32_t to_flags = 0; - if (ht->table_size > 1) { - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, ht->table_size - 1); - to_flags = cluster->flags & 0x80000000; + assert(ht->empty_record); + ht->empty_slot = LIGHT(get_empty_next)(ht->empty_record); + if (ht->empty_slot == LIGHT(end)) { + ht->empty_record = 0; + } else { + ht->empty_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, ht->empty_slot); + LIGHT(set_empty_prev)(ht->empty_record, LIGHT(end)); } - uint32_t to_slot; - struct LH(light_cluster) *to_cluster = (struct LH(light_cluster) *) - matras_alloc(&ht->mtable, &to_slot); - __builtin_prefetch(to_cluster, 1); - if (ht->cover_mask < ht->table_size) - ht->cover_mask = (ht->cover_mask << 1) | 1; - ht->table_size++; - hash_t split_slot = to_slot & (ht->cover_mask >> 1); - struct LH(light_cluster) *split_cluster = LH(light_cluster)(ht, split_slot); - __builtin_prefetch(split_cluster, 1); - hash_t split_diff_shift = __builtin_ctz(~(ht->cover_mask >> 1)); - hash_t mask = 0; - for (int i = 0; i < 5; i++) { - uint32_t match = (split_cluster->hash[i] / 5) >> split_diff_shift; - uint32_t chain = split_cluster->flags >> (i * 6 + 5); - mask |= (match & (~chain) & 1) << (i * 6); +} + +inline void +LIGHT(detach_empty)(struct LIGHT(core) *ht, struct LIGHT(record) *record) +{ + uint32_t prev_slot = LIGHT(get_empty_prev)(record); + uint32_t next_slot = LIGHT(get_empty_next)(record); + if (prev_slot == LIGHT(end)) { + ht->empty_slot = next_slot; + if (next_slot == LIGHT(end)) + ht->empty_record = 0; + else + ht->empty_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, next_slot); + } else { + struct LIGHT(record) *prev_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, prev_slot); + LIGHT(set_empty_next)(prev_record, next_slot); } - mask *= 077; - - *to_cluster = *split_cluster; - split_cluster->flags &= ~mask; - to_cluster->flags &= mask; - to_cluster->flags |= to_flags; - - uint32_t dst_slot = to_slot; - uint32_t hash_flags = 020; - while (split_cluster->flags >> 31) { - split_slot++; - if (split_slot == dst_slot) - break; - split_cluster = LH(light_cluster)(ht, split_slot); - uint32_t test = (split_cluster->flags & 02020202020) & ((split_cluster->flags & 04040404040) >> 1) ; - while (test) { - uint32_t bit = __builtin_ctz(test); - test ^= (1u << bit); - uint32_t pos = bit / 6; - if (LH(light_slot)(ht, split_cluster->hash[pos]) == dst_slot) { - LH(light_clr_value)(split_cluster, pos); - - uint32_t slot = split_slot; - struct LH(light_cluster) *cluster = split_cluster; - while (!(cluster->flags & 024040404040)) { - if (slot == 0) - slot = ht->table_size; - slot--; - cluster = LH(light_cluster)(ht, slot); - if (!(cluster->flags >> 31)) - break; - cluster->flags &= 0x7FFFFFFF; - } + if (next_slot != LIGHT(end)) { + struct LIGHT(record) *next_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, next_slot); + LIGHT(set_empty_prev)(next_record, prev_slot); + } +} + +inline bool +LIGHT(prepare_first_insert)(struct LIGHT(core) *ht) +{ + assert(ht->count == 0); + assert(ht->table_size == 0); + assert(ht->mtable.block_count == 0); + + uint32_t slot; + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_alloc_range(&ht->mtable, &slot, ht->GROW_INCREMENT); + if (!record) + return false; + assert(slot == 0); + ht->table_size = ht->GROW_INCREMENT; + ht->cover_mask = ht->GROW_INCREMENT - 1; + ht->empty_slot = 0; + ht->empty_record = record; + for (int i = 0; i < ht->GROW_INCREMENT; i++) { + record[i].next = i; + LIGHT(set_empty_prev)(record + i, i - 1); + LIGHT(set_empty_next)(record + i, i + 1); + } + LIGHT(set_empty_prev)(record, LIGHT(end)); + LIGHT(set_empty_next)(record + ht->GROW_INCREMENT - 1, LIGHT(end)); + return true; +} + +inline bool +LIGHT(grow)(struct LIGHT(core) *ht) +{ + uint32_t new_slot; + struct LIGHT(record) *new_record = (struct LIGHT(record) *) + matras_alloc_range(&ht->mtable, &new_slot, ht->GROW_INCREMENT); + if (!new_record) /* memory failure */ + return false; + ht->table_size += ht->GROW_INCREMENT; + + if (ht->cover_mask < ht->table_size - 1) + ht->cover_mask = (ht->cover_mask << 1) | (uint32_t)1; + + uint32_t split_comm_mask = (ht->cover_mask >> 1); + uint32_t split_diff_mask = ht->cover_mask ^ split_comm_mask; + + uint32_t susp_slot = new_slot & split_comm_mask; + struct LIGHT(record) *susp_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, susp_slot); + + for (int i = 0; i < ht->GROW_INCREMENT; i++, susp_slot++, susp_record++, new_slot++, new_record++) { + if (susp_record->next == susp_slot) { + /* Suspicious slot is empty, nothing to split */ + LIGHT(enqueue_empty)(ht, new_slot, new_record); + continue; + } + if ((susp_record->hash & split_comm_mask) != susp_slot) { + /* Another chain in suspicious slot, nothing to split */ + LIGHT(enqueue_empty)(ht, new_slot, new_record); + continue; + } - while ((to_cluster->flags & 02020202020) == 02020202020) { - to_cluster->flags |= (1 << 31); - hash_flags = 060; - to_slot++; - if (to_slot >= ht->table_size) - to_slot = 0; - to_cluster = (struct LH(light_cluster) *) - matras_get(&ht->mtable, to_slot); + uint32_t chain_head_slot[2] = {susp_slot, new_slot}; + struct LIGHT(record) *chain_head[2] = {susp_record, new_record}; + struct LIGHT(record) *chain_tail[2] = {0, 0}; + uint32_t shift = __builtin_ctz(split_diff_mask); + assert(split_diff_mask == (((uint32_t)1) << shift)); + + uint32_t last_empty_slot = new_slot; + struct LIGHT(record) *last_empty_record = new_record; + uint32_t prev_flag = 0; + struct LIGHT(record) *test_record = susp_record; + uint32_t test_slot = susp_slot; + struct LIGHT(record) *prev_record = 0; + while (1) { + uint32_t test_flag = (test_record->hash >> shift) & ((uint32_t)1); + if (test_flag != prev_flag) { + chain_tail[prev_flag] = prev_record; + if (chain_tail[test_flag]) { + chain_tail[test_flag]->next = test_slot; + } else { + *chain_head[test_flag] = *test_record; + last_empty_slot = test_slot; + last_empty_record = test_record; + test_record = chain_head[test_flag]; + test_slot = chain_head_slot[test_flag]; } - uint32_t to_pos = __builtin_ctz(((~to_cluster->flags) & 02020202020)) / 6; - LH(light_set_value)(to_cluster, to_pos, hash_flags, split_cluster->hash[pos], split_cluster->data[pos]); + prev_flag = test_flag; } + test_slot = test_record->next; + if (test_slot == LIGHT(end)) + break; + prev_record = test_record; + test_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, test_slot); } + prev_flag = prev_flag ^ ((uint32_t)1); + if (chain_tail[prev_flag]) + chain_tail[prev_flag]->next = LIGHT(end); + + LIGHT(enqueue_empty)(ht, last_empty_slot, last_empty_record); } + return true; } inline uint32_t -LH(light_insert)(struct LH(light) *ht, uint32_t hash, LIGHT_DATA_TYPE data) +LIGHT(insert)(struct LIGHT(core) *ht, uint32_t hash, LIGHT_DATA_TYPE value) { - if (ht->table_size == 0) { - ht->table_size = 1; - uint32_t slot; - struct LH(light_cluster) *cluster = (struct LH(light_cluster) *) - matras_alloc(&ht->mtable, &slot); - cluster->flags = 0; + if (ht->table_size == 0) + if (!LIGHT(prepare_first_insert)(ht)) + return LIGHT(end); + if (!ht->empty_record) + if (!LIGHT(grow)(ht)) + return LIGHT(end); + + ht->count++; + uint32_t slot = LIGHT(slot)(ht, hash); + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); + + if (record->next == slot) { + /* Inserting to an empty slot */ + LIGHT(detach_empty)(ht, record); + record->value = value; + record->hash = hash; + record->next = LIGHT(end); + return slot; } - if (ht->count >= 1 * ht->table_size) - LH(light_grow)(ht); - - uint32_t slot = LH(light_slot)(ht, hash); - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, slot); - uint32_t hash_flags = 020; - while ((cluster->flags & 02020202020) == 02020202020) { - cluster->flags |= (1 << 31); - hash_flags = 060; - slot++; - if (slot >= ht->table_size) - slot = 0; - cluster = LH(light_cluster)(ht, slot); + + uint32_t empty_slot = ht->empty_slot; + struct LIGHT(record) *empty_record = ht->empty_record; + LIGHT(detach_first_empty)(ht); + + uint32_t chain_slot = LIGHT(slot)(ht, record->hash); + if (chain_slot == slot) { + /* add to existing chain */ + empty_record->value = value; + empty_record->hash = hash; + empty_record->next = record->next; + record->next = empty_slot; + return empty_slot; + } else { + /* create new chain */ + struct LIGHT(record) *chain = (struct LIGHT(record) *) + matras_get(&ht->mtable, chain_slot); + while (chain->next != slot) { + chain_slot = chain->next; + chain = (struct LIGHT(record) *) + matras_get(&ht->mtable, chain_slot); + } + *empty_record = *record; + chain->next = empty_slot; + record->value = value; + record->hash = hash; + record->next = LIGHT(end); + return slot; } - uint32_t pos = __builtin_ctz(((~cluster->flags) & 02020202020)) / 6; - LH(light_set_value)(cluster, pos, hash_flags, hash, data); - ht->count++; - return slot * 5 + pos; } inline void -LH(light_delete)(struct LH(light) *ht, uint32_t slotpos) +LIGHT(delete)(struct LIGHT(core) *ht, uint32_t slot) { - uint32_t slot = slotpos / 5; - uint32_t pos = slotpos % 5; - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, slot); - uint32_t was_chain = cluster->flags & (040 << (pos * 6)); - LH(light_clr_value)(cluster, pos); + assert(slot < ht->table_size); + uint32_t empty_slot; + struct LIGHT(record) *empty_record; + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); + assert(record->next != slot); + if (record->next != LIGHT(end)) { + empty_slot = record->next; + empty_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, empty_slot); + *record = *empty_record; + } else { + empty_slot = slot; + empty_record = record; + uint32_t chain_slot = LIGHT(slot)(ht, record->hash); + if (chain_slot != slot) { + /* deleting a last record of chain */ + struct LIGHT(record) *chain = (struct LIGHT(record) *) + matras_get(&ht->mtable, chain_slot); + uint32_t chain_next_slot = chain->next; + assert(chain_next_slot != LIGHT(end)); + while (chain_next_slot != slot) { + chain_slot = chain_next_slot; + chain = (struct LIGHT(record) *) + matras_get(&ht->mtable, chain_next_slot); + chain_next_slot = chain->next; + assert(chain_next_slot != LIGHT(end)); + } + chain->next = LIGHT(end); + } + } + LIGHT(enqueue_empty)(ht, empty_slot, empty_record); ht->count--; +} - if (was_chain) { - while (!(cluster->flags & 024040404040)) { - if (slot == 0) - slot = ht->table_size; - slot--; - cluster = LH(light_cluster)(ht, slot); - if (!(cluster->flags >> 31)) - break; - cluster->flags &= 0x7FFFFFFF; - } +inline void +LIGHT(delete_value)(struct LIGHT(core) *ht, uint32_t hash, LIGHT_DATA_TYPE value) +{ + if (ht->count == 0) + return; + uint32_t slot = LIGHT(slot)(ht, hash); + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); + if (record->next == slot) + return; + struct LIGHT(record) *prev_record = 0; + while (1) { + if (record->hash == hash && LIGHT_EQUAL((record->value), (value), (ht->arg))) + break; + slot = record->next; + if (slot == LIGHT(end)) + return; + prev_record = record; + record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); + } + ht->count--; + if (prev_record) { + prev_record->next = record->next; + LIGHT(enqueue_empty)(ht, slot, record); + return; + } + if (record->next == LIGHT(end)) { + LIGHT(enqueue_empty)(ht, slot, record); + return; } + uint32_t next_slot = record->next; + struct LIGHT(record) *next_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, next_slot); + *record = *next_record; + LIGHT(enqueue_empty)(ht, next_slot, next_record); } inline LIGHT_DATA_TYPE -LH(light_get)(struct LH(light) *ht, uint32_t slotpos) +LIGHT(get)(struct LIGHT(core) *ht, uint32_t slotpos) { - uint32_t slot = slotpos / 5; - uint32_t pos = slotpos % 5; - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, slot); - return cluster->data[pos]; + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slotpos); + return record->value; } /** @@ -530,62 +675,97 @@ LH(light_get)(struct LH(light) *ht, uint32_t slotpos) * @param slotpos - ID of an record */ inline bool -LH(light_pos_valid)(struct LH(light) *ht, uint32_t slotpos) +LIGHT(pos_valid)(LIGHT(core) *ht, uint32_t slotpos) { - uint32_t slot = slotpos / 5; - uint32_t pos = slotpos % 5; - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, slot); - return cluster->flags & (020 << (pos * 6)); + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slotpos); + return record->next != slotpos; } -inline uint32_t -LH(light_selfcheck)(const struct LH(light) *ht) +inline int +LIGHT(selfcheck)(const struct LIGHT(core) *ht) { - uint32_t res = 0; - uint32_t total_count = 0; - for (uint32_t slot = 0; slot < ht->table_size; slot++) { - struct LH(light_cluster) *cluster = LH(light_cluster)(ht, slot); - uint32_t flags = cluster->flags; - total_count += __builtin_popcount(flags & 02020202020); - for (uint32_t pos = 0; pos < 5u; pos++) { - if (flags & (020 << (pos * 6))) { - uint32_t hint1 = (flags >> (pos * 6)) & 15; - uint32_t hint2 = cluster->hash[pos] & 15; - if (hint1 != hint2) - res |= 1; - - uint32_t from_slot = LH(light_slot)(ht, cluster->hash[pos]); - bool here1 = from_slot == slot; - bool here2 = !(flags & (040 << (6 * pos))); - if (here1 != here2) - res |= 2; - - while (from_slot != slot) { - struct LH(light_cluster) *from_cluster = LH(light_cluster)(ht, from_slot); - if (!(from_cluster->flags >> 31)) { - res |= 4; - break; - } - from_slot++; - if (from_slot >= ht->table_size) - from_slot = 0; - } + int res = 0; + if (ht->table_size != ht->mtable.block_count) + res |= 64; + uint32_t empty_slot = ht->empty_slot; + if (empty_slot == LIGHT(end)) { + if (ht->empty_record) + res |= 512; + } else { + struct LIGHT(record) *should_be = (struct LIGHT(record) *) + matras_get(&ht->mtable, empty_slot); + if (ht->empty_record != should_be) + res |= 1024; + } + uint32_t prev_empty_slot = LIGHT(end); + while (empty_slot != LIGHT(end)) { + struct LIGHT(record) *empty_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, empty_slot); + if (empty_record->next != empty_slot) + res |= 2048; + if (LIGHT(get_empty_prev)(empty_record) != prev_empty_slot) + res |= 4096; + prev_empty_slot = empty_slot; + empty_slot = LIGHT(get_empty_next)(empty_record); + } + for (uint32_t i = 0; i < ht->table_size; i++) { + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, i); + if (record->next == i) { + uint32_t empty_slot = ht->empty_slot; + while (empty_slot != LIGHT(end) && empty_slot != i) { + struct LIGHT(record) *empty_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, empty_slot); + empty_slot = LIGHT(get_empty_next)(empty_record); } + if (empty_slot != i) + res |= 256; + continue; + } + uint32_t slot = LIGHT(slot)(ht, record->hash); + if (slot != i) { + bool found = false; + uint32_t chain_slot = slot; + uint32_t chain_start_slot = slot; + do { + struct LIGHT(record) *chain_record = (struct LIGHT(record) *) + matras_get(&ht->mtable, chain_slot); + chain_slot = chain_record->next; + if (chain_slot >= ht->table_size) { + res |= 16; /* out of bounds (1) */ + break; + } + if (chain_slot == i) { + found = true; + break; + } + if (chain_slot == chain_start_slot) { + res |= 4; /* cycles in chain (1) */ + break; + } + } while (chain_slot != LIGHT(end)); + if (!found) + res |= 1; /* slot is out of chain */ + } else { + do { + struct LIGHT(record) *record = (struct LIGHT(record) *) + matras_get(&ht->mtable, slot); + if (LIGHT(slot)(ht, record->hash) != i) + res |= 2; /* wrong value in chain */ + slot = record->next; + if (slot != LIGHT(end) && slot >= ht->table_size) { + res |= 32; /* out of bounds (2) */ + break; + } + if (slot == i) { + res |= 8; /* cycles in chain (2) */ + break; + } + } while (slot != LIGHT(end)); } } - if (ht->count != total_count) - res |= 256; - - uint32_t cover = ht->cover_mask + 1; - if (ht->cover_mask & cover) - res |= 512; - - if (ht->table_size && cover < ht->table_size) - res |= 1024; - if (ht->table_size && cover / 2 >= ht->table_size) - res |= 2048; return res; } -#undef LH diff --git a/src/lib/small/matras.c b/src/lib/small/matras.c index 95cdd196c973d4d3c27f6206b1f4974a5ad7e6a0..09929583788745bbff44b1819878ab25958de91b 100644 --- a/src/lib/small/matras.c +++ b/src/lib/small/matras.c @@ -4,6 +4,13 @@ #include "matras.h" #include <limits.h> +#ifdef WIN32 +#include <intrin.h> +#pragma intrinsic (_BitScanReverse) +#ifndef _DEBUG +#define __OPTIMIZE__ 1 +#endif +#endif /* * Binary logarithm of value (exact if the value is a power of 2, @@ -13,8 +20,15 @@ static matras_id_t pt_log2(matras_id_t val) { assert(val > 0); - return sizeof(unsigned long) * CHAR_BIT - - __builtin_clzl((unsigned long) val) - 1; +#ifdef WIN32 + unsigned long res = 0; + unsigned char nonzero = _BitScanReverse(&res, val); + assert(nonzero); (void)nonzero; + return (matras_id_t)res; +#else + return sizeof(unsigned int) * CHAR_BIT - + __builtin_clz((unsigned int) val) - 1; +#endif } /** @@ -92,7 +106,7 @@ matras_destroy(struct matras *m) m->block_count = 0; } #ifndef __OPTIMIZE__ - m->extent = (void *)0xDEADBEEF; + m->extent = (void *)(long long)0xDEADBEEF; #endif } @@ -221,6 +235,40 @@ matras_dealloc(struct matras *m) } } +/** + * Allocate a range_count of blocks. Return both, first block pointer + * and first block id. This method only works if current number of blocks and + * number of blocks in one extent are divisible by range_count. + * range_count must also be less or equal to number of blocks in one extent. + * + * @retval NULL failed to allocate memory + */ +void * +matras_alloc_range(struct matras *m, matras_id_t *id, matras_id_t range_count) +{ + assert(m->block_count % range_count == 0); + assert(m->extent_size / m->block_size % range_count == 0); + void *res = matras_alloc(m, id); + if (res) + m->block_count += (range_count - 1); + return res; +} + +/* + * Deallocate last range_count of blocks (blocks with maximum ID) + * This method only works if current number of blocks and + * number of blocks in one extent are divisible by range_count. + * range_count must also be less or equal to number of blocks in one extent. + */ +void +matras_dealloc_range(struct matras *m, matras_id_t range_count) +{ + assert(m->block_count % range_count == 0); + assert(m->extent_size / m->block_size % range_count == 0); + m->block_count -= (range_count - 1); + matras_dealloc(m); +} + /** * Return the number of allocated extents (of size m->extent_size each) */ diff --git a/src/lib/small/matras.h b/src/lib/small/matras.h index 06bd34f7400328a36026a7ff47eef02323f9441d..24595995a65ded6134ead750b8c537477337ee74 100644 --- a/src/lib/small/matras.h +++ b/src/lib/small/matras.h @@ -82,7 +82,13 @@ */ /* }}} */ +#ifdef WIN32 +typedef unsigned __int32 matras_id_t; +#else #include <stdint.h> +typedef uint32_t matras_id_t; +#endif + #include <assert.h> #if defined(__cplusplus) @@ -92,7 +98,6 @@ extern "C" { /** * Type of a block ID. */ -typedef uint32_t matras_id_t; /** * Type of the extent allocator (the allocator for regions @@ -109,7 +114,7 @@ typedef void (*prov_free_func)(void *); struct matras { /* Pointer to the root extent of matras */ void *extent; - /* Block size (M) */ + /* Block size (N) */ matras_id_t block_size; /* Extent size (M) */ matras_id_t extent_size; @@ -186,6 +191,26 @@ matras_alloc(struct matras *m, matras_id_t *id); void matras_dealloc(struct matras *m); +/** + * Allocate a range_count of blocks. Return both, first block pointer + * and first block id. This method only works if current number of blocks and + * number of blocks in one extent are divisible by range_count. + * range_count must also be less or equal to number of blocks in one extent. + * + * @retval NULL failed to allocate memory + */ +void * +matras_alloc_range(struct matras *m, matras_id_t *id, matras_id_t range_count); + +/* + * Deallocate last range_count of blocks (blocks with maximum ID) + * This method only works if current number of blocks and + * number of blocks in one extent are divisible by range_count. + * range_count must also be less or equal to number of blocks in one extent. + */ +void +matras_dealloc_range(struct matras *m, matras_id_t range_count); + /** * Convert block id into block address. */ @@ -215,8 +240,6 @@ matras_get(const struct matras *m, matras_id_t id) return (((char***)m->extent)[n1][n2] + n3 * m->block_size); } - - #if defined(__cplusplus) } /* extern "C" */ #endif /* defined(__cplusplus) */ diff --git a/test/box/access_misc.result b/test/box/access_misc.result index 52e788ab82c63a65cb52272990db158e47bc9738..35c5509b940fb21e6c6ee9d66f7be87381e72e47 100644 --- a/test/box/access_misc.result +++ b/test/box/access_misc.result @@ -178,8 +178,8 @@ session.su('admin') ... s:select() --- -- - [3] - - [2] +- - [2] + - [3] ... -- -- Create user with universe read&write grants @@ -244,8 +244,8 @@ box.schema.user.drop('uniuser_testus') -- box.space.admin_space:select() --- -- - [3] - - [2] +- - [2] + - [3] ... box.space._user:select(1) --- diff --git a/test/unit/light.cc b/test/unit/light.cc index 39b037231ff0f2fb9b98b88338baca1fc6ce28bf..2dee5dcba3040cb2260857dfaea5e4ac08d5ce19 100644 --- a/test/unit/light.cc +++ b/test/unit/light.cc @@ -62,7 +62,7 @@ simple_test() { header(); - struct light ht; + struct light_core ht; light_create(&ht, light_extent_size, my_light_alloc, my_light_free, 0); std::vector<bool> vect; size_t count = 0; @@ -125,7 +125,7 @@ collision_test() { header(); - struct light ht; + struct light_core ht; light_create(&ht, light_extent_size, my_light_alloc, my_light_free, 0); std::vector<bool> vect; size_t count = 0;