diff --git a/README.md b/README.md index 6d72a6d2b97fa3482fcd1ff5badb5f877a327b32..eeede13bffbdee0447f40d8c8b4faa6a1d80f85b 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ To start the server, try: This will start Tarantool in interactive mode. To run Tarantool regression tests (test/test-run.py), -a few additional Python modules are ncessary: +a few additional Python modules are necessary: * daemon * pyyaml * msgpack-python diff --git a/cmake/BuildLibEV.cmake b/cmake/BuildLibEV.cmake index 3adb93ba5159933026ba0bfc2c26e97d8b4e7d68..c9e3a90bdce956dabcd346e6e233f4b70e2d4d34 100644 --- a/cmake/BuildLibEV.cmake +++ b/cmake/BuildLibEV.cmake @@ -29,7 +29,7 @@ macro(libev_build) # # Enable Linux-specific event notification API (man inotify) set(ev_compile_flags "${ev_compile_flags} -DEV_USE_INOTIFY") - elseif (TARGET_OS_FREEBSD) + elseif (TARGET_OS_FREEBSD OR TARGET_OS_DARWIN) # # On FreeBSD build libev loop on top of set(ev_compile_flags "${ev_compile_flags} -DEV_USE_KQUEUE") diff --git a/cmake/compiler.cmake b/cmake/compiler.cmake index e5d16ae9158f4dacd2d7b3076a5286dbf8f47617..55ea1a382fa634d956395a3aa8627025f348814a 100644 --- a/cmake/compiler.cmake +++ b/cmake/compiler.cmake @@ -41,7 +41,14 @@ endif() if((NOT HAVE_STD_C11 AND NOT HAVE_STD_GNU99) OR (NOT HAVE_STD_CXX11 AND NOT HAVE_STD_GNUXX0X)) set(CMAKE_REQUIRED_FLAGS "-std=c11") - check_c_source_compiles("int main(void) { return 0; }" HAVE_STD_C11) + check_c_source_compiles(" + /* + * FreeBSD 10 ctype.h header fail to compile on gcc4.8 in c11 mode. + * Make sure we aren't affected. + */ + #include <ctype.h> + int main(void) { return 0; } + " HAVE_STD_C11) set(CMAKE_REQUIRED_FLAGS "-std=gnu99") check_c_source_compiles("int main(void) { return 0; }" HAVE_STD_GNU99) set(CMAKE_REQUIRED_FLAGS "-std=c++11") @@ -73,10 +80,9 @@ set(CMAKE_REQUIRED_FLAGS "") # # Perform build type specific configuration. # -if (CMAKE_COMPILER_IS_GNUCC) +check_c_compiler_flag("-ggdb" CC_HAS_GGDB) +if (CC_HAS_GGDB) set (CC_DEBUG_OPT "-ggdb") -else() - set (CC_DEBUG_OPT "-g") endif() set (CMAKE_C_FLAGS_DEBUG diff --git a/cmake/luajit.cmake b/cmake/luajit.cmake index 06ac79d6a16d1737c3e259c8eceb6bb0e7052d87..39b32410589bec01ba17b45bcc70fd9b1eff6f37 100644 --- a/cmake/luajit.cmake +++ b/cmake/luajit.cmake @@ -170,6 +170,7 @@ macro(luajit_build) CFLAGS="" CXXFLAGS="" XCFLAGS="${luajit_xcflags}" + CC="${luajit_host_cc}" HOST_CC="${luajit_host_cc}" TARGET_CC="${luajit_target_cc}" CCOPT="${luajit_copt}") diff --git a/cmake/module.cmake b/cmake/module.cmake index 8d6eafbcafe20fcdfff3a82bf68a13e751b80a2f..25cb8f4934ca5d650869ce533a2b80979f35c14e 100644 --- a/cmake/module.cmake +++ b/cmake/module.cmake @@ -2,6 +2,7 @@ function(rebuild_module_api) set (dstfile "${CMAKE_CURRENT_BINARY_DIR}/tarantool.h") set (tmpfile "${dstfile}.new") + set (errcodefile "${CMAKE_CURRENT_BINARY_DIR}/errcode.i") set (headers) # Get absolute path for header files (required of out-of-source build) foreach (header ${ARGN}) @@ -12,16 +13,24 @@ function(rebuild_module_api) endif() endforeach() + set (cflags ${CMAKE_C_FLAGS}) + separate_arguments(cflags) + # Pass sysroot settings on OSX + if (NOT "${CMAKE_OSX_SYSROOT}" STREQUAL "") + set (cflags ${cflags} ${CMAKE_C_SYSROOT_FLAG} ${CMAKE_OSX_SYSROOT}) + endif() add_custom_command(OUTPUT ${dstfile} COMMAND cat ${CMAKE_CURRENT_SOURCE_DIR}/tarantool_header.h > ${tmpfile} COMMAND cat ${headers} | ${CMAKE_SOURCE_DIR}/extra/apigen >> ${tmpfile} COMMAND ${CMAKE_C_COMPILER} + ${cflags} -I ${CMAKE_SOURCE_DIR}/src -I ${CMAKE_BINARY_DIR}/src - -E ${CMAKE_SOURCE_DIR}/src/box/errcode.h | - grep "enum box_error_code" >> ${tmpfile} + -E ${CMAKE_SOURCE_DIR}/src/box/errcode.h > ${errcodefile} + COMMAND + grep "enum box_error_code" ${errcodefile} >> ${tmpfile} COMMAND cat ${CMAKE_CURRENT_SOURCE_DIR}/tarantool_footer.h >> ${tmpfile} COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmpfile} ${dstfile} - COMMAND ${CMAKE_COMMAND} -E remove ${tmpfile} + COMMAND ${CMAKE_COMMAND} -E remove ${errcodefile} ${tmpfile} DEPENDS ${srcfiles} ${CMAKE_SOURCE_DIR}/src/box/errcode.h ${CMAKE_CURRENT_SOURCE_DIR}/tarantool_header.h ${CMAKE_CURRENT_SOURCE_DIR}/tarantool_footer.h diff --git a/doc/sphinx/dev_guide/box-protocol.rst b/doc/sphinx/dev_guide/box-protocol.rst index 121ee6c354a267ff5fb16bb31db4acb39b717ed4..f3a4bff721162da7923c62e05a932c552207c6ba 100644 --- a/doc/sphinx/dev_guide/box-protocol.rst +++ b/doc/sphinx/dev_guide/box-protocol.rst @@ -109,6 +109,7 @@ Let's list them here too: <function_name> ::= 0x22 <username> ::= 0x23 <expression> ::= 0x27 + <def_tuple> ::= 0x28 <data> ::= 0x30 <error> ::= 0x31 @@ -124,6 +125,7 @@ Let's list them here too: <call> ::= 0x06 <auth> ::= 0x07 <eval> ::= 0x08 + <upsert> ::= 0x09 -- Admin command codes <ping> ::= 0x40 @@ -378,6 +380,48 @@ It's an error to specify an argument of a type that differs from expected type. MP_MAP +* UPSERT: CODE - 0x09 + Update tuple if it would be found elsewhere try to insert tuple. Always use primary index for key. + + + +.. code-block:: bash + + UPSERT BODY: + + +==================+==========================+ + | | | + | 0x10: SPACE_ID | 0x20: KEY | + | MP_INT: MP_INT | MP_INT: MP_ARRAY | + | | | + +==================+==========================+ + | | +~~~~~~~~~~+ | + | | | | | + | | (DEF_TUPLE) | OP | | + | 0x21: TUPLE | 0x28: | | | + | MP_INT: MP_ARRAY | MP_INT: +~~~~~~~~~~+ | + | | MP_ARRAY | + +==================+==========================+ + MP_MAP + + Operations structure same as for UPDATE operation. + 0 2 + +-----------+==========+==========+ + | | | | + | OP | FIELD_NO | ARGUMENT | + | MP_FIXSTR | MP_INT | MP_INT | + | | | | + +-----------+==========+==========+ + MP_ARRAY + + Support operations: + '+' - add a value to a numeric field. If the filed is not numeric, it's changed to 0 first. If the field does not exist, the operation is skipped. There is no error in case of overflow either, the value simply wraps around in C style. The range of the integer is MsgPack: from -2^63 to 2^64-1 + '-' - same as the previous, but subtract a value + '=' - assign a field to a value. The field must exist, if it does not exist, the operation is skipped. + '!' - insert a field. It's only possible to insert a field if this create no nil "gaps" between fields. E.g. it's possible to add a field between existing fields or as the last field of the tuple. + '#' - delete a field. If the field does not exist, the operation is skipped. It's not possible to change with update operations a part of the primary key (this is validated before performing upsert). + + ================================================================================ Response packet structure ================================================================================ diff --git a/src/box/alter.cc b/src/box/alter.cc index 380156e659089a60ca1ed0f4db0bf21b29953386..6187135588e0dae9f8fa2cecb9d76bd89af7c104 100644 --- a/src/box/alter.cc +++ b/src/box/alter.cc @@ -105,52 +105,66 @@ access_check_ddl(uint32_t owner_uid) } } +const char * +map_field_get(const char *map, const char *needle, int needle_type, + int map_field_no) +{ + if (map == NULL || mp_typeof(*map) != MP_MAP) { + tnt_raise(ClientError, ER_FIELD_TYPE, + map_field_no + INDEX_OFFSET, "map"); + } + uint32_t map_size = mp_decode_map(&map); + for (uint32_t i = 0; i < map_size; i++) { + const char *key; + uint32_t len; + if (mp_typeof(*map) != MP_STR) { + tnt_raise(ClientError, ER_FIELD_TYPE, + map_field_no + INDEX_OFFSET, + "map key type"); + } + key = mp_decode_str(&map, &len); + if (len != strlen(needle) || + strncasecmp(key, needle, len) != 0) { + + mp_next(&map); + continue; + } + if (mp_typeof(*map) != needle_type) { + tnt_raise(ClientError, ER_FIELD_TYPE, + map_field_no + INDEX_OFFSET, + "map value type"); + } + return map; + } + return NULL; +} /** Fill key_opts structure from opts field in tuple of space _index */ void -key_opts_create_from_field(struct key_opts *opts, const char *field) +key_opts_create_from_field(struct key_opts *opts, const char *map) { - key_opts_create(opts); - if (field == NULL) + *opts = key_opts_default; + if (map == NULL) return; - if (mp_typeof(*field) != MP_MAP) { - tnt_raise(ClientError, ER_FIELD_TYPE, INDEX_OPTS + INDEX_OFFSET, - "map"); - } - uint32_t map_size = mp_decode_map(&field); - for (uint32_t i = 0; i < map_size; i++) { - const char *key; + const char *field; + + if ((field = map_field_get(map, "unique", MP_BOOL, INDEX_OPTS))) + opts->is_unique = mp_decode_bool(&field); + if ((field = map_field_get(map, "dimension", MP_UINT, INDEX_OPTS))) + opts->dimension = mp_decode_uint(&field); + if ((field = map_field_get(map, "distance", MP_STR, INDEX_OPTS))) { uint32_t len; - if (mp_typeof(*field) != MP_STR) { + const char *distance = mp_decode_str(&field, &len); + distance = tuple_field_to_cstr(distance, len); + + enum rtree_index_distance_type distance_type = + STR2ENUM(rtree_index_distance_type, distance); + if (distance_type == rtree_index_distance_type_MAX) { tnt_raise(ClientError, - ER_FIELD_TYPE, INDEX_OPTS + INDEX_OFFSET, - "string"); - } - key = mp_decode_str(&field, &len); - if (len == strlen("unique") && - memcmp(key, "unique", len) == 0) { - - - if (mp_typeof(*field) != MP_BOOL) { - tnt_raise(ClientError, - ER_FIELD_TYPE, - INDEX_OPTS + INDEX_OFFSET, - "bool"); - } - opts->is_unique = mp_decode_bool(&field); - } else if (len == strlen("dimension") && - memcmp(key, "dimension", len) == 0) { - - if (mp_typeof(*field) != MP_UINT) { - tnt_raise(ClientError, - ER_FIELD_TYPE, - INDEX_OPTS + INDEX_OFFSET, - "unsigned"); - } - opts->dimension = mp_decode_uint(&field); - } else { - mp_next(&field); + ER_UNKNOWN_RTREE_INDEX_DISTANCE_TYPE, + distance); } + opts->distance = distance_type; } } @@ -186,10 +200,16 @@ key_def_new_from_tuple(struct tuple *tuple) part_count = mp_decode_array(&parts); } else { /* 1.6.5 _index space structure */ - key_opts_create(&opts); + opts = key_opts_default; opts.is_unique = tuple_field_u32(tuple, INDEX_165_IS_UNIQUE); part_count = tuple_field_u32(tuple, INDEX_165_PART_COUNT); } + /** + * XXX this is an ugly clutch in absence of Lua-level + * index-type specific defaults. + */ + if (opts.dimension == 0 && type == RTREE) + opts.dimension = 2; key_def = key_def_new(id, index_id, name, type, &opts, part_count); auto scoped_guard = make_scoped_guard([=] { key_def_delete(key_def); }); diff --git a/src/box/box.cc b/src/box/box.cc index f98318e9427e12980ae063e2157ecc3e64e67e0b..d07b56a8467fd91aaf2418b996e0f309221aaa9b 100644 --- a/src/box/box.cc +++ b/src/box/box.cc @@ -602,6 +602,7 @@ box_free(void) tuple_free(); port_free(); engine_shutdown(); + rmean_delete(rmean_error); rmean_delete(rmean_box); } } @@ -626,6 +627,25 @@ engine_init() engine_register(sophia); } +/** + * @brief Reduce the current number of threads in the thread pool to the + * bare minimum. Doesn't prevent the pool from spawning new threads later + * if demand mounts. + */ +static void +thread_pool_trim() +{ + /* + * Trim OpenMP thread pool. + * Though we lack the direct control the workaround below works for + * GNU OpenMP library. The library stops surplus threads on entering + * a parallel region. Can't go below 2 threads due to the + * implementation quirk. + */ +#pragma omp parallel num_threads(2) + ; +} + static inline void box_init(void) { @@ -635,6 +655,7 @@ box_init(void) cfg_getd("slab_alloc_factor")); rmean_box = rmean_new(iproto_type_strs, IPROTO_TYPE_STAT_MAX); + rmean_error = rmean_new(rmean_error_strings, RMEAN_ERROR_LAST); engine_init(); @@ -696,6 +717,11 @@ box_init(void) engine_end_recovery(); + /* + * Recovery inflates the thread pool quite a bit (due to parallel sort). + */ + thread_pool_trim(); + rmean_cleanup(rmean_box); if (recovery_has_replica(recovery)) diff --git a/src/box/errcode.h b/src/box/errcode.h index 13a776a7242f59cba8fcc84c4f939ec0f5041597..57008e1b1f1405ed50fac53c4c0732f0d91b762d 100644 --- a/src/box/errcode.h +++ b/src/box/errcode.h @@ -156,6 +156,7 @@ struct errcode_record { /*100 */_(ER_FUNCTION_LANGUAGE, 2, "Unsupported language '%s' specified for function '%s'") \ /*101 */_(ER_RTREE_RECT, 2, "RTree: %s must be an array with %u (point) or %u (rectangle/box) numeric coordinates") \ /*102 */_(ER_PROC_C, 2, "%s") \ + /*103 */_(ER_UNKNOWN_RTREE_INDEX_DISTANCE_TYPE, 2, "Unknown RTREE index distance type %s") \ /* * !IMPORTANT! Please follow instructions at start of the file diff --git a/src/box/error.cc b/src/box/error.cc index 789ffc6f6287941fb7fbb7e0d06b9f9180d14e6f..d50c0f33d2614e7e21afd80c9cabc316b92f110b 100644 --- a/src/box/error.cc +++ b/src/box/error.cc @@ -32,6 +32,11 @@ #include <stdio.h> #include <fiber.h> +struct rmean *rmean_error = NULL; + +const char *rmean_error_strings[RMEAN_ERROR_LAST] = { + "ERROR" +}; static struct method clienterror_methods[] = { make_method(&type_ClientError, "code", &ClientError::errcode), @@ -49,6 +54,8 @@ ClientError::ClientError(const char *file, unsigned line, va_start(ap, errcode); vsnprintf(m_errmsg, sizeof(m_errmsg), tnt_errcode_desc(m_errcode), ap); + if (rmean_error) + rmean_collect(rmean_error, RMEAN_ERROR, 1); va_end(ap); } @@ -59,6 +66,8 @@ ClientError::ClientError(const char *file, unsigned line, const char *msg, m_errcode = errcode; strncpy(m_errmsg, msg, sizeof(m_errmsg) - 1); m_errmsg[sizeof(m_errmsg) - 1] = 0; + if (rmean_error) + rmean_collect(rmean_error, RMEAN_ERROR, 1); } void diff --git a/src/box/error.h b/src/box/error.h index 95e41ca0b3a624a6d94fd5ff263ae4f23fbc83f2..9aa96b4ffaf29a9884251059850216c03ee62000 100644 --- a/src/box/error.h +++ b/src/box/error.h @@ -32,6 +32,15 @@ */ #include "errcode.h" #include "exception.h" +#include "rmean.h" + +extern struct rmean *rmean_error; + +enum rmean_error_name { + RMEAN_ERROR, + RMEAN_ERROR_LAST +}; +extern const char *rmean_error_strings[RMEAN_ERROR_LAST]; extern const struct type type_ClientError; class ClientError: public Exception { diff --git a/src/box/index.cc b/src/box/index.cc index a07f41d0ea063bb03abf8d1616e455911190268c..b0519d774590ee3df1699d1e81f3f71d1f236820 100644 --- a/src/box/index.cc +++ b/src/box/index.cc @@ -34,6 +34,8 @@ #include "schema.h" #include "user_def.h" #include "space.h" +#include "iproto_constants.h" +#include "request.h" const char *iterator_type_strs[] = { /* [ITER_EQ] = */ "EQ", @@ -305,6 +307,9 @@ box_index_get(uint32_t space_id, uint32_t index_id, const char *key, uint32_t part_count = key ? mp_decode_array(&key) : 0; primary_key_validate(index->key_def, key, part_count); struct tuple *tuple = index->findByKey(key, part_count); + /* Count statistics */ + rmean_collect(rmean_box, IPROTO_SELECT, 1); + *result = tuple_bless_null(tuple); return 0; } catch (Exception *) { diff --git a/src/box/iproto.cc b/src/box/iproto.cc index 79867f03fa0907be3d245c27268b1280bb3004d5..be8026c38526d8fd0632e376becffe64549ca244 100644 --- a/src/box/iproto.cc +++ b/src/box/iproto.cc @@ -55,6 +55,7 @@ /* {{{ iproto_msg - declaration */ + /** * A single msg from io thread. All requests * from all connections are queued into a single queue @@ -513,6 +514,9 @@ iproto_connection_on_input(ev_loop *loop, struct ev_io *watcher, iproto_connection_close(con); return; } + /* Count statistics */ + rmean_collect(rmean_net, RMEAN_NET_RECEIVED, nrd); + /* Update the read position and connection state. */ in->wpos += nrd; con->parse_size += nrd; @@ -564,6 +568,9 @@ iproto_flush(struct iobuf *iobuf, struct iproto_connection *con) iov[iovcnt-1].iov_len = end->iov_len - begin->iov_len * (iovcnt == 1); ssize_t nwr = sio_writev(fd, iov, iovcnt); + + /* Count statistics */ + rmean_collect(rmean_net, RMEAN_NET_SENT, nwr); if (nwr > 0) { if (begin->used + nwr == end->used) { if (ibuf_used(&iobuf->in) == 0) { @@ -789,8 +796,11 @@ net_send_greeting(struct cmsg *m) if (msg->close_connection) { struct obuf *out = &msg->iobuf->out; try { - sio_writev(con->output.fd, out->iov, - obuf_iovcnt(out)); + int64_t nwr = sio_writev(con->output.fd, out->iov, + obuf_iovcnt(out)); + + /* Count statistics */ + rmean_collect(rmean_net, RMEAN_NET_SENT, nwr); } catch (Exception *e) { e->log(); } @@ -863,13 +873,27 @@ net_cord_f(va_list /* ap */) evio_service_init(loop(), &binary, "binary", iproto_on_accept, NULL); + + /* Init statistics counter */ + rmean_net = rmean_new(rmean_net_strings, RMEAN_NET_LAST); + + if (rmean_net == NULL) + tnt_raise(OutOfMemory, + sizeof(*rmean_net) + + RMEAN_NET_LAST * sizeof(stats), + "rmean", "struct rmean"); + + cbus_join(&net_tx_bus, &net_pipe); + /* * Nothing to do in the fiber so far, the service * will take care of creating events for incoming * connections. */ fiber_yield(); + + rmean_delete(rmean_net); } /** Initialize the iproto subsystem and start network io thread */ @@ -890,6 +914,7 @@ iproto_init() if (cord_costart(&net_cord, "iproto", net_cord_f, NULL)) panic("failed to initialize iproto thread"); + cbus_join(&net_tx_bus, &tx_pipe); } diff --git a/src/box/key_def.cc b/src/box/key_def.cc index eb1343e3348e337f3b9a7bffd67eb7f1b2ca87e8..c898674434005151c77de2c412ee3f642c544352 100644 --- a/src/box/key_def.cc +++ b/src/box/key_def.cc @@ -37,6 +37,8 @@ const char *field_type_strs[] = {"UNKNOWN", "NUM", "STR", "ARRAY", "NUMBER", ""}; STRS(index_type, ENUM_INDEX_TYPE); +const char *rtree_index_distance_type_strs[] = { "EUCLID", "MANHATTAN" }; + const char *func_language_strs[] = {"LUA", "C"}; const uint32_t key_mp_type[] = { @@ -47,6 +49,10 @@ const uint32_t key_mp_type[] = { /* [NUMBER] = */ (1U << MP_UINT) | (1U << MP_INT) | (1U << MP_FLOAT) | (1U << MP_DOUBLE), }; +const struct key_opts key_opts_default = { + true, 0, RTREE_INDEX_DISTANCE_TYPE_EUCLID +}; + enum schema_object_type schema_object_type(const char *name) { diff --git a/src/box/key_def.h b/src/box/key_def.h index f8db51cad1f4461054f22229cbda9323b8deb59c..c9e6e74976b4dcec197c9ac8420a44d7ddc83910 100644 --- a/src/box/key_def.h +++ b/src/box/key_def.h @@ -99,6 +99,15 @@ field_type_maxlen(enum field_type type) ENUM(index_type, ENUM_INDEX_TYPE); extern const char *index_type_strs[]; +enum rtree_index_distance_type { + /* Euclid distance, sqrt(dx*dx + dy*dy) */ + RTREE_INDEX_DISTANCE_TYPE_EUCLID, + /* Manhattan distance, fabs(dx) + fabs(dy) */ + RTREE_INDEX_DISTANCE_TYPE_MANHATTAN, + rtree_index_distance_type_MAX +}; +extern const char *rtree_index_distance_type_strs[]; + /** Descriptor of a single part in a multipart key. */ struct key_part { uint32_t fieldno; @@ -116,22 +125,23 @@ struct key_opts { * RTREE index dimension. */ uint32_t dimension; + /** + * RTREE distance type. + */ + enum rtree_index_distance_type distance; }; -static inline void -key_opts_create(struct key_opts *opts) -{ - opts->is_unique = true; - opts->dimension = 2; -} +extern const struct key_opts key_opts_default; static inline int key_opts_cmp(const struct key_opts *o1, const struct key_opts *o2) { if (o1->is_unique != o2->is_unique) return o1->is_unique < o2->is_unique ? -1 : 1; - if (o2->dimension != o1->dimension) - return o1->dimension < o2-> dimension ? -1 : 1; + if (o1->dimension != o2->dimension) + return o1->dimension < o2->dimension ? -1 : 1; + if (o1->distance != o2->distance) + return o1->distance < o2->distance ? -1 : 1; return 0; } diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua index 9c9994c5b2360daa746c1bffb0f132feed4a2f58..308d1842d41a65cb8a842d85be82e77cc6fed368 100644 --- a/src/box/lua/schema.lua +++ b/src/box/lua/schema.lua @@ -289,7 +289,7 @@ end -- space format - the metadata about space fields function box.schema.space.format(id, format) - _space = box.space._space + local _space = box.space._space check_param(id, 'id', 'number') check_param(format, 'format', 'table') if format == nil then @@ -379,6 +379,7 @@ box.schema.index.create = function(space_id, name, options) id = 'number', if_not_exists = 'boolean', dimension = 'number', + distance = 'string', } local options_defaults = { type = 'tree', @@ -417,7 +418,8 @@ box.schema.index.create = function(space_id, name, options) for i = 1, #options.parts, 2 do table.insert(parts, {options.parts[i], options.parts[i + 1]}) end - local key_opts = { dimension = options.dimension, unique = options.unique } + local key_opts = { dimension = options.dimension, + unique = options.unique, distance = options.distance } _index:insert{space_id, iid, name, options.type, key_opts, parts} return box.space[space_id].index[name] end @@ -460,6 +462,7 @@ box.schema.index.alter = function(space_id, index_id, options) parts = 'table', unique = 'boolean', dimension = 'number', + distance = 'string', } check_param_table(options, options_template) @@ -526,6 +529,9 @@ box.schema.index.alter = function(space_id, index_id, options) if options.dimension ~= nil then key_opts.dimension = options.dimension end + if options.distance ~= nil then + key_opts.distance = options.distance + end if options.parts ~= nil then check_index_parts(options.parts) options.parts = update_index_parts(options.parts) diff --git a/src/box/lua/stat.cc b/src/box/lua/stat.cc index 7aa56497429d432b4280cad414d1269ee5b04016..efdc1d3121de07c0fda45aa7239361c53ad7187b 100644 --- a/src/box/lua/stat.cc +++ b/src/box/lua/stat.cc @@ -34,6 +34,8 @@ #include <string.h> #include <rmean.h> #include <box/request.h> +#include <cbus.h> +#include <box/error.h> extern "C" { #include <lua.h> @@ -91,7 +93,10 @@ static int lbox_stat_index(struct lua_State *L) { luaL_checkstring(L, -1); - return rmean_foreach(rmean_box, seek_stat_item, L); + int res = rmean_foreach(rmean_box, seek_stat_item, L); + if (res) + return res; + return rmean_foreach(rmean_error, seek_stat_item, L); } static int @@ -99,6 +104,22 @@ lbox_stat_call(struct lua_State *L) { lua_newtable(L); rmean_foreach(rmean_box, set_stat_item, L); + rmean_foreach(rmean_error, set_stat_item, L); + return 1; +} + +static int +lbox_stat_net_index(struct lua_State *L) +{ + luaL_checkstring(L, -1); + return rmean_foreach(rmean_net, seek_stat_item, L); +} + +static int +lbox_stat_net_call(struct lua_State *L) +{ + lua_newtable(L); + rmean_foreach(rmean_net, set_stat_item, L); return 1; } @@ -108,7 +129,13 @@ static const struct luaL_reg lbox_stat_meta [] = { {NULL, NULL} }; -/** Initialize bos.stat package. */ +static const struct luaL_reg lbox_stat_net_meta [] = { + {"__index", lbox_stat_net_index}, + {"__call", lbox_stat_net_call}, + {NULL, NULL} +}; + +/** Initialize box.stat package. */ void box_lua_stat_init(struct lua_State *L) { @@ -121,7 +148,14 @@ box_lua_stat_init(struct lua_State *L) lua_newtable(L); luaL_register(L, NULL, lbox_stat_meta); lua_setmetatable(L, -2); - lua_pop(L, 1); /* stat module */ + + + luaL_register_module(L, "box.stat.net", statlib); + + lua_newtable(L); + luaL_register(L, NULL, lbox_stat_net_meta); + lua_setmetatable(L, -2); + lua_pop(L, 1); /* stat net module */ } diff --git a/src/box/memtx_rtree.cc b/src/box/memtx_rtree.cc index ecd0472877b5bb5f8dfc7df4268affa22ffd0601..6c773d89e61b90c09aa7c2170c02aa519ef81282 100644 --- a/src/box/memtx_rtree.cc +++ b/src/box/memtx_rtree.cc @@ -38,6 +38,10 @@ /* {{{ Utilities. *************************************************/ +/** + * Extract coordinates of rectangle from message packed string. + * There must be <count> or <count * 2> numbers in that string. + */ inline int mp_decode_rect(struct rtree_rect *rect, unsigned dimension, const char *mp, unsigned count) @@ -64,6 +68,11 @@ mp_decode_rect(struct rtree_rect *rect, unsigned dimension, return 0; } +/** + * Extract rectangle from message packed string. + * There must be an array with appropriate number of coordinates in + * that string. + */ inline int mp_decode_rect(struct rtree_rect *rect, unsigned dimension, const char *mp) @@ -72,6 +81,23 @@ mp_decode_rect(struct rtree_rect *rect, unsigned dimension, return mp_decode_rect(rect, dimension, mp, size); } +/** + * Extract rectangle from message packed key. + * Due to historical issues, + * in key a rectangle could be written in two variants: + * a)array with appropriate number of coordinates + * b)array with on element - array with appropriate number of coordinates + */ +inline int +mp_decode_rect_from_key(struct rtree_rect *rect, unsigned dimension, + const char *mp, uint32_t part_count) +{ + if (part_count != 1) /* variant a */ + return mp_decode_rect(rect, dimension, mp, part_count); + else /* variant b */ + return mp_decode_rect(rect, dimension, mp); +} + inline void extract_rectangle(struct rtree_rect *rect, const struct tuple *tuple, struct key_def *key_def) @@ -137,8 +163,13 @@ MemtxRTree::MemtxRTree(struct key_def *key_def) } memtx_index_arena_init(); + assert((int)RTREE_EUCLID == (int)RTREE_INDEX_DISTANCE_TYPE_EUCLID); + assert((int)RTREE_MANHATTAN == (int)RTREE_INDEX_DISTANCE_TYPE_MANHATTAN); + enum rtree_distance_type distance_type = + (enum rtree_distance_type)(int)key_def->opts.distance; rtree_init(&m_tree, m_dimension, MEMTX_EXTENT_SIZE, - memtx_index_extent_alloc, memtx_index_extent_free); + memtx_index_extent_alloc, memtx_index_extent_free, + distance_type); } size_t @@ -160,7 +191,7 @@ MemtxRTree::findByKey(const char *key, uint32_t part_count) const rtree_iterator_init(&iterator); rtree_rect rect; - if (mp_decode_rect(&rect, m_dimension, key, part_count)) + if (mp_decode_rect_from_key(&rect, m_dimension, key, part_count)) assert(false); struct tuple *result = NULL; @@ -217,7 +248,8 @@ MemtxRTree::initIterator(struct iterator *iterator, enum iterator_type type, "It is possible to omit " "key only for ITER_ALL"); } - } else if (mp_decode_rect(&rect, m_dimension, key, part_count)) { + } else if (mp_decode_rect_from_key(&rect, m_dimension, + key, part_count)) { tnt_raise(ClientError, ER_RTREE_RECT, "Key", m_dimension, m_dimension * 2); } diff --git a/src/box/schema.cc b/src/box/schema.cc index 10d22afdaa63e70050988ef17732940c0fa112cc..c6133cad13c12c3959acddc1ca636078564aab91 100644 --- a/src/box/schema.cc +++ b/src/box/schema.cc @@ -250,7 +250,7 @@ schema_init() struct space_def def = { BOX_SCHEMA_ID, ADMIN, 0, "_schema", "memtx", false }; - struct key_opts opts = { true /* is_unique */, 0 /* dimension */ }; + struct key_opts opts = key_opts_default; struct key_def *key_def = key_def_new(def.id, 0 /* index id */, "primary", /* name */ diff --git a/src/box/sophia_engine.cc b/src/box/sophia_engine.cc index 561f528217913f7d6d78baa65f0250e96d1ffe33..a70423b6aa351fe5e6da2e43d09ae2c14c30861f 100644 --- a/src/box/sophia_engine.cc +++ b/src/box/sophia_engine.cc @@ -320,7 +320,7 @@ sophia_join_key_def(void *env, void *db) uint32_t id = sp_getint(db, "id"); uint32_t count = sp_getint(db, "key-count"); struct key_def *key_def; - struct key_opts key_opts = { true /* is_unique */, 0 /* dimension */ }; + struct key_opts key_opts = key_opts_default; key_def = key_def_new(id, 0, "sophia_join", TREE, &key_opts, count); int i = 0; while (i < count) { diff --git a/src/box/tuple.cc b/src/box/tuple.cc index 25836fde3ca7d2daf4a2bb9743e9df65954b5a8f..6aa2f2df5ec7dfd81ee9141cca9e4fd639636aa2 100644 --- a/src/box/tuple.cc +++ b/src/box/tuple.cc @@ -365,11 +365,15 @@ tuple_next(struct tuple_iterator *it) extern inline uint32_t tuple_next_u32(struct tuple_iterator *it); -static const char * +const char * tuple_field_to_cstr(const char *field, uint32_t len) { - static __thread char buf[256]; - len = MIN(len, sizeof(buf) - 1); + enum { MAX_STR_BUFS = 3, MAX_BUF_LEN = 256 }; + static __thread char bufs[MAX_STR_BUFS][MAX_BUF_LEN]; + static __thread int i = 0; + char *buf = bufs[i]; + i = (i + 1) % MAX_STR_BUFS; + len = MIN(len, MAX_BUF_LEN - 1); memcpy(buf, field, len); buf[len] = '\0'; return buf; diff --git a/src/box/tuple.h b/src/box/tuple.h index 315f6eb410d853d31908cfa9bb678a7b27ce784d..98396201365543f3c9ed78658a66a7c9f6547828 100644 --- a/src/box/tuple.h +++ b/src/box/tuple.h @@ -390,6 +390,10 @@ tuple_field_num(const struct tuple* tuple, uint32_t field_no) const char * tuple_field_cstr(struct tuple *tuple, uint32_t i); +/** Helper method for the above function. */ +const char * +tuple_field_to_cstr(const char *field, uint32_t len); + /** * @brief Tuple Interator */ diff --git a/src/cbus.cc b/src/cbus.cc index 131a7f9d57c4400423d5d934d71bb7daae0c5f49..fd4a3f002ddc8615e47a425aec3277f2b9b372fd 100644 --- a/src/cbus.cc +++ b/src/cbus.cc @@ -31,6 +31,14 @@ #include "cbus.h" #include "scoped_guard.h" +struct rmean *rmean_net = NULL; +const char *rmean_net_strings[RMEAN_NET_LAST] = { + "EVENTS", + "LOCKS", + "RECEIVED", + "SENT" +}; + static void cbus_flush_cb(ev_loop * /* loop */, struct ev_async *watcher, int /* events */); @@ -183,9 +191,14 @@ cbus_flush_cb(ev_loop * /* loop */, struct ev_async *watcher, STAILQ_CONCAT(&peer->output, &peer->pipe); cbus_unlock(pipe->bus); + pipe->n_input = 0; - if (pipe_was_empty) + if (pipe_was_empty) { + /* Count statistics */ + rmean_collect(rmean_net, RMEAN_NET_EVENTS, 1); + ev_async_send(pipe->consumer, &pipe->fetch_output); + } if (peer_output_was_empty && !STAILQ_EMPTY(&peer->output)) ev_feed_event(peer->consumer, &peer->fetch_output, EV_CUSTOM); } @@ -200,6 +213,8 @@ cpipe_peek_impl(struct cpipe *pipe) assert(peer->producer == loop()); bool peer_pipe_was_empty = false; + + cbus_lock(pipe->bus); STAILQ_CONCAT(&pipe->output, &pipe->pipe); if (! STAILQ_EMPTY(&peer->input)) { @@ -209,8 +224,12 @@ cpipe_peek_impl(struct cpipe *pipe) cbus_unlock(pipe->bus); peer->n_input = 0; - if (peer_pipe_was_empty) + if (peer_pipe_was_empty) { + /* Count statistics */ + rmean_collect(rmean_net, RMEAN_NET_EVENTS, 1); + ev_async_send(peer->consumer, &peer->fetch_output); + } return STAILQ_FIRST(&pipe->output); } diff --git a/src/cbus.h b/src/cbus.h index 4985a36e3e1b15cc9d28d6d339846b22e52c9464..3b717d369950e79ea007dd7211e3a98b420ab343 100644 --- a/src/cbus.h +++ b/src/cbus.h @@ -32,6 +32,7 @@ */ #include "fiber.h" #include "coio.h" +#include "rmean.h" /** cbus, cmsg - inter-cord bus and messaging */ @@ -39,6 +40,20 @@ struct cmsg; struct cpipe; typedef void (*cmsg_f)(struct cmsg *); +/** rmean_net - network statistics (iproto & cbus) */ +extern struct rmean *rmean_net; + +enum rmean_net_name { + RMEAN_NET_EVENTS, + RMEAN_NET_LOCKS, + RMEAN_NET_RECEIVED, + RMEAN_NET_SENT, + + RMEAN_NET_LAST +}; + +extern const char *rmean_net_strings[RMEAN_NET_LAST]; + /** * One hop in a message travel route. A message may need to be * delivered to many destinations before it can be dispensed with. @@ -364,6 +379,10 @@ cbus_join(struct cbus *bus, struct cpipe *pipe); static inline void cbus_lock(struct cbus *bus) { + /* Count statistics */ + if (rmean_net) + rmean_collect(rmean_net, RMEAN_NET_LOCKS, 1); + tt_pthread_mutex_lock(&bus->mutex); } diff --git a/src/lib/salad/rtree.c b/src/lib/salad/rtree.c index 5901168ab515058e2f912e93eabebb1ece3790b4..af6c86968a339900b1023c799ad9736b95fcf15f 100644 --- a/src/lib/salad/rtree.c +++ b/src/lib/salad/rtree.c @@ -131,6 +131,28 @@ rtree_set2dp(struct rtree_rect *rect, coord_t x, coord_t y) rect->coords[3] = y; } +/* Manhattan distance */ +static sq_coord_t +rtree_rect_neigh_distance(const struct rtree_rect *rect, + const struct rtree_rect *neigh_rect, + unsigned dimension) +{ + sq_coord_t result = 0; + for (int i = dimension; --i >= 0; ) { + const coord_t *coords = &rect->coords[2 * i]; + coord_t neigh_coord = neigh_rect->coords[2 * i]; + if (neigh_coord < coords[0]) { + sq_coord_t diff = (sq_coord_t)(neigh_coord - coords[0]); + result += -diff; + } else if (neigh_coord > coords[1]) { + sq_coord_t diff = (sq_coord_t)(neigh_coord - coords[1]); + result += diff; + } + } + return result; +} + +/* Euclid distance, squared */ static sq_coord_t rtree_rect_neigh_distance2(const struct rtree_rect *rect, const struct rtree_rect *neigh_rect, @@ -855,8 +877,13 @@ rtree_iterator_process_neigh(struct rtree_iterator *itr, for (int i = 0, n = pg->n; i < n; i++) { struct rtree_page_branch *b; b = rtree_branch_get(itr->tree, pg, i); - coord_t distance = - rtree_rect_neigh_distance2(&b->rect, &itr->rect, d); + coord_t distance; + if (itr->tree->distance_type == RTREE_EUCLID) + distance = rtree_rect_neigh_distance2(&b->rect, + &itr->rect, d); + else + distance = rtree_rect_neigh_distance(&b->rect, + &itr->rect, d); struct rtree_neighbor *neigh = rtree_iterator_new_neighbor(itr, b->data.page, distance, level - 1); @@ -919,7 +946,8 @@ rtree_iterator_next(struct rtree_iterator *itr) int rtree_init(struct rtree *tree, unsigned dimension, uint32_t extent_size, - rtree_extent_alloc_t extent_alloc, rtree_extent_free_t extent_free) + rtree_extent_alloc_t extent_alloc, rtree_extent_free_t extent_free, + enum rtree_distance_type distance_type) { tree->n_records = 0; tree->height = 0; @@ -929,6 +957,7 @@ rtree_init(struct rtree *tree, unsigned dimension, uint32_t extent_size, tree->free_pages = 0; tree->dimension = dimension; + tree->distance_type = distance_type; tree->page_branch_size = (RTREE_BRANCH_DATA_SIZE + dimension * 2 * sizeof(coord_t)); tree->page_size = RTREE_OPTIMAL_BRANCHES_IN_PAGE * @@ -1071,9 +1100,15 @@ rtree_search(const struct rtree *tree, const struct rtree_rect *rect, if (tree->root) { struct rtree_rect cover; rtree_page_cover(tree, tree->root, &cover); - sq_coord_t distance = + sq_coord_t distance; + if (tree->distance_type == RTREE_EUCLID) + distance = rtree_rect_neigh_distance2(&cover, rect, tree->dimension); + else + distance = + rtree_rect_neigh_distance(&cover, rect, + tree->dimension); struct rtree_neighbor *n = rtree_iterator_new_neighbor(itr, tree->root, distance, diff --git a/src/lib/salad/rtree.h b/src/lib/salad/rtree.h index 4f5fdc8d1ce7cad279a140f320038c3c7804457d..72a87d37b062b2a71e54521862ce2d900656574c 100644 --- a/src/lib/salad/rtree.h +++ b/src/lib/salad/rtree.h @@ -114,6 +114,12 @@ typedef bool (*rtree_comparator_t)(const struct rtree_rect *rt1, const struct rtree_rect *rt2, unsigned dimension); +/* Type distance comparison */ +enum rtree_distance_type { + RTREE_EUCLID = 0, /* Euclid distance, sqrt(dx*dx + dy*dy) */ + RTREE_MANHATTAN = 1 /* Manhattan distance, fabs(dx) + fabs(dy) */ +}; + /* Main rtree struct */ struct rtree { @@ -144,6 +150,8 @@ struct rtree struct matras mtab; /* List of free pages */ void *free_pages; + /* Distance type */ + enum rtree_distance_type distance_type; }; /* Struct for iteration and retrieving rtree values */ @@ -227,7 +235,8 @@ rtree_set2dp(struct rtree_rect *rect, coord_t x, coord_t y); */ int rtree_init(struct rtree *tree, unsigned dimension, uint32_t extent_size, - rtree_extent_alloc_t extent_alloc, rtree_extent_free_t extent_free); + rtree_extent_alloc_t extent_alloc, rtree_extent_free_t extent_free, + enum rtree_distance_type distance_type); /** * @brief Destroy a tree diff --git a/src/rmean.cc b/src/rmean.cc index d7d781e18b20b72709be9e2a22602df0504b3816..0720bf01749f66c56cf92a1b00ce969d04a153bc 100644 --- a/src/rmean.cc +++ b/src/rmean.cc @@ -38,6 +38,8 @@ void rmean_collect(struct rmean *rmean, size_t name, int64_t value) { + assert(name < rmean->stats_n); + rmean->stats[name].value[0] += value; rmean->stats[name].total += value; } @@ -115,10 +117,10 @@ rmean_new(const char **name, size_t n) void rmean_delete(struct rmean *rmean) { - if (rmean) { - ev_timer_stop(loop(), &rmean->timer); - free(rmean); - } + + ev_timer_stop(loop(), &rmean->timer); + free(rmean); + rmean = 0; } void diff --git a/src/rmean.h b/src/rmean.h index 5b980b620c06d03694ef658ca9a5b4de54203275..b6dd5848a933aa40a14abf8775a19ffcd094a037 100644 --- a/src/rmean.h +++ b/src/rmean.h @@ -39,6 +39,7 @@ #define PERF_SECS 5 + struct stats { const char *name; int64_t value[PERF_SECS + 1]; diff --git a/test/box/misc.result b/test/box/misc.result index 24a3d4705eb3004b0849050db3f0f2dbce7285fb..03daae50f2ebdc57bdd98d77c9b7104cbb3c87d3 100644 --- a/test/box/misc.result +++ b/test/box/misc.result @@ -146,6 +146,7 @@ t; - REPLACE - UPSERT - AUTH + - ERROR - UPDATE - total - rps @@ -249,6 +250,7 @@ t; - 'box.error.CREATE_FUNCTION : 50' - 'box.error.SOPHIA : 60' - 'box.error.NO_SUCH_INDEX : 35' + - 'box.error.UNKNOWN_RTREE_INDEX_DISTANCE_TYPE : 103' - 'box.error.TUPLE_IS_TOO_LONG : 27' - 'box.error.UNKNOWN_SERVER : 62' - 'box.error.FUNCTION_EXISTS : 52' diff --git a/test/box/net.box.result b/test/box/net.box.result index 7f0da9fe206261d2c832c9d4f401df1dc68993de..391de7170253fe668104fd5ca2bcb49a99b32339 100644 --- a/test/box/net.box.result +++ b/test/box/net.box.result @@ -771,6 +771,12 @@ cn:ping() cn:close() --- ... +box.schema.user.revoke('netbox', 'read, write, execute', 'universe'); +--- +... +box.schema.user.drop('netbox') +--- +... -- #594: bad argument #1 to 'setmetatable' (table expected, got number) --# setopt delimiter ';' function gh594() diff --git a/test/box/net.box.test.lua b/test/box/net.box.test.lua index 5942779adbd246d12bcaf679b7e9050bfbc8e92b..28281fafa9527c78f18a76fd39d8d8b6c125ba7f 100644 --- a/test/box/net.box.test.lua +++ b/test/box/net.box.test.lua @@ -302,6 +302,9 @@ cn = remote.new(uri, { password = 'test' }) cn:ping() cn:close() +box.schema.user.revoke('netbox', 'read, write, execute', 'universe'); +box.schema.user.drop('netbox') + -- #594: bad argument #1 to 'setmetatable' (table expected, got number) --# setopt delimiter ';' function gh594() diff --git a/test/box/rtree_misc.result b/test/box/rtree_misc.result index 9b5183d5fd4bbe726ed64d9a48c79aefcc88f114..fa15112896e2c505e8ddc1f1b381180131304885 100644 --- a/test/box/rtree_misc.result +++ b/test/box/rtree_misc.result @@ -262,11 +262,6 @@ box.snapshot() --- - ok ... -for i = 11,20 do s:insert{i, i, i + 1, i + 1} ---- -- error: '[string "for i = 11,20 do s:insert{i, i, i + 1, i + 1} "]:1: ''end'' expected - near ''<eof>''' -... i:select({0, 0}, {iterator = 'neighbor'}) --- - - [1, 1, [1, 1, 2, 2]] @@ -304,3 +299,185 @@ i:select({0, 0}, {iterator = 'neighbor'}) s:drop() --- ... +s = box.schema.space.create('spatial') +--- +... +i = s:create_index('primary', { type = 'tree', parts = {1, 'num'}}) +--- +... +i = s:create_index('spatial', { type = 'rtree', unique = false, parts = {3, 'array'}, dimension = 4}) +--- +... +for i = 1,10 do s:insert{i, i, {i, i, i, i, i + 1, i + 1, i + 1, i + 1}} end +--- +... +box.snapshot() +--- +- ok +... +i:select({0, 0, 0, 0}, {iterator = 'neighbor'}) +--- +- - [1, 1, [1, 1, 1, 1, 2, 2, 2, 2]] + - [2, 2, [2, 2, 2, 2, 3, 3, 3, 3]] + - [3, 3, [3, 3, 3, 3, 4, 4, 4, 4]] + - [4, 4, [4, 4, 4, 4, 5, 5, 5, 5]] + - [5, 5, [5, 5, 5, 5, 6, 6, 6, 6]] + - [6, 6, [6, 6, 6, 6, 7, 7, 7, 7]] + - [7, 7, [7, 7, 7, 7, 8, 8, 8, 8]] + - [8, 8, [8, 8, 8, 8, 9, 9, 9, 9]] + - [9, 9, [9, 9, 9, 9, 10, 10, 10, 10]] + - [10, 10, [10, 10, 10, 10, 11, 11, 11, 11]] +... +--# stop server default +--# start server default +s = box.space.spatial +--- +... +i = s.index.spatial +--- +... +i:select({0, 0, 0, 0}, {iterator = 'neighbor'}) +--- +- - [1, 1, [1, 1, 1, 1, 2, 2, 2, 2]] + - [2, 2, [2, 2, 2, 2, 3, 3, 3, 3]] + - [3, 3, [3, 3, 3, 3, 4, 4, 4, 4]] + - [4, 4, [4, 4, 4, 4, 5, 5, 5, 5]] + - [5, 5, [5, 5, 5, 5, 6, 6, 6, 6]] + - [6, 6, [6, 6, 6, 6, 7, 7, 7, 7]] + - [7, 7, [7, 7, 7, 7, 8, 8, 8, 8]] + - [8, 8, [8, 8, 8, 8, 9, 9, 9, 9]] + - [9, 9, [9, 9, 9, 9, 10, 10, 10, 10]] + - [10, 10, [10, 10, 10, 10, 11, 11, 11, 11]] +... +s:drop() +--- +... +-- distance type +iopts = { type = 'rtree', unique = false, parts = {2, 'array'} } +--- +... +iopts['distance'] = 'euclid' +--- +... +s = box.schema.space.create('spatial') +--- +... +i = s:create_index('primary', { type = 'tree', parts = {1, 'num'}}) +--- +... +i = s:create_index('spatial', iopts) +--- +... +s:insert{1, {0, 5}} +--- +- [1, [0, 5]] +... +s:insert{2, {5, 0}} +--- +- [2, [5, 0]] +... +s:insert{3, {5, 5}} +--- +- [3, [5, 5]] +... +s:insert{4, {8, 0}} +--- +- [4, [8, 0]] +... +s:insert{5, {0, 8}} +--- +- [5, [0, 8]] +... +s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) +--- +- - [1, [0, 5]] + - [2, [5, 0]] + - [3, [5, 5]] + - [4, [8, 0]] + - [5, [0, 8]] +... +s:drop() +--- +... +iopts = { type = 'rtree', unique = false, parts = {2, 'array'} } +--- +... +iopts['distance'] = 'manhattan' +--- +... +s = box.schema.space.create('spatial') +--- +... +i = s:create_index('primary', { type = 'tree', parts = {1, 'num'}}) +--- +... +i = s:create_index('spatial', iopts) +--- +... +s:insert{1, {0, 5}} +--- +- [1, [0, 5]] +... +s:insert{2, {5, 0}} +--- +- [2, [5, 0]] +... +s:insert{3, {5, 5}} +--- +- [3, [5, 5]] +... +s:insert{4, {8, 0}} +--- +- [4, [8, 0]] +... +s:insert{5, {0, 8}} +--- +- [5, [0, 8]] +... +s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) +--- +- - [1, [0, 5]] + - [2, [5, 0]] + - [4, [8, 0]] + - [5, [0, 8]] + - [3, [5, 5]] +... +--# stop server default +--# start server default +s = box.space.spatial +--- +... +i = s.index.spatial +--- +... +s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) +--- +- - [1, [0, 5]] + - [2, [5, 0]] + - [4, [8, 0]] + - [5, [0, 8]] + - [3, [5, 5]] +... +box.snapshot() +--- +- ok +... +--# stop server default +--# start server default +s = box.space.spatial +--- +... +i = s.index.spatial +--- +... +s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) +--- +- - [1, [0, 5]] + - [2, [5, 0]] + - [4, [8, 0]] + - [5, [0, 8]] + - [3, [5, 5]] +... +s:drop() +--- +... diff --git a/test/box/rtree_misc.test.lua b/test/box/rtree_misc.test.lua index fdc21ce307428d219d3683352d8e22a6144750d6..20cee17dc5c55548e1e896315deb85307c335c58 100644 --- a/test/box/rtree_misc.test.lua +++ b/test/box/rtree_misc.test.lua @@ -103,7 +103,6 @@ i = s:create_index('primary', { type = 'tree', parts = {1, 'num'}}) i = s:create_index('spatial', { type = 'rtree', unique = false, parts = {3, 'array'}}) for i = 1,10 do s:insert{i, i, {i, i, i + 1, i + 1}} end box.snapshot() -for i = 11,20 do s:insert{i, i, i + 1, i + 1} i:select({0, 0}, {iterator = 'neighbor'}) --# stop server default --# start server default @@ -112,3 +111,53 @@ i = s.index.spatial i:select({0, 0}, {iterator = 'neighbor'}) s:drop() +s = box.schema.space.create('spatial') +i = s:create_index('primary', { type = 'tree', parts = {1, 'num'}}) +i = s:create_index('spatial', { type = 'rtree', unique = false, parts = {3, 'array'}, dimension = 4}) +for i = 1,10 do s:insert{i, i, {i, i, i, i, i + 1, i + 1, i + 1, i + 1}} end +box.snapshot() +i:select({0, 0, 0, 0}, {iterator = 'neighbor'}) +--# stop server default +--# start server default +s = box.space.spatial +i = s.index.spatial +i:select({0, 0, 0, 0}, {iterator = 'neighbor'}) +s:drop() + +-- distance type +iopts = { type = 'rtree', unique = false, parts = {2, 'array'} } +iopts['distance'] = 'euclid' +s = box.schema.space.create('spatial') +i = s:create_index('primary', { type = 'tree', parts = {1, 'num'}}) +i = s:create_index('spatial', iopts) +s:insert{1, {0, 5}} +s:insert{2, {5, 0}} +s:insert{3, {5, 5}} +s:insert{4, {8, 0}} +s:insert{5, {0, 8}} +s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) +s:drop() + +iopts = { type = 'rtree', unique = false, parts = {2, 'array'} } +iopts['distance'] = 'manhattan' +s = box.schema.space.create('spatial') +i = s:create_index('primary', { type = 'tree', parts = {1, 'num'}}) +i = s:create_index('spatial', iopts) +s:insert{1, {0, 5}} +s:insert{2, {5, 0}} +s:insert{3, {5, 5}} +s:insert{4, {8, 0}} +s:insert{5, {0, 8}} +s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) +--# stop server default +--# start server default +s = box.space.spatial +i = s.index.spatial +s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) +box.snapshot() +--# stop server default +--# start server default +s = box.space.spatial +i = s.index.spatial +s.index.spatial:select({{0, 0}}, {iterator = 'neighbor'}) +s:drop() diff --git a/test/box/stat.result b/test/box/stat.result index f497bce2950ac9ed7361cea9924209f0c141f798..4ea136fccd32e24696f54bc0557b7b12015f7c29 100644 --- a/test/box/stat.result +++ b/test/box/stat.result @@ -21,6 +21,10 @@ box.stat.SELECT.total --- - 0 ... +box.stat.ERROR.total +--- +- 0 +... space = box.schema.space.create('tweedledum') --- ... @@ -50,6 +54,15 @@ box.stat.REPLACE.total ... box.stat.SELECT.total --- +- 2 +... +-- check exceptions +space:get('Impossible value') +--- +- error: 'Supplied key type of part 0 does not match index part type: expected NUM' +... +box.stat.ERROR.total +--- - 1 ... --# stop server default @@ -75,6 +88,10 @@ box.stat.SELECT.total --- - 0 ... +box.stat.ERROR.total +--- +- 0 +... -- cleanup box.space.tweedledum:drop() --- diff --git a/test/box/stat.test.lua b/test/box/stat.test.lua index 5ff384a4d19608c9320d71fc863e4dd644691b95..7484c781cfd41becb98fb988e7ff48d8e4f8cc24 100644 --- a/test/box/stat.test.lua +++ b/test/box/stat.test.lua @@ -6,6 +6,7 @@ box.stat.DELETE.total box.stat.UPDATE.total box.stat.REPLACE.total box.stat.SELECT.total +box.stat.ERROR.total space = box.schema.space.create('tweedledum') index = space:create_index('primary', { type = 'hash' }) @@ -19,6 +20,10 @@ box.stat.UPDATE.total box.stat.REPLACE.total box.stat.SELECT.total +-- check exceptions +space:get('Impossible value') +box.stat.ERROR.total + --# stop server default --# start server default @@ -28,6 +33,7 @@ box.stat.DELETE.total box.stat.UPDATE.total box.stat.REPLACE.total box.stat.SELECT.total +box.stat.ERROR.total -- cleanup box.space.tweedledum:drop() diff --git a/test/box/stat_net.result b/test/box/stat_net.result new file mode 100644 index 0000000000000000000000000000000000000000..56f1a373d8ff0adc29dceec07c39097a9809a203 --- /dev/null +++ b/test/box/stat_net.result @@ -0,0 +1,60 @@ +-- clear statistics +--# stop server default +--# start server default +box.stat.net.SENT -- zero +--- +- total: 0 + rps: 0 +... +box.stat.net.RECEIVED -- zero +--- +- total: 0 + rps: 0 +... +space = box.schema.space.create('tweedledum') +--- +... +box.schema.user.grant('guest','read,write,execute','universe') +--- +... +index = space:create_index('primary', { type = 'hash' }) +--- +... +remote = require 'net.box' +--- +... +LISTEN = require('uri').parse(box.cfg.listen) +--- +... +cn = remote:new(LISTEN.host, LISTEN.service) +--- +... +cn.space.tweedledum:select() --small request +--- +- [] +... +box.stat.net.SENT.total > 0 +--- +- true +... +box.stat.net.RECEIVED.total > 0 +--- +- true +... +box.stat.net.EVENTS.total > 0 +--- +- true +... +box.stat.net.LOCKS.total > 0 +--- +- true +... +space:drop() +--- +... +cn:close() +--- +... +box.schema.user.revoke('guest','read,write,execute','universe') +--- +... diff --git a/test/box/stat_net.test.lua b/test/box/stat_net.test.lua new file mode 100644 index 0000000000000000000000000000000000000000..334bb149fc88552f7a052701c5b588db2e1a14ed --- /dev/null +++ b/test/box/stat_net.test.lua @@ -0,0 +1,25 @@ +-- clear statistics +--# stop server default +--# start server default + +box.stat.net.SENT -- zero +box.stat.net.RECEIVED -- zero + +space = box.schema.space.create('tweedledum') +box.schema.user.grant('guest','read,write,execute','universe') +index = space:create_index('primary', { type = 'hash' }) +remote = require 'net.box' + +LISTEN = require('uri').parse(box.cfg.listen) +cn = remote:new(LISTEN.host, LISTEN.service) + +cn.space.tweedledum:select() --small request + +box.stat.net.SENT.total > 0 +box.stat.net.RECEIVED.total > 0 +box.stat.net.EVENTS.total > 0 +box.stat.net.LOCKS.total > 0 + +space:drop() +cn:close() +box.schema.user.revoke('guest','read,write,execute','universe') diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 384fc7674ffa4fd93e752fe4cbf54a9ecb3bce75..c86a062ad7d52fce7758d5b0d33995a342e1032e 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -60,7 +60,8 @@ target_link_libraries(light.test small) add_executable(vclock.test vclock.cc unit.c ${CMAKE_SOURCE_DIR}/src/box/vclock.c ${CMAKE_SOURCE_DIR}/src/box/errcode.c - ${CMAKE_SOURCE_DIR}/src/box/error.cc) + ${CMAKE_SOURCE_DIR}/src/box/error.cc + ${CMAKE_SOURCE_DIR}/src/rmean.cc) target_link_libraries(vclock.test core small) add_executable(quota.test quota.cc unit.c) target_link_libraries(quota.test pthread) diff --git a/test/unit/rtree.cc b/test/unit/rtree.cc index 3469fe2d1ec705936d0e73e5e9bb324fa9d04a95..bd6a6f64d5eb141da9b372429c0ed453c045294b 100644 --- a/test/unit/rtree.cc +++ b/test/unit/rtree.cc @@ -36,7 +36,8 @@ simple_check() header(); struct rtree tree; - rtree_init(&tree, 2, extent_size, extent_alloc, extent_free); + rtree_init(&tree, 2, extent_size, extent_alloc, extent_free, + RTREE_EUCLID); printf("Insert 1..X, remove 1..X\n"); for (size_t i = 1; i <= rounds; i++) { @@ -228,7 +229,8 @@ neighbor_test() for (size_t i = 0; i <= test_count; i++) { struct rtree tree; - rtree_init(&tree, 2, extent_size, extent_alloc, extent_free); + rtree_init(&tree, 2, extent_size, extent_alloc, extent_free, + RTREE_EUCLID); rtree_test_build(&tree, arr, i); diff --git a/test/unit/rtree_itr.cc b/test/unit/rtree_itr.cc index 088edd657e1ca3e929247251656a931037ce263d..3c6c3c3fdc1e7d46dc59a759da1a26b3fe9180c6 100644 --- a/test/unit/rtree_itr.cc +++ b/test/unit/rtree_itr.cc @@ -30,7 +30,8 @@ itr_check() header(); struct rtree tree; - rtree_init(&tree, 2, extent_size, extent_alloc, extent_free); + rtree_init(&tree, 2, extent_size, extent_alloc, extent_free, + RTREE_EUCLID); /* Filling tree */ const size_t count1 = 10000; @@ -211,7 +212,8 @@ itr_invalidate_check() del_cnt = test_size - del_pos; } struct rtree tree; - rtree_init(&tree, 2, extent_size, extent_alloc, extent_free); + rtree_init(&tree, 2, extent_size, extent_alloc, extent_free, + RTREE_EUCLID); struct rtree_iterator iterators[test_size]; for (size_t i = 0; i < test_size; i++) rtree_iterator_init(iterators + i); @@ -255,7 +257,8 @@ itr_invalidate_check() size_t ins_cnt = rand() % max_insert_count + 1; struct rtree tree; - rtree_init(&tree, 2, extent_size, extent_alloc, extent_free); + rtree_init(&tree, 2, extent_size, extent_alloc, extent_free, + RTREE_EUCLID); struct rtree_iterator iterators[test_size]; for (size_t i = 0; i < test_size; i++) rtree_iterator_init(iterators + i); diff --git a/test/unit/rtree_multidim.cc b/test/unit/rtree_multidim.cc index 4d67179e47d47437180bf4fb6e1770bf27ffaaa2..70c11b2b9f3b35f31f608a8f4d169276bec73a93 100644 --- a/test/unit/rtree_multidim.cc +++ b/test/unit/rtree_multidim.cc @@ -139,6 +139,20 @@ struct CBox { } return res; } + coord_t DistanceMan(const CBox<DIMENSION> &point) const + { + coord_t res = 0; + for (unsigned i = 0; i < DIMENSION; i++) { + if (point.pairs[i].a < pairs[i].a) { + coord_t d = pairs[i].a - point.pairs[i].a; + res += d; + } else if (point.pairs[i].a > pairs[i].b) { + coord_t d = point.pairs[i].a - pairs[i].b; + res += d; + } + } + return res; + } }; template<unsigned DIMENSION> @@ -215,6 +229,8 @@ struct CBoxSet { } void SelectNeigh(const CBox<DIMENSION> &point, vector<CBoxSetEntry<DIMENSION> > &result) const; + void SelectNeighMan(const CBox<DIMENSION> &point, + vector<CBoxSetEntry<DIMENSION> > &result) const; }; template<unsigned DIMENSION> @@ -230,9 +246,22 @@ struct CEntryByDistance { } }; +template<unsigned DIMENSION> +struct CEntryByDistanceMan { + const CBox<DIMENSION> &point; + CEntryByDistanceMan(const CBox<DIMENSION> &point_) : point(point_) {} + bool operator()(const CBoxSetEntry<DIMENSION> &a, + const CBoxSetEntry<DIMENSION> &b) const + { + coord_t da = a.box.DistanceMan(point); + coord_t db = b.box.DistanceMan(point); + return da < db ? true : da > db ? false : a.id < b.id; + } +}; + template<unsigned DIMENSION> void CBoxSet<DIMENSION>::SelectNeigh(const CBox<DIMENSION> &point, - vector<CBoxSetEntry<DIMENSION> > &result) const + vector<CBoxSetEntry<DIMENSION> > &result) const { result.clear(); CEntryByDistance<DIMENSION> comp(point); @@ -243,6 +272,8 @@ void CBoxSet<DIMENSION>::SelectNeigh(const CBox<DIMENSION> &point, continue; set.insert(entries[i]); } + if (set.empty()) + return; coord_t max_d = set.rbegin()->box.Distance2(point); for (; i < entries.size(); i++) { if (!entries[i].used) @@ -260,6 +291,38 @@ void CBoxSet<DIMENSION>::SelectNeigh(const CBox<DIMENSION> &point, result.push_back(itr); } +template<unsigned DIMENSION> +void CBoxSet<DIMENSION>::SelectNeighMan(const CBox<DIMENSION> &point, + vector<CBoxSetEntry<DIMENSION> > &result) const +{ + result.clear(); + CEntryByDistanceMan<DIMENSION> comp(point); + set<CBoxSetEntry<DIMENSION>, CEntryByDistanceMan<DIMENSION> > set(comp); + size_t i = 0; + for (; i < entries.size() && set.size() < NEIGH_COUNT; i++) { + if (!entries[i].used) + continue; + set.insert(entries[i]); + } + if (set.empty()) + return; + coord_t max_d = set.rbegin()->box.DistanceMan(point); + for (; i < entries.size(); i++) { + if (!entries[i].used) + continue; + coord_t d = entries[i].box.DistanceMan(point); + if (d < max_d) { + auto itr = set.end(); + --itr; + set.erase(itr); + set.insert(entries[i]); + max_d = set.rbegin()->box.DistanceMan(point); + } + } + for (auto itr : set) + result.push_back(itr); +} + template<unsigned DIMENSION> static void test_select_neigh(const CBoxSet<DIMENSION> &set, const struct rtree *tree) @@ -291,10 +354,51 @@ test_select_neigh(const CBoxSet<DIMENSION> &set, const struct rtree *tree) } else { for (size_t i = 0; i < res1.size(); i++) if (res1[i].id != res2[i].id && - res1[i].box.Distance2(box) != - res2[i].box.Distance2(box)) + res1[i].box.Distance2(box) != + res2[i].box.Distance2(box)) + printf("%s result differ!\n", __func__); + } + rtree_iterator_destroy(&iterator); + +} + +template<unsigned DIMENSION> +static void +test_select_neigh_man(const CBoxSet<DIMENSION> &set, struct rtree *tree) +{ + CBox<DIMENSION> box; + box.RandomizeBig(); + vector<CBoxSetEntry<DIMENSION> > res1; + set.SelectNeighMan(box, res1); + + struct rtree_rect rt; + box.FillRTreeRect(&rt); + struct rtree_iterator iterator; + rtree_iterator_init(&iterator); + vector<CBoxSetEntry<DIMENSION> > res2; + tree->distance_type = RTREE_MANHATTAN; /* dirty hack */ + if (rtree_search(tree, &rt, SOP_NEIGHBOR, &iterator)) { + void *record; + while((record = rtree_iterator_next(&iterator))) { + CBoxSetEntry<DIMENSION> entry; + entry.id = ((unsigned)(uintptr_t)record) - 1; + entry.box = set.entries[entry.id].box; + res2.push_back(entry); + if (res2.size() == NEIGH_COUNT) + break; + } + } + if (res1.size() != res2.size()) { + printf("%s result size differ %d %d\n", __func__, + (int)res1.size(), (int)res2.size()); + } else { + for (size_t i = 0; i < res1.size(); i++) + if (res1[i].id != res2[i].id && + res1[i].box.DistanceMan(box) != + res2[i].box.DistanceMan(box)) printf("%s result differ!\n", __func__); } + tree->distance_type = RTREE_EUCLID; /* dirty hack */ rtree_iterator_destroy(&iterator); } @@ -382,7 +486,8 @@ rand_test() CBoxSet<DIMENSION> set; struct rtree tree; - rtree_init(&tree, DIMENSION, extent_size, extent_alloc, extent_free); + rtree_init(&tree, DIMENSION, extent_size, extent_alloc, extent_free, + RTREE_EUCLID); printf("\tDIMENSION: %u, page size: %u, max fill: %u\n", DIMENSION, tree.page_size, tree.page_max_fill); @@ -412,6 +517,7 @@ rand_test() } assert(set.boxCount == tree.n_records); test_select_neigh<DIMENSION>(set, &tree); + test_select_neigh_man<DIMENSION>(set, &tree); test_select_in<DIMENSION>(set, &tree); test_select_strict_in<DIMENSION>(set, &tree); }