From c4a12d786efbda76b770d927c50de4748b842252 Mon Sep 17 00:00:00 2001
From: Konstantin Osipov <kostja.osipov@gmail.com>
Date: Thu, 6 Oct 2011 22:26:48 +0400
Subject: [PATCH] Lua: implement a range select and truncate namespace.
Implement iterators, range select, namespace truncate.
---
core/tarantool_lua.m | 5 +-
mod/box/box.lua | 39 +++++++++++
mod/box/box_lua.m | 127 +++++++++++++++++++++++++++++++++++-
test/box/lua.result | 10 ++-
test/box/lua.test | 1 +
test/box_big/lua.result | 21 ++++++
test/box_big/lua.test | 11 ++++
test/box_big/sql.result | 60 +++++++++++++++++
test/box_big/sql.test | 10 +++
test/box_big/tree_pk.result | 7 ++
test/box_big/tree_pk.test | 2 +
11 files changed, 287 insertions(+), 6 deletions(-)
diff --git a/core/tarantool_lua.m b/core/tarantool_lua.m
index d19390a9b7..db75c078bd 100644
--- a/core/tarantool_lua.m
+++ b/core/tarantool_lua.m
@@ -164,6 +164,7 @@ lbox_unpack(struct lua_State *L)
const char *format = luaL_checkstring(L, 1);
int i = 2; /* first arg comes second */
int nargs = lua_gettop(L);
+ size_t size;
u32 u32buf;
while (*format) {
@@ -171,7 +172,9 @@ lbox_unpack(struct lua_State *L)
luaL_error(L, "box.unpack: argument count does not match the format");
switch (*format) {
case 'i':
- u32buf = * (u32 *) lua_tostring(L, i);
+ u32buf = * (u32 *) lua_tolstring(L, i, &size);
+ if (size != sizeof(u32))
+ luaL_error(L, "box.unpack('%c'): got %d bytes (expected: 4)", *format, (int) size);
lua_pushnumber(L, u32buf);
break;
default:
diff --git a/mod/box/box.lua b/mod/box/box.lua
index c0609d3163..099f939485 100644
--- a/mod/box/box.lua
+++ b/mod/box/box.lua
@@ -13,6 +13,16 @@ function box.select(space, index, ...)
#key, -- key cardinality
unpack(key))))
end
+
+--
+-- Select a range of tuples in a given namespace via a given
+-- index. If key is NULL, starts from the beginning, otherwise
+-- starts from the key.
+--
+function box.select_range(sno, ino, limit, ...)
+ return box.space[tonumber(sno)].index[tonumber(ino)]:range(tonumber(limit), ...)
+end
+
--
-- delete can be done only by the primary key, whose
-- index is always 0. It doesn't accept compound keys
@@ -61,10 +71,29 @@ end
function box.on_reload_configuration()
index_mt = {}
+ -- __len and __index
index_mt.len = function(index) return #index.idx end
index_mt.__newindex = function(table, index)
return error('Attempt to modify a read-only table') end
index_mt.__index = index_mt
+ -- min and max
+ index_mt.min = function(index) return index.idx:min() end
+ index_mt.max = function(index) return index.idx:max() end
+ -- iteration
+ index_mt.pairs = function(index)
+ return index.idx.next, index.idx, nil end
+ --
+ index_mt.range = function(index, limit, ...)
+ range = {}
+ for k, v in index.idx.next, index.idx, ... do
+ if #range >= limit then
+ break
+ end
+ table.insert(range, v)
+ end
+ return unpack(range)
+ end
+ --
space_mt = {}
space_mt.len = function(space) return space.index[0]:len() end
space_mt.__newindex = index_mt.__newindex
@@ -73,6 +102,16 @@ function box.on_reload_configuration()
space_mt.update = function(space, ...) return box.update(space.n, ...) end
space_mt.replace = function(space, ...) return box.replace(space.n, ...) end
space_mt.delete = function(space, ...) return box.delete(space.n, ...) end
+ space_mt.truncate = function(space)
+ while true do
+ k, v = space.index[0].idx:next()
+ if v == nil then
+ break
+ end
+ space:delete(v[0])
+ end
+ end
+ space_mt.pairs = function(space) return space.index[0]:pairs() end
space_mt.__index = space_mt
for i, space in pairs(box.space) do
rawset(space, 'n', i)
diff --git a/mod/box/box_lua.m b/mod/box/box_lua.m
index d5e33aba5b..78f8d9f6bb 100644
--- a/mod/box/box_lua.m
+++ b/mod/box/box_lua.m
@@ -78,9 +78,10 @@ lua_checktuple(struct lua_State *L, int narg)
static inline struct box_tuple *
lua_istuple(struct lua_State *L, int narg)
{
- struct box_tuple *tuple = 0;
- lua_getmetatable(L, narg);
+ if (lua_getmetatable(L, narg) == 0)
+ return NULL;
luaL_getmetatable(L, tuplelib_name);
+ struct box_tuple *tuple = 0;
if (lua_equal(L, -1, -2))
tuple = * (void **) lua_touserdata(L, narg);
lua_pop(L, 2);
@@ -217,11 +218,133 @@ lbox_index_max(struct lua_State *L)
return 1;
}
+/**
+ * Convert an element on Lua stack to a part of an index
+ * key.
+ *
+ * Lua type system has strings, numbers, booleans, tables,
+ * userdata objects. Tarantool indexes only support 32/64-bit
+ * integers and strings.
+ *
+ * Instead of considering each Tarantool <-> Lua type pair,
+ * here we follow the approach similar to one in lbox_pack()
+ * (see tarantool_lua.m):
+ *
+ * Lua numbers are converted to 32 or 64 bit integers,
+ * if key part is integer. In all other cases,
+ * Lua types are converted to Lua strings, and these
+ * strings are used as key parts.
+ */
+
+void append_key_part(struct lua_State *L, int i,
+ struct tbuf *tbuf, enum field_data_type type)
+{
+ const char *str;
+ size_t size;
+ u32 v_u32;
+ u64 v_u64;
+
+ if (lua_type(L, i) == LUA_TNUMBER) {
+ if (type == NUM64) {
+ v_u64 = (u64) lua_tonumber(L, i);
+ str = (char *) &v_u64;
+ size = sizeof(u64);
+ } else {
+ v_u32 = (u32) lua_tointeger(L, i);
+ str = (char *) &v_u32;
+ size = sizeof(u32);
+ }
+ } else {
+ str = luaL_checklstring(L, i, &size);
+ }
+ write_varint32(tbuf, size);
+ tbuf_append(tbuf, str, size);
+}
+
+/**
+ * Lua iterator over a Taratnool/Box index.
+ *
+ * (iteration_state, tuple) = index.next(index, [iteration_state])
+ *
+ * When [iteration_state] is absent or nil
+ * returns a pointer to a new iterator and
+ * to the first tuple (or nil, if the index is
+ * empty).
+ *
+ * When [iteration_state] is a userdata,
+ * i.e. we're inside an iteration loop, retrieves
+ * the next tuple from the iterator.
+ *
+ * Otherwise, [iteration_state] can be used to seed
+ * the iterator with one or several Lua scalars
+ * (numbers, strings) and start iteration from an
+ * offset.
+ *
+ * @todo/FIXME: Currently we always store iteration
+ * state within index. This limits the total
+ * amount of active iterators to 1.
+ */
+static int
+lbox_index_next(struct lua_State *L)
+{
+ struct index *index = lua_checkindex(L, 1);
+ int argc = lua_gettop(L) - 1;
+ if (argc == 0 || (argc == 1 && lua_type(L, 2) == LUA_TNIL)) {
+ /*
+ * If there is nothing or nil on top of the stack,
+ * start iteration from the beginning.
+ */
+ index->iterator_init(index, 0, NULL);
+ } else if (argc > 1 || !lua_islightuserdata(L, 2)) {
+ /*
+ * We've got something different from iterator's
+ * light userdata: must be a key
+ * to start iteration from an offset. Seed the
+ * iterator with this key.
+ */
+ int cardinality;
+ void *key;
+
+ if (argc == 1 && lua_type(L, 2) == LUA_TUSERDATA) {
+ /* Searching by tuple. */
+ struct box_tuple *tuple = lua_checktuple(L, 2);
+ key = tuple->data;
+ cardinality = tuple->cardinality;
+ } else {
+ /* Single or multi- part key. */
+ cardinality = argc;
+ struct tbuf *data = tbuf_alloc(fiber->gc_pool);
+ for (int i = 0; i < argc; ++i)
+ append_key_part(L, i + 2, data,
+ index->key_field[i].type);
+ key = data->data;
+ }
+ /*
+ * We allow partially specified keys for TREE
+ * indexes. HASH indexes can only use single-part
+ * keys.
+ */
+ assert(cardinality != 0);
+ if (cardinality > index->key_cardinality)
+ luaL_error(L, "index.next(): key part count (%d) "
+ "does not match index cardinality (%d)",
+ cardinality, index->key_cardinality);
+ index->iterator_init(index, cardinality, key);
+ }
+ struct box_tuple *tuple = index->iterator.next(index);
+ if (tuple)
+ lua_pushlightuserdata(L, &index->iterator);
+ /* If tuple is NULL, pushes nil as end indicator. */
+ lbox_pushtuple(L, tuple);
+ return tuple ? 2 : 1;
+}
+
static const struct luaL_reg lbox_index_meta[] = {
{"__tostring", lbox_index_tostring},
{"__len", lbox_index_len},
{"min", lbox_index_min},
{"max", lbox_index_max},
+ {"next", lbox_index_next},
{NULL, NULL}
};
diff --git a/test/box/lua.result b/test/box/lua.result
index 91cfbb8cba..cc822e853a 100644
--- a/test/box/lua.result
+++ b/test/box/lua.result
@@ -15,15 +15,16 @@ lua for n in pairs(box) do print(' - box.', n) end
- box.space
- box.cfg
- box.on_reload_configuration
+ - box.update
- box.process
- box.delete
+ - box.insert
- box.replace
- - box.select
- box.index
- box.unpack
+ - box.select
+ - box.select_range
- box.pack
- - box.update
- - box.insert
...
lua box.pack()
---
@@ -226,6 +227,9 @@ Found 1 tuple:
call box.select(0, 0, 'pass')
Found 1 tuple:
[1936941424, 'new', 1684234849]
+call box.select_range(0, 0, 1, 'pass')
+Found 1 tuple:
+[1936941424, 'new', 1684234849]
call box.update(0, 'miss', '+p', 2, '���')
No match
call box.update(0, 'pass', '+p', 2, '���')
diff --git a/test/box/lua.test b/test/box/lua.test
index f1591502a7..c457b4b1e6 100644
--- a/test/box/lua.test
+++ b/test/box/lua.test
@@ -76,6 +76,7 @@ exec sql "call box.insert(0, 'test', 'old', 'abcd')"
exec sql "call box.insert(0, 'test', 'old', 'abcd')"
exec sql "call box.update(0, 'test', '=p=p', 0, 'pass', 1, 'new')"
exec sql "call box.select(0, 0, 'pass')"
+exec sql "call box.select_range(0, 0, 1, 'pass')"
exec sql "call box.update(0, 'miss', '+p', 2, '\1\0\0\0')"
exec sql "call box.update(0, 'pass', '+p', 2, '\1\0\0\0')"
exec admin "lua box.update(0, 'pass', '+p', 2, 1)"
diff --git a/test/box_big/lua.result b/test/box_big/lua.result
index c21c4e7a1f..5d6f808e4a 100644
--- a/test/box_big/lua.result
+++ b/test/box_big/lua.result
@@ -1,8 +1,29 @@
insert into t1 values ('brave', 'new', 'world')
Insert OK, 1 row affected
+lua box.space[1].index[1]:min()
+---
+ - 'brave': {'new', 'world'}
+...
+lua box.space[1].index[1]:max()
+---
+ - 'brave': {'new', 'world'}
+...
call box.select(1, 1, 'new', 'world')
Found 1 tuple:
['brave', 'new', 'world']
call box.delete(1, 'brave')
Found 1 tuple:
['brave', 'new', 'world']
+insert into t5 values ('01234567', 'new', 'world')
+Insert OK, 1 row affected
+insert into t5 values ('00000000', 'of', 'puppets')
+Insert OK, 1 row affected
+insert into t5 values ('00000001', 'of', 'might', 'and', 'magic')
+Insert OK, 1 row affected
+call box.select_range(5, 1, 2, 'of')
+Found 2 tuples:
+['00000001', 'of', 'might', 'and', 'magic']
+['00000000', 'of', 'puppets']
+lua box.space[5]:truncate()
+---
+...
diff --git a/test/box_big/lua.test b/test/box_big/lua.test
index 8f75aee511..220fba9982 100644
--- a/test/box_big/lua.test
+++ b/test/box_big/lua.test
@@ -1,4 +1,15 @@
# encoding: tarantool
exec sql "insert into t1 values ('brave', 'new', 'world')"
+exec admin "lua box.space[1].index[1]:min()"
+exec admin "lua box.space[1].index[1]:max()"
exec sql "call box.select(1, 1, 'new', 'world')"
exec sql "call box.delete(1, 'brave')"
+
+#
+# Check range scan over multipart keys
+#
+exec sql "insert into t5 values ('01234567', 'new', 'world')"
+exec sql "insert into t5 values ('00000000', 'of', 'puppets')"
+exec sql "insert into t5 values ('00000001', 'of', 'might', 'and', 'magic')"
+exec sql "call box.select_range(5, 1, 2, 'of')"
+exec admin "lua box.space[5]:truncate()"
diff --git a/test/box_big/sql.result b/test/box_big/sql.result
index c8ef39f981..9fcdc6dba2 100644
--- a/test/box_big/sql.result
+++ b/test/box_big/sql.result
@@ -75,6 +75,12 @@ No match
select * from t0 where k1='Britney'
Found 1 tuple:
['Spears', 'Britney']
+call box.select_range(0, 0, 100, 'Spears')
+Found 1 tuple:
+['Spears', 'Britney']
+call box.select_range(0, 1, 100, 'Britney')
+Found 1 tuple:
+['Spears', 'Britney']
delete from t0 where k0='Spears'
Delete OK, 1 row affected
#
@@ -86,6 +92,12 @@ insert into t1 values ('key2', 'part1', 'part2_a')
Insert OK, 1 row affected
insert into t1 values ('key3', 'part1', 'part2_b')
Insert OK, 1 row affected
+lua for k, v in box.space[1]:pairs() do print(v) end
+---
+830039403: {'part1', 'part2'}
+863593835: {'part1', 'part2_b'}
+846816619: {'part1', 'part2_a'}
+...
select * from t1 where k0='key1'
Found 1 tuple:
[830039403, 'part1', 'part2']
@@ -100,6 +112,19 @@ Found 3 tuples:
[830039403, 'part1', 'part2']
[846816619, 'part1', 'part2_a']
[863593835, 'part1', 'part2_b']
+call box.select_range(1, 1, 100, 'part1')
+Found 3 tuples:
+[830039403, 'part1', 'part2']
+[846816619, 'part1', 'part2_a']
+[863593835, 'part1', 'part2_b']
+call box.select_range(1, 0, 100, 'key2')
+Found 1 tuple:
+[846816619, 'part1', 'part2_a']
+call box.select_range(1, 1, 100, 'part1', 'part2_a')
+Found 3 tuples:
+[830039403, 'part1', 'part2']
+[846816619, 'part1', 'part2_a']
+[863593835, 'part1', 'part2_b']
insert into t5 values ('01234567', 'part1', 'part2')
Insert OK, 1 row affected
insert into t5 values ('11234567', 'part1', 'part2')
@@ -110,6 +135,14 @@ insert into t5 values ('31234567', 'part1_a', 'part2')
Insert OK, 1 row affected
insert into t5 values ('41234567', 'part1_a', 'part2_a')
Insert OK, 1 row affected
+lua for k, v in box.space[5]:pairs() do print(v) end
+---
+'01234567': {'part1', 'part2'}
+'11234567': {'part1', 'part2'}
+'21234567': {'part1', 'part2_a'}
+'31234567': {'part1_a', 'part2'}
+'41234567': {'part1_a', 'part2_a'}
+...
select * from t5 where k0='01234567'
Found 1 tuple:
['01234567', 'part1', 'part2']
@@ -176,6 +209,9 @@ delete from t5 where k0='31234567'
Delete OK, 1 row affected
delete from t5 where k0='41234567'
Delete OK, 1 row affected
+lua for k, v in box.space[5]:pairs() do print(v) end
+---
+...
#
# A test case for: http://bugs.launchpad.net/bugs/735140
@@ -248,6 +284,24 @@ Found 5 tuples:
[13, 'duplicate three']
[14, 'duplicate three']
[15, 'duplicate three']
+lua for k, v in box.space[4]:pairs() do print(v) end
+---
+10: {'duplicate two'}
+13: {'duplicate three'}
+4: {'duplicate one'}
+14: {'duplicate three'}
+3: {'duplicate one'}
+8: {'duplicate two'}
+1: {'duplicate one'}
+15: {'duplicate three'}
+2: {'duplicate one'}
+9: {'duplicate two'}
+11: {'duplicate three'}
+7: {'duplicate two'}
+6: {'duplicate two'}
+5: {'duplicate one'}
+12: {'duplicate three'}
+...
delete from t4 where k0=1
Delete OK, 1 row affected
delete from t4 where k0=2
@@ -284,6 +338,12 @@ insert into t4 values(2, 'Bilimbi')
Insert OK, 1 row affected
insert into t4 values(3, 'Creature')
Insert OK, 1 row affected
+lua for k, v in box.space[4]:pairs() do print(v) end
+---
+1: {'Aardvark'}
+2: {'Bilimbi'}
+3: {'Creature'}
+...
lua box.space[4].index[0].idx:min()
---
error: 'Unsupported'
diff --git a/test/box_big/sql.test b/test/box_big/sql.test
index dbbcf485fb..76afa77bb9 100644
--- a/test/box_big/sql.test
+++ b/test/box_big/sql.test
@@ -49,6 +49,8 @@ exec sql "insert into t0 values ('Spears', 'Britney')"
exec sql "select * from t0 where k0='Spears'"
exec sql "select * from t0 where k1='Anything'"
exec sql "select * from t0 where k1='Britney'"
+exec sql "call box.select_range(0, 0, 100, 'Spears')"
+exec sql "call box.select_range(0, 1, 100, 'Britney')"
exec sql "delete from t0 where k0='Spears'"
print """#
# Test composite keys with trees
@@ -56,16 +58,21 @@ print """#
exec sql "insert into t1 values ('key1', 'part1', 'part2')"
exec sql "insert into t1 values ('key2', 'part1', 'part2_a')"
exec sql "insert into t1 values ('key3', 'part1', 'part2_b')"
+exec admin "lua for k, v in box.space[1]:pairs() do print(v) end"
exec sql "select * from t1 where k0='key1'"
exec sql "select * from t1 where k0='key2'"
exec sql "select * from t1 where k0='key3'"
exec sql "select * from t1 where k1='part1'"
+exec sql "call box.select_range(1, 1, 100, 'part1')"
+exec sql "call box.select_range(1, 0, 100, 'key2')"
+exec sql "call box.select_range(1, 1, 100, 'part1', 'part2_a')"
# check non-unique multipart keys
exec sql "insert into t5 values ('01234567', 'part1', 'part2')"
exec sql "insert into t5 values ('11234567', 'part1', 'part2')"
exec sql "insert into t5 values ('21234567', 'part1', 'part2_a')"
exec sql "insert into t5 values ('31234567', 'part1_a', 'part2')"
exec sql "insert into t5 values ('41234567', 'part1_a', 'part2_a')"
+exec admin "lua for k, v in box.space[5]:pairs() do print(v) end"
exec sql "select * from t5 where k0='01234567'"
exec sql "select * from t5 where k0='11234567'"
exec sql "select * from t5 where k0='21234567'"
@@ -94,6 +101,7 @@ exec sql "delete from t5 where k0='11234567'"
exec sql "delete from t5 where k0='21234567'"
exec sql "delete from t5 where k0='31234567'"
exec sql "delete from t5 where k0='41234567'"
+exec admin "lua for k, v in box.space[5]:pairs() do print(v) end"
print """
#
@@ -135,6 +143,7 @@ sql.sort = True
exec sql "select * from t4 where k1='duplicate one'"
exec sql "select * from t4 where k1='duplicate two'"
exec sql "select * from t4 where k1='duplicate three'"
+exec admin "lua for k, v in box.space[4]:pairs() do print(v) end"
sql.sort = False
exec sql "delete from t4 where k0=1"
exec sql "delete from t4 where k0=2"
@@ -157,6 +166,7 @@ exec sql "delete from t4 where k0=15"
exec sql "insert into t4 values(1, 'Aardvark')"
exec sql "insert into t4 values(2, 'Bilimbi')"
exec sql "insert into t4 values(3, 'Creature')"
+exec admin "lua for k, v in box.space[4]:pairs() do print(v) end"
exec admin "lua box.space[4].index[0].idx:min()"
exec admin "lua box.space[4].index[0].idx:max()"
exec admin "lua box.space[4].index[1].idx:min()"
diff --git a/test/box_big/tree_pk.result b/test/box_big/tree_pk.result
index 151dc6a152..4a3e468081 100644
--- a/test/box_big/tree_pk.result
+++ b/test/box_big/tree_pk.result
@@ -39,6 +39,13 @@ save snapshot
---
ok
...
+call box.select_range(3, 0, 100, 'second')
+Found 1 tuple:
+['second', 'tuple 2']
+call box.select_range(3, 0, 100, 'identifier')
+Found 2 tuples:
+['identifier', 'tuple']
+['second', 'tuple 2']
insert into t3 values ('third', 'tuple 3')
Insert OK, 1 row affected
select * from t3 where k0 = 'identifier'
diff --git a/test/box_big/tree_pk.test b/test/box_big/tree_pk.test
index a4e77e1a59..3792956c9f 100644
--- a/test/box_big/tree_pk.test
+++ b/test/box_big/tree_pk.test
@@ -20,6 +20,8 @@ exec sql "insert into t3 values ('identifier', 'tuple')"
exec admin "save snapshot"
exec sql "insert into t3 values ('second', 'tuple 2')"
exec admin "save snapshot"
+exec sql "call box.select_range(3, 0, 100, 'second')"
+exec sql "call box.select_range(3, 0, 100, 'identifier')"
exec sql "insert into t3 values ('third', 'tuple 3')"
exec sql "select * from t3 where k0 = 'identifier'"
--
GitLab