Lua: implement a range select and truncate namespace.

Implement iterators, range select, namespace truncate.

core/tarantool_lua.m

@@ -164,6 +164,7 @@ lbox_unpack(struct lua_State *L)
 	const char *format = luaL_checkstring(L, 1);
 	int i = 2; /* first arg comes second */
 	int nargs = lua_gettop(L);
+	size_t size;
 	u32 u32buf;
 
 	while (*format) {
@@ -171,7 +172,9 @@ lbox_unpack(struct lua_State *L)
 			luaL_error(L, "box.unpack: argument count does not match the format");
 		switch (*format) {
 		case 'i':
-			u32buf = * (u32 *) lua_tostring(L, i);
+			u32buf = * (u32 *) lua_tolstring(L, i, &size);
+			if (size != sizeof(u32))
+				luaL_error(L, "box.unpack('%c'): got %d bytes (expected: 4)", *format, (int) size);
 			lua_pushnumber(L, u32buf);
 			break;
 		default:

mod/box/box.lua

@@ -13,6 +13,16 @@ function box.select(space, index, ...)
                                  #key, -- key cardinality
                                  unpack(key))))
 end
+
+--
+-- Select a range of tuples in a given namespace via a given
+-- index. If key is NULL, starts from the beginning, otherwise
+-- starts from the key.
+--
+function box.select_range(sno, ino, limit, ...)
+    return box.space[tonumber(sno)].index[tonumber(ino)]:range(tonumber(limit), ...)
+end
+
 --
 -- delete can be done only by the primary key, whose
 -- index is always 0. It doesn't accept compound keys
@@ -61,10 +71,29 @@ end
 
 function box.on_reload_configuration()
     index_mt = {}
+    -- __len and __index
     index_mt.len = function(index) return #index.idx end
     index_mt.__newindex = function(table, index)
         return error('Attempt to modify a read-only table') end
     index_mt.__index = index_mt
+    -- min and max
+    index_mt.min = function(index) return index.idx:min() end
+    index_mt.max = function(index) return index.idx:max() end
+    -- iteration
+    index_mt.pairs = function(index)
+        return index.idx.next, index.idx, nil end
+    --
+    index_mt.range = function(index, limit, ...)
+        range = {}
+        for k, v in index.idx.next, index.idx, ... do
+            if #range >= limit then
+                break
+            end
+            table.insert(range, v)
+        end
+        return unpack(range)
+    end
+    --
     space_mt = {}
     space_mt.len = function(space) return space.index[0]:len() end
     space_mt.__newindex = index_mt.__newindex
@@ -73,6 +102,16 @@ function box.on_reload_configuration()
     space_mt.update = function(space, ...) return box.update(space.n, ...) end
     space_mt.replace = function(space, ...) return box.replace(space.n, ...) end
     space_mt.delete = function(space, ...) return box.delete(space.n, ...) end
+    space_mt.truncate = function(space)
+        while true do
+            k, v = space.index[0].idx:next()
+            if v == nil then
+                break
+            end
+            space:delete(v[0])
+        end
+    end
+    space_mt.pairs = function(space) return space.index[0]:pairs() end
     space_mt.__index = space_mt
     for i, space in pairs(box.space) do
         rawset(space, 'n', i)

mod/box/box_lua.m

@@ -78,9 +78,10 @@ lua_checktuple(struct lua_State *L, int narg)
 static inline struct box_tuple *
 lua_istuple(struct lua_State *L, int narg)
 {
-	struct box_tuple *tuple = 0;
-	lua_getmetatable(L, narg);
+	if (lua_getmetatable(L, narg) == 0)
+		return NULL;
 	luaL_getmetatable(L, tuplelib_name);
+	struct box_tuple *tuple = 0;
 	if (lua_equal(L, -1, -2))
 		tuple = * (void **) lua_touserdata(L, narg);
 	lua_pop(L, 2);
@@ -217,11 +218,133 @@ lbox_index_max(struct lua_State *L)
 	return 1;
 }
 
+/**
+ * Convert an element on Lua stack to a part of an index
+ * key.
+ *
+ * Lua type system has strings, numbers, booleans, tables,
+ * userdata objects. Tarantool indexes only support 32/64-bit
+ * integers and strings.
+ *
+ * Instead of considering each Tarantool <-> Lua type pair,
+ * here we follow the approach similar to one in lbox_pack()
+ * (see tarantool_lua.m):
+ *
+ * Lua numbers are converted to 32 or 64 bit integers,
+ * if key part is integer. In all other cases,
+ * Lua types are converted to Lua strings, and these
+ * strings are used as key parts.
+ */
+
+void append_key_part(struct lua_State *L, int i,
+		     struct tbuf *tbuf, enum field_data_type type)
+{
+	const char *str;
+	size_t size;
+	u32 v_u32;
+	u64 v_u64;
+
+	if (lua_type(L, i) == LUA_TNUMBER) {
+		if (type == NUM64) {
+			v_u64 = (u64) lua_tonumber(L, i);
+			str = (char *) &v_u64;
+			size = sizeof(u64);
+		} else {
+			v_u32 = (u32) lua_tointeger(L, i);
+			str = (char *) &v_u32;
+			size = sizeof(u32);
+		}
+	} else {
+		str = luaL_checklstring(L, i, &size);
+	}
+	write_varint32(tbuf, size);
+	tbuf_append(tbuf, str, size);
+}
+
+/**
+ * Lua iterator over a Taratnool/Box index.
+ *
+ *	(iteration_state, tuple) = index.next(index, [iteration_state])
+ *
+ * When [iteration_state] is absent or nil
+ * returns a pointer to a new iterator and
+ * to the first tuple (or nil, if the index is
+ * empty).
+ *
+ * When [iteration_state] is a userdata,
+ * i.e. we're inside an iteration loop, retrieves
+ * the next tuple from the iterator.
+ *
+ * Otherwise, [iteration_state] can be used to seed
+ * the iterator with one or several Lua scalars
+ * (numbers, strings) and start iteration from an
+ * offset.
+ *
+ * @todo/FIXME: Currently we always store iteration
+ * state within index. This limits the total
+ * amount of active iterators to 1.
+ */
+static int
+lbox_index_next(struct lua_State *L)
+{
+	struct index *index = lua_checkindex(L, 1);
+	int argc = lua_gettop(L) - 1;
+	if (argc == 0 || (argc == 1 && lua_type(L, 2) == LUA_TNIL)) {
+		/*
+		 * If there is nothing or nil on top of the stack,
+		 * start iteration from the beginning.
+		 */
+		index->iterator_init(index, 0, NULL);
+	} else if (argc > 1 || !lua_islightuserdata(L, 2)) {
+		/*
+		 * We've got something different from iterator's
+		 * light userdata: must be a key
+		 * to start iteration from an offset. Seed the
+		 * iterator with this key.
+		 */
+		int cardinality;
+		void *key;
+
+		if (argc == 1 && lua_type(L, 2) == LUA_TUSERDATA) {
+			/* Searching by tuple. */
+			struct box_tuple *tuple = lua_checktuple(L, 2);
+			key = tuple->data;
+			cardinality = tuple->cardinality;
+		} else {
+			/* Single or multi- part key. */
+			cardinality = argc;
+			struct tbuf *data = tbuf_alloc(fiber->gc_pool);
+			for (int i = 0; i < argc; ++i)
+				append_key_part(L, i + 2, data,
+						index->key_field[i].type);
+			key = data->data;
+		}
+		/*
+		 * We allow partially specified keys for TREE
+		 * indexes. HASH indexes can only use single-part
+		 * keys.
+		*/
+		assert(cardinality != 0);
+		if (cardinality > index->key_cardinality)
+			luaL_error(L, "index.next(): key part count (%d) "
+				   "does not match index cardinality (%d)",
+				   cardinality, index->key_cardinality);
+		index->iterator_init(index, cardinality, key);
+	}
+	struct box_tuple *tuple = index->iterator.next(index);
+	if (tuple)
+		lua_pushlightuserdata(L, &index->iterator);
+	/* If tuple is NULL, pushes nil as end indicator. */
+	lbox_pushtuple(L, tuple);
+	return tuple ? 2 : 1;
+}
+
 static const struct luaL_reg lbox_index_meta[] = {
 	{"__tostring", lbox_index_tostring},
 	{"__len", lbox_index_len},
 	{"min", lbox_index_min},
 	{"max", lbox_index_max},
+	{"next", lbox_index_next},
 	{NULL, NULL}
 };
 

test/box/lua.test

@@ -76,6 +76,7 @@ exec sql "call box.insert(0, 'test', 'old', 'abcd')"
 exec sql "call box.insert(0, 'test', 'old', 'abcd')"
 exec sql "call box.update(0, 'test', '=p=p', 0, 'pass', 1, 'new')"
 exec sql "call box.select(0, 0, 'pass')"
+exec sql "call box.select_range(0, 0, 1, 'pass')"
 exec sql "call box.update(0, 'miss', '+p', 2, '\1\0\0\0')"
 exec sql "call box.update(0, 'pass', '+p', 2, '\1\0\0\0')"
 exec admin "lua box.update(0, 'pass', '+p', 2, 1)"

test/box_big/lua.result

 insert into t1 values ('brave', 'new', 'world')
 Insert OK, 1 row affected
+lua box.space[1].index[1]:min()
+---
+ - 'brave': {'new', 'world'}
+...
+lua box.space[1].index[1]:max()
+---
+ - 'brave': {'new', 'world'}
+...
 call box.select(1, 1, 'new', 'world')
 Found 1 tuple:
 ['brave', 'new', 'world']
 call box.delete(1, 'brave')
 Found 1 tuple:
 ['brave', 'new', 'world']
+insert into t5 values ('01234567', 'new', 'world')
+Insert OK, 1 row affected
+insert into t5 values ('00000000', 'of', 'puppets')
+Insert OK, 1 row affected
+insert into t5 values ('00000001', 'of', 'might', 'and', 'magic')
+Insert OK, 1 row affected
+call box.select_range(5, 1, 2, 'of')
+Found 2 tuples:
+['00000001', 'of', 'might', 'and', 'magic']
+['00000000', 'of', 'puppets']
+lua box.space[5]:truncate()
+---
+...

test/box_big/lua.test

 # encoding: tarantool
 exec sql "insert into t1 values ('brave', 'new', 'world')"
+exec admin "lua box.space[1].index[1]:min()"
+exec admin "lua box.space[1].index[1]:max()"
 exec sql "call box.select(1, 1, 'new', 'world')"
 exec sql "call box.delete(1, 'brave')"
+
+#
+# Check range scan over multipart keys
+#
+exec sql "insert into t5 values ('01234567', 'new', 'world')"
+exec sql "insert into t5 values ('00000000', 'of', 'puppets')"
+exec sql "insert into t5 values ('00000001', 'of', 'might', 'and', 'magic')"
+exec sql "call box.select_range(5, 1, 2, 'of')"
+exec admin "lua box.space[5]:truncate()"

test/box_big/sql.result

@@ -75,6 +75,12 @@ No match
 select * from t0 where k1='Britney'
 Found 1 tuple:
 ['Spears', 'Britney']
+call box.select_range(0, 0, 100, 'Spears')
+Found 1 tuple:
+['Spears', 'Britney']
+call box.select_range(0, 1, 100, 'Britney')
+Found 1 tuple:
+['Spears', 'Britney']
 delete from t0 where k0='Spears'
 Delete OK, 1 row affected
 #
@@ -86,6 +92,12 @@ insert into t1 values ('key2', 'part1', 'part2_a')
 Insert OK, 1 row affected
 insert into t1 values ('key3', 'part1', 'part2_b')
 Insert OK, 1 row affected
+lua for k, v in box.space[1]:pairs() do print(v) end
+---
+830039403: {'part1', 'part2'}
+863593835: {'part1', 'part2_b'}
+846816619: {'part1', 'part2_a'}
+...
 select * from t1 where k0='key1'
 Found 1 tuple:
 [830039403, 'part1', 'part2']
@@ -100,6 +112,19 @@ Found 3 tuples:
 [830039403, 'part1', 'part2']
 [846816619, 'part1', 'part2_a']
 [863593835, 'part1', 'part2_b']
+call box.select_range(1, 1, 100, 'part1')
+Found 3 tuples:
+[830039403, 'part1', 'part2']
+[846816619, 'part1', 'part2_a']
+[863593835, 'part1', 'part2_b']
+call box.select_range(1, 0, 100, 'key2')
+Found 1 tuple:
+[846816619, 'part1', 'part2_a']
+call box.select_range(1, 1, 100, 'part1', 'part2_a')
+Found 3 tuples:
+[830039403, 'part1', 'part2']
+[846816619, 'part1', 'part2_a']
+[863593835, 'part1', 'part2_b']
 insert into t5 values ('01234567', 'part1', 'part2')
 Insert OK, 1 row affected
 insert into t5 values ('11234567', 'part1', 'part2')
@@ -110,6 +135,14 @@ insert into t5 values ('31234567', 'part1_a', 'part2')
 Insert OK, 1 row affected
 insert into t5 values ('41234567', 'part1_a', 'part2_a')
 Insert OK, 1 row affected
+lua for k, v in box.space[5]:pairs() do print(v) end
+---
+'01234567': {'part1', 'part2'}
+'11234567': {'part1', 'part2'}
+'21234567': {'part1', 'part2_a'}
+'31234567': {'part1_a', 'part2'}
+'41234567': {'part1_a', 'part2_a'}
+...
 select * from t5 where k0='01234567'
 Found 1 tuple:
 ['01234567', 'part1', 'part2']
@@ -176,6 +209,9 @@ delete from t5 where k0='31234567'
 Delete OK, 1 row affected
 delete from t5 where k0='41234567'
 Delete OK, 1 row affected
+lua for k, v in box.space[5]:pairs() do print(v) end
+---
+...
 
 #
 # A test case for: http://bugs.launchpad.net/bugs/735140
@@ -248,6 +284,24 @@ Found 5 tuples:
 [13, 'duplicate three']
 [14, 'duplicate three']
 [15, 'duplicate three']
+lua for k, v in box.space[4]:pairs() do print(v) end
+---
+10: {'duplicate two'}
+13: {'duplicate three'}
+4: {'duplicate one'}
+14: {'duplicate three'}
+3: {'duplicate one'}
+8: {'duplicate two'}
+1: {'duplicate one'}
+15: {'duplicate three'}
+2: {'duplicate one'}
+9: {'duplicate two'}
+11: {'duplicate three'}
+7: {'duplicate two'}
+6: {'duplicate two'}
+5: {'duplicate one'}
+12: {'duplicate three'}
+...
 delete from t4 where k0=1
 Delete OK, 1 row affected
 delete from t4 where k0=2
@@ -284,6 +338,12 @@ insert into t4 values(2, 'Bilimbi')
 Insert OK, 1 row affected
 insert into t4 values(3, 'Creature')
 Insert OK, 1 row affected
+lua for k, v in box.space[4]:pairs() do print(v) end
+---
+1: {'Aardvark'}
+2: {'Bilimbi'}
+3: {'Creature'}
+...
 lua box.space[4].index[0].idx:min()
 ---
 error: 'Unsupported'

test/box_big/sql.test

@@ -49,6 +49,8 @@ exec sql "insert into t0 values ('Spears', 'Britney')"
 exec sql "select * from t0 where k0='Spears'"
 exec sql "select * from t0 where k1='Anything'"
 exec sql "select * from t0 where k1='Britney'"
+exec sql "call box.select_range(0, 0, 100, 'Spears')"
+exec sql "call box.select_range(0, 1, 100, 'Britney')"
 exec sql "delete from t0 where k0='Spears'"
 print """#
 # Test composite keys with trees
@@ -56,16 +58,21 @@ print """#
 exec sql "insert into t1 values ('key1', 'part1', 'part2')"
 exec sql "insert into t1 values ('key2', 'part1', 'part2_a')"
 exec sql "insert into t1 values ('key3', 'part1', 'part2_b')"
+exec admin "lua for k, v in box.space[1]:pairs() do print(v) end"
 exec sql "select * from t1 where k0='key1'"
 exec sql "select * from t1 where k0='key2'"
 exec sql "select * from t1 where k0='key3'"
 exec sql "select * from t1 where k1='part1'"
+exec sql "call box.select_range(1, 1, 100, 'part1')"
+exec sql "call box.select_range(1, 0, 100, 'key2')"
+exec sql "call box.select_range(1, 1, 100, 'part1', 'part2_a')"
 # check non-unique multipart keys
 exec sql "insert into t5 values ('01234567', 'part1', 'part2')"
 exec sql "insert into t5 values ('11234567', 'part1', 'part2')"
 exec sql "insert into t5 values ('21234567', 'part1', 'part2_a')"
 exec sql "insert into t5 values ('31234567', 'part1_a', 'part2')"
 exec sql "insert into t5 values ('41234567', 'part1_a', 'part2_a')"
+exec admin "lua for k, v in box.space[5]:pairs() do print(v) end"
 exec sql "select * from t5 where k0='01234567'"
 exec sql "select * from t5 where k0='11234567'"
 exec sql "select * from t5 where k0='21234567'"
@@ -94,6 +101,7 @@ exec sql "delete from t5 where k0='11234567'"
 exec sql "delete from t5 where k0='21234567'"
 exec sql "delete from t5 where k0='31234567'"
 exec sql "delete from t5 where k0='41234567'"
+exec admin "lua for k, v in box.space[5]:pairs() do print(v) end"
 
 print """
 #
@@ -135,6 +143,7 @@ sql.sort = True
 exec sql "select * from t4 where k1='duplicate one'"
 exec sql "select * from t4 where k1='duplicate two'"
 exec sql "select * from t4 where k1='duplicate three'"
+exec admin "lua for k, v in box.space[4]:pairs() do print(v) end"
 sql.sort = False
 exec sql "delete from t4 where k0=1"
 exec sql "delete from t4 where k0=2"
@@ -157,6 +166,7 @@ exec sql "delete from t4 where k0=15"
 exec sql "insert into t4 values(1, 'Aardvark')"
 exec sql "insert into t4 values(2, 'Bilimbi')"
 exec sql "insert into t4 values(3, 'Creature')"
+exec admin "lua for k, v in box.space[4]:pairs() do print(v) end"
 exec admin "lua box.space[4].index[0].idx:min()"
 exec admin "lua box.space[4].index[0].idx:max()"
 exec admin "lua box.space[4].index[1].idx:min()"

test/box_big/tree_pk.result

@@ -39,6 +39,13 @@ save snapshot
 ---
 ok
 ...
+call box.select_range(3, 0, 100, 'second')
+Found 1 tuple:
+['second', 'tuple 2']
+call box.select_range(3, 0, 100, 'identifier')
+Found 2 tuples:
+['identifier', 'tuple']
+['second', 'tuple 2']
 insert into t3 values ('third', 'tuple 3')
 Insert OK, 1 row affected
 select * from t3 where k0 = 'identifier'

test/box_big/tree_pk.test

@@ -20,6 +20,8 @@ exec sql "insert into t3 values ('identifier', 'tuple')"
 exec admin "save snapshot"
 exec sql "insert into t3 values ('second', 'tuple 2')"
 exec admin "save snapshot"
+exec sql "call box.select_range(3, 0, 100, 'second')"
+exec sql "call box.select_range(3, 0, 100, 'identifier')"
 
 exec sql "insert into t3 values ('third', 'tuple 3')"
 exec sql "select * from t3 where k0 = 'identifier'"