fix gh-1638: box.rollback() crashes JIT-compiler

txn_rollback started to yield recently. Since it was invoked via a ffi
call, which must never yield, all sorts of bad things started to happen.

src/box/lua/init.c

@@ -81,6 +81,14 @@ lbox_commit(lua_State *L)
 	return 0;
 }
 
+static int
+lbox_rollback(lua_State *L)
+{
+	(void)L;
+	box_txn_rollback();
+	return 0;
+}
+
 static int
 lbox_snapshot(struct lua_State *L)
 {
@@ -95,8 +103,9 @@ lbox_snapshot(struct lua_State *L)
 }
 
 static const struct luaL_reg boxlib[] = {
-	{"snapshot", lbox_snapshot},
 	{"commit", lbox_commit},
+	{"rollback", lbox_rollback},
+	{"snapshot", lbox_snapshot},
 	{NULL, NULL}
 };
 

src/box/lua/schema.lua

@@ -55,8 +55,6 @@ ffi.cdef[[
     /** \cond public */
     int
     box_txn_begin();
-    void
-    box_txn_rollback();
     /** \endcond public */
 
     struct port
@@ -242,8 +240,7 @@ box.begin = function()
     end
 end
 -- box.commit yields, so it's defined as Lua/C binding
-
-box.rollback = builtin.box_txn_rollback;
+-- box.rollback yields as well
 
 box.schema.space = {}
 box.schema.space.create = function(name, options)

test/box/transaction.result

@@ -414,3 +414,14 @@ box.space.test:len()
 box.space.test:drop()
 ---
 ...
+--
+-- gh-1638: box.rollback on a JIT-ed code path crashes LuaJIT
+-- (ffi call + yield don't mix well, rollback started to yield recently)
+-- Note: don't remove gh_1638(), it's necessary to trigger JIT-compilation.
+--
+function gh_1638() box.begin(); box.rollback() end
+---
+...
+for i = 1, 1000 do fiber.create(function() gh_1638() end) end
+---
+...

test/box/transaction.test.lua

@@ -187,3 +187,11 @@ _ = box.space.test:create_index('primary');
 tx_limit(10000)
 box.space.test:len()
 box.space.test:drop()
+
+--
+-- gh-1638: box.rollback on a JIT-ed code path crashes LuaJIT
+-- (ffi call + yield don't mix well, rollback started to yield recently)
+-- Note: don't remove gh_1638(), it's necessary to trigger JIT-compilation.
+--
+function gh_1638() box.begin(); box.rollback() end
+for i = 1, 1000 do fiber.create(function() gh_1638() end) end