From 96938fafb2f56d5176ecacb56aaa7b679349642d Mon Sep 17 00:00:00 2001
From: Georgy Kirichenko <georgy@tarantool.org>
Date: Tue, 15 Aug 2017 10:33:26 +0300
Subject: [PATCH] Add hot function reload for C procedures

This patch adds ability to reload C procedures on the fly without
downtime. To achive that, Tarantool loads a new copy of shared
library and starts routing all new request to the new version.
The previous version remains active until all started calls
are finished. All shared libraries are loaded with RTLD_LOCAL,
therefore two or more copies can co-exist without any problems.
From now box loads all external modules via an unique symlink to
avoid caching inside dlopen().

If one of some module function is reloaded then all other functions
from this module will be reloaded.

Reviewed and heavily patched by Roman Tsisyk.

Closes #910
---
 src/box/box.cc                |   4 +
 src/box/call.cc               |  13 ++
 src/box/call.h                |  10 ++
 src/box/func.c                | 294 +++++++++++++++++++++++++++++++---
 src/box/func.h                |  43 ++++-
 src/box/lua/call.c            |  10 ++
 src/box/lua/schema.lua        |   2 +
 test/box/CMakeLists.txt       |   4 +-
 test/box/func_reload.result   | 250 +++++++++++++++++++++++++++++
 test/box/func_reload.test.lua |  86 ++++++++++
 test/box/function1.c          |   9 ++
 test/box/function1.result     |  23 +++
 test/box/function1.test.lua   |   9 ++
 test/box/reload1.c            |  57 +++++++
 test/box/reload2.c            |  49 ++++++
 15 files changed, 840 insertions(+), 23 deletions(-)
 create mode 100644 test/box/func_reload.result
 create mode 100644 test/box/func_reload.test.lua
 create mode 100644 test/box/reload1.c
 create mode 100644 test/box/reload2.c

diff --git a/src/box/box.cc b/src/box/box.cc
index dc83b513ae..784f20e33e 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -68,6 +68,7 @@
 #include "checkpoint.h"
 #include "systemd.h"
 #include "call.h"
+#include "func.h"
 
 static char status[64] = "unknown";
 
@@ -1214,6 +1215,7 @@ box_free(void)
 		replication_free();
 		user_cache_free();
 		schema_free();
+		module_free();
 		tuple_free();
 		port_free();
 #endif
@@ -1403,6 +1405,8 @@ box_cfg_xc(void)
 
 	gc_init();
 	engine_init();
+	if (module_init() != 0)
+		diag_raise();
 	schema_init();
 	replication_init();
 	port_init();
diff --git a/src/box/call.cc b/src/box/call.cc
index c8fbf3d33c..b2706c1e2c 100644
--- a/src/box/call.cc
+++ b/src/box/call.cc
@@ -85,6 +85,7 @@ box_c_call(struct func *func, struct call_request *request, struct obuf *out)
 
 	/* Call function from the shared library */
 	int rc = func_call(func, &ctx, request->args, request->args_end);
+	func = NULL; /* May be deleted by DDL */
 	if (rc != 0) {
 		if (diag_last_error(&fiber()->diag) == NULL) {
 			/* Stored procedure forget to set diag  */
@@ -128,6 +129,18 @@ box_c_call(struct func *func, struct call_request *request, struct obuf *out)
 	return -1;
 }
 
+int
+box_func_reload(const char *name)
+{
+	size_t name_len = strlen(name);
+	struct func *func = access_check_func(name, name_len);
+	if (func->def->language != FUNC_LANGUAGE_C || func->func == NULL)
+		return 0; /* Nothing to do */
+	if (func_reload(func) == 0)
+		return 0;
+	return -1;
+}
+
 void
 box_process_call(struct call_request *request, struct obuf *out)
 {
diff --git a/src/box/call.h b/src/box/call.h
index b44a696f1c..9e105087a9 100644
--- a/src/box/call.h
+++ b/src/box/call.h
@@ -33,6 +33,16 @@
 
 #include <stdint.h>
 
+#if defined(__cplusplus)
+extern "C" {
+#endif /* defined(__cplusplus) */
+int
+box_func_reload(const char *name);
+
+#if defined(__cplusplus)
+} /* extern "C" */
+#endif /* defined(__cplusplus) */
+
 struct obuf;
 
 struct box_function_ctx {
diff --git a/src/box/func.c b/src/box/func.c
index b17a00583e..2cc276f9cd 100644
--- a/src/box/func.c
+++ b/src/box/func.c
@@ -32,6 +32,8 @@
 
 #include <dlfcn.h>
 
+#include "assoc.h"
+
 #include "lua/utils.h"
 
 /**
@@ -102,7 +104,14 @@ luaT_module_find(lua_State *L)
 	if (lua_isnil(L, -1))
 		return luaL_error(L, "module not found in package.cpath");
 
-	snprintf(ctx->path, ctx->path_len, "%s", lua_tostring(L, -1));
+	/* Convert path to absolute */
+	char resolved[PATH_MAX];
+	if (realpath(lua_tostring(L, -1), resolved) == NULL) {
+		diag_set(SystemError, "realpath");
+		return luaT_error(L);
+	}
+
+	snprintf(ctx->path, ctx->path_len, "%s", resolved);
 	return 0;
 }
 
@@ -133,6 +142,224 @@ module_find(const char *package, const char *package_end, char *path,
 	return 0;
 }
 
+static struct mh_strnptr_t *modules = NULL;
+
+static void
+module_gc(struct module *module);
+
+int
+module_init(void)
+{
+	modules = mh_strnptr_new();
+	if (modules == NULL) {
+		diag_set(OutOfMemory, sizeof(*modules), "malloc",
+			  "modules hash table");
+		return -1;
+	}
+	return 0;
+}
+
+void
+module_free(void)
+{
+	while (mh_size(modules) > 0) {
+		mh_int_t i = mh_first(modules);
+		struct module *module =
+			(struct module *) mh_strnptr_node(modules, i)->val;
+		/* Can't delete modules if they have active calls */
+		module_gc(module);
+	}
+	mh_strnptr_delete(modules);
+}
+
+/**
+ * Look up a module in the modules cache.
+ */
+static struct module *
+module_cache_find(const char *name, const char *name_end)
+{
+	mh_int_t i = mh_strnptr_find_inp(modules, name, name_end - name);
+	if (i == mh_end(modules))
+		return NULL;
+	return (struct module *)mh_strnptr_node(modules, i)->val;
+}
+
+/**
+ * Save module to the module cache.
+ */
+static inline int
+module_cache_put(const char *name, const char *name_end, struct module *module)
+{
+	size_t name_len = name_end - name;
+	uint32_t name_hash = mh_strn_hash(name, name_len);
+	const struct mh_strnptr_node_t strnode = {
+		name, name_len, name_hash, module};
+
+	if (mh_strnptr_put(modules, &strnode, NULL, NULL) == mh_end(modules)) {
+		diag_set(OutOfMemory, sizeof(strnode), "malloc", "modules");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * Delete a module from the module cache
+ */
+static void
+module_cache_del(const char *name, const char *name_end)
+{
+	mh_int_t i = mh_strnptr_find_inp(modules, name, name_end - name);
+	if (i == mh_end(modules))
+		return;
+	mh_strnptr_del(modules, i, NULL);
+}
+
+/*
+ * Load a dso.
+ * Create a new symlink based on temporary directory and try to
+ * load via this symink to load a dso twice for cases of a function
+ * reload.
+ */
+static struct module *
+module_load(const char *package, const char *package_end)
+{
+	char path[PATH_MAX];
+	if (module_find(package, package_end, path, sizeof(path)) != 0)
+		return NULL;
+
+	struct module *module = (struct module *) malloc(sizeof(*module));
+	if (module == NULL) {
+		diag_set(OutOfMemory, sizeof(struct module), "malloc",
+			 "struct module");
+		return NULL;
+	}
+	rlist_create(&module->funcs);
+	module->calls = 0;
+	module->is_unloading = false;
+	char dir_name[] = "/tmp/tntXXXXXX";
+	if (mkdtemp(dir_name) == NULL) {
+		diag_set(SystemError, "failed to create unique dir name");
+		goto error;
+	}
+	char load_name[PATH_MAX + 1];
+	snprintf(load_name, sizeof(load_name), "%s/%*s", dir_name,
+		 (int)(package_end - package), package);
+	if (symlink(path, load_name) < 0) {
+		diag_set(SystemError, "failed to create dso link");
+		goto error;
+	}
+	module->handle = dlopen(load_name, RTLD_NOW | RTLD_LOCAL);
+	if (unlink(load_name) != 0)
+		say_warn("failed to unlink dso link %s", load_name);
+	if (rmdir(dir_name) != 0)
+		say_warn("failed to delete temporary dir %s", dir_name);
+	if (module->handle == NULL) {
+		int package_len = (int) (package_end - package_end);
+		diag_set(ClientError, ER_LOAD_MODULE, package_len,
+			  package, dlerror());
+		goto error;
+	}
+
+	return module;
+error:
+	free(module);
+	return NULL;
+}
+
+static void
+module_delete(struct module *module)
+{
+	dlclose(module->handle);
+	TRASH(module);
+	free(module);
+}
+
+/*
+ * Check if a dso is unused and can be closed.
+ */
+static void
+module_gc(struct module *module)
+{
+	if (!module->is_unloading || !rlist_empty(&module->funcs) ||
+	     module->calls != 0)
+		return;
+	module_delete(module);
+}
+
+/*
+ * Import a function from the module.
+ */
+static box_function_f
+module_sym(struct module *module, const char *name)
+{
+	box_function_f f = (box_function_f)dlsym(module->handle, name);
+	if (f == NULL) {
+		diag_set(ClientError, ER_LOAD_FUNCTION, name, dlerror());
+		return NULL;
+	}
+	return f;
+}
+
+/*
+ * Reload a dso.
+ */
+int
+module_reload(const char *package, const char *package_end, struct module **module)
+{
+	struct module *old_module = module_cache_find(package, package_end);
+	if (old_module == NULL) {
+		/* Module wasn't loaded - do nothing. */
+		*module = NULL;
+		return 0;
+	}
+
+	struct module *new_module = module_load(package, package_end);
+	if (new_module == NULL)
+		return -1;
+
+	struct func *func, *tmp_func;
+	rlist_foreach_entry_safe(func, &old_module->funcs, item, tmp_func) {
+		struct func_name name;
+		func_split_name(func->def->name, &name);
+		func->func = module_sym(new_module, name.sym);
+		if (func->func == NULL)
+			goto restore;
+		func->module = new_module;
+		rlist_move(&new_module->funcs, &func->item);
+	}
+	module_cache_del(package, package_end);
+	if (module_cache_put(package, package_end, new_module) != 0)
+		goto restore;
+	old_module->is_unloading = true;
+	module_gc(old_module);
+	*module = new_module;
+	return 0;
+restore:
+	/*
+	 * Some old-dso func can't be load from new module, restore old
+	 * functions.
+	 */
+	do {
+		struct func_name name;
+		func_split_name(func->def->name, &name);
+		func->func = module_sym(old_module, name.sym);
+		if (func->func == NULL) {
+			/*
+			 * Something strange was happen, an early loaden
+			 * function was not found in an old dso.
+			 */
+			panic("Can't restore module function, "
+			      "server state is inconsistent");
+		}
+		func->module = old_module;
+		rlist_move(&old_module->funcs, &func->item);
+	} while (func != rlist_first_entry(&old_module->funcs,
+					   struct func, item));
+	assert(rlist_empty(&new_module->funcs));
+	module_delete(new_module);
+	return -1;
+}
+
 struct func *
 func_new(struct func_def *def)
 {
@@ -157,16 +384,23 @@ func_new(struct func_def *def)
 	 */
 	func->owner_credentials.auth_token = BOX_USER_MAX; /* invalid value */
 	func->func = NULL;
-	func->dlhandle = NULL;
+	func->module = NULL;
 	return func;
 }
 
 static void
 func_unload(struct func *func)
 {
-	if (func->dlhandle)
-		dlclose(func->dlhandle);
-	func->dlhandle = NULL;
+	if (func->module) {
+		rlist_del(&func->item);
+		if (rlist_empty(&func->module->funcs)) {
+			struct func_name name;
+			func_split_name(func->def->name, &name);
+			module_cache_del(name.package, name.package_end);
+		}
+		module_gc(func->module);
+	}
+	func->module = NULL;
 	func->func = NULL;
 }
 
@@ -182,22 +416,34 @@ func_load(struct func *func)
 	struct func_name name;
 	func_split_name(func->def->name, &name);
 
-	char path[PATH_MAX];
-	if (module_find(name.package, name.package_end, path, sizeof(path)))
-		return -1;
+	struct module *module = module_cache_find(name.package,
+						  name.package_end);
+	if (module == NULL) {
+		/* Try to find loaded module in the cache */
+		module = module_load(name.package, name.package_end);
+		if (module == NULL)
+			diag_raise();
+		if (module_cache_put(name.package, name.package_end, module)) {
+			module_delete(module);
+			diag_raise();
+		}
+	}
 
-	func->dlhandle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
-	if (func->dlhandle == NULL) {
-		int package_len = (int) (name.package_end - name.package_end);
-		diag_set(ClientError, ER_LOAD_MODULE, package_len,
-			  name.package, dlerror());
-		diag_log();
+	func->func = module_sym(module, name.sym);
+	if (func->func == NULL)
 		return -1;
-	}
-	func->func = (box_function_f) dlsym(func->dlhandle, name.sym);
-	if (func->func == NULL) {
-		diag_set(ClientError, ER_LOAD_FUNCTION, func->def->name,
-			  dlerror());
+	func->module = module;
+	rlist_add(&module->funcs, &func->item);
+	return 0;
+}
+
+int
+func_reload(struct func *func)
+{
+	struct func_name name;
+	func_split_name(func->def->name, &name);
+	struct module *module = NULL;
+	if (module_reload(name.package, name.package_end, &module) != 0) {
 		diag_log();
 		return -1;
 	}
@@ -213,7 +459,15 @@ func_call(struct func *func, box_function_ctx_t *ctx, const char *args,
 			return -1;
 	}
 
-	return func->func(ctx, args, args_end);
+	/* Module can be changed after function reload. */
+	struct module *module = func->module;
+	if (module != NULL)
+		++module->calls;
+	int rc = func->func(ctx, args, args_end);
+	if (module != NULL)
+		--module->calls;
+	module_gc(module);
+	return rc;
 }
 
 void
diff --git a/src/box/func.h b/src/box/func.h
index 144b031f67..b152a2285b 100644
--- a/src/box/func.h
+++ b/src/box/func.h
@@ -30,17 +30,42 @@
  * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#include <small/rlist.h>
+
 #include "key_def.h"
 
 #if defined(__cplusplus)
 extern "C" {
 #endif /* defined(__cplusplus) */
 
+/**
+ * Dynamic shared module.
+ */
+struct module {
+	/** Module dlhandle. */
+	void *handle;
+	/** List of imported functions. */
+	struct rlist funcs;
+	/** Count of active calls. */
+	size_t calls;
+	/** True if module is being unloaded. */
+	bool is_unloading;
+};
+
 /**
  * Stored function.
  */
 struct func {
 	struct func_def *def;
+	/**
+	 * Anchor for module membership.
+	 */
+	struct rlist item;
 	/**
 	 * For C functions, the body of the function.
 	 */
@@ -49,7 +74,7 @@ struct func {
 	 * Each stored function keeps a handle to the
 	 * dynamic library for the C callback.
 	 */
-	void *dlhandle;
+	struct module *module;
 	/**
 	 * Authentication id of the owner of the function,
 	 * used for set-user-id functions.
@@ -61,6 +86,18 @@ struct func {
 	struct access access[BOX_USER_MAX];
 };
 
+/**
+ * Initialize modules subsystem.
+ */
+int
+module_init(void);
+
+/**
+ * Cleanup modules subsystem.
+ */
+void
+module_free(void);
+
 struct func *
 func_new(struct func_def *def);
 
@@ -77,9 +114,11 @@ int
 func_call(struct func *func, box_function_ctx_t *ctx, const char *args,
 	  const char *args_end);
 
+int
+func_reload(struct func *func);
+
 #if defined(__cplusplus)
 } /* extern "C" */
-
 #endif /* defined(__cplusplus) */
 
 #endif /* TARANTOOL_BOX_FUNC_H_INCLUDED */
diff --git a/src/box/lua/call.c b/src/box/lua/call.c
index f0bc7863de..0063707e2b 100644
--- a/src/box/lua/call.c
+++ b/src/box/lua/call.c
@@ -414,8 +414,18 @@ box_lua_eval(struct call_request *request, struct obuf *out)
 	return box_process_lua(request, out, execute_lua_eval);
 }
 
+static int
+lbox_func_reload(lua_State *L)
+{
+	const char *name = luaL_optstring(L, 1, "function name");
+	if (box_func_reload(name) != 0)
+		return luaT_error(L);
+	return 0;
+}
+
 static const struct luaL_Reg boxlib_internal[] = {
 	{"call_loadproc",  lbox_call_loadproc},
+	{"func_reload", lbox_func_reload},
 	{NULL, NULL}
 };
 
diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua
index 62964ee346..92c29ea7c2 100644
--- a/src/box/lua/schema.lua
+++ b/src/box/lua/schema.lua
@@ -1280,6 +1280,8 @@ function box.schema.func.exists(name_or_id)
     return tuple ~= nil
 end
 
+box.schema.func.reload = internal.func_reload
+
 box.schema.user = {}
 
 box.schema.user.password = function(password)
diff --git a/test/box/CMakeLists.txt b/test/box/CMakeLists.txt
index ebacc96adb..4f859d1ce7 100644
--- a/test/box/CMakeLists.txt
+++ b/test/box/CMakeLists.txt
@@ -1,4 +1,6 @@
 include_directories(${MSGPUCK_INCLUDE_DIRS})
 build_module(function1 function1.c)
-target_link_libraries(function1 ${MSGPUCK_LIBRARIES})
+build_module(reload1 reload1.c)
+build_module(reload2 reload2.c)
+target_link_libraries(function1 reload1 reload2 ${MSGPUCK_LIBRARIES})
 build_module(tuple_bench tuple_bench.c)
diff --git a/test/box/func_reload.result b/test/box/func_reload.result
new file mode 100644
index 0000000000..00cde1d3bd
--- /dev/null
+++ b/test/box/func_reload.result
@@ -0,0 +1,250 @@
+fio  = require('fio')
+---
+...
+net = require('net.box')
+---
+...
+fiber = require('fiber')
+---
+...
+ext = (jit.os == "OSX" and "dylib" or "so")
+---
+...
+build_path = os.getenv("BUILDDIR")
+---
+...
+reload1_path = build_path..'/test/box/reload1.'..ext
+---
+...
+reload2_path = build_path..'/test/box/reload2.'..ext
+---
+...
+reload_path = "reload."..ext
+---
+...
+_ = fio.unlink(reload_path)
+---
+...
+c = net.connect(os.getenv("LISTEN"))
+---
+...
+box.schema.func.create('reload.foo', {language = "C"})
+---
+...
+box.schema.user.grant('guest', 'execute', 'function', 'reload.foo')
+---
+...
+_ = box.schema.space.create('test')
+---
+...
+_ = box.space.test:create_index('primary', {parts = {1, "integer"}})
+---
+...
+box.schema.user.grant('guest', 'read,write', 'space', 'test')
+---
+...
+_ = fio.unlink(reload_path)
+---
+...
+fio.symlink(reload1_path, reload_path)
+---
+- true
+...
+--check not fail on non-load func
+box.schema.func.reload("reload.foo")
+---
+...
+-- test of usual case reload. No hanging calls
+box.space.test:insert{0}
+---
+- [0]
+...
+c:call("reload.foo", {1})
+---
+- []
+...
+box.space.test:delete{0}
+---
+- [0]
+...
+_ = fio.unlink(reload_path)
+---
+...
+fio.symlink(reload2_path, reload_path)
+---
+- true
+...
+box.schema.func.reload("reload.foo")
+---
+...
+c:call("reload.foo")
+---
+- []
+...
+box.space.test:select{}
+---
+- - [-1]
+  - [0]
+  - [1]
+...
+box.space.test:truncate()
+---
+...
+-- test case with hanging calls
+_ = fio.unlink(reload_path)
+---
+...
+fio.symlink(reload1_path, reload_path)
+---
+- true
+...
+box.schema.func.reload("reload.foo")
+---
+...
+fibers = 10
+---
+...
+for i = 1, fibers do fiber.create(function() c:call("reload.foo", {i}) end) end
+---
+...
+while box.space.test:count() < fibers do fiber.sleep(0.001) end
+---
+...
+-- double reload doesn't fail waiting functions
+box.schema.func.reload("reload.foo")
+---
+...
+_ = fio.unlink(reload_path)
+---
+...
+fio.symlink(reload2_path, reload_path)
+---
+- true
+...
+box.schema.func.reload("reload.foo")
+---
+...
+c:call("reload.foo")
+---
+- []
+...
+while box.space.test:count() < 2 * fibers + 1 do fiber.sleep(0.001) end
+---
+...
+box.space.test:select{}
+---
+- - [-10]
+  - [-9]
+  - [-8]
+  - [-7]
+  - [-6]
+  - [-5]
+  - [-4]
+  - [-3]
+  - [-2]
+  - [-1]
+  - [0]
+  - [1]
+  - [2]
+  - [3]
+  - [4]
+  - [5]
+  - [6]
+  - [7]
+  - [8]
+  - [9]
+  - [10]
+...
+box.schema.func.drop("reload.foo")
+---
+...
+box.space.test:drop()
+---
+...
+_ = fio.unlink(reload_path)
+---
+...
+fio.symlink(reload1_path, reload_path)
+---
+- true
+...
+box.schema.func.create('reload.test_reload', {language = "C"})
+---
+...
+box.schema.user.grant('guest', 'execute', 'function', 'reload.test_reload')
+---
+...
+ch = fiber.channel(2)
+---
+...
+-- call first time to load function
+c:call("reload.test_reload")
+---
+- [[1]]
+...
+_ = fiber.create(function() ch:put(c:call("reload.test_reload")) end)
+---
+...
+_ = fio.unlink(reload_path)
+---
+...
+fio.symlink(reload2_path, reload_path)
+---
+- true
+...
+box.schema.func.reload("reload.test_reload")
+---
+...
+_ = fiber.create(function() ch:put(c:call("reload.test_reload")) end)
+---
+...
+ch:get()
+---
+- [[1]]
+...
+ch:get()
+---
+- [[2]]
+...
+box.schema.func.create('reload.test_reload_fail', {language = "C"})
+---
+...
+box.schema.user.grant('guest', 'execute', 'function', 'reload.test_reload_fail')
+---
+...
+c:call("reload.test_reload_fail")
+---
+- [[2]]
+...
+_ = fio.unlink(reload_path)
+---
+...
+fio.symlink(reload1_path, reload_path)
+---
+- true
+...
+s, e = pcall(box.schema.func.reload, "reload.test_reload")
+---
+...
+s, string.find(tostring(e), 'test_reload_fail') ~= nil
+---
+- false
+- true
+...
+c:call("reload.test_reload")
+---
+- [[2]]
+...
+c:call("reload.test_reload_fail")
+---
+- [[2]]
+...
+box.schema.func.drop("reload.test_reload")
+---
+...
+box.schema.func.drop("reload.test_reload_fail")
+---
+...
+_ = fio.unlink(reload_path)
+---
+...
diff --git a/test/box/func_reload.test.lua b/test/box/func_reload.test.lua
new file mode 100644
index 0000000000..728ee470f1
--- /dev/null
+++ b/test/box/func_reload.test.lua
@@ -0,0 +1,86 @@
+fio  = require('fio')
+net = require('net.box')
+fiber = require('fiber')
+
+ext = (jit.os == "OSX" and "dylib" or "so")
+build_path = os.getenv("BUILDDIR")
+reload1_path = build_path..'/test/box/reload1.'..ext
+reload2_path = build_path..'/test/box/reload2.'..ext
+reload_path = "reload."..ext
+_ = fio.unlink(reload_path)
+
+c = net.connect(os.getenv("LISTEN"))
+
+box.schema.func.create('reload.foo', {language = "C"})
+box.schema.user.grant('guest', 'execute', 'function', 'reload.foo')
+_ = box.schema.space.create('test')
+_ = box.space.test:create_index('primary', {parts = {1, "integer"}})
+box.schema.user.grant('guest', 'read,write', 'space', 'test')
+_ = fio.unlink(reload_path)
+fio.symlink(reload1_path, reload_path)
+
+--check not fail on non-load func
+box.schema.func.reload("reload.foo")
+
+-- test of usual case reload. No hanging calls
+box.space.test:insert{0}
+c:call("reload.foo", {1})
+box.space.test:delete{0}
+_ = fio.unlink(reload_path)
+fio.symlink(reload2_path, reload_path)
+
+box.schema.func.reload("reload.foo")
+c:call("reload.foo")
+box.space.test:select{}
+box.space.test:truncate()
+
+-- test case with hanging calls
+_ = fio.unlink(reload_path)
+fio.symlink(reload1_path, reload_path)
+box.schema.func.reload("reload.foo")
+
+fibers = 10
+for i = 1, fibers do fiber.create(function() c:call("reload.foo", {i}) end) end
+
+while box.space.test:count() < fibers do fiber.sleep(0.001) end
+-- double reload doesn't fail waiting functions
+box.schema.func.reload("reload.foo")
+
+_ = fio.unlink(reload_path)
+fio.symlink(reload2_path, reload_path)
+box.schema.func.reload("reload.foo")
+c:call("reload.foo")
+
+while box.space.test:count() < 2 * fibers + 1 do fiber.sleep(0.001) end
+box.space.test:select{}
+box.schema.func.drop("reload.foo")
+box.space.test:drop()
+_ = fio.unlink(reload_path)
+
+fio.symlink(reload1_path, reload_path)
+box.schema.func.create('reload.test_reload', {language = "C"})
+box.schema.user.grant('guest', 'execute', 'function', 'reload.test_reload')
+ch = fiber.channel(2)
+-- call first time to load function
+c:call("reload.test_reload")
+_ = fiber.create(function() ch:put(c:call("reload.test_reload")) end)
+_ = fio.unlink(reload_path)
+fio.symlink(reload2_path, reload_path)
+box.schema.func.reload("reload.test_reload")
+_ = fiber.create(function() ch:put(c:call("reload.test_reload")) end)
+ch:get()
+ch:get()
+
+box.schema.func.create('reload.test_reload_fail', {language = "C"})
+box.schema.user.grant('guest', 'execute', 'function', 'reload.test_reload_fail')
+c:call("reload.test_reload_fail")
+_ = fio.unlink(reload_path)
+fio.symlink(reload1_path, reload_path)
+s, e = pcall(box.schema.func.reload, "reload.test_reload")
+s, string.find(tostring(e), 'test_reload_fail') ~= nil
+c:call("reload.test_reload")
+c:call("reload.test_reload_fail")
+
+box.schema.func.drop("reload.test_reload")
+box.schema.func.drop("reload.test_reload_fail")
+_ = fio.unlink(reload_path)
diff --git a/test/box/function1.c b/test/box/function1.c
index a72381b28b..8b9ea144c8 100644
--- a/test/box/function1.c
+++ b/test/box/function1.c
@@ -143,3 +143,12 @@ errors(box_function_ctx_t *ctx, const char *args, const char *args_end)
 
 	return -1; /* raises "Unknown procedure error" */
 }
+
+int
+test_yield(box_function_ctx_t *ctx, const char *args, const char *args_end)
+{
+	say_info("-- yield -  called --");
+	fiber_sleep(0.001);
+	printf("ok - yield\n");
+	return 0;
+}
diff --git a/test/box/function1.result b/test/box/function1.result
index b792739799..a70e601a0a 100644
--- a/test/box/function1.result
+++ b/test/box/function1.result
@@ -214,6 +214,29 @@ c:call(name)
 box.schema.func.drop(name)
 ---
 ...
+-- Drop function while executing gh-910
+box.schema.func.create('function1.test_yield', {language = "C"})
+---
+...
+box.schema.user.grant('guest', 'execute', 'function', 'function1.test_yield')
+---
+...
+fiber = require('fiber')
+---
+...
+ch = fiber.channel(1)
+---
+...
+_ = fiber.create(function() c:call('function1.test_yield') ch:put(true) end)
+---
+...
+box.schema.func.drop('function1.test_yield')
+---
+...
+ch:get()
+---
+- true
+...
 c:close()
 ---
 ...
diff --git a/test/box/function1.test.lua b/test/box/function1.test.lua
index 746b11a59f..30b4190da7 100644
--- a/test/box/function1.test.lua
+++ b/test/box/function1.test.lua
@@ -69,4 +69,13 @@ box.schema.user.grant('guest', 'execute', 'function', name)
 c:call(name)
 box.schema.func.drop(name)
 
+-- Drop function while executing gh-910
+box.schema.func.create('function1.test_yield', {language = "C"})
+box.schema.user.grant('guest', 'execute', 'function', 'function1.test_yield')
+fiber = require('fiber')
+ch = fiber.channel(1)
+_ = fiber.create(function() c:call('function1.test_yield') ch:put(true) end)
+box.schema.func.drop('function1.test_yield')
+ch:get()
+
 c:close()
diff --git a/test/box/reload1.c b/test/box/reload1.c
new file mode 100644
index 0000000000..06baf39621
--- /dev/null
+++ b/test/box/reload1.c
@@ -0,0 +1,57 @@
+#include "module.h"
+
+#include <stdio.h>
+#include <msgpuck.h>
+
+int
+foo(box_function_ctx_t *ctx, const char *args, const char *args_end)
+{
+	static const char *SPACE_TEST_NAME = "test";
+	static const char *INDEX_NAME = "primary";
+	uint32_t space_test_id = box_space_id_by_name(SPACE_TEST_NAME,
+			strlen(SPACE_TEST_NAME));
+	uint32_t index_id = box_index_id_by_name(space_test_id, INDEX_NAME,
+		strlen(INDEX_NAME));
+	if (space_test_id == BOX_ID_NIL || index_id == BOX_ID_NIL) {
+		return box_error_set(__FILE__, __LINE__, ER_PROC_C,
+			"Can't find index %s in space %s",
+			INDEX_NAME, SPACE_TEST_NAME);
+	}
+	mp_decode_array(&args);
+	uint32_t num = mp_decode_uint(&args);
+
+	char buf[16];
+	char *end = buf;
+	end = mp_encode_array(end, 1);
+	end = mp_encode_uint(end, num);
+	if (box_insert(space_test_id, buf, end, NULL) < 0) {
+		return box_error_set(__FILE__, __LINE__, ER_PROC_C,
+			"Can't insert in space %s", SPACE_TEST_NAME);
+	}
+	end = buf;
+	end = mp_encode_array(end, 1);
+	end = mp_encode_uint(end, 0);
+	while (box_index_count(space_test_id, index_id, ITER_EQ, buf, end) <= 0) {
+		fiber_sleep(0.001);
+	}
+	end = buf;
+	end = mp_encode_array(end, 1);
+	end = mp_encode_int(end, -((int)num));
+	if (box_insert(space_test_id, buf, end, NULL) < 0) {
+		return box_error_set(__FILE__, __LINE__, ER_PROC_C,
+			"Can't insert in space %s", SPACE_TEST_NAME);
+	}
+	return 0;
+}
+
+int
+test_reload(box_function_ctx_t *ctx, const char *args, const char *args_end)
+{
+	fiber_sleep(0.001);
+	char tuple_buf[64];
+	char *tuple_end = tuple_buf;
+	tuple_end = mp_encode_array(tuple_end, 1);
+	tuple_end = mp_encode_uint(tuple_end, 1);
+	struct tuple *tuple = box_tuple_new(box_tuple_format_default(), tuple_buf, tuple_end);
+	return box_return_tuple(ctx, tuple);
+}
diff --git a/test/box/reload2.c b/test/box/reload2.c
new file mode 100644
index 0000000000..30a359171d
--- /dev/null
+++ b/test/box/reload2.c
@@ -0,0 +1,49 @@
+#include "module.h"
+
+#include <stdio.h>
+#include <msgpuck.h>
+
+int
+foo(box_function_ctx_t *ctx, const char *args, const char *args_end)
+{
+	static const char *SPACE_TEST_NAME = "test";
+	uint32_t space_test_id = box_space_id_by_name(SPACE_TEST_NAME,
+			strlen(SPACE_TEST_NAME));
+	if (space_test_id == BOX_ID_NIL) {
+		return box_error_set(__FILE__, __LINE__, ER_PROC_C,
+			"Can't find space %s", SPACE_TEST_NAME);
+	}
+	char buf[16];
+	char *end = buf;
+	end = mp_encode_array(end, 1);
+	end = mp_encode_uint(end, 0);
+	if (box_insert(space_test_id, buf, end, NULL) < 0) {
+		return box_error_set(__FILE__, __LINE__, ER_PROC_C,
+			"Can't insert in space %s", SPACE_TEST_NAME);
+	}
+	return 0;
+}
+
+int
+test_reload(box_function_ctx_t *ctx, const char *args, const char *args_end)
+{
+	fiber_sleep(0.001);
+	char tuple_buf[64];
+	char *tuple_end = tuple_buf;
+	tuple_end = mp_encode_array(tuple_end, 1);
+	tuple_end = mp_encode_uint(tuple_end, 2);
+	struct tuple *tuple = box_tuple_new(box_tuple_format_default(), tuple_buf, tuple_end);
+	return box_return_tuple(ctx, tuple);
+}
+
+int
+test_reload_fail(box_function_ctx_t *ctx, const char *args, const char *args_end)
+{
+	char tuple_buf[64];
+	char *tuple_end = tuple_buf;
+	tuple_end = mp_encode_array(tuple_end, 1);
+	tuple_end = mp_encode_uint(tuple_end, 2);
+	struct tuple *tuple = box_tuple_new(box_tuple_format_default(), tuple_buf, tuple_end);
+	return box_return_tuple(ctx, tuple);
+
+}
-- 
GitLab