app: fix crash on logrotate and immediate exit

`say_logrotate` does not rotate logs synchronously. It posts tasks to
coio which executes them in it's pool thread. On application exit we
destroy logs calling `log_destroy`. This function waits for rotate task
to finish because if it will free resources immediately then unfinished
rotate task can have use-after-free issues. Waiting crashes because at
this moment event loop is not running which is required for
`fiber_cond_wait` to work.

Note that if there is no crash then we will hang forever waiting in
`log_destroy` because event loop is not running and
`log_rotate_async_cb` will never be executed.

Let's use mutexes and conditions instead. It solves both problems with
crash and hang. Thanks Vladimir Davydov for idea.

Fixes #4450

NO_DOC=bugfix

(cherry picked from commit eed09192)

changelogs/unreleased/gh-4450-fix-crash-on-logrotate-and-exit.md

+## bugfix/core
+
+* Fixed crash on log rotate and immediate application exit (gh-4450).

src/lib/core/say.c

@@ -330,10 +330,15 @@ static int
 logrotate_cb(struct coio_task *ptr)
 {
 	struct rotate_task *task = (struct rotate_task *) ptr;
-	if (log_rotate(task->log) < 0) {
+	struct log *log = task->log;
+	if (log_rotate(log) < 0)
 		diag_log();
-	}
-	ev_async_send(task->loop, &task->log->log_async);
+	tt_pthread_mutex_lock(&log->rotate_mutex);
+	assert(log->rotating_threads > 0);
+	log->rotating_threads--;
+	if (log->rotating_threads == 0)
+		tt_pthread_cond_signal(&log->rotate_cond);
+	tt_pthread_mutex_unlock(&log->rotate_mutex);
 	return 0;
 }
 
@@ -346,16 +351,6 @@ logrotate_cleanup_cb(struct coio_task *ptr)
 	return 0;
 }
 
-static void
-log_rotate_async_cb(struct ev_loop *loop, struct ev_async *watcher, int events)
-{
-	(void)loop;
-	(void)events;
-	struct log *log = container_of(watcher, struct log, log_async);
-	log->rotating_threads--;
-	fiber_cond_signal(&log->rotate_cond);
-}
-
 void
 say_logrotate(struct ev_loop *loop, struct ev_signal *w, int revents)
 {
@@ -373,8 +368,9 @@ say_logrotate(struct ev_loop *loop, struct ev_signal *w, int revents)
 			diag_log();
 			continue;
 		}
-		ev_async_start(loop(), &log->log_async);
+		tt_pthread_mutex_lock(&log->rotate_mutex);
 		log->rotating_threads++;
+		tt_pthread_mutex_unlock(&log->rotate_mutex);
 		coio_task_create(&task->base, logrotate_cb, logrotate_cleanup_cb);
 		task->log = log;
 		task->loop = loop();
@@ -661,8 +657,8 @@ log_create(struct log *log, const char *init_str, int nonblock)
 	log->level = S_INFO;
 	log->rotating_threads = 0;
 	log->on_log = NULL;
-	fiber_cond_create(&log->rotate_cond);
-	ev_async_init(&log->log_async, log_rotate_async_cb);
+	tt_pthread_mutex_init(&log->rotate_mutex, NULL);
+	tt_pthread_cond_init(&log->rotate_cond, NULL);
 	setvbuf(stderr, NULL, _IONBF, 0);
 
 	if (init_str != NULL) {
@@ -1241,8 +1237,10 @@ void
 log_destroy(struct log *log)
 {
 	assert(log != NULL);
+	tt_pthread_mutex_lock(&log->rotate_mutex);
 	while(log->rotating_threads > 0)
-		fiber_cond_wait(&log->rotate_cond);
+		tt_pthread_cond_wait(&log->rotate_cond, &log->rotate_mutex);
+	tt_pthread_mutex_unlock(&log->rotate_mutex);
 	pm_atomic_store(&log->type, SAY_LOGGER_BOOT);
 
 	if (log->fd != -1)
@@ -1250,8 +1248,8 @@ log_destroy(struct log *log)
 	free(log->syslog_ident);
 	free(log->path);
 	rlist_del_entry(log, in_log_list);
-	ev_async_stop(loop(), &log->log_async);
-	fiber_cond_destroy(&log->rotate_cond);
+	tt_pthread_mutex_destroy(&log->rotate_mutex);
+	tt_pthread_cond_destroy(&log->rotate_cond);
 }
 
 int

src/lib/core/say.h

@@ -168,16 +168,15 @@ struct log {
 	/* Application identifier used to group syslog messages. */
 	char *syslog_ident;
 	/**
-	 * Used to wake up the main logger thread from a eio thread.
+	 * Counter identifying number of threads executing log_rotate.
+	 * Protected by rotate_mutex as it is accessed from different
+	 * threads.
 	 */
-	ev_async log_async;
-	/**
-	 * Conditional variable securing variable below
-	 * from concurrent usage.
-	 */
-	struct fiber_cond rotate_cond;
-	/** Counter identifying number of threads executing log_rotate. */
 	int rotating_threads;
+	/** Mutex for accessing rotating_threads field. */
+	pthread_mutex_t rotate_mutex;
+	/** Condition that all rotation tasks are finished. */
+	pthread_cond_t rotate_cond;
 	enum syslog_facility syslog_facility;
 	struct rlist in_log_list;
 	/** Callback called on log event. */

test/unit/say.c

@@ -271,6 +271,13 @@ int main()
 		ok(strstr(line, "<131>") != NULL, "syslog line");
 	}
 	log_destroy(&test_log);
+
+	/* Test gh-4450. */
+	log_create(&test_log, tmp_filename, false);
+	say_logrotate(NULL, NULL, 0);
+	coio_shutdown();
+	log_destroy(&test_log);
+
 	fiber_free();
 	memory_free();
 	unlink(tmp_filename);