Test-runner: add signal handler for SIGCHLD

If Tarantool server segfaults when a test is
running, it used to hand in <defunct> state,
since there was no wait() call in the parent
to collect child process state. All file descriptors,
as well as open sockets, of a <defunct> process were
left open. This lead to the test runner cycling
in an infinite loop in recv() from the server,
and the whole test suite to, ultimately, hanging
in a busy wait.

This looks more like a bug in python pexpect,
but install a SIGCHLD handler to work it around,
and collect child data immediately as soon as
child terminates.

core/util.c

@@ -107,8 +107,9 @@ xrealloc(void *ptr, size_t size)
 #ifdef ENABLE_BACKTRACE
 
 /*
- * we use global static buffer because it is too late to do
- * any allocation when we are printing bactrace and fiber stack is small
+ * We use a global static buffer because it is too late to do any
+ * allocation when we are printing backtrace and fiber stack is
+ * small.
  */
 
 static char backtrace_buf[4096 * 4];

test/lib/tarantool_silverbox_server.py

@@ -9,6 +9,8 @@ import time
 import socket
 import daemon
 import glob
+import signal
+import traceback
 
 def wait_until_connected(host, port):
   """Wait until the server is started and accepting connections"""
@@ -66,6 +68,16 @@ class TarantoolSilverboxServer:
     self.abspath_to_exe = None
     self.is_started = False
 
+    if self.args.gdb or self.args.start_and_exit:
+      self.sigchld_handler = signal.SIG_DFL
+    else:
+      def sigchld_handler(signo, frame):
+        os.wait()
+        traceback.print_stack(frame)
+
+      self.sigchld_handler = sigchld_handler
+
+
   def install(self, silent = False):
     """Start server instance: check if the old one exists, kill it
     if necessary, create necessary directories and files, start
@@ -125,6 +137,11 @@ class TarantoolSilverboxServer:
     if not silent:
       print "Starting {0} {1}.".format(os.path.basename(self.abspath_to_exe),
                                        version)
+  def setup_sigchld_handler(self):
+    signal.signal(signal.SIGCHLD, self.sigchld_handler)
+
+  def clear_sigchld_handler(self):
+    signal.signal(signal.SIGCHLD, signal.SIG_DFL)
 
   def start(self, silent = False):
 
@@ -152,7 +169,7 @@ class TarantoolSilverboxServer:
         os.wait()
       else:
         with daemon.DaemonContext(working_directory = self.args.vardir):
-	  os.execvp(args[0], args)
+          os.execvp(args[0], args)
     else:
       self.server = pexpect.spawn(args[0], args[1:], cwd = self.args.vardir)
       if self.args.start_and_exit:
@@ -166,6 +183,8 @@ class TarantoolSilverboxServer:
     else:
       wait_until_connected(self.suite_ini["host"], self.suite_ini["port"])
 
+    self.setup_sigchld_handler()
+
 # Set is_started flag, to nicely support cleanup during an exception.
     self.is_started = True
 
@@ -179,6 +198,8 @@ class TarantoolSilverboxServer:
         print "Stopping the server..."
       if self.args.gdb:
         self.kill_old_server(True)
+      else:
+        self.clear_sigchld_handler()
       self.server.terminate()
       self.server.expect(pexpect.EOF)
       self.is_started = False
@@ -190,13 +211,15 @@ class TarantoolSilverboxServer:
     self.start(True)
 
   def test_option(self, option_list_str):
-      args = [self.abspath_to_exe] + option_list_str.split()
-      print " ".join([os.path.basename(self.abspath_to_exe)] + args[1:])
-      output = subprocess.Popen(args,
-                                cwd = self.args.vardir,
-                                stdout = subprocess.PIPE,
-                                stderr = subprocess.STDOUT).stdout.read()
-      print output
+    args = [self.abspath_to_exe] + option_list_str.split()
+    print " ".join([os.path.basename(self.abspath_to_exe)] + args[1:])
+    self.clear_sigchld_handler()
+    output = subprocess.Popen(args,
+                          cwd = self.args.vardir,
+                          stdout = subprocess.PIPE,
+                          stderr = subprocess.STDOUT).stdout.read()
+    self.setup_sigchld_handler()
+    print output
 
 
   def find_exe(self):

test/lib/test_suite.py

@@ -104,10 +104,11 @@ class Test:
     sql = DataConnection(self.suite_ini["host"],
                          self.suite_ini["port"])
     server = self.suite_ini["server"]
+    stdout_redirect = FilteredStream(self.tmp_result)
     try:
       admin.connect()
       sql.connect()
-      sys.stdout = FilteredStream(self.tmp_result)
+      sys.stdout = stdout_redirect
       server = self.suite_ini["server"]
       vardir = self.suite_ini["vardir"]
       execfile(self.name, globals(), locals())