test: add test for graceful shutdown

test/conftest.py

@@ -194,6 +194,7 @@ class Instance:
     env: dict[str, str] = field(default_factory=dict)
     process: subprocess.Popen | None = None
     raft_id: int = INVALID_RAFT_ID
+    _on_output_callbacks: list[Callable[[str], None]] = field(default_factory=list)
 
     @property
     def listen(self):
@@ -290,6 +291,11 @@ class Instance:
                 out.write(prefix)
                 out.write(line)
                 out.flush()
+                for cb in self._on_output_callbacks:
+                    cb(line)
+
+    def on_output_line(self, cb: Callable[[str], None]):
+        self._on_output_callbacks.append(cb)
 
     def start(self, peers=[]):
         if self.process:

test/int/test_couple.py

@@ -197,3 +197,30 @@ def test_gl119_panic_in_on_shutdown(cluster2: Cluster):
     # wait for the follower to start acquiring the read barrier
     sleep(1)
     assert i2.terminate() == 0
+
+
+# it's 2022 and i have to work around a mypy bug reported in 2018
+on_shutdown_timed_out: bool
+
+
+def test_gl127_graceul_shutdown(cluster2: Cluster):
+    i1, i2 = cluster2.instances
+
+    # make sure i1 is leader
+    i1.promote_or_fail()
+    i2.wait_ready()
+
+    global on_shutdown_timed_out
+    on_shutdown_timed_out = False
+
+    def check_log_line(log):
+        if "on_shutdown triggers are timed out" in log:
+            global on_shutdown_timed_out
+            on_shutdown_timed_out = True
+
+    i1.on_output_line(check_log_line)
+    # on_shutdown triggers will timeout after 3sec
+    # so we must wait longer than # that
+    i1.terminate(kill_after_seconds=10)
+
+    assert not on_shutdown_timed_out