Fix #1264: multi-instance management using systemd

This patch also enables daemon supervision when used with systemd.
A prerequisite for Fedora/EPEL7 packages.

.gitignore

@@ -28,6 +28,7 @@ extra/bin2c
 extra/dist/tarantoolctl.1
 extra/dist/tarantool.logrotate
 extra/dist/tarantool.service
+extra/dist/tarantool@.service
 extra/dist/tarantool.tmpfiles.conf
 cmake_install.cmake
 config.mk
@@ -47,7 +48,6 @@ doc/tnt.ent
 rpm/tarantool-1*.spec
 extra/rpm.spec
 extra/rpm/tarantool.rpm.spec
-extra/dist/tarantool.service
 include/tarantool/config.h
 src/trivia/config.h
 install_manifest.txt

README.systemd.md

+# Notes for Systemd Users
+
+Tarantool package fully supports **systemd** for managing instances and
+supervising database daemons.
+
+## Instance Management
+
+Package was designed to have multiple running instances of Tarantool on
+the same machine. Please use
+`systemctl {start|stop|restart|status} tarantool@${MYAPP}` to manage your
+databases and Lua applications.
+
+### Creating Instances
+
+Simple put your Lua configuration to
+`/etc/tarantool/instances.available/${MYAPP}.lua`:
+
+    box.cfg {
+        slab_alloc_arena = 1.0; -- 1Gb
+        listen = 3313;
+    }
+
+    require('myappcode').start()
+
+Tarantool ships with `example.lua` script which can be used as a start point.
+
+### Starting Instances
+
+Use `systemctl start tarantool@${MYAPP}` to start `${MYAPP}` instance:
+
+    # systemctl start tarantool@example
+    # ps axuf|grep exampl[e]
+    taranto+  5350  1.3  0.3 1448872 7736 ?        Ssl  20:05   0:28 tarantool example.lua <running>
+
+Use `systemctl enable tarantool@${MYAPP}` to enable `${MYAPP}` instance
+for auto-load during system startup.
+
+### Monitoring Instances
+
+Use `systemctl status tarantool@${MYAPP}` to check information about
+`${MYAPP}` instance:
+
+    # systemctl status tarantool@example
+    ● tarantool@example.service - Tarantool Database Server
+       Loaded: loaded (/etc/systemd/system/tarantool@.service; disabled; vendor preset: disabled)
+       Active: active (running)
+         Docs: man:tarantool(1)
+      Process: 5346 ExecStart=/usr/bin/tarantoolctl start %I (code=exited, status=0/SUCCESS)
+     Main PID: 5350 (tarantool)
+        Tasks: 11 (limit: 512)
+       CGroup: /system.slice/system-tarantool.slice/tarantool@example.service
+               + 5350 tarantool example.lua <running>
+
+Use `journalctl -u tarantool@${MYAPP}` to check boot log:
+
+    journalctl -u tarantool@example -n 5
+    -- Logs begin at Fri 2016-01-08 12:21:53 MSK, end at Thu 2016-01-21 21:17:47 MSK. --
+    Jan 21 21:17:47 localhost.localdomain systemd[1]: Stopped Tarantool Database Server.
+    Jan 21 21:17:47 localhost.localdomain systemd[1]: Starting Tarantool Database Server...
+    Jan 21 21:17:47 localhost.localdomain tarantoolctl[5969]: /usr/bin/tarantoolctl: Found example.lua in /etc/tarantool/instances.available
+    Jan 21 21:17:47 localhost.localdomain tarantoolctl[5969]: /usr/bin/tarantoolctl: Starting instance...
+    Jan 21 21:17:47 localhost.localdomain systemd[1]: Started Tarantool Database Server
+
+
+### Attaching to Instances
+
+It is possible to attach to a running Tarantool instance and evaluate some
+Lua code using `tarantoolctl` utility:
+
+    # tarantoolctl enter example
+    /bin/tarantoolctl: Found example.lua in /etc/tarantool/instances.available
+    /bin/tarantoolctl: Connecting to /var/run/tarantool/example.control
+    /bin/tarantoolctl: connected to unix/:/var/run/tarantool/example.control
+    unix/:/var/run/tarantool/example.control> 1 + 1
+    ---
+    - 2
+    ...
+    unix/:/var/run/tarantool/example.control>
+
+
+### Checking Logs
+
+Tarantool log important events to `/var/log/tarantool/${MYAPP}.log`.
+
+Let's write something to the log file:
+
+    # tarantoolctl enter example
+    /bin/tarantoolctl: Found example.lua in /etc/tarantool/instances.available
+    /bin/tarantoolctl: Connecting to /var/run/tarantool/example.control
+    /bin/tarantoolctl: connected to unix/:/var/run/tarantool/example.control
+    unix/:/var/run/tarantool/example.control> require('log').info("Hello for README.systemd readers")
+    ---
+    ...
+
+Then check the logs:
+
+    # tail /var/log/tarantool/example.log
+    2016-01-21 21:09:45.982 [5914] iproto I> binary: started
+    2016-01-21 21:09:45.982 [5914] iproto I> binary: bound to 0.0.0.0:3301
+    2016-01-21 21:09:45.983 [5914] main/101/tarantoolctl I> ready to accept requests
+    2016-01-21 21:09:45.983 [5914] main/101/example I> Run console at /var/run/tarantool/example.control
+    2016-01-21 21:09:45.984 [5914] main/101/example I> tcp_server: remove dead UNIX socket: /var/run/tarantool/example.control
+    2016-01-21 21:09:45.984 [5914] main/104/console/unix/:/var/run/tarant I> started
+    2016-01-21 21:09:45.985 [5914] main C> entering the event loop
+    2016-01-21 21:14:43.320 [5914] main/105/console/unix/: I> client unix/: connected
+    2016-01-21 21:15:07.115 [5914] main/105/console/unix/: I> Hello for README.systemd readers
+    2016-01-21 21:15:09.250 [5914] main/105/console/unix/: I> client unix/: disconnected
+
+Log rotation is enabled by default if you have `logrotate` installed.
+
+Please tweak `/etc/logrotate.d/tarantool` to change the default behavior.
+
+### Stopping Instance
+
+Use `systemctl stop tarantool@${MYAPP}` to see information about running
+`${MYAPP}` instance.
+
+    # systemctl stop tarantool@example
+
+## Daemon Supervision
+
+All instances are automatically restarted by `systemd` in case of failure.
+
+Let's try to destroy an instance:
+
+    # systemctl status tarantool@example|grep PID
+     Main PID: 5885 (tarantool)
+    # tarantoolctl enter example
+    /bin/tarantoolctl: Found example.lua in /etc/tarantool/instances.available
+    /bin/tarantoolctl: Connecting to /var/run/tarantool/example.control
+    /bin/tarantoolctl: connected to unix/:/var/run/tarantool/example.control
+    unix/:/var/run/tarantool/example.control> os.exit(-1)
+    /bin/tarantoolctl: unix/:/var/run/tarantool/example.control: Remote host closed connection
+
+`systemd` has revived our Tarantool:
+
+    # systemctl status tarantool@example|grep PID
+     Main PID: 5914 (tarantool)
+
+Let's check the boot logs:
+
+    # journalctl -u tarantool@example -n 8
+    -- Logs begin at Fri 2016-01-08 12:21:53 MSK, end at Thu 2016-01-21 21:09:45 MSK. --
+    Jan 21 21:09:45 localhost.localdomain systemd[1]: tarantool@example.service: Unit entered failed state.
+    Jan 21 21:09:45 localhost.localdomain systemd[1]: tarantool@example.service: Failed with result 'exit-code'.
+    Jan 21 21:09:45 localhost.localdomain systemd[1]: tarantool@example.service: Service hold-off time over, scheduling restart.
+    Jan 21 21:09:45 localhost.localdomain systemd[1]: Stopped Tarantool Database Server.
+    Jan 21 21:09:45 localhost.localdomain systemd[1]: Starting Tarantool Database Server...
+    Jan 21 21:09:45 localhost.localdomain tarantoolctl[5910]: /usr/bin/tarantoolctl: Found example.lua in /etc/tarantool/instances.available
+    Jan 21 21:09:45 localhost.localdomain tarantoolctl[5910]: /usr/bin/tarantoolctl: Starting instance...
+    Jan 21 21:09:45 localhost.localdomain systemd[1]: Started Tarantool Database Server.
+
+## Customizing Service File
+
+Please don't modify `tarantool@.service` file in-place, because it will be
+overwrriten during package upgrades. It is recommended to copy this file to
+`/etc/systemd/system` and then modify the chosen settings. Alternatively,
+one can create a directory named `unit.d/` within `/etc/systemd/system` and
+place a drop-in file name.conf there that only changes the specific
+settings one is interested in. Please see systemd.unit(5) manual page for
+additional information.
+
+## Precautions
+
+* Please don't use `tarantoolctl {start,stop,restart}` to control instances
+  started by systemd. It is still possible to use `tarantoolctl` to start and
+  stop instances from your local directories (e.g. `${HOME}`) without
+  obtaining `ROOT` access.
+
+* `tarantoolctl` is configured to work properly with **systemd**. Please don't
+   modify system-wide settings of `tarantoolctl`, such as paths, directory
+   permissions and usernames. Otherwise, you have a chance to shoot yourself
+   in the foot.
+
+* systemd scripts are maintained by Tarantool Team (http://tarantool.org).
+  Please file tickets directly to the upstream's bug tracker rather than to
+  your Linux distribution.

extra/dist/CMakeLists.txt

@@ -15,6 +15,11 @@ set(TARANTOOL_LOGDIR "${CMAKE_INSTALL_FULL_LOCALSTATEDIR}/log/tarantool")
 message (STATUS "tarantoolctl logdir: ${TARANTOOL_LOGDIR}")
 set(TARANTOOL_RUNDIR "${CMAKE_INSTALL_FULL_LOCALSTATEDIR}/run/tarantool")
 message (STATUS "tarantoolctl rundir: ${TARANTOOL_RUNDIR}")
+set(TARANTOOL_USER "tarantool")
+set(SYSCONFIG_AVAILABLEDIR "tarantool/instances.available")
+set(SYSCONFIG_ENABLEDDIR "tarantool/instances.enabled")
+set(TARANTOOL_AVAILABLEDIR "${CMAKE_INSTALL_FULL_SYSCONFDIR}/${SYSCONFIG_AVAILABLEDIR}")
+set(TARANTOOL_ENABLEDDIR "${CMAKE_INSTALL_FULL_SYSCONFDIR}/${SYSCONFIG_ENABLEDDIR}")
 
 # config file for tarantoolctl
 if (TARGET_OS_FREEBSD)
@@ -44,15 +49,17 @@ install (FILES tarantoolctl DESTINATION ${CMAKE_INSTALL_BINDIR}
 
 # directories in /etc/ for tarantoolctl
 install(DIRECTORY DESTINATION
-    ${CMAKE_INSTALL_SYSCONFDIR}/tarantool/instances.enabled
-)
-install(DIRECTORY DESTINATION
-    ${CMAKE_INSTALL_SYSCONFDIR}/tarantool/instances.available
+    ${CMAKE_INSTALL_SYSCONFDIR}/${SYSCONFIG_AVAILABLEDIR}
 )
+if (WITH_SYSVINIT)
+    install(DIRECTORY DESTINATION
+        ${CMAKE_INSTALL_SYSCONFDIR}/${SYSCONFIG_ENABLEDDIR}
+    )
+endif()
 
 # an example instance script for tarantoolctl
 install (FILES example.lua DESTINATION
-        ${CMAKE_INSTALL_SYSCONFDIR}/tarantool/instances.available
+        ${CMAKE_INSTALL_SYSCONFDIR}/${SYSCONFIG_AVAILABLEDIR}
     PERMISSIONS
     OWNER_READ OWNER_WRITE OWNER_READ
     GROUP_READ WORLD_READ
@@ -93,18 +100,13 @@ if (WITH_SYSTEMD)
     # architecture, but tarantool-common is noarch package.
     set(SYSV_INITD_DIR ${CMAKE_INSTALL_PREFIX}/lib/tarantool)
 
-    configure_file("tarantool.service.in" "tarantool.service" @ONLY)
-    install (FILES ${PROJECT_BINARY_DIR}/extra/dist/tarantool.service
+    configure_file("tarantool@.service.in" "tarantool@.service" @ONLY)
+    install (FILES ${PROJECT_BINARY_DIR}/extra/dist/tarantool@.service
         DESTINATION ${SYSTEMD_UNIT_DIR}
         PERMISSIONS
         OWNER_READ OWNER_WRITE
         GROUP_READ GROUP_READ
         WORLD_READ WORLD_READ)
-    install (FILES tarantool.init DESTINATION ${SYSV_INITD_DIR}
-        PERMISSIONS
-        OWNER_READ OWNER_WRITE
-        GROUP_READ GROUP_EXECUTE
-        WORLD_READ WORLD_EXECUTE)
     configure_file(tarantool.tmpfiles.conf.in tarantool.tmpfiles.conf @ONLY)
     install (FILES "${PROJECT_BINARY_DIR}/extra/dist/tarantool.tmpfiles.conf"
         DESTINATION "${SYSTEMD_TMPFILES_DIR}"
@@ -115,7 +117,7 @@ if (WITH_SYSTEMD)
         WORLD_READ WORLD_READ)
 endif()
 
-if(WITH_SYSVINIT)
+if (WITH_SYSVINIT)
     message (STATUS "Using scripts for sysvinit")
     install (FILES tarantool.init DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/init.d/
         RENAME tarantool
@@ -123,10 +125,4 @@ if(WITH_SYSVINIT)
         OWNER_READ OWNER_WRITE
         GROUP_READ GROUP_EXECUTE
         WORLD_READ WORLD_EXECUTE)
-    install (FILES ${PROJECT_BINARY_DIR}/extra/dist/default/tarantool DESTINATION
-        ${CMAKE_INSTALL_SYSCONFDIR}/${SYSCONFIG_DEFAULT}/
-        PERMISSIONS
-        OWNER_READ OWNER_WRITE
-        GROUP_READ
-        WORLD_READ)
 endif()

extra/dist/default/tarantool.in

@@ -19,9 +19,10 @@ default_cfg = {
     snap_dir   = "@TARANTOOL_DATADIR@", -- @TARANTOOL_DATADIR@/${INSTANCE}
     sophia_dir = "@TARANTOOL_DATADIR@", -- @TARANTOOL_DATADIR@/${INSTANCE}
     logger     = "@TARANTOOL_LOGDIR@", -- @TARANTOOL_LOGDIR@/${INSTANCE}.log
-    username   = "tarantool",
+    username   = "@TARANTOOL_USER@",
 }
 
-instance_dir = "@CMAKE_INSTALL_FULL_SYSCONFDIR@/tarantool/instances.enabled"
-
+-- instances.available - all available instances
+-- instances.enabled - instances to autostart by sysvinit
+instance_dir = "@TARANTOOL_AVAILABLEDIR@"
 -- vim: set ft=lua :

extra/dist/tarantool.logrotate.in

@@ -5,7 +5,7 @@
     rotate 10
     compress
     delaycompress
-    create 0640 tarantool adm
+    create 0640 @TARANTOOL_USER@ adm
     postrotate
         @CMAKE_INSTALL_FULL_BINDIR@/tarantoolctl logrotate `basename ${1%%.*}`
     endscript

extra/dist/tarantool.service.in

-# It's not recommended to modify this file in-place, because it will be
-# overwritten during package upgrades. If you want to customize there're
-# number of ways:
-
-# Recommended way:
-# 1) Use "/etc/sysconfig/tarantool" or "/etc/default/tarantool" -
-# They're supported by our start-stop utility - tarantoolctl
-
-# Usual way for RPM-based distros
-# 2) Create a file "/etc/systemd/system/tarantool.service",
-# containing
-#   .include /usr/lib/systemd/system/tarantool.service
-#   # Here're your changes
-#
-# For example, if you want to change CONF_DIR create
-# "/etc/systemd/system/tarantool.service" containing:
-#   .include /usr/lib/systemd/system/tarantool.service
-#   [Service]
-#   Environment=CONF_DIR=/etc/tarantool/instances.other
-# This will override the settings appearing below
-
-[Unit]
-Description=Tarantool instances
-After=network.target
-Documentation=man:tarantool(1)
-
-[Service]
-Type=forking
-User=tarantool
-Group=tarantool
-OOMScoreAdjust=-1000
-
-ExecStart=@SYSV_INITD_DIR@/tarantool.init start
-ExecStop=@SYSV_INITD_DIR@/tarantool.init stop
-
-TimeoutSec=300

extra/dist/tarantool.tmpfiles.conf.in

-d @TARANTOOL_RUNDIR@ 0755 tarantool tarantool -
+d @TARANTOOL_RUNDIR@ 0755 @TARANTOOL_USER@ @TARANTOOL_USER@ -

extra/dist/tarantool@.service.in

+# Please don't modify this file in-place, because it will be overwrriten
+# during package upgrades. It is recommended to copy this file to
+# /etc/systemd/system and then modify the chosen settings. Alternatively,
+# one can create a directory named unit.d/ within /etc/systemd/system and
+# place a drop-in file name.conf there that only changes the specific
+# settings one is interested in.
+#
+# Please see README.systemd.md for additional information.
+#
+#
+[Unit]
+Description=Tarantool Database Server
+After=network.target
+Documentation=man:tarantool(1)
+
+# Created by package
+AssertPathIsReadWrite=@TARANTOOL_DATADIR@
+AssertPathIsReadWrite=@TARANTOOL_LOGDIR@
+# Created by systemd-tmpfiles
+AssertPathIsReadWrite=@TARANTOOL_RUNDIR@
+
+[Service]
+Type=forking
+User=@TARANTOOL_USER@
+Group=@TARANTOOL_USER@
+# Disable OOM killer
+OOMScoreAdjust=-1000
+
+ExecStart=@CMAKE_INSTALL_BINDIR@/tarantoolctl start %I
+ExecStop=@CMAKE_INSTALL_BINDIR@/tarantoolctl stop %I
+## NYI: https://github.com/tarantool/tarantool/issues/1229
+#ExecReload=@CMAKE_INSTALL_BINDIR@/tarantoolctl reload %I
+
+# Give a reasonable amount of time to close xlogs
+TimeoutStopSec=10s
+Restart=on-failure
+RestartSec=100ms
+
+[Install]
+WantedBy=multi-user.target

rpm/tarantool.spec

@@ -129,7 +129,7 @@ rm -rf %{buildroot}%{_datarootdir}/doc/tarantool/
 %post common
 %if %{with systemd}
 %tmpfiles_create tarantool.conf
-%systemd_post tarantool.service
+%systemd_post tarantool@.service
 %else
 chkconfig --add tarantool || :
 service tarantool start || :
@@ -137,7 +137,7 @@ service tarantool start || :
 
 %preun common
 %if %{with systemd}
-%systemd_preun tarantool.service
+%systemd_preun tarantool@.service
 %else
 service tarantool stop
 chkconfig --del tarantool
@@ -145,7 +145,7 @@ chkconfig --del tarantool
 
 %postun common
 %if %{with systemd}
-%systemd_postun_with_restart tarantool.service
+%systemd_postun_with_restart tarantool@.service
 %endif
 
 %files
@@ -170,7 +170,6 @@ chkconfig --del tarantool
 %{_mandir}/man1/tarantoolctl.1*
 %config(noreplace) %{_sysconfdir}/sysconfig/tarantool
 %dir %{_sysconfdir}/tarantool
-%dir %{_sysconfdir}/tarantool/instances.enabled
 %dir %{_sysconfdir}/tarantool/instances.available
 %config(noreplace) %{_sysconfdir}/tarantool/instances.available/example.lua
 # Use 0750 for database files
@@ -179,15 +178,18 @@ chkconfig --del tarantool
 %config(noreplace) %{_sysconfdir}/logrotate.d/tarantool
 
 %if %{with systemd}
-%{_unitdir}/tarantool.service
-%{_prefix}/lib/tarantool/tarantool.init
+%doc README.systemd.md
+%{_unitdir}/tarantool@.service
 %{_tmpfilesdir}/tarantool.conf
 %else
 %{_sysconfdir}/init.d/tarantool
+%dir %{_sysconfdir}/tarantool/instances.enabled
 %attr(-,tarantool,tarantool) %dir %{_localstatedir}/run/tarantool/
 %endif
 
 %changelog
+* Thu Jan 21 2016 Roman Tsisyk <roman@tarantool.org> 1.6.8.376-1
+- Implement proper support of multi-instance management using systemd
 * Sat Jan 9 2016 Roman Tsisyk <roman@tarantool.org> 1.6.8.0-1
 - Change naming scheme to include a postrelease number to Version
 - Fix arch-specific paths in tarantool-common