test: add stress tests with k6 to ci

.gitlab-ci.yml

@@ -2,10 +2,12 @@
 variables:
     GIT_SUBMODULE_STRATEGY: none
     SBROAD_DEV_IMAGE: docker-public.binary.picodata.io/sbroad-builder:0.7.0
+    CACHE_ARCHIVE: /data/gitlab-runner/shared-storage/sbroad
 
 stages:
     - build
     - test
+    - stress-test
     - deploy
 
 cache:
@@ -45,6 +47,61 @@ integration:
         - sbroad-cartridge/test_app/tmp/tarantool.log
       expire_in: 1 week
 
+.stress:
+    tags:
+        - shell_sbrd
+    image: docker/compose
+    stage: stress-test
+    when:
+        manual
+    resource_group: stress
+    before_script:
+        - test=$STRESS_TEST docker-compose down
+    script:
+        - make stress test=$STRESS_TEST
+        - >
+            if [ $CI_PIPELINE_SOURCE == "main" ]; then
+                mkdir -p $CACHE_ARCHIVE/$STRESS_TEST/$CI_COMMIT_SHORT_SHA
+                mkdir -p $CACHE_ARCHIVE/$STRESS_TEST/main
+                cp $PWD/sbroad-cartridge/stress-test/$STRESS_TEST/k6_summary.json $CACHE_ARCHIVE/$STRESS_TEST/$CI_COMMIT_SHORT_SHA/k6_summary.json
+                cp $PWD/sbroad-cartridge/stress-test/$STRESS_TEST/k6_summary.json $CACHE_ARCHIVE/$STRESS_TEST/main/k6_summary.json
+            else
+                docker run --rm \
+                -v $PWD/sbroad-cartridge/stress-test/compare.lua:/tmp/compare.lua \
+                -v $CACHE_ARCHIVE/$STRESS_TEST/main/k6_summary.json:/tmp/left.json \
+                -v $PWD/sbroad-cartridge/stress-test/$STRESS_TEST/k6_summary.json:/tmp/right.json \
+                ${SBROAD_DEV_IMAGE} \
+                bash -c "tarantool /tmp/compare.lua /tmp/left.json /tmp/right.json"
+            fi
+    after_script:
+        - test=$STRESS_TEST docker-compose down
+    artifacts:
+      paths:
+        - sbroad-cartridge/test_app/tmp/log/*
+        - sbroad-cartridge/stress-test/$STRESS_TEST/k6_summary.json
+      expire_in: 1 week
+      when: always
+
+stress_projection:
+    extends: .stress
+    variables:
+        STRESS_TEST: projection
+
+stress_projection_wide:
+    extends: .stress
+    variables:
+        STRESS_TEST: projection_wide
+
+stress_groupby:
+    extends: .stress
+    variables:
+        STRESS_TEST: groupby
+
+stress_insert:
+    extends: .stress
+    variables:
+        STRESS_TEST: insert
+
 bench:
     stage: test
     script:

Makefile

@@ -54,3 +54,15 @@ release_rock:
 	&& tarantoolctl rocks pack sbroad-${CI_COMMIT_TAG}-1.rockspec \
 	&& mv sbroad*rock .. \
 	&& rm -rf sbroad-${CI_COMMIT_TAG}-1.rockspec
+
+stress:
+	test=$(test) docker-compose -f docker-compose.yml down
+	test=$(test) docker-compose -f docker-compose.yml up --abort-on-container-exit --exit-code-from k6
+	test=$(test) docker-compose -f docker-compose.yml down
+
+stress_all:
+	$(MAKE) stress test=projection
+	$(MAKE) stress test=projection_wide
+	$(MAKE) stress test=insert
+	$(MAKE) stress test=groupby
+

docker-compose.yml

+---
+version: '2.1'
+services:
+
+  tarantool:
+    image: docker-public.binary.picodata.io/sbroad-builder:0.7.0
+    command: bash -c "cd /sbroad && make start && make stress_init test=${test} && touch /etc/OK && sleep infinity"
+    volumes:
+      - "${PWD}:/sbroad"
+    healthcheck:
+        test: ["CMD-SHELL", "cat /etc/OK"]
+        interval: 30s
+        timeout: 10s
+        retries: 20
+    environment:
+      - test=${test}
+
+  k6:
+    image: docker-public.binary.picodata.io/k6_tnt:latest
+    command:
+      - k6
+      - run
+      - -u
+      - "10"
+      - -d
+      - "1m"
+      - -e
+      - HOST=tarantool
+      - /stress-test/${test}/k6.js
+      - --summary-export
+      - /stress-test/${test}/k6_summary.json
+    volumes:
+      - "${PWD}/sbroad-cartridge/stress-test/:/stress-test/"
+    depends_on:
+      tarantool:
+        condition: service_healthy
+    environment:
+      - test=${test}
+
+networks:
+  default:

sbroad-cartridge/Makefile

@@ -13,6 +13,8 @@ TARGET_ROOT?=../target
 CARTRIDGE_MODULE?=.
 CORE_MODULE=$(CARTRIDGE_MODULE)/../sbroad-core
 TEST_APP=$(CARTRIDGE_MODULE)/test_app
+STRESS_TEST=$(CARTRIDGE_MODULE)/stress-test
+K6_PATH=$(STRESS_TEST)/k6
 
 build_cartridge_engine:
 	cargo build -p sbroad-cartridge --release
@@ -42,3 +44,36 @@ install_release:
 
 test_integration:
 	$(MAKE) build_integration && $(MAKE) run_integration
+
+clean_integration:
+	$(MAKE) stop
+	rm -rf $(TEST_APP)/.rocks
+	rm -rf $(TEST_APP)/tmp
+
+setup:
+	cd $(TEST_APP) && cartridge replicasets setup --bootstrap-vshard && cd ..
+
+start:
+	$(MAKE) clean_integration
+	$(MAKE) build_integration && cd $(TEST_APP) && cartridge start -d
+	$(MAKE) setup
+
+stop:
+	cd $(TEST_APP) && cartridge stop &&  while  cartridge status 2>&1 | grep "RUNNING" -q; do sleep 1; done
+
+stress_init:
+	@if [ -z "${test}" ]; then echo "test variable must be set"; exit 1; fi
+	tarantool $(CARTRIDGE_MODULE)/stress-test/$(test)/init.lua 1000
+
+stress_local:
+	@if [ -z "${test}" ]; then echo "test variable must be set"; exit 1; fi
+	$(MAKE) start
+	$(MAKE) stress_init test=$(test)
+	$(K6_PATH) run -u 10 -d 1m $(STRESS_TEST)/$(test)/k6.js --summary-export $(STRESS_TEST)/$(test)/k6_summary_local.json
+	$(MAKE) stop
+
+stress_all_local:
+	$(MAKE) stress_local test=projection
+	$(MAKE) stress_local test=projection_wide
+	$(MAKE) stress_local test=insert
+	$(MAKE) stress_local test=groupby

sbroad-cartridge/stress-test/README.md

 # Load testing
-## Using docker-compose
-Run from sbroad directory
+
+## Navigation
+* [General information](#general)
+* [How to run using docker-compose](#run-docker-compose)
+* [How to run locally](#run-locally)
+   * [using makefile](#make-rules)
+   * [using manual commands](#manually)
+* [How to use results](#results-comparison)
+
+## <a name="general"></a>General information
+We use [k6](https://k6.io/docs/getting-started/running-k6/) for load testing with tarantool go package that allows to connect to tarantool instance. At the moment we have several scenarios that can be run separately or together (consequentially) using docker-compose or with local tools.
+
+For each scenario, the result will be a file named `k6_summary.json` or `k6_summary_local.json` containing summarized metrics within the chosen stress test folder.
+## <a name="run-docker-compose"></a>Run using docker-compose
+* Run from `sbroad` directory
+   ```bash
+   make stress test=projection
+   ```
+   Parameter `test` is type of stress test (corresponding to the name of folder). To run all stress tests consequentially, use:
+      ```bash
+      make stress_all
+      ```
+* You can find summarized metrics by `stress-test/[test]/k6_summary.json` path.
+
+## <a name="run-locally"></a>Run using k6 and tarantool
+* Firstly, build [k6](https://k6.io/docs/getting-started/running-k6/) with tarantool module
+   ```bash
+   xk6 build --with github.com/WeCodingNow/xk6-tarantool --output sbroad/sbroad-cartridge/stress-test/k6
+   ```
+* You can run stress tests locally (without docker-compose) by using make rules or running them manually. Summarized metrics will be placed in `stress-test/[test]/k6_summary_local.json`.
+
+### <a name="make-rules"></a>run by make rules
+Run a chosen stress test from the `sbroad` or `sbroad/sbroad-cartridge` directory. You can specify the path to your k6 binary. If the parameter `K6_PATH` is not set, the default path `sbroad/sbroad-cartridge/stress-test/k6` will be used:
+```bash
+make stress_local test=projection K6_PATH=path_to_k6_binary
 ```
-make stress test=projection
+If you want to run all stress tests use
+```bash
+make stress_all_local K6_PATH=path_to_k6_binary
 ```
-Parameter `test` is type of stress test (corresponding to the name of folder). As result you will see file `k6_summary.lua` with summarized metrics inside chosen stress test folder.
-## Using k6 and tarantool
+### <a name="manually"></a>run manually
 1. Run [test application](../test_app)
     ```bash
     cd ../test_app && cartridge start
@@ -19,25 +53,22 @@ Parameter `test` is type of stress test (corresponding to the name of folder). A
     ```bash
     ./init.lua 1000
     ```
-1. Build [k6](https://k6.io/docs/getting-started/running-k6/)
-   ```bash
-   xk6 build --with github.com/WeCodingNow/xk6-tarantool
-   ```
 
 1. Run the [k6](https://k6.io/docs/getting-started/running-k6/) script
     ```bash
-   ./k6 run -u 10 -d 1m k6.js --summary-export k6_summary.json
+   ./k6 run -u 10 -d 1m k6.js --summary-export k6_summary_local.json
    ```
    **Note:**
 If you run stress tests sequentially, you may need to do `cartridge clean` before next test run because tests might have
 conflicting schemas.
 
-## Compare results
-If you need to compare results serveral test, you may run
-```
-lua compare.lua path_to_k6_summary_1.json path_to_k6_summary_2.json
-```
-It considered that first result must have lower rps to get 0 exit code. Example of output:
+## <a name="results-comparison"></a>Compare results
+   If you need to compare results serveral test, you may run script `compare.lua` with lua or tarantool
+   ```bash
+   [lua|tarantool] compare.lua path_to_k6_summary_1.json path_to_k6_summary_2.json
+   ```
+It is assumed that the first result will have a lower or equal RPS and will result in an exit code of 0. Otherwise, it prints error message and return exit code equals to 1. An example of the output is as follows:
+
 ```
 $ lua compare.lua path_to_k6_summary_2.json path_to_k6_summary_1.json
 +----------------+-----------------+------------------+

sbroad-cartridge/stress-test/groupby/k6.js

 import tarantool from "k6/x/tarantool";
-import { callTarantool } from '../metrics.js';
-
-let host = "localhost";
-if (__ENV.HOST) {
-    host = __ENV.HOST;
-}
+import { callTarantool, HOST } from '../metrics.js';
 
 const clients = [
-    tarantool.connect([host + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
+    tarantool.connect([HOST + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
 ]
 
 const pattern = `SELECT "a0"+"a1"+"a2", "a1", "a1"+"a2", "a3", "a4", "a5" + "a1", "a1"+"a6", "a7"*"a9", "a8", "a9"

sbroad-cartridge/stress-test/insert/k6.js

 import tarantool from "k6/x/tarantool";
 import {uuidv4} from 'https://jslib.k6.io/k6-utils/1.1.0/index.js';
-import { callTarantool } from '../metrics.js';
-
-let host = "localhost";
-if (__ENV.HOST) {
-    host = __ENV.HOST;
-}
+import { callTarantool, HOST } from '../metrics.js';
 
 const clients = [
-    tarantool.connect([host + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
-    tarantool.connect([host + ":3306"], {"user": "admin", pass: "app-cluster-cookie"}),
-    tarantool.connect([host + ":3307"], {"user": "admin", pass: "app-cluster-cookie"}),
-    tarantool.connect([host + ":3308"], {"user": "admin", pass: "app-cluster-cookie"})
+    tarantool.connect([HOST + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
+    tarantool.connect([HOST + ":3306"], {"user": "admin", pass: "app-cluster-cookie"}),
+    tarantool.connect([HOST + ":3307"], {"user": "admin", pass: "app-cluster-cookie"}),
+    tarantool.connect([HOST + ":3308"], {"user": "admin", pass: "app-cluster-cookie"})
 ]
 
 export let current_server = 0

sbroad-cartridge/stress-test/metrics.js

 import { Rate } from 'k6/metrics';
 import tarantool from "k6/x/tarantool";
 
+export let HOST = "localhost";
+if (__ENV.HOST) {
+  HOST = __ENV.HOST;
+}
+
 export const successRate = new Rate('success');
 
 export function updateSuccessRate(resp) {

sbroad-cartridge/stress-test/projection/k6.js

 import tarantool from "k6/x/tarantool";
 import {randomItem} from 'https://jslib.k6.io/k6-utils/1.1.0/index.js';
-import { callTarantool } from '../metrics.js';
+import { callTarantool, HOST } from '../metrics.js';
 
-let host = "localhost";
-if (__ENV.HOST) {
-    host = __ENV.HOST;
-}
-
-const client = tarantool.connect([host + ":3301"], {"user": "admin", pass: "app-cluster-cookie"})
+const client = tarantool.connect([HOST + ":3301"], {"user": "admin", pass: "app-cluster-cookie"})
 
 let ids = Array.from(
     {

sbroad-cartridge/stress-test/projection_wide/k6.js

 import tarantool from "k6/x/tarantool";
 import {randomItem} from 'https://jslib.k6.io/k6-utils/1.1.0/index.js';
-import { callTarantool } from '../metrics.js';
-
-let host = "localhost";
-if (__ENV.HOST) {
-    host = __ENV.HOST;
-}
+import { callTarantool, HOST } from '../metrics.js';
 
 const clients = [
-    tarantool.connect([host + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
-    tarantool.connect([host + ":3306"], {"user": "admin", pass: "app-cluster-cookie"}),
-    tarantool.connect([host + ":3307"], {"user": "admin", pass: "app-cluster-cookie"}),
-    tarantool.connect([host + ":3308"], {"user": "admin", pass: "app-cluster-cookie"})
+    tarantool.connect([HOST + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
+    tarantool.connect([HOST + ":3306"], {"user": "admin", pass: "app-cluster-cookie"}),
+    tarantool.connect([HOST + ":3307"], {"user": "admin", pass: "app-cluster-cookie"}),
+    tarantool.connect([HOST + ":3308"], {"user": "admin", pass: "app-cluster-cookie"})
 ]
 
 export let current_server = 0