diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0aa9331a43875ac11e46a64995a9d8bd29a3f0d3..6f24f6efc66e4b3e8d2ce2110b807dd98ce4752d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,10 +2,12 @@
variables:
GIT_SUBMODULE_STRATEGY: none
SBROAD_DEV_IMAGE: docker-public.binary.picodata.io/sbroad-builder:0.7.0
+ CACHE_ARCHIVE: /data/gitlab-runner/shared-storage/sbroad
stages:
- build
- test
+ - stress-test
- deploy
cache:
@@ -45,6 +47,61 @@ integration:
- sbroad-cartridge/test_app/tmp/tarantool.log
expire_in: 1 week
+.stress:
+ tags:
+ - shell_sbrd
+ image: docker/compose
+ stage: stress-test
+ when:
+ manual
+ resource_group: stress
+ before_script:
+ - test=$STRESS_TEST docker-compose down
+ script:
+ - make stress test=$STRESS_TEST
+ - >
+ if [ $CI_PIPELINE_SOURCE == "main" ]; then
+ mkdir -p $CACHE_ARCHIVE/$STRESS_TEST/$CI_COMMIT_SHORT_SHA
+ mkdir -p $CACHE_ARCHIVE/$STRESS_TEST/main
+ cp $PWD/sbroad-cartridge/stress-test/$STRESS_TEST/k6_summary.json $CACHE_ARCHIVE/$STRESS_TEST/$CI_COMMIT_SHORT_SHA/k6_summary.json
+ cp $PWD/sbroad-cartridge/stress-test/$STRESS_TEST/k6_summary.json $CACHE_ARCHIVE/$STRESS_TEST/main/k6_summary.json
+ else
+ docker run --rm \
+ -v $PWD/sbroad-cartridge/stress-test/compare.lua:/tmp/compare.lua \
+ -v $CACHE_ARCHIVE/$STRESS_TEST/main/k6_summary.json:/tmp/left.json \
+ -v $PWD/sbroad-cartridge/stress-test/$STRESS_TEST/k6_summary.json:/tmp/right.json \
+ ${SBROAD_DEV_IMAGE} \
+ bash -c "tarantool /tmp/compare.lua /tmp/left.json /tmp/right.json"
+ fi
+ after_script:
+ - test=$STRESS_TEST docker-compose down
+ artifacts:
+ paths:
+ - sbroad-cartridge/test_app/tmp/log/*
+ - sbroad-cartridge/stress-test/$STRESS_TEST/k6_summary.json
+ expire_in: 1 week
+ when: always
+
+stress_projection:
+ extends: .stress
+ variables:
+ STRESS_TEST: projection
+
+stress_projection_wide:
+ extends: .stress
+ variables:
+ STRESS_TEST: projection_wide
+
+stress_groupby:
+ extends: .stress
+ variables:
+ STRESS_TEST: groupby
+
+stress_insert:
+ extends: .stress
+ variables:
+ STRESS_TEST: insert
+
bench:
stage: test
script:
diff --git a/Makefile b/Makefile
index b865d606d9c93e1dd47aff9e91d64acbddf4bfd9..b9acc13fcece4aa45956e3e90af2933d88363b3d 100644
--- a/Makefile
+++ b/Makefile
@@ -54,3 +54,15 @@ release_rock:
&& tarantoolctl rocks pack sbroad-${CI_COMMIT_TAG}-1.rockspec \
&& mv sbroad*rock .. \
&& rm -rf sbroad-${CI_COMMIT_TAG}-1.rockspec
+
+stress:
+ test=$(test) docker-compose -f docker-compose.yml down
+ test=$(test) docker-compose -f docker-compose.yml up --abort-on-container-exit --exit-code-from k6
+ test=$(test) docker-compose -f docker-compose.yml down
+
+stress_all:
+ $(MAKE) stress test=projection
+ $(MAKE) stress test=projection_wide
+ $(MAKE) stress test=insert
+ $(MAKE) stress test=groupby
+
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..eaa2a072d22ae335986a463daffc47b84866d858
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,41 @@
+---
+version: '2.1'
+services:
+
+ tarantool:
+ image: docker-public.binary.picodata.io/sbroad-builder:0.7.0
+ command: bash -c "cd /sbroad && make start && make stress_init test=${test} && touch /etc/OK && sleep infinity"
+ volumes:
+ - "${PWD}:/sbroad"
+ healthcheck:
+ test: ["CMD-SHELL", "cat /etc/OK"]
+ interval: 30s
+ timeout: 10s
+ retries: 20
+ environment:
+ - test=${test}
+
+ k6:
+ image: docker-public.binary.picodata.io/k6_tnt:latest
+ command:
+ - k6
+ - run
+ - -u
+ - "10"
+ - -d
+ - "1m"
+ - -e
+ - HOST=tarantool
+ - /stress-test/${test}/k6.js
+ - --summary-export
+ - /stress-test/${test}/k6_summary.json
+ volumes:
+ - "${PWD}/sbroad-cartridge/stress-test/:/stress-test/"
+ depends_on:
+ tarantool:
+ condition: service_healthy
+ environment:
+ - test=${test}
+
+networks:
+ default:
diff --git a/sbroad-cartridge/Makefile b/sbroad-cartridge/Makefile
index ef09c2bfbe3ba8aef98ecf63545db6880edc2371..7311491a5b4fcee2d5ccdc5bd2625d94c6fd45e6 100644
--- a/sbroad-cartridge/Makefile
+++ b/sbroad-cartridge/Makefile
@@ -13,6 +13,8 @@ TARGET_ROOT?=../target
CARTRIDGE_MODULE?=.
CORE_MODULE=$(CARTRIDGE_MODULE)/../sbroad-core
TEST_APP=$(CARTRIDGE_MODULE)/test_app
+STRESS_TEST=$(CARTRIDGE_MODULE)/stress-test
+K6_PATH=$(STRESS_TEST)/k6
build_cartridge_engine:
cargo build -p sbroad-cartridge --release
@@ -42,3 +44,36 @@ install_release:
test_integration:
$(MAKE) build_integration && $(MAKE) run_integration
+
+clean_integration:
+ $(MAKE) stop
+ rm -rf $(TEST_APP)/.rocks
+ rm -rf $(TEST_APP)/tmp
+
+setup:
+ cd $(TEST_APP) && cartridge replicasets setup --bootstrap-vshard && cd ..
+
+start:
+ $(MAKE) clean_integration
+ $(MAKE) build_integration && cd $(TEST_APP) && cartridge start -d
+ $(MAKE) setup
+
+stop:
+ cd $(TEST_APP) && cartridge stop && while cartridge status 2>&1 | grep "RUNNING" -q; do sleep 1; done
+
+stress_init:
+ @if [ -z "${test}" ]; then echo "test variable must be set"; exit 1; fi
+ tarantool $(CARTRIDGE_MODULE)/stress-test/$(test)/init.lua 1000
+
+stress_local:
+ @if [ -z "${test}" ]; then echo "test variable must be set"; exit 1; fi
+ $(MAKE) start
+ $(MAKE) stress_init test=$(test)
+ $(K6_PATH) run -u 10 -d 1m $(STRESS_TEST)/$(test)/k6.js --summary-export $(STRESS_TEST)/$(test)/k6_summary_local.json
+ $(MAKE) stop
+
+stress_all_local:
+ $(MAKE) stress_local test=projection
+ $(MAKE) stress_local test=projection_wide
+ $(MAKE) stress_local test=insert
+ $(MAKE) stress_local test=groupby
diff --git a/sbroad-cartridge/stress-test/README.md b/sbroad-cartridge/stress-test/README.md
index aff1e9cf420c13d28f5c2664f2e72ba34efc610b..e34845d52936daf148298519185f69daedbdb317 100644
--- a/sbroad-cartridge/stress-test/README.md
+++ b/sbroad-cartridge/stress-test/README.md
@@ -1,11 +1,45 @@
# Load testing
-## Using docker-compose
-Run from sbroad directory
+
+## Navigation
+* [General information](#general)
+* [How to run using docker-compose](#run-docker-compose)
+* [How to run locally](#run-locally)
+ * [using makefile](#make-rules)
+ * [using manual commands](#manually)
+* [How to use results](#results-comparison)
+
+## <a name="general"></a>General information
+We use [k6](https://k6.io/docs/getting-started/running-k6/) for load testing with tarantool go package that allows to connect to tarantool instance. At the moment we have several scenarios that can be run separately or together (consequentially) using docker-compose or with local tools.
+
+For each scenario, the result will be a file named `k6_summary.json` or `k6_summary_local.json` containing summarized metrics within the chosen stress test folder.
+## <a name="run-docker-compose"></a>Run using docker-compose
+* Run from `sbroad` directory
+ ```bash
+ make stress test=projection
+ ```
+ Parameter `test` is type of stress test (corresponding to the name of folder). To run all stress tests consequentially, use:
+ ```bash
+ make stress_all
+ ```
+* You can find summarized metrics by `stress-test/[test]/k6_summary.json` path.
+
+## <a name="run-locally"></a>Run using k6 and tarantool
+* Firstly, build [k6](https://k6.io/docs/getting-started/running-k6/) with tarantool module
+ ```bash
+ xk6 build --with github.com/WeCodingNow/xk6-tarantool --output sbroad/sbroad-cartridge/stress-test/k6
+ ```
+* You can run stress tests locally (without docker-compose) by using make rules or running them manually. Summarized metrics will be placed in `stress-test/[test]/k6_summary_local.json`.
+
+### <a name="make-rules"></a>run by make rules
+Run a chosen stress test from the `sbroad` or `sbroad/sbroad-cartridge` directory. You can specify the path to your k6 binary. If the parameter `K6_PATH` is not set, the default path `sbroad/sbroad-cartridge/stress-test/k6` will be used:
+```bash
+make stress_local test=projection K6_PATH=path_to_k6_binary
```
-make stress test=projection
+If you want to run all stress tests use
+```bash
+make stress_all_local K6_PATH=path_to_k6_binary
```
-Parameter `test` is type of stress test (corresponding to the name of folder). As result you will see file `k6_summary.lua` with summarized metrics inside chosen stress test folder.
-## Using k6 and tarantool
+### <a name="manually"></a>run manually
1. Run [test application](../test_app)
```bash
cd ../test_app && cartridge start
@@ -19,25 +53,22 @@ Parameter `test` is type of stress test (corresponding to the name of folder). A
```bash
./init.lua 1000
```
-1. Build [k6](https://k6.io/docs/getting-started/running-k6/)
- ```bash
- xk6 build --with github.com/WeCodingNow/xk6-tarantool
- ```
1. Run the [k6](https://k6.io/docs/getting-started/running-k6/) script
```bash
- ./k6 run -u 10 -d 1m k6.js --summary-export k6_summary.json
+ ./k6 run -u 10 -d 1m k6.js --summary-export k6_summary_local.json
```
**Note:**
If you run stress tests sequentially, you may need to do `cartridge clean` before next test run because tests might have
conflicting schemas.
-## Compare results
-If you need to compare results serveral test, you may run
-```
-lua compare.lua path_to_k6_summary_1.json path_to_k6_summary_2.json
-```
-It considered that first result must have lower rps to get 0 exit code. Example of output:
+## <a name="results-comparison"></a>Compare results
+ If you need to compare results serveral test, you may run script `compare.lua` with lua or tarantool
+ ```bash
+ [lua|tarantool] compare.lua path_to_k6_summary_1.json path_to_k6_summary_2.json
+ ```
+It is assumed that the first result will have a lower or equal RPS and will result in an exit code of 0. Otherwise, it prints error message and return exit code equals to 1. An example of the output is as follows:
+
```
$ lua compare.lua path_to_k6_summary_2.json path_to_k6_summary_1.json
+----------------+-----------------+------------------+
diff --git a/sbroad-cartridge/stress-test/groupby/k6.js b/sbroad-cartridge/stress-test/groupby/k6.js
index 8eb17bfad9213e418435fac8bf80f0ac18958e89..cc1fcbb1bdbe961855989a130cbd50b192cb6122 100644
--- a/sbroad-cartridge/stress-test/groupby/k6.js
+++ b/sbroad-cartridge/stress-test/groupby/k6.js
@@ -1,13 +1,8 @@
import tarantool from "k6/x/tarantool";
-import { callTarantool } from '../metrics.js';
-
-let host = "localhost";
-if (__ENV.HOST) {
- host = __ENV.HOST;
-}
+import { callTarantool, HOST } from '../metrics.js';
const clients = [
- tarantool.connect([host + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
+ tarantool.connect([HOST + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
]
const pattern = `SELECT "a0"+"a1"+"a2", "a1", "a1"+"a2", "a3", "a4", "a5" + "a1", "a1"+"a6", "a7"*"a9", "a8", "a9"
diff --git a/sbroad-cartridge/stress-test/insert/k6.js b/sbroad-cartridge/stress-test/insert/k6.js
index fcf004d96b126ae8aa5d67bfa483b4f61d63c648..a61adcdf35a0233647edeed3ee922cc23eae5a85 100644
--- a/sbroad-cartridge/stress-test/insert/k6.js
+++ b/sbroad-cartridge/stress-test/insert/k6.js
@@ -1,17 +1,12 @@
import tarantool from "k6/x/tarantool";
import {uuidv4} from 'https://jslib.k6.io/k6-utils/1.1.0/index.js';
-import { callTarantool } from '../metrics.js';
-
-let host = "localhost";
-if (__ENV.HOST) {
- host = __ENV.HOST;
-}
+import { callTarantool, HOST } from '../metrics.js';
const clients = [
- tarantool.connect([host + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
- tarantool.connect([host + ":3306"], {"user": "admin", pass: "app-cluster-cookie"}),
- tarantool.connect([host + ":3307"], {"user": "admin", pass: "app-cluster-cookie"}),
- tarantool.connect([host + ":3308"], {"user": "admin", pass: "app-cluster-cookie"})
+ tarantool.connect([HOST + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
+ tarantool.connect([HOST + ":3306"], {"user": "admin", pass: "app-cluster-cookie"}),
+ tarantool.connect([HOST + ":3307"], {"user": "admin", pass: "app-cluster-cookie"}),
+ tarantool.connect([HOST + ":3308"], {"user": "admin", pass: "app-cluster-cookie"})
]
export let current_server = 0
diff --git a/sbroad-cartridge/stress-test/metrics.js b/sbroad-cartridge/stress-test/metrics.js
index c6d4f9c449d893d8eb9898fa5637db9ba6754c61..628b4d688c1070b0828bf05b918949f6c928cb9e 100644
--- a/sbroad-cartridge/stress-test/metrics.js
+++ b/sbroad-cartridge/stress-test/metrics.js
@@ -1,6 +1,11 @@
import { Rate } from 'k6/metrics';
import tarantool from "k6/x/tarantool";
+export let HOST = "localhost";
+if (__ENV.HOST) {
+ HOST = __ENV.HOST;
+}
+
export const successRate = new Rate('success');
export function updateSuccessRate(resp) {
diff --git a/sbroad-cartridge/stress-test/projection/k6.js b/sbroad-cartridge/stress-test/projection/k6.js
index b7c97add6b5fddcfd12a9283e46f0b608bec65d6..11e0317219c40e6729930ba0e581c5150db9a520 100644
--- a/sbroad-cartridge/stress-test/projection/k6.js
+++ b/sbroad-cartridge/stress-test/projection/k6.js
@@ -1,13 +1,8 @@
import tarantool from "k6/x/tarantool";
import {randomItem} from 'https://jslib.k6.io/k6-utils/1.1.0/index.js';
-import { callTarantool } from '../metrics.js';
+import { callTarantool, HOST } from '../metrics.js';
-let host = "localhost";
-if (__ENV.HOST) {
- host = __ENV.HOST;
-}
-
-const client = tarantool.connect([host + ":3301"], {"user": "admin", pass: "app-cluster-cookie"})
+const client = tarantool.connect([HOST + ":3301"], {"user": "admin", pass: "app-cluster-cookie"})
let ids = Array.from(
{
diff --git a/sbroad-cartridge/stress-test/projection_wide/k6.js b/sbroad-cartridge/stress-test/projection_wide/k6.js
index 17e2d2e72fa4806bcfbb51a913853c6b9743e385..337b04e5b4ed2716879db68145fd712fa2c26e94 100644
--- a/sbroad-cartridge/stress-test/projection_wide/k6.js
+++ b/sbroad-cartridge/stress-test/projection_wide/k6.js
@@ -1,17 +1,12 @@
import tarantool from "k6/x/tarantool";
import {randomItem} from 'https://jslib.k6.io/k6-utils/1.1.0/index.js';
-import { callTarantool } from '../metrics.js';
-
-let host = "localhost";
-if (__ENV.HOST) {
- host = __ENV.HOST;
-}
+import { callTarantool, HOST } from '../metrics.js';
const clients = [
- tarantool.connect([host + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
- tarantool.connect([host + ":3306"], {"user": "admin", pass: "app-cluster-cookie"}),
- tarantool.connect([host + ":3307"], {"user": "admin", pass: "app-cluster-cookie"}),
- tarantool.connect([host + ":3308"], {"user": "admin", pass: "app-cluster-cookie"})
+ tarantool.connect([HOST + ":3301"], {"user": "admin", pass: "app-cluster-cookie"}),
+ tarantool.connect([HOST + ":3306"], {"user": "admin", pass: "app-cluster-cookie"}),
+ tarantool.connect([HOST + ":3307"], {"user": "admin", pass: "app-cluster-cookie"}),
+ tarantool.connect([HOST + ":3308"], {"user": "admin", pass: "app-cluster-cookie"})
]
export let current_server = 0