From 19ac10c99db502638001be2f3c217a32a3e9cb5a Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov.dev@gmail.com>
Date: Tue, 31 Oct 2017 14:23:52 +0300
Subject: [PATCH] vinyl: ignore quota timeout during bootstrap from master
During initial join, a replica receives all data accumulated on the
master for its whole lifetime, which may be quota a lot. If the network
connection is fast enough, the replica might fail to keep up with dumps,
in which case replication fails with ER_VY_QUOTA_TIMEOUT. To avoid that,
let's ignore quota timeout until bootstrap is complete.
Note, replication may still fail during the 'subscribe' stage for the
same reason, but it's unlikely, because the rate at which the master
sends data is limited by the number of requests served by the master per
a unit of time, and it should become nearly impossible once throttling
is introduced (See #1862).
Closes #2873
---
src/box/vinyl.c | 11 +++++++++--
test/vinyl/join_quota.lua | 1 +
test/vinyl/replica_quota.result | 7 +++++++
test/vinyl/replica_quota.test.lua | 7 +++++++
4 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 59289f9321..405c39cde8 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -2029,14 +2029,21 @@ vy_begin(struct vy_env *env)
int
vy_prepare(struct vy_env *env, struct vy_tx *tx)
{
+ /*
+ * A replica receives a lot of data during initial join.
+ * If the network connection is fast enough, it might fail
+ * to keep up with dumps. To avoid replication failure due
+ * to this, we ignore the quota timeout during bootstrap.
+ */
+ double timeout = (env->status == VINYL_ONLINE ?
+ env->timeout : TIMEOUT_INFINITY);
/*
* Reserve quota needed by the transaction before allocating
* memory. Since this may yield, which opens a time window for
* the transaction to be sent to read view or aborted, we call
* it before checking for conflicts.
*/
- if (vy_quota_use(&env->quota, tx->write_size,
- env->timeout) != 0) {
+ if (vy_quota_use(&env->quota, tx->write_size, timeout) != 0) {
diag_set(ClientError, ER_VY_QUOTA_TIMEOUT);
return -1;
}
diff --git a/test/vinyl/join_quota.lua b/test/vinyl/join_quota.lua
index 14250a93b8..d371abf14f 100644
--- a/test/vinyl/join_quota.lua
+++ b/test/vinyl/join_quota.lua
@@ -4,6 +4,7 @@ box.cfg({
listen = os.getenv("LISTEN"),
replication = os.getenv("MASTER"),
vinyl_memory = 1024 * 1024,
+ vinyl_timeout = 0.001,
})
require('console').listen(os.getenv('ADMIN'))
diff --git a/test/vinyl/replica_quota.result b/test/vinyl/replica_quota.result
index d2f3c650a4..485efde79f 100644
--- a/test/vinyl/replica_quota.result
+++ b/test/vinyl/replica_quota.result
@@ -29,6 +29,13 @@ for i = 1001,2000 do s:insert{i, pad} end
...
-- Replica has memory limit set to 1 MB so replication would hang
-- if the scheduler didn't work on the destination.
+--
+-- Also check that quota timeout isn't taken into account while
+-- the replica is joining (see gh-2873). To do that, we set
+-- vinyl_timeout to 1 ms on the replica, which isn't enough for
+-- a dump to complete and hence would result in bootstrap failure
+-- were the timeout not ignored.
+--
_ = test_run:cmd("create server replica with rpl_master=default, script='vinyl/join_quota.lua'")
---
...
diff --git a/test/vinyl/replica_quota.test.lua b/test/vinyl/replica_quota.test.lua
index f9215c2203..bc6cfb0d71 100644
--- a/test/vinyl/replica_quota.test.lua
+++ b/test/vinyl/replica_quota.test.lua
@@ -14,6 +14,13 @@ for i = 1001,2000 do s:insert{i, pad} end
-- Replica has memory limit set to 1 MB so replication would hang
-- if the scheduler didn't work on the destination.
+--
+-- Also check that quota timeout isn't taken into account while
+-- the replica is joining (see gh-2873). To do that, we set
+-- vinyl_timeout to 1 ms on the replica, which isn't enough for
+-- a dump to complete and hence would result in bootstrap failure
+-- were the timeout not ignored.
+--
_ = test_run:cmd("create server replica with rpl_master=default, script='vinyl/join_quota.lua'")
_ = test_run:cmd("start server replica")
_ = test_run:wait_lsn('replica', 'default')
--
GitLab