diff --git a/src/box/applier.cc b/src/box/applier.cc index cdbd2762cd12658f9ca01e7a7ae7324ccfe6e761..e18c34f01407bdd0c9ab092467a3c55a39276adc 100644 --- a/src/box/applier.cc +++ b/src/box/applier.cc @@ -127,11 +127,15 @@ applier_connect(struct applier *applier) /* Don't display previous error messages in box.info.replication */ diag_clear(&fiber()->diag); + applier_set_state(applier, APPLIER_CONNECTED); + + /* Detect connection to itself */ + if (tt_uuid_is_equal(&applier->uuid, &SERVER_UUID)) + tnt_raise(ClientError, ER_CONNECTION_TO_SELF); + /* Perform authentication if user provided at least login */ - if (!uri->login) { - applier_set_state(applier, APPLIER_CONNECTED); + if (!uri->login) return; - } /* Authenticate */ applier_set_state(applier, APPLIER_AUTH); @@ -156,9 +160,6 @@ applier_connect(struct applier *applier) static void applier_join(struct applier *applier) { - say_info("bootstraping replica from %s", - sio_strfaddr(&applier->addr, applier->addr_len)); - /* Send JOIN request */ struct ev_io *coio = &applier->io; struct iobuf *iobuf = applier->iobuf; @@ -317,8 +318,6 @@ applier_subscribe(struct applier *applier) static inline void applier_log_error(struct applier *applier, struct error *e) { - if (type_cast(FiberIsCancelled, e)) - return; if (applier->warning_said) return; switch (applier->state) { @@ -346,10 +345,8 @@ applier_log_error(struct applier *applier, struct error *e) } static inline void -applier_disconnect(struct applier *applier, struct error *e, - enum applier_state state) +applier_disconnect(struct applier *applier, enum applier_state state) { - applier_log_error(applier, e); coio_close(loop(), &applier->io); iobuf_reset(applier->iobuf); applier_set_state(applier, state); @@ -382,15 +379,32 @@ applier_f(va_list ap) assert(0); return 0; } catch (ClientError *e) { - /* log logical error which caused replica to stop */ - e->log(); - applier_disconnect(applier, e, APPLIER_STOPPED); - throw; + if (e->errcode() == ER_CONNECTION_TO_SELF && + tt_uuid_is_equal(&applier->uuid, &SERVER_UUID)) { + /* Connection to itself, stop applier */ + applier_disconnect(applier, APPLIER_OFF); + return 0; + } else if (e->errcode() != ER_LOADING) { + applier_log_error(applier, e); + applier_disconnect(applier, APPLIER_STOPPED); + throw; + } + assert(e->errcode() == ER_LOADING); + /* + * Ignore ER_LOADING + */ + if (!applier->warning_said) { + say_info("bootstrap in progress..."); + applier->warning_said = true; + } + applier_disconnect(applier, APPLIER_DISCONNECTED); + /* fall through, try again later */ } catch (FiberIsCancelled *e) { - applier_disconnect(applier, e, APPLIER_OFF); + applier_disconnect(applier, APPLIER_OFF); throw; } catch (SocketError *e) { - applier_disconnect(applier, e, APPLIER_DISCONNECTED); + applier_log_error(applier, e); + applier_disconnect(applier, APPLIER_DISCONNECTED); /* fall through */ } /* Put fiber_sleep() out of catch block. diff --git a/src/box/box.cc b/src/box/box.cc index c31b1230c1050121dea51cce85fb6bb0bebb5b90..6be2207c39c3d72d760603af269941c09c185ad9 100644 --- a/src/box/box.cc +++ b/src/box/box.cc @@ -62,6 +62,7 @@ #include "iproto_port.h" #include "xrow.h" #include "xrow_io.h" +#include "authentication.h" static char status[64] = "unknown"; @@ -427,29 +428,6 @@ box_set_listen(void) iproto_set_listen(uri); } -/** - * Check if (host, port) in box.cfg.listen is equal to (host, port) in uri. - * Used to determine that an uri from box.cfg.replication_source is - * actually points to the same address as box.cfg.listen. - */ -static bool -box_cfg_listen_eq(struct uri *what) -{ - const char *listen = cfg_gets("listen"); - if (listen == NULL) - return false; - - struct uri uri; - int rc = uri_parse(&uri, listen); - assert(rc == 0 && uri.service); - (void) rc; - - return (uri.service_len == what->service_len && - uri.host_len == what->host_len && - memcmp(uri.service, what->service, uri.service_len) == 0 && - memcmp(uri.host, what->host, uri.host_len) == 0); -} - extern "C" void box_set_log_level(void) { @@ -784,6 +762,13 @@ box_truncate(uint32_t space_id) } } +static inline void +box_register_server(uint32_t id, const struct tt_uuid *uuid) +{ + boxk(IPROTO_INSERT, BOX_CLUSTER_ID, "%u%s", + (unsigned) id, tt_uuid_str(uuid)); + assert(vclock_has(&recovery->vclock, id)); +} /** * @brief Called when recovery/replication wants to add a new server @@ -796,6 +781,9 @@ static void box_on_cluster_join(const tt_uuid *server_uuid) { box_check_writable(); + struct server *server = server_by_uuid(server_uuid); + if (server != NULL) + return; /* nothing to do - already registered */ /** Find the largest existing server id. */ struct space *space = space_cache_find(BOX_CLUSTER_ID); @@ -810,8 +798,7 @@ box_on_cluster_join(const tt_uuid *server_uuid) break; server_id++; } - boxk(IPROTO_INSERT, BOX_CLUSTER_ID, "%u%s", - (unsigned) server_id, tt_uuid_str(server_uuid)); + box_register_server(server_id, server_uuid); } static inline struct func * @@ -968,6 +955,22 @@ box_process_eval(struct request *request, struct obuf *out) } } +void +box_process_auth(struct request *request, struct obuf *out) +{ + assert(request->type == IPROTO_AUTH); + + /* Check that bootstrap has been finished */ + if (wal == NULL) + tnt_raise(ClientError, ER_LOADING); + + const char *user = request->key; + uint32_t len = mp_decode_strl(&user); + authenticate(user, len, request->tuple, request->tuple_end); + assert(request->header != NULL); + iproto_reply_ok(out, request->header->sync); +} + void box_process_join(struct ev_io *io, struct xrow_header *header) { @@ -1012,6 +1015,18 @@ box_process_join(struct ev_io *io, struct xrow_header *header) assert(header->type == IPROTO_JOIN); + /* Decode JOIN request */ + struct tt_uuid server_uuid = uuid_nil; + xrow_decode_join(header, &server_uuid); + + /* Check that bootstrap has been finished */ + if (wal == NULL) + tnt_raise(ClientError, ER_LOADING); + + /* Forbid connection to itself */ + if (tt_uuid_is_equal(&server_uuid, &SERVER_UUID)) + tnt_raise(ClientError, ER_CONNECTION_TO_SELF); + /* Check permissions */ access_check_universe(PRIV_R); access_check_space(space_cache_find(BOX_CLUSTER_ID), PRIV_W); @@ -1019,10 +1034,6 @@ box_process_join(struct ev_io *io, struct xrow_header *header) /* Check that we actually can register a new replica */ box_check_writable(); - /* Decode JOIN request */ - struct tt_uuid server_uuid = uuid_nil; - xrow_decode_join(header, &server_uuid); - /* Remember start vclock. */ struct vclock start_vclock; recovery_last_checkpoint(&start_vclock); @@ -1073,8 +1084,11 @@ box_process_join(struct ev_io *io, struct xrow_header *header) void box_process_subscribe(struct ev_io *io, struct xrow_header *header) { - /* Check permissions */ - access_check_universe(PRIV_R); + assert(header->type == IPROTO_SUBSCRIBE); + + /* Check that bootstrap has been finished */ + if (wal == NULL) + tnt_raise(ClientError, ER_LOADING); struct tt_uuid cluster_uuid = uuid_nil, replica_uuid = uuid_nil; struct vclock replica_clock; @@ -1082,6 +1096,17 @@ box_process_subscribe(struct ev_io *io, struct xrow_header *header) xrow_decode_subscribe(header, &cluster_uuid, &replica_uuid, &replica_clock); + /* Check that bootstrap has been finished */ + if (wal == NULL) + tnt_raise(ClientError, ER_LOADING); + + /* Forbid connection to itself */ + if (tt_uuid_is_equal(&replica_uuid, &SERVER_UUID)) + tnt_raise(ClientError, ER_CONNECTION_TO_SELF); + + /* Check permissions */ + access_check_universe(PRIV_R); + /** * Check that the given UUID matches the UUID of the * cluster this server belongs to. Used to handshake @@ -1132,29 +1157,6 @@ box_process_subscribe(struct ev_io *io, struct xrow_header *header) relay_subscribe(io->fd, header->sync, server, &replica_clock); } -/** Replace the current server id in _cluster */ -static void -box_set_server_uuid() -{ - struct recovery *r = recovery; - - assert(r->server_id == 0); - - /* Unregister local server if it was registered by bootstrap.bin */ - boxk(IPROTO_DELETE, BOX_CLUSTER_ID, "%u", 1); - - /* Register local server */ - tt_uuid_create(&SERVER_UUID); - boxk(IPROTO_INSERT, BOX_CLUSTER_ID, "%u%s", 1, - tt_uuid_str(&SERVER_UUID)); - assert(r->server_id == 1); - - /* Ugly hack: bootstrap always starts from scratch */ - vclock_create(&r->vclock); - vclock_add_server(&r->vclock, 1); - assert(vclock_sum(&r->vclock) == 0); -} - /** Insert a new cluster into _schema */ static void box_set_cluster_uuid() @@ -1227,11 +1229,33 @@ bootstrap_cluster(void) xstream_create(&bootstrap_stream, apply_row); recovery_bootstrap(recovery, &bootstrap_stream); + uint32_t server_id = 1; + + /* Unregister local server if it was registered by bootstrap.bin */ + assert(recovery->server_id == 0); + boxk(IPROTO_DELETE, BOX_CLUSTER_ID, "%u", 1); + + /* Register local server */ + box_register_server(server_id, &SERVER_UUID); + assert(recovery->server_id == 1); + + /* Register other cluster members */ + server_foreach(server) { + if (tt_uuid_is_equal(&server->uuid, &SERVER_UUID)) + continue; + assert(server->applier != NULL); + box_register_server(++server_id, &server->uuid); + assert(server->id == server_id); + } + /* Generate UUID of a new cluster */ box_set_cluster_uuid(); - /* Generate Server-UUID */ - box_set_server_uuid(); + /* Ugly hack: bootstrap always starts from scratch */ + vclock_create(&recovery->vclock); + server_foreach(server) + vclock_add_server(&recovery->vclock, server->id); + assert(vclock_sum(&recovery->vclock) == 0); } /** @@ -1246,13 +1270,15 @@ bootstrap_from_master(struct server *master) assert(applier != NULL); assert(applier->state == APPLIER_CONNECTED); + say_info("bootstraping replica from %s", + sio_strfaddr(&applier->addr, applier->addr_len)); + /* * Send JOIN request to master * See box_process_join(). */ - /* Generate Server-UUID */ - tt_uuid_create(&SERVER_UUID); + assert(!tt_uuid_is_nil(&SERVER_UUID)); applier_resume_to_state(applier, APPLIER_INITIAL_JOIN, TIMEOUT_INFINITY); /* @@ -1290,7 +1316,7 @@ bootstrap(void) struct server *master = server_first(); assert(master == NULL || master->applier != NULL); - if (master != NULL && !box_cfg_listen_eq(&master->applier->uri)) { + if (master != NULL && !tt_uuid_is_equal(&master->uuid, &SERVER_UUID)) { bootstrap_from_master(master); } else { bootstrap_cluster(); @@ -1330,15 +1356,11 @@ box_init(void) title("loading"); box_set_too_long_threshold(); - - /* - * Initialize the cluster registry using replication_source, - * but don't start replication right now. - */ + struct wal_stream wal_stream; + wal_stream_create(&wal_stream, cfg_geti64("rows_per_wal")); xstream_create(&initial_join_stream, apply_initial_join_row); xstream_create(&final_join_stream, apply_row); xstream_create(&subscribe_stream, apply_subscribe_row); - box_sync_replication_source(); struct vclock checkpoint_vclock; int64_t lsn = recovery_last_checkpoint(&checkpoint_vclock); @@ -1353,27 +1375,36 @@ box_init(void) /* Replace server vclock using the data from snapshot */ vclock_copy(&recovery->vclock, &checkpoint_vclock); engine_begin_wal_recovery(); + title("orphan"); + recovery_follow_local(recovery, &wal_stream.base, "hot_standby", + cfg_getd("wal_dir_rescan_delay")); + title("hot_standby"); + + /* Start network */ + assert(!tt_uuid_is_nil(&SERVER_UUID)); + port_init(); + iproto_init(); + box_set_listen(); + box_sync_replication_source(); } else { /* TODO: don't create recovery for this case */ vclock_create(&checkpoint_vclock); recovery = recovery_new(cfg_gets("wal_dir"), cfg_geti("panic_on_wal_error"), &checkpoint_vclock); + + /* Start network */ + tt_uuid_create(&SERVER_UUID); + port_init(); + iproto_init(); + box_set_listen(); + box_sync_replication_source(); + + /* Bootstrap cluster */ bootstrap(); } fiber_gc(); - title("orphan"); - struct wal_stream wal_stream; - wal_stream_create(&wal_stream, cfg_geti64("rows_per_wal")); - recovery_follow_local(recovery, &wal_stream.base, "hot_standby", - cfg_getd("wal_dir_rescan_delay")); - title("hot_standby"); - - port_init(); - iproto_init(); - box_set_listen(); - int64_t rows_per_wal = box_check_rows_per_wal(cfg_geti64("rows_per_wal")); enum wal_mode wal_mode = box_check_wal_mode(cfg_gets("wal_mode")); recovery_finalize(recovery, &wal_stream.base, wal_mode, rows_per_wal); diff --git a/src/box/box.h b/src/box/box.h index 556caa2448deaa133a79c23cb55186cdd2657e10..16058be41e9ece797fd021864a943a6c18c6147b 100644 --- a/src/box/box.h +++ b/src/box/box.h @@ -83,7 +83,7 @@ int box_snapshot(void); const char *box_status(void); void -box_process_auth(struct request *request); +box_process_auth(struct request *request, struct obuf *out); void box_process_call(struct request *request, struct obuf *out); diff --git a/src/box/errcode.h b/src/box/errcode.h index aced9fab40d08e1b6d7887c71fc7b2fb7b409a3d..f5499f035b046aff1fda252bc469a6e705e91662 100644 --- a/src/box/errcode.h +++ b/src/box/errcode.h @@ -169,6 +169,8 @@ struct errcode_record { /*113 */_(ER_VIEW_IS_RO, 2, "View '%s' is read-only") \ /*114 */_(ER_SERVER_UUID_MISMATCH, 2, "Remote UUID mismatch: expected %s, got %s") \ /*115 */_(ER_SYSTEM, 2, "%s") \ + /*116 */_(ER_LOADING, 2, "Server bootstrap hasn't finished yet") \ + /*117 */_(ER_CONNECTION_TO_SELF, 2, "Connection to self") \ /* * !IMPORTANT! Please follow instructions at start of the file diff --git a/src/box/iproto.cc b/src/box/iproto.cc index 3d051dcd7a7c7fc18622614d2c910edf10a8086e..0b9451e233b6658257144b37f765b718533cc540 100644 --- a/src/box/iproto.cc +++ b/src/box/iproto.cc @@ -57,7 +57,6 @@ #include "schema.h" /* sc_version */ #include "cluster.h" /* server_uuid */ #include "iproto_constants.h" -#include "authentication.h" #include "rmean.h" /* The number of iproto messages in flight */ @@ -816,15 +815,10 @@ tx_process_misc(struct cmsg *m) box_process_eval(&msg->request, out); break; case IPROTO_AUTH: - { - assert(msg->request.type == msg->header.type); - const char *user = msg->request.key; - uint32_t len = mp_decode_strl(&user); - authenticate(user, len, msg->request.tuple, - msg->request.tuple_end); - iproto_reply_ok(out, msg->header.sync); - break; - } + assert(msg->request.type == msg->header.type); + rmean_collect(rmean_box, msg->request.type, 1); + box_process_auth(&msg->request, out); + break; case IPROTO_PING: iproto_reply_ok(out, msg->header.sync); break; diff --git a/test/box/misc.result b/test/box/misc.result index 456a4cadea60a3108036dc210b5136e66ceddf08..157c8623e7a0c31a4d702c987a7d27835bcd88b4 100644 --- a/test/box/misc.result +++ b/test/box/misc.result @@ -293,11 +293,13 @@ t; - 'box.error.KEY_PART_TYPE : 18' - 'box.error.CREATE_FUNCTION : 50' - 'box.error.SOPHIA : 60' + - 'box.error.injection : table: <address> + - 'box.error.CONNECTION_TO_SELF : 117' - 'box.error.NO_SUCH_INDEX : 35' - 'box.error.UNKNOWN_RTREE_INDEX_DISTANCE_TYPE : 103' - 'box.error.TUPLE_IS_TOO_LONG : 27' - 'box.error.VIEW_IS_RO : 113' - - 'box.error.injection : table: <address> + - 'box.error.WRONG_SCHEMA_VERSION : 109' - 'box.error.ROLE_GRANTED : 90' - 'box.error.UNKNOWN_SERVER : 62' - 'box.error.FUNCTION_EXISTS : 52' @@ -308,7 +310,7 @@ t; - 'box.error.WRONG_INDEX_PARTS : 107' - 'box.error.ROLE_LOOP : 87' - 'box.error.TUPLE_NOT_FOUND : 4' - - 'box.error.WRONG_SCHEMA_VERSION : 109' + - 'box.error.LOADING : 116' - 'box.error.NO_SUCH_ROLE : 82' - 'box.error.SLAB_ALLOC_MAX : 110' - 'box.error.WRONG_INDEX_RECORD : 106' diff --git a/test/replication/autobootstrap.lua b/test/replication/autobootstrap.lua new file mode 100644 index 0000000000000000000000000000000000000000..a72a619deab04b2def1d33e093e5a59a23d747a9 --- /dev/null +++ b/test/replication/autobootstrap.lua @@ -0,0 +1,31 @@ +#!/usr/bin/env tarantool + +-- get instance name from filename (autobootstrap1.lua => autobootstrap1) +local INSTANCE_ID = string.match(arg[0], "%d") +local USER = 'cluster' +local PASSWORD = 'somepassword' +local SOCKET_DIR = require('fio').cwd() +local function instance_uri(instance_id) + --return 'localhost:'..(3310 + instance_id) + return SOCKET_DIR..'/autobootstrap'..instance_id..'.sock'; +end + +-- start console first +require('console').listen(os.getenv('ADMIN')) + +box.cfg({ + listen = instance_uri(INSTANCE_ID); + log_level = 7; + replication_source = { + USER..':'..PASSWORD..'@'..instance_uri(1); + USER..':'..PASSWORD..'@'..instance_uri(2); + USER..':'..PASSWORD..'@'..instance_uri(3); + }; +}) + +box.once("bootstrap", function() + box.schema.user.create(USER, { password = PASSWORD }) + box.schema.user.grant(USER, 'replication') + box.schema.space.create('test') + box.space.test:create_index('primary') +end) diff --git a/test/replication/autobootstrap.result b/test/replication/autobootstrap.result new file mode 100644 index 0000000000000000000000000000000000000000..5ec855b3bbffcfcec28fa46c2f0fb190fe9fe6eb --- /dev/null +++ b/test/replication/autobootstrap.result @@ -0,0 +1,131 @@ +env = require('test_run') +--- +... +test_run = env.new() +--- +... +SERVERS = { 'autobootstrap1', 'autobootstrap2', 'autobootstrap3' } +--- +... +-- +-- Start servers +-- +test_run:create_cluster(SERVERS) +--- +... +-- +-- Wait for full mesh +-- +test_run:wait_fullmesh(SERVERS) +--- +... +-- +-- Print vclock +-- +_ = test_run:cmd("switch autobootstrap1") +--- +... +box.info.vclock +--- +- {1: 7, 2: 0, 3: 0} +... +_ = test_run:cmd("switch autobootstrap2") +--- +... +box.info.vclock +--- +- {1: 7, 2: 0, 3: 0} +... +_ = test_run:cmd("switch autobootstrap3") +--- +... +box.info.vclock +--- +- {1: 7, 2: 0, 3: 0} +... +_ = test_run:cmd("switch default") +--- +... +vclock = test_run:get_vclock('autobootstrap1') +--- +... +vclock +--- +- {2: 0, 3: 0, 1: 7} +... +-- +-- Insert rows on each server +-- +_ = test_run:cmd("switch autobootstrap1") +--- +... +_ = box.space.test:insert({box.info.server.id}) +--- +... +_ = test_run:cmd("switch autobootstrap2") +--- +... +_ = box.space.test:insert({box.info.server.id}) +--- +... +_ = test_run:cmd("switch autobootstrap3") +--- +... +_ = box.space.test:insert({box.info.server.id}) +--- +... +_ = test_run:cmd("switch default") +--- +... +-- +-- Synchronize +-- +for i, v in ipairs(vclock) do vclock[i] = v + 1 end +--- +... +vclock +--- +- {2: 1, 3: 1, 1: 8} +... +for _, name in pairs(SERVERS) do test_run:wait_vclock(name, vclock) end +--- +... +-- +-- Check result +-- +_ = test_run:cmd("switch autobootstrap1") +--- +... +box.space.test:select() +--- +- - [1] + - [2] + - [3] +... +_ = test_run:cmd("switch autobootstrap2") +--- +... +box.space.test:select() +--- +- - [1] + - [2] + - [3] +... +_ = test_run:cmd("switch autobootstrap3") +--- +... +box.space.test:select() +--- +- - [1] + - [2] + - [3] +... +_ = test_run:cmd("switch default") +--- +... +-- +-- Stop servers +-- +test_run:drop_cluster(SERVERS) +--- +... diff --git a/test/replication/autobootstrap.test.lua b/test/replication/autobootstrap.test.lua new file mode 100644 index 0000000000000000000000000000000000000000..616a7f64f987eeab76e72306c3eab3d61d9bdbfd --- /dev/null +++ b/test/replication/autobootstrap.test.lua @@ -0,0 +1,63 @@ +env = require('test_run') +test_run = env.new() + +SERVERS = { 'autobootstrap1', 'autobootstrap2', 'autobootstrap3' } + +-- +-- Start servers +-- +test_run:create_cluster(SERVERS) + +-- +-- Wait for full mesh +-- +test_run:wait_fullmesh(SERVERS) + +-- +-- Print vclock +-- +_ = test_run:cmd("switch autobootstrap1") +box.info.vclock +_ = test_run:cmd("switch autobootstrap2") +box.info.vclock +_ = test_run:cmd("switch autobootstrap3") +box.info.vclock +_ = test_run:cmd("switch default") + +vclock = test_run:get_vclock('autobootstrap1') +vclock + +-- +-- Insert rows on each server +-- +_ = test_run:cmd("switch autobootstrap1") +_ = box.space.test:insert({box.info.server.id}) +_ = test_run:cmd("switch autobootstrap2") +_ = box.space.test:insert({box.info.server.id}) +_ = test_run:cmd("switch autobootstrap3") +_ = box.space.test:insert({box.info.server.id}) +_ = test_run:cmd("switch default") + +-- +-- Synchronize +-- + +for i, v in ipairs(vclock) do vclock[i] = v + 1 end +vclock +for _, name in pairs(SERVERS) do test_run:wait_vclock(name, vclock) end + +-- +-- Check result +-- +_ = test_run:cmd("switch autobootstrap1") +box.space.test:select() +_ = test_run:cmd("switch autobootstrap2") +box.space.test:select() +_ = test_run:cmd("switch autobootstrap3") +box.space.test:select() +_ = test_run:cmd("switch default") + +-- +-- Stop servers +-- +test_run:drop_cluster(SERVERS) diff --git a/test/replication/autobootstrap1.lua b/test/replication/autobootstrap1.lua new file mode 120000 index 0000000000000000000000000000000000000000..600cd4c168398e00d57d96f93fbaf1b615247ba4 --- /dev/null +++ b/test/replication/autobootstrap1.lua @@ -0,0 +1 @@ +autobootstrap.lua \ No newline at end of file diff --git a/test/replication/autobootstrap2.lua b/test/replication/autobootstrap2.lua new file mode 120000 index 0000000000000000000000000000000000000000..600cd4c168398e00d57d96f93fbaf1b615247ba4 --- /dev/null +++ b/test/replication/autobootstrap2.lua @@ -0,0 +1 @@ +autobootstrap.lua \ No newline at end of file diff --git a/test/replication/autobootstrap3.lua b/test/replication/autobootstrap3.lua new file mode 120000 index 0000000000000000000000000000000000000000..600cd4c168398e00d57d96f93fbaf1b615247ba4 --- /dev/null +++ b/test/replication/autobootstrap3.lua @@ -0,0 +1 @@ +autobootstrap.lua \ No newline at end of file