From 6137c197f27d8e9a23fbbc8b69dab898f7f02974 Mon Sep 17 00:00:00 2001
From: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Mon, 29 Apr 2019 19:08:29 +0300
Subject: [PATCH] swim: implement and expose transport-level encryption
SWIM is going to be used in and between datacenters, which means,
that its packets will go through public networks. Therefore raw
SWIM packets are vulnerable to attacks.
An attacker can do any and all of the following things:
1) Extract secret information from member payloads, like
credentials to Tarantool binary ports;
2) Change UUIDs and addresses in the packets and break a
topology;
3) Catch the packets and pretend being a Tarantool instance,
which could lead to undefined behaviour depending on an
application logic.
SWIM packets need a protection layer. This commit introduces it.
SWIM transport level allows to choose an encryption algorithm
with a private key to encrypt each packet with that key.
Besides, each packet is encrypted using a random public key
prepended to the packet.
SWIM now provides a public API to choose an encryption algorithm
and a private key.
Part of #3234
---
src/lib/swim/CMakeLists.txt | 2 +-
src/lib/swim/swim.c | 8 +++
src/lib/swim/swim.h | 18 +++++
src/lib/swim/swim_ev.h | 2 +
src/lib/swim/swim_io.c | 139 +++++++++++++++++++++++++++++++-----
src/lib/swim/swim_io.h | 27 ++++++-
src/lib/swim/swim_proto.h | 5 ++
test/unit/swim.c | 50 ++++++++++++-
test/unit/swim.result | 9 ++-
test/unit/swim_test_utils.c | 16 +++--
test/unit/swim_test_utils.h | 8 +++
11 files changed, 257 insertions(+), 27 deletions(-)
diff --git a/src/lib/swim/CMakeLists.txt b/src/lib/swim/CMakeLists.txt
index 8ba99d8038..11202dce38 100644
--- a/src/lib/swim/CMakeLists.txt
+++ b/src/lib/swim/CMakeLists.txt
@@ -5,7 +5,7 @@ set(lib_swim_ev_sources swim_ev.c)
set_source_files_compile_flags(${lib_swim_sources} ${lib_swim_udp_sources}
${lib_swim_ev_sources})
add_library(swim STATIC ${lib_swim_sources})
-target_link_libraries(swim core misc uuid)
+target_link_libraries(swim core misc uuid crypto)
add_library(swim_udp STATIC ${lib_swim_udp_sources})
target_link_libraries(swim_udp core)
add_library(swim_ev STATIC ${lib_swim_ev_sources})
diff --git a/src/lib/swim/swim.c b/src/lib/swim/swim.c
index 9d67c4cb88..1283520d3b 100644
--- a/src/lib/swim/swim.c
+++ b/src/lib/swim/swim.c
@@ -1821,6 +1821,14 @@ swim_cfg(struct swim *swim, const char *uri, double heartbeat_rate,
return 0;
}
+int
+swim_set_codec(struct swim *swim, enum crypto_algo algo, enum crypto_mode mode,
+ const char *key, int key_size)
+{
+ return swim_scheduler_set_codec(&swim->scheduler, algo, mode,
+ key, key_size);
+}
+
bool
swim_is_configured(const struct swim *swim)
{
diff --git a/src/lib/swim/swim.h b/src/lib/swim/swim.h
index 5f3134cc4c..6db12ba9e4 100644
--- a/src/lib/swim/swim.h
+++ b/src/lib/swim/swim.h
@@ -32,6 +32,7 @@
*/
#include <stdbool.h>
#include <stdint.h>
+#include "crypto/crypto.h"
#include "swim_constants.h"
#if defined(__cplusplus)
@@ -104,6 +105,23 @@ swim_cfg(struct swim *swim, const char *uri, double heartbeat_rate,
int
swim_set_payload(struct swim *swim, const char *payload, int payload_size);
+/**
+ * Set SWIM codec to encrypt/decrypt messages.
+ * @param swim SWIM instance to set codec for.
+ * @param algo Cipher algorithm.
+ * @param mode Algorithm mode.
+ * @param key Private key of the chosen algorithm. It is used to
+ * encrypt/decrypt messages, and should be the same on all
+ * cluster nodes. Note that it can be changed, but it shall
+ * be done on all cluster nodes. Otherwise the nodes will
+ * not understand each other. There is also a public key
+ * usually, but it is generated randomly inside SWIM.
+ * @param key_size Key size in bytes.
+ */
+int
+swim_set_codec(struct swim *swim, enum crypto_algo algo, enum crypto_mode mode,
+ const char *key, int key_size);
+
/**
* Stop listening and broadcasting messages, cleanup all internal
* structures, free memory.
diff --git a/src/lib/swim/swim_ev.h b/src/lib/swim/swim_ev.h
index fe261ff38b..1bd81306f3 100644
--- a/src/lib/swim/swim_ev.h
+++ b/src/lib/swim/swim_ev.h
@@ -66,4 +66,6 @@ swim_ev_timer_stop(struct ev_loop *loop, struct ev_timer *watcher);
#define swim_ev_io_set ev_io_set
+#define swim_ev_set_cb ev_set_cb
+
#endif /* TARANTOOL_SWIM_EV_H_INCLUDED */
diff --git a/src/lib/swim/swim_io.c b/src/lib/swim/swim_io.c
index 39aaaf83c8..c55c276cb7 100644
--- a/src/lib/swim/swim_io.c
+++ b/src/lib/swim/swim_io.c
@@ -269,31 +269,20 @@ swim_scheduler_fd(const struct swim_scheduler *scheduler)
return scheduler->transport.fd;
}
-/**
- * Dispatch a next output event. Build packet meta and send the
- * packet.
- */
-static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events);
-
-/**
- * Dispatch a next input event. Unpack meta, forward a packet or
- * propagate further to protocol logic.
- */
-static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events);
-
void
swim_scheduler_create(struct swim_scheduler *scheduler,
swim_scheduler_on_input_f on_input)
{
- swim_ev_init(&scheduler->output, swim_scheduler_on_output);
scheduler->output.data = (void *) scheduler;
- swim_ev_init(&scheduler->input, swim_scheduler_on_input);
scheduler->input.data = (void *) scheduler;
rlist_create(&scheduler->queue_output);
scheduler->on_input = on_input;
swim_transport_create(&scheduler->transport);
+ scheduler->codec = NULL;
+ int rc = swim_scheduler_set_codec(scheduler, CRYPTO_ALGO_NONE,
+ CRYPTO_MODE_ECB, NULL, 0);
+ assert(rc == 0);
+ (void) rc;
}
int
@@ -338,6 +327,42 @@ swim_scheduler_destroy(struct swim_scheduler *scheduler)
swim_scheduler_stop_input(scheduler);
}
+/**
+ * Encrypt data and prepend it with a fresh crypto algorithm's
+ * initial vector.
+ */
+static inline int
+swim_encrypt(struct crypto_codec *c, const char *in, int in_size,
+ char *out, int out_size)
+{
+ assert(out_size >= crypto_codec_iv_size(c));
+ int iv_size = crypto_codec_gen_iv(c, out, out_size);
+ char *iv = out;
+ out += iv_size;
+ out_size -= iv_size;
+ int rc = crypto_codec_encrypt(c, iv, in, in_size, out, out_size);
+ if (rc < 0)
+ return -1;
+ return rc + iv_size;
+}
+
+/** Decrypt data prepended with an initial vector. */
+static inline int
+swim_decrypt(struct crypto_codec *c, const char *in, int in_size,
+ char *out, int out_size)
+{
+ int iv_size = crypto_codec_iv_size(c);
+ if (in_size < iv_size) {
+ diag_set(SwimError, "too small message, can't extract IV for "\
+ "decryption");
+ return -1;
+ }
+ const char *iv = in;
+ in += iv_size;
+ in_size -= iv_size;
+ return crypto_codec_decrypt(c, iv, in, in_size, out, out_size);
+}
+
/**
* Begin packet transmission. Prepare a next task in the queue to
* send its packet: build a meta header, pop the task from the
@@ -411,8 +436,34 @@ swim_complete_send(struct swim_scheduler *scheduler, struct swim_task *task,
task->complete(task, scheduler, size);
}
+/**
+ * On a new EV_WRITE event send a next packet from the queue
+ * encrypted with the currently chosen algorithm.
+ */
+static void
+swim_on_encrypted_output(struct ev_loop *loop, struct ev_io *io, int events)
+{
+ struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+ const struct sockaddr_in *dst;
+ struct swim_task *task = swim_begin_send(scheduler, loop, io, events,
+ &dst);
+ if (task == NULL)
+ return;
+ char *buf = static_alloc(UDP_PACKET_SIZE);
+ assert(buf != NULL);
+ ssize_t size = swim_encrypt(scheduler->codec, task->packet.buf,
+ task->packet.pos - task->packet.buf,
+ buf, UDP_PACKET_SIZE);
+ if (size > 0)
+ size = swim_do_send(scheduler, buf, size, dst);
+ swim_complete_send(scheduler, task, size);
+}
+
+/**
+ * On a new EV_WRITE event send a next packet without encryption.
+ */
static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events)
+swim_on_plain_output(struct ev_loop *loop, struct ev_io *io, int events)
{
struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
const struct sockaddr_in *dst;
@@ -515,8 +566,35 @@ swim_complete_recv(struct swim_scheduler *scheduler, const char *buf,
diag_log();
}
+/**
+ * On a new EV_READ event receive an encrypted packet from the
+ * network.
+ */
+static void
+swim_on_encrypted_input(struct ev_loop *loop, struct ev_io *io, int events)
+{
+ struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+ /*
+ * Buffer for decrypted data is on stack, not on static
+ * memory, because the SWIM code uses static memory as
+ * well and can accidentally rewrite the packet data.
+ */
+ char buf[UDP_PACKET_SIZE];
+ swim_begin_recv(scheduler, loop, io, events);
+
+ char *ibuf = static_alloc(UDP_PACKET_SIZE);
+ assert(ibuf != NULL);
+ ssize_t size = swim_do_recv(scheduler, ibuf, UDP_PACKET_SIZE);
+ if (size > 0) {
+ size = swim_decrypt(scheduler->codec, ibuf, size,
+ buf, UDP_PACKET_SIZE);
+ }
+ swim_complete_recv(scheduler, buf, size);
+}
+
+/** On a new EV_READ event receive a packet from the network. */
static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
+swim_on_plain_input(struct ev_loop *loop, struct ev_io *io, int events)
{
struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
char buf[UDP_PACKET_SIZE];
@@ -525,6 +603,31 @@ swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
swim_complete_recv(scheduler, buf, size);
}
+int
+swim_scheduler_set_codec(struct swim_scheduler *scheduler,
+ enum crypto_algo algo, enum crypto_mode mode,
+ const char *key, int key_size)
+{
+ if (algo == CRYPTO_ALGO_NONE) {
+ if (scheduler->codec != NULL) {
+ crypto_codec_delete(scheduler->codec);
+ scheduler->codec = NULL;
+ }
+ swim_ev_set_cb(&scheduler->output, swim_on_plain_output);
+ swim_ev_set_cb(&scheduler->input, swim_on_plain_input);
+ return 0;
+ }
+ struct crypto_codec *newc = crypto_codec_new(algo, mode, key, key_size);
+ if (newc == NULL)
+ return -1;
+ if (scheduler->codec != NULL)
+ crypto_codec_delete(scheduler->codec);
+ scheduler->codec = newc;
+ swim_ev_set_cb(&scheduler->output, swim_on_encrypted_output);
+ swim_ev_set_cb(&scheduler->input, swim_on_encrypted_input);
+ return 0;
+}
+
const char *
swim_inaddr_str(const struct sockaddr_in *addr)
{
diff --git a/src/lib/swim/swim_io.h b/src/lib/swim/swim_io.h
index 42d94fc92b..6bf42cb05a 100644
--- a/src/lib/swim/swim_io.h
+++ b/src/lib/swim/swim_io.h
@@ -33,6 +33,7 @@
#include "tt_static.h"
#include "small/rlist.h"
#include "salad/stailq.h"
+#include "crypto/crypto.h"
#include "swim_transport.h"
#include "tarantool_ev.h"
#include "uuid/tt_uuid.h"
@@ -57,13 +58,24 @@ enum {
* configuration.
*/
UDP_PACKET_SIZE = 1472,
+ /**
+ * Data can be encrypted, which usually makes it slightly
+ * bigger in size. Also, to decode data the receiver needs
+ * two keys: private key and public initial vector. Public
+ * initial vector is generated randomly for each packet
+ * and prepends the data. This is why maximal data size is
+ * reduced by one block and IV sizes.
+ */
+ MAX_PACKET_SIZE = UDP_PACKET_SIZE - CRYPTO_MAX_BLOCK_SIZE -
+ CRYPTO_MAX_IV_SIZE,
+
};
/**
* UDP packet. Works as an allocator, allowing to fill its body
* gradually, while preserving prefix for metadata.
*
- * < - - - -UDP_PACKET_SIZE- - - - ->
+ * < - - - -MAX_PACKET_SIZE- - - - ->
* +--------+-----------------------+
* | meta | body | *free* |
* +--------+-----------------------+
@@ -86,7 +98,7 @@ struct swim_packet {
*/
char meta[0];
/** Packet body buffer. */
- char buf[UDP_PACKET_SIZE];
+ char buf[MAX_PACKET_SIZE];
/**
* Pointer to the end of the buffer. Just sugar to do not
* write 'buf + sizeof(buf)' each time.
@@ -147,6 +159,11 @@ typedef void (*swim_scheduler_on_input_f)(struct swim_scheduler *scheduler,
struct swim_scheduler {
/** Transport to send/receive packets. */
struct swim_transport transport;
+ /**
+ * Codec to encode messages before sending, and decode
+ * before lifting up to the SWIM core logic.
+ */
+ struct crypto_codec *codec;
/**
* Function called when a packet is received. It takes
* packet body, while meta is handled by transport level
@@ -180,6 +197,12 @@ int
swim_scheduler_bind(struct swim_scheduler *scheduler,
const struct sockaddr_in *addr);
+/** Set a new codec to encrypt/decrypt messages. */
+int
+swim_scheduler_set_codec(struct swim_scheduler *scheduler,
+ enum crypto_algo algo, enum crypto_mode mode,
+ const char *key, int key_size);
+
/** Stop accepting new packets from the network. */
void
swim_scheduler_stop_input(struct swim_scheduler *scheduler);
diff --git a/src/lib/swim/swim_proto.h b/src/lib/swim/swim_proto.h
index beb9bb1fed..482d79fb19 100644
--- a/src/lib/swim/swim_proto.h
+++ b/src/lib/swim/swim_proto.h
@@ -47,6 +47,11 @@ enum {
* SWIM binary protocol structures and helpers. Below is a picture
* of a SWIM message template:
*
+ * +-----------------Public data, not encrypted------------------+
+ * | |
+ * | Initial vector, size depends on chosen algorithm. |
+ * | Next data is encrypted. |
+ * | |
* +----------Meta section, handled by transport level-----------+
* | { |
* | SWIM_META_TARANTOOL_VERSION: uint, Tarantool version ID,|
diff --git a/test/unit/swim.c b/test/unit/swim.c
index c6ef1eebc0..6467aa35ec 100644
--- a/test/unit/swim.c
+++ b/test/unit/swim.c
@@ -31,6 +31,7 @@
#include "memory.h"
#include "fiber.h"
+#include "random.h"
#include "uuid/tt_uuid.h"
#include "unit.h"
#include "uri/uri.h"
@@ -888,10 +889,54 @@ swim_test_indirect_ping(void)
swim_finish_test();
}
+static void
+swim_test_encryption(void)
+{
+ swim_start_test(3);
+ struct swim_cluster *cluster = swim_cluster_new(2);
+ const char *key = "1234567812345678";
+ swim_cluster_set_codec(cluster, CRYPTO_ALGO_AES128, CRYPTO_MODE_CBC,
+ key, CRYPTO_AES128_KEY_SIZE);
+ swim_cluster_add_link(cluster, 0, 1);
+
+ is(swim_cluster_wait_fullmesh(cluster, 2), 0,
+ "cluster works with encryption");
+ swim_cluster_delete(cluster);
+ /*
+ * Test that the instances can not interact with different
+ * encryption keys.
+ */
+ cluster = swim_cluster_new(2);
+ struct swim *s1 = swim_cluster_member(cluster, 0);
+ int rc = swim_set_codec(s1, CRYPTO_ALGO_AES128, CRYPTO_MODE_CBC,
+ key, CRYPTO_AES128_KEY_SIZE);
+ fail_if(rc != 0);
+ struct swim *s2 = swim_cluster_member(cluster, 1);
+ key = "8765432187654321";
+ rc = swim_set_codec(s2, CRYPTO_ALGO_AES128, CRYPTO_MODE_CBC,
+ key, CRYPTO_AES128_KEY_SIZE);
+ fail_if(rc != 0);
+ swim_cluster_add_link(cluster, 0, 1);
+ swim_run_for(2);
+ ok(! swim_cluster_is_fullmesh(cluster),
+ "different encryption keys - can't interact");
+
+ rc = swim_set_codec(s1, CRYPTO_ALGO_NONE, CRYPTO_MODE_ECB, NULL, 0);
+ fail_if(rc != 0);
+ rc = swim_set_codec(s2, CRYPTO_ALGO_NONE, CRYPTO_MODE_ECB, NULL, 0);
+ fail_if(rc != 0);
+ is(swim_cluster_wait_fullmesh(cluster, 2), 0,
+ "cluster works after encryption has been disabled");
+
+ swim_cluster_delete(cluster);
+
+ swim_finish_test();
+}
+
static int
main_f(va_list ap)
{
- swim_start_test(18);
+ swim_start_test(19);
(void) ap;
swim_test_ev_init();
@@ -915,6 +960,7 @@ main_f(va_list ap)
swim_test_payload_basic();
swim_test_payload_refutation();
swim_test_indirect_ping();
+ swim_test_encryption();
swim_test_transport_free();
swim_test_ev_free();
@@ -927,6 +973,7 @@ main_f(va_list ap)
int
main()
{
+ random_init();
time_t seed = time(NULL);
srand(seed);
memory_init();
@@ -951,6 +998,7 @@ main()
say_logger_free();
fiber_free();
memory_free();
+ random_free();
return test_result;
}
\ No newline at end of file
diff --git a/test/unit/swim.result b/test/unit/swim.result
index 587f66c7a9..4093ecb931 100644
--- a/test/unit/swim.result
+++ b/test/unit/swim.result
@@ -1,5 +1,5 @@
*** main_f ***
-1..18
+1..19
*** swim_test_one_link ***
1..6
ok 1 - no rounds - no fullmesh
@@ -188,4 +188,11 @@ ok 17 - subtests
ok 2 - as well as S2 - they communicated via S3
ok 18 - subtests
*** swim_test_indirect_ping: done ***
+ *** swim_test_encryption ***
+ 1..3
+ ok 1 - cluster works with encryption
+ ok 2 - different encryption keys - can't interact
+ ok 3 - cluster works after encryption has been disabled
+ok 19 - subtests
+ *** swim_test_encryption: done ***
*** main_f: done ***
diff --git a/test/unit/swim_test_utils.c b/test/unit/swim_test_utils.c
index f55388055c..ffd42cbd0b 100644
--- a/test/unit/swim_test_utils.c
+++ b/test/unit/swim_test_utils.c
@@ -227,9 +227,9 @@ swim_cluster_new(int size)
return res;
}
-#define swim_cluster_set_cfg(cluster, ...) ({ \
+#define swim_cluster_set_cfg(cluster, func, ...) ({ \
for (int i = 0; i < cluster->size; ++i) { \
- int rc = swim_cfg(cluster->node[i].swim, __VA_ARGS__); \
+ int rc = func(cluster->node[i].swim, __VA_ARGS__); \
assert(rc == 0); \
(void) rc; \
} \
@@ -238,14 +238,22 @@ swim_cluster_new(int size)
void
swim_cluster_set_ack_timeout(struct swim_cluster *cluster, double ack_timeout)
{
- swim_cluster_set_cfg(cluster, NULL, -1, ack_timeout, -1, NULL);
+ swim_cluster_set_cfg(cluster, swim_cfg, NULL, -1, ack_timeout, -1, NULL);
cluster->ack_timeout = ack_timeout;
}
+void
+swim_cluster_set_codec(struct swim_cluster *cluster, enum crypto_algo algo,
+ enum crypto_mode mode, const char *key, int key_size)
+{
+ swim_cluster_set_cfg(cluster, swim_set_codec, algo, mode,
+ key, key_size);
+}
+
void
swim_cluster_set_gc(struct swim_cluster *cluster, enum swim_gc_mode gc_mode)
{
- swim_cluster_set_cfg(cluster, NULL, -1, -1, gc_mode, NULL);
+ swim_cluster_set_cfg(cluster, swim_cfg, NULL, -1, -1, gc_mode, NULL);
cluster->gc_mode = gc_mode;
}
diff --git a/test/unit/swim_test_utils.h b/test/unit/swim_test_utils.h
index b786dfd790..0e6f29d80f 100644
--- a/test/unit/swim_test_utils.h
+++ b/test/unit/swim_test_utils.h
@@ -48,6 +48,14 @@ swim_cluster_new(int size);
void
swim_cluster_set_ack_timeout(struct swim_cluster *cluster, double ack_timeout);
+/**
+ * Set an encryption algorithm and a key for each instance in
+ * @a cluster.
+ */
+void
+swim_cluster_set_codec(struct swim_cluster *cluster, enum crypto_algo algo,
+ enum crypto_mode mode, const char *key, int key_size);
+
/**
* Change number of unacknowledged pings to delete a dead member
* of all the instances in the cluster.
--
GitLab