diff --git a/changelogs/unreleased/gh-4763-introduce-map-to-sql.md b/changelogs/unreleased/gh-4763-introduce-map-to-sql.md new file mode 100644 index 0000000000000000000000000000000000000000..bc078b4c2869794354b674b425b78d22f806834b --- /dev/null +++ b/changelogs/unreleased/gh-4763-introduce-map-to-sql.md @@ -0,0 +1,4 @@ +## feature/sql + + * Field type MAP is now available in SQL. The syntax has also been implemented + to allow the creation of MAP values (gh-4763). diff --git a/src/box/sql/expr.c b/src/box/sql/expr.c index 6fd0885eeaf94e3549cf2dd999ef2b60979128f0..decb42b973d5e436f9d995abcec5a2dd360638bc 100644 --- a/src/box/sql/expr.c +++ b/src/box/sql/expr.c @@ -3411,6 +3411,51 @@ expr_code_array(struct Parse *parser, struct Expr *expr, int reg) sqlVdbeAddOp3(vdbe, OP_Array, count, reg, values_reg); } +/** Generate opcodes that will create a MAP from an expression. */ +static void +expr_code_map(struct Parse *parser, struct Expr *expr, int reg) +{ + struct Vdbe *vdbe = parser->pVdbe; + struct ExprList *list = expr->x.pList; + if (list == NULL) { + sqlVdbeAddOp3(vdbe, OP_Map, 0, reg, 0); + return; + } + assert(list->nExpr % 2 == 0); + int count = list->nExpr / 2; + for (int i = 0; i < count; ++i) { + struct Expr *expr = list->a[2 * i].pExpr; + enum field_type type = sql_expr_type(expr); + if (expr->op != TK_VARIABLE && type != FIELD_TYPE_INTEGER && + type != FIELD_TYPE_UNSIGNED && type != FIELD_TYPE_STRING && + type != FIELD_TYPE_UUID) { + diag_set(ClientError, ER_SQL_PARSER_GENERIC, "Only " + "integer, string and uuid can be keys in map"); + parser->is_aborted = true; + return; + } + } + int len = 0; + int result_reg = parser->nMem + 1; + for (int i = 0; i < count; ++i) { + struct Expr *key = list->a[2 * i].pExpr; + int j; + for (j = i + 1; j < count; ++j) { + struct Expr *tmp_key = list->a[2 * j].pExpr; + if (sqlExprCompare(key, tmp_key, -1) == 0) + break; + } + if (j < count) + continue; + + ++len; + struct Expr *value = list->a[2 * i + 1].pExpr; + sqlExprCodeFactorable(parser, key, ++parser->nMem); + sqlExprCodeFactorable(parser, value, ++parser->nMem); + } + sqlVdbeAddOp3(vdbe, OP_Map, len, reg, result_reg); +} + /* * Erase column-cache entry number i */ @@ -3866,6 +3911,10 @@ sqlExprCodeTarget(Parse * pParse, Expr * pExpr, int target) expr_code_array(pParse, pExpr, target); break; + case TK_MAP: + expr_code_map(pParse, pExpr, target); + return target; + case TK_LT: case TK_LE: case TK_GT: diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c index fa50b2183a67562138eecbb73db0bba81738aab2..b8a9d39ca476e1614113765943b822a570da4ecc 100644 --- a/src/box/sql/mem.c +++ b/src/box/sql/mem.c @@ -3245,6 +3245,45 @@ mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, return array; } +char * +mem_encode_map(const struct Mem *mems, uint32_t count, uint32_t *size, + struct region *region) +{ + size_t used = region_used(region); + bool is_error = false; + struct mpstream stream; + mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb, + set_encode_error, &is_error); + mpstream_encode_map(&stream, count); + for (uint32_t i = 0; i < count; ++i) { + const struct Mem *key = &mems[2 * i]; + const struct Mem *value = &mems[2 * i + 1]; + if (mem_is_metatype(key) || + (key->type & (MEM_TYPE_UINT | MEM_TYPE_INT | MEM_TYPE_UUID | + MEM_TYPE_STR)) == 0) { + diag_set(ClientError, ER_SQL_TYPE_MISMATCH, + mem_str(key), "integer, string or uuid"); + goto error; + } + mem_to_mpstream(key, &stream); + mem_to_mpstream(value, &stream); + } + mpstream_flush(&stream); + if (is_error) { + diag_set(OutOfMemory, stream.pos - stream.buf, + "mpstream_flush", "stream"); + goto error; + } + *size = region_used(region) - used; + char *map = region_join(region, *size); + if (map != NULL) + return map; + diag_set(OutOfMemory, *size, "region_join", "map"); +error: + region_truncate(region, used); + return NULL; +} + /** * Allocate a sequence of initialized vdbe memory registers * on region. diff --git a/src/box/sql/mem.h b/src/box/sql/mem.h index ca68b8d488874d8188b2e56ac56d5185461b07d4..70da203dd66d6dc348c927e3fe08dbe606aa28ae 100644 --- a/src/box/sql/mem.h +++ b/src/box/sql/mem.h @@ -919,3 +919,19 @@ mem_to_mpstream(const struct Mem *var, struct mpstream *stream); char * mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, struct region *region); + +/** + * Encode array of MEMs as msgpack map on region. Values in even position are + * treated as keys in MAP, values in odd position are treated as values in MAP. + * number of MEMs should be even. + * + * @param mems array of MEMs to encode. + * @param count number of elements in the array. + * @param[out] size Size of encoded msgpack map. + * @param region Region to use. + * @retval NULL on error, diag message is set. + * @retval Pointer to valid msgpack map on success. + */ +char * +mem_encode_map(const struct Mem *mems, uint32_t count, uint32_t *size, + struct region *region); diff --git a/src/box/sql/parse.y b/src/box/sql/parse.y index 971f9e30a7ff4b1e2b10171f40fba0982ec85d37..1788ac55ad136f2b3d3c237f19b94b55709b5606 100644 --- a/src/box/sql/parse.y +++ b/src/box/sql/parse.y @@ -1073,11 +1073,11 @@ expr(A) ::= VARNUM(X). { A.pExpr = expr_new_variable(pParse, &X, NULL); spanSet(&A, &X, &X); } -expr(A) ::= VARIABLE(X) id(Y). { +expr(A) ::= COLON|VARIABLE(X) id(Y). { A.pExpr = expr_new_variable(pParse, &X, &Y); spanSet(&A, &X, &Y); } -expr(A) ::= VARIABLE(X) INTEGER(Y). { +expr(A) ::= COLON|VARIABLE(X) INTEGER(Y). { A.pExpr = expr_new_variable(pParse, &X, &Y); spanSet(&A, &X, &Y); } @@ -1111,6 +1111,39 @@ expr(A) ::= LB(X) exprlist(Y) RB(E). { spanSet(&A, &X, &E); } +expr(A) ::= LCB(X) maplist(Y) RCB(E). { + struct sql *db = pParse->db; + struct Expr *expr = sql_expr_new_anon(db, TK_MAP); + if (expr == NULL) { + sql_expr_list_delete(db, Y); + pParse->is_aborted = true; + return; + } + expr->x.pList = Y; + expr->type = FIELD_TYPE_MAP; + sqlExprSetHeightAndFlags(pParse, expr); + A.pExpr = expr; + spanSet(&A, &X, &E); +} + +maplist(A) ::= nmaplist(A). +maplist(A) ::= . { + A = NULL; +} +nmaplist(A) ::= nmaplist(A) COMMA expr(X) COLON expr(Y). { + A = sql_expr_list_append(pParse->db, A, X.pExpr); + A = sql_expr_list_append(pParse->db, A, Y.pExpr); +} +nmaplist(A) ::= expr(X) COLON expr(Y). { + A = sql_expr_list_append(pParse->db, NULL, X.pExpr); + A = sql_expr_list_append(pParse->db, A, Y.pExpr); +} + +%type maplist {ExprList *} +%destructor maplist {sql_expr_list_delete(pParse->db, $$);} +%type nmaplist {ExprList *} +%destructor nmaplist {sql_expr_list_delete(pParse->db, $$);} + expr(A) ::= TRIM(X) LP trim_operands(Y) RP(E). { A.pExpr = sqlExprFunction(pParse, Y, &X); spanSet(&A, &X, &E); diff --git a/src/box/sql/tokenize.c b/src/box/sql/tokenize.c index 8bc519b9da8f9502ff33e0fa44408b64e2ff1e1f..9e85801a354a26fd4bf990578cc741328edf653e 100644 --- a/src/box/sql/tokenize.c +++ b/src/box/sql/tokenize.c @@ -58,7 +58,9 @@ #define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ #define CC_ID 2 /* unicode characters usable in IDs */ #define CC_DIGIT 3 /* Digits */ -/** SQL variables: '@', '#', ':', and '$'. */ +/** Character ':'. */ +#define CC_COLON 4 +/** SQL variable special characters: '@', '#', and '$'. */ #define CC_VARALPHA 5 #define CC_VARNUM 6 /* '?'. Numeric SQL variables */ #define CC_SPACE 7 /* Space characters */ @@ -85,17 +87,21 @@ #define CC_LINEFEED 28 /* '\n' */ #define CC_LB 29 /* '[' */ #define CC_RB 30 /* ']' */ +/** Character '{'. */ +#define CC_LCB 31 +/** Character '}'. */ +#define CC_RCB 32 static const char sql_ascii_class[] = { /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ /* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 28, 7, 7, 7, 27, 27, /* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, /* 2x */ 7, 15, 9, 5, 5, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, -/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 19, 12, 14, 13, 6, /* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 29, 27, 30, 27, 1, /* 6x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 31, 10, 32, 25, 27, /* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -228,6 +234,12 @@ sql_token(const char *z, int *type, bool *is_reserved) case CC_RB: *type = TK_RB; return 1; + case CC_LCB: + *type = TK_LCB; + return 1; + case CC_RCB: + *type = TK_RCB; + return 1; case CC_SEMI: *type = TK_SEMI; return 1; @@ -371,6 +383,9 @@ sql_token(const char *z, int *type, bool *is_reserved) case CC_VARNUM: *type = TK_VARNUM; return 1; + case CC_COLON: + *type = TK_COLON; + return 1; case CC_VARALPHA: *type = TK_VARIABLE; return 1; diff --git a/src/box/sql/vdbe.c b/src/box/sql/vdbe.c index 51ed1219549557c5eccc3f2cad1f422895a09da2..2f323c252bcb947a3a70a1e3fd9e085eded49fdf 100644 --- a/src/box/sql/vdbe.c +++ b/src/box/sql/vdbe.c @@ -1442,6 +1442,27 @@ case OP_Array: { break; } +/** + * Opcode: Map P1 P2 P3 * * + * Synopsis: r[P2] = map(P3@P1) + * + * Construct an MAP value from P1 registers starting at reg(P3). + */ +case OP_Map: { + pOut = &aMem[pOp->p2]; + + uint32_t size; + struct region *region = &fiber()->gc; + size_t svp = region_used(region); + char *val = mem_encode_map(&aMem[pOp->p3], pOp->p1, &size, region); + if (val == NULL || mem_copy_map(pOut, val, size) != 0) { + region_truncate(region, svp); + goto abort_due_to_error; + } + region_truncate(region, svp); + break; +} + /* Opcode: Eq P1 P2 P3 P4 P5 * Synopsis: IF r[P3]==r[P1] * diff --git a/test/sql-luatest/map_test.lua b/test/sql-luatest/map_test.lua new file mode 100644 index 0000000000000000000000000000000000000000..217d09bd0dd3c1940496329825c96e6598ef47a6 --- /dev/null +++ b/test/sql-luatest/map_test.lua @@ -0,0 +1,352 @@ +local server = require('test.luatest_helpers.server') +local t = require('luatest') +local g = t.group() + +g.before_all(function() + g.server = server:new({alias = 'map'}) + g.server:start() +end) + +g.after_all(function() + g.server:stop() +end) + +--- Make sure syntax for MAP values works as intended. +g.test_map_1_1 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : 123};]] + local res = {{{a = 123}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_2 = function() + g.server:exec(function() + local t = require('luatest') + local dec = require('decimal') + local sql = [[SELECT {'a' : 123.1};]] + local res = {{{a = dec.new('123.1')}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_3 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : 123.1e0};]] + local res = {{{a = 123.1}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_4 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : CAST(123 AS NUMBER)};]] + local res = {{{a = 123}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_5 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : '123'};]] + local res = {{{a = '123'}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_6 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : x'313233'};]] + local res = {{{a = '123'}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_7 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : true};]] + local res = {{{a = true}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_8 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : CAST(123 AS SCALAR)};]] + local res = {{{a = 123}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_9 = function() + g.server:exec(function() + local t = require('luatest') + local uuid = require('uuid') + local sql = [[SELECT {'a' : ]].. + [[CAST('11111111-1111-1111-1111-111111111111' AS UUID)};]] + local res = {a = uuid.fromstr('11111111-1111-1111-1111-111111111111')} + t.assert_equals(box.execute(sql).rows[1][1], res) + end) +end + +g.test_map_1_10 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : [1, 2, 3]};]] + local res = {{{a = {1, 2, 3}}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_1_11 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : {'a': 1, 'b' : 2, 'c' : 3}};]] + local res = {{{a = {a = 1, b = 2, c = 3}}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +-- Make sure MAP() accepts only INTEGER, STRING and UUID as keys. +g.test_map_2_1 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {123 : 123};]] + local res = {{{[123] = 123}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_2_2 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {123.1 : 123};]] + local res = [[Only integer, string and uuid can be keys in map]] + local _, err = box.execute(sql) + t.assert_equals(err.message, res) + end) +end + +g.test_map_2_3 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {123.1e0 : 123};]] + local res = [[Only integer, string and uuid can be keys in map]] + local _, err = box.execute(sql) + t.assert_equals(err.message, res) + end) +end + +g.test_map_2_4 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {CAST(123 AS NUMBER) : 123};]] + local res = [[Only integer, string and uuid can be keys in map]] + local _, err = box.execute(sql) + t.assert_equals(err.message, res) + end) +end + +g.test_map_2_5 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : 123};]] + local res = {{{a = 123}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_2_6 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {x'313233' : 123};]] + local res = [[Only integer, string and uuid can be keys in map]] + local _, err = box.execute(sql) + t.assert_equals(err.message, res) + end) +end + +g.test_map_2_7 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {true : 123};]] + local res = [[Only integer, string and uuid can be keys in map]] + local _, err = box.execute(sql) + t.assert_equals(err.message, res) + end) +end + +g.test_map_2_8 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {CAST(123 AS SCALAR) : 123};]] + local res = [[Only integer, string and uuid can be keys in map]] + local _, err = box.execute(sql) + t.assert_equals(err.message, res) + end) +end + +g.test_map_2_9 = function() + g.server:exec(function() + local t = require('luatest') + local uuid = require('uuid') + local sql = [[SELECT {CAST('11111111-1111-1111-1111-111111111111' ]].. + [[AS UUID) : 123}]] + local res = uuid.fromstr('11111111-1111-1111-1111-111111111111') + local val, _ = next(box.execute(sql).rows[1][1]) + t.assert_equals(val, res); + end) +end + +g.test_map_2_10 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {[1, 2, 3]: 123};]] + local res = [[Only integer, string and uuid can be keys in map]] + local _, err = box.execute(sql) + t.assert_equals(err.message, res) + end) +end + +g.test_map_2_11 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {{'a': 1, 'b' : 2, 'c' : 3}: 123};]] + local res = [[Only integer, string and uuid can be keys in map]] + local _, err = box.execute(sql) + t.assert_equals(err.message, res) + end) +end + +-- Make sure expressions can be used as a key or a value in the MAP constructor. +g.test_map_3_1 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : a} FROM (SELECT 123 AS a);]] + local res = {{{a = 123}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_3_2 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {a : 123} FROM (SELECT 123 AS a);]] + local res = {{{[123] = 123}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_3_3 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {1 + 2 : 123};]] + local res = {{{[3] = 123}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +g.test_map_3_4 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : 1 + 2};]] + local res = {{{a = 3}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +-- Make sure TYPEOF() properly works with the MAP constructor. +g.test_map_4 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT typeof({'a' : 123});]] + local res = {{'map'}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +-- Make sure PRINTF() properly works with the MAP constructor. +g.test_map_5 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT printf({});]] + local res = {{'{}'}} + t.assert_equals(box.execute(sql).rows, res) + end) +end + +-- Make sure that the MAP constructor can create big MAP values. +g.test_map_6 = function() + g.server:exec(function() + local t = require('luatest') + local map = {['c0'] = 0} + local str = "'c0': 0" + for i = 1, 1000 do + map['c'..tostring(i)] = i + str = str .. string.format(", 'c%d': %d", i, i) + end + local sql = [[SELECT {]]..str..[[};]] + t.assert_equals(box.execute(sql).rows[1][1], map) + end) +end + +-- Make sure symbol ':' is properly processed by parser. +g.test_map_7_1 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {:name};]] + local res = [[Syntax error at line 1 near '}']] + local _, err = box.execute(sql, {{[':name'] = 1}}) + t.assert_equals(err.message, res) + end) +end + +g.test_map_7_2 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {:name : 5};]] + local res = {{{[1] = 5}}} + t.assert_equals(box.execute(sql, {{[':name'] = 1}}).rows, res) + end) +end + +g.test_map_7_3 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {5::name};]] + local res = {{{[5] = 1}}} + t.assert_equals(box.execute(sql, {{[':name'] = 1}}).rows, res) + end) +end + +-- Make sure that multiple row with maps can be selected properly. +g.test_map_8 = function() + g.server:exec(function() + local t = require('luatest') + local maps = {{{a1 = 1}}} + local strs = "({'a1': 1})" + for i = 2, 1000 do + maps[i] = {{['a'..tostring(i)] = i}} + strs = strs .. string.format(", ({'a%d': %d})", i, i) + end + local sql = [[SELECT * FROM (VALUES]]..strs..[[);]] + t.assert_equals(box.execute(sql).rows, maps) + end) +end + +-- Make sure that the last of values with the same key is set. +g.test_map_9 = function() + g.server:exec(function() + local t = require('luatest') + local sql = [[SELECT {'a' : 1, 'a' : 2, 'b' : 3, 'a' : 4};]] + local res = {{{b = 3, a = 4}}} + t.assert_equals(box.execute(sql).rows, res) + end) +end