Next step in new preprocessor for tests

test/box/alter_limits.result

@@ -632,17 +632,17 @@ s.index.primary
 ...
 s.index.pk.type
 ---
-- error: '[string "return s.index.pk.type"]:1: attempt to index field ''pk'' (a nil
+- error: '[string "return s.index.pk.type "]:1: attempt to index field ''pk'' (a nil
     value)'
 ...
 s.index.pk.unique
 ---
-- error: '[string "return s.index.pk.unique"]:1: attempt to index field ''pk'' (a
+- error: '[string "return s.index.pk.unique "]:1: attempt to index field ''pk'' (a
     nil value)'
 ...
 s.index.pk:rename('primary')
 ---
-- error: '[string "return s.index.pk:rename(''primary'')"]:1: attempt to index field
+- error: '[string "return s.index.pk:rename(''primary'') "]:1: attempt to index field
     ''pk'' (a nil value)'
 ...
 s:create_index('second', 'tree', { parts = {  1, 'str' } })

test/box/session.test.lua

@@ -65,7 +65,7 @@ type(box.session.on_disconnect(nil))
 box.session.on_connect(function() box.space['tweedledum']:insert(box.session.id()) end)
 box.session.on_disconnect(function() box.space['tweedledum']:delete(box.session.id()) end)
 
---# create connection con_three to default 
+--# create connection con_three to default
 --# set connection con_three
 space:select(0, box.session.id())[0] == box.session.id()
 --# set connection default

test/lib/preprocessor.py

+import os
+import sys
 import shlex
+import socket
+
 from collections import deque
 
-from lib.tarantool_server import LuaPreprocessorException
+from lib.admin_connection import AdminConnection
+
+class LuaPreprocessorException(Exception):
+    def __init__(self, val):
+        super(LuaPreprocessorException, self).__init__()
+        self.value = val
+    def __str__(self):
+        return "lua preprocessor error: " + repr(self.value)
 
 class State(object):
-    def __init__(self, suite_ini):
+    def __init__(self, suite_ini, curcon, server):
         self.delimiter = ''
         self.suite_ini = suite_ini
+        self.curcon = [curcon]
+        self.tarantool_server = server
 
-    def parse(self, string):
+    def parse_preprocessor(self, string):
         token_store = deque()
         lexer = shlex.shlex(string)
         lexer.commenters = []
@@ -16,14 +29,20 @@ class State(object):
         if not token:
             return
         if token == 'setopt':
-            return OPTIONS(lexer)
+            option = lexer.get_token()
+            if not option:
+                raise LuaPreprocessorException() #TODO
+            value = lexer.get_token()
+            if not value:
+                raise LuaPreprocessorException() #TODO
+            return self.options(option, value)
         token_store.append(token)
         token = lexer.get_token()
         if token == 'server':
             stype = token_store.popleft()
             sname = lexer.get_token()
             if not sname:
-                raise LuaPreprocessorException()
+                raise LuaPreprocessorException() #TODO
             options = {}
             temp = lexer.get_token()
             if not temp:
@@ -37,22 +56,29 @@ class State(object):
                     if v == '=':
                         v = lexer.get_token()
                     options[k] = v
+                    lexer.get_token()
             else:
-                raise LuaPreprocessorException()
+                raise LuaPreprocessorException() #TODO
             return self.server(stype, sname, options)
         elif token == 'connection':
             ctype = token_store.popleft()
-            cname = lexer.get_token()
-            if not cname:
-                raise LuaPreprocessorException()
+            cname = [lexer.get_token()]
+            if not cname[0]:
+                raise LuaPreprocessorException() #TODO
             cargs = None
             temp = lexer.get_token()
-            if not temp:
-                cargs = None
-            elif temp == 'to':
+            if temp == 'to':
                 cargs = lexer.get_token()
-            else:
-                raise LuaPreprocessorException()
+            elif temp == ',':
+                while True:
+                    a = lexer.get_token()
+                    if not a:
+                        break
+                    if a == ',':
+                        continue
+                    cname.append(a)
+            elif temp:
+                raise LuaPreprocessorException() #TODO
             return self.connection(ctype, cname, cargs)
         elif token == 'filter':
             ftype = token_store.popleft()
@@ -62,32 +88,125 @@ class State(object):
             if temp:
                 ref = temp
                 if lexer.get_token() != 'to':
-                    raise LuaPreprocessorException()
+                    raise LuaPreprocessorException() #TODO
                 temp = lexer.get_token()
                 if not temp:
-                    raise LuaPreprocessorException()
+                    raise LuaPreprocessorException() #TODO
                 ret = temp
             return self.filter(ftype, ref, ret)
         else:
-            raise LuaPreprocessorException()
+            raise LuaPreprocessorException() #TODO
 
-    def options(lexer):
-        option = lexer.get_token()
-        if option == 'delimiter':
-            value = lexer.get_token()
-            if not value:
-                raise LuaPreprocessorException()
-            self.delimiter = value
+    def options(self, key, value):
+        if key == 'delimiter':
+            self.delimiter = value[1:-1]
         else:
-            raise LuaPreprocessorException()
+            raise LuaPreprocessorException() #TODO
 
-    def server(ctype, sname, opts):
-        pass
-    def connection(ctype, cname, sname):
-        pass
-    def filter(ctype, ref, ret):
-        pass
+    def server(self, ctype, sname, opts):
+        if ctype == 'create':
+            temp = self.tarantool_server()
+            if 'configuration' in opts:
+                temp.config = opts['configuration'][1:-1]
+            else:
+                temp.cofnfig = self.suite_ini['config']
+            if 'need_init' in opts:
+                temp.need_init = True if opts['need_init'] == 'True' else False
+            if 'init' in opts:
+                temp.init_lua = params['init'][1:-1]
+            temp.vardir = os.path.join(self.suite_ini['vardir'], sname)
+            temp.binary = temp.find_exe(self.suite_ini['builddir'])
+            self.suite_ini['servers'][sname] = temp
+            temp.configure(temp.config)
+            temp.install(temp.binary,
+                    temp.vardir, temp.mem, True)
+            if temp.need_init:
+                temp.init()
+        elif ctype == 'start':
+            if not (sname in self.suite_ini['servers']):
+                raise LuaPreprocessprException() #TODO
+            self.suite_ini['servers'][sname].start(silent=True)
+            self.suite_ini['connections'][sname] = [self.suite_ini['servers'][sname].admin, sname]
+            try:
+                self.suite_ini['connections'][sname][0]('print()', silent=True)
+            except socket.error as e:
+                LuaPreprocessorException() #TODO
+        elif ctype == 'stop':
+            self.suite_ini['servers'][sname].stop()
+            for cname in [k for k, v in self.suite_ini['connections'].iteritems() if v[1] == 'sname']:
+                self.suite_ini['connections'][cname].disconnect()
+                self.suite_ini['connections'].pop(cname)
+        elif ctype == 'deploy':
+            pass
+        elif ctype == 'reconfigure':
+            if not (sname in self.suite_ini['servers']):
+                raise LuaPreprocessorException() #TODO
+            temp = self.suite_ini['servers'][sname]
+            if 'configuration' in opts:
+                temp.reconfigure(opts['configuration'][1:-1], silent = True)
+            else:
+                temp.cofnfig = self.suite_ini['config']
+                if temp.init_lua != None:
+                    var_init_lua = os.path.join(temp.vardir, temp.default_init_lua_name)
+                    if os.path.exists(var_init_lua):
+                        os.path.remove(var_init_lua)
+                if 'init' in params:
+                    temp.init_lua = params['init'][1:-1]
+                    var_init_lua = os.path.join(temp.vardir, temp.default_init_lua_name)
+                    shutil.copy(temp.init_lua, var_init_lua)
+                    temp.restart()
+        elif ctype == 'cleanup':
+            if sname not in self.suite_ini['servers']:
+                raise LuaPreprocessorException() #TODO
+            self.suite_ini['servers'][sname].cleanup()
+        else:
+            raise LuaPreprocessorException() #TODO
+
+    def connection(self, ctype, cname, sname):
+        if ctype == 'create':
+            if sname not in self.suite_ini['servers']:
+                raise LuaPreprocessorException() #TODO
+            if cname[0] in self.suite_ini['connections']:
+                raise LuaPreprocessorException() #TODO
+            self.suite_ini['connections'][cname[0]] = [AdminConnection('localhost',
+                    self.suite_ini['servers'][sname].port), sname]
+            self.suite_ini['connections'][cname[0]][0].connect()
+        elif ctype == 'drop':
+            if cname[0] not in self.suite_ini['connections']:
+                raise LuaPreprocessorException() #TODO
+            self.suite_ini['connections'][cname[0]][0].disconnect()
+            self.suite_ini['connections'].pop(cname[0])
+        elif ctype == 'set':
+            for i in cname:
+                if not i in self.suite_ini['connections']:
+                    raise LuaPreprocessorException() #TODO
+            self.curcon = [self.suite_ini['connections'][i][0] for i in cname]
+        else:
+            raise LuaPreprocessorException() #TOD#O
+
+    def filter(self, ctype, ref, ret):
+        if ctype == 'push':
+            sys.stdout.push_filter(ref[1:], ret[:-1])
+        elif ctype == 'pop':
+            sys.stdout.pop_filter()
+        elif ctype == 'clear':
+            sys.stdout.clear_all_filters()
+        else:
+            raise LuaPreprocessorException("Wrong command for filters: " + repr(ctype))
 
     def __call__(self, string):
         string = string[3:].strip()
-        self.parse(string)
+        self.parse_preprocessor(string)
+
+    def flush(self):
+        sys.stdout.clear_all_filters()
+        a = self.suite_ini['servers']['default']
+        self.suite_ini['servers'].pop('default')
+        for k, v in self.suite_ini['servers'].iteritems():
+            v.stop(silent=True)
+            v.cleanup()
+            for cname in [name for name, tup in self.suite_ini['connections'].iteritems() if tup[1] == 'sname']:
+                self.suite_ini['connections'][cname].disconnect()
+                self.suite_ini['connections'].pop(cname)
+        self.suite_ini['servers']['default'] = a
+

test/lib/tarantool_server.py

@@ -21,6 +21,8 @@ from lib.test_suite import FilteredStream, Test, check_libs
 from lib.admin_connection import AdminConnection
 from lib.memcached_connection import MemcachedConnection
 
+from lib.preprocessor import State
+
 try:
     import cStringIO as StringIO
 except ImportError:
@@ -62,190 +64,50 @@ class FuncTest(Test):
     def execute(self, server):
         execfile(self.name, dict(locals(), **server.__dict__))
 
-class LuaPreprocessorException(Exception):
-    def __init__(self, val):
-        super(LuaPreprocessorException, self).__init__()
-        self.value = val
-    def __str__(self):
-        return "lua preprocessor error: " + repr(self.value)
 
 class LuaTest(FuncTest):
     def execute(self, server):
-        delimiter = ''
+        ts = State(self.suite_ini, server.admin, TarantoolServer)
         cmd = None
-        curcon = server.admin
+
+        def send_command(command):
+            result = ts.curcon[0](command, silent=True)
+            for conn in ts.curcon[1:]:
+                conn(command, silent=True)
+            return result
+
         for line in open(self.name, 'r'):
             if not cmd:
                 cmd = StringIO.StringIO()
             if line.find('--#') == 0:
-                line = line[3:].strip()
-                matched1 = re.match(r"setopt\s+(.*)", line)
-                matched2 = re.match(r"(.*)\s+server\s+(.*)", line)
-                matched3 = re.match(r"(.*)\s+connection\s+(.*)", line)
-                matched4 = re.match(r"(.*)\s+filter[s]?(\s+.*)?", line)
-                if matched1:
-                    command = re.split(r'\s*=\s*|\s+|\s*,\s*', matched1.group(1))
-                    command = { command[i] : command[i + 1] for i in xrange(0, len(command) - 1, 2) }
-                    if 'delimiter' in command:
-                        delimiter = command['delimiter'].strip()[1:-1]
-                        command.pop('delimiter')
-                    if command:
-                        raise LuaPreprocessorException("Wrong setopt options - " + str(command))
-                elif matched2:
-                    try:
-                        if matched2.group(1) == 'create':
-                            name, params = re.match(r"(.*)\s+with\s+(.*)", matched2.group(2)).groups()
-                            params = re.split(r'\s*=\s*|\s+|\s*,\s*', params)
-                            params = { params[i] : params[i + 1] for i in xrange(0, len(params) - 1, 2) }
-                            temp_server = TarantoolServer()
-                            if 'configuration' in params:
-                                temp_server.config = params['configuration'][1:-1]
-                            else:
-                                temp_server.config = self.suite_ini['config']
-                            if 'need_init' in params:
-                                temp_server.need_init = False
-                            if 'init' in params:
-                                temp_server.init_lua = params['init'][1:-1]
-                                        
-                            temp_server.vardir = os.path.join(self.suite_ini['vardir'], name)
-                            temp_server.binary = temp_server.find_exe(self.suite_ini['builddir'])
-                            self.suite_ini['servers'][name] = temp_server
-                            temp_server.configure(temp_server.config)
-                            temp_server.install(temp_server.binary,
-                                    temp_server.vardir, temp_server.mem, True)
-                            if temp_server.need_init:
-                                temp_server.init()
-                        elif matched2.group(1) == 'start':
-                            name = matched2.group(2)
-                            if name in self.suite_ini['servers']:
-                                self.suite_ini['servers'][name].start(silent=True)
-                                self.suite_ini['connections'][name] = self.suite_ini['servers'][name].admin
-                                try:
-                                    self.suite_ini['connections'][name]('print( 1 )', silent=True)
-                                except socket.error as e:
-                                    raise LuaPreprocessorException("Can't connect to server with errno " + str(e.errno))
-                            else:
-                                raise LuaPreprocessorException("Wrong server name: " + name)
-                        elif matched2.group(1) == 'stop':
-                            name = matched2.group(2)
-                            if name in self.suite_ini['servers']:
-                                self.suite_ini['servers'][name].stop()
-                                self.suite_ini['connections'].pop(name)
-                            else:
-                                raise LuaPreprocessorException("Wrong server name: " + name)
-                        elif matched2.group(1) == 'deploy':
-                            name = matched2.group(2)
-                            if name in self.suite_ini['servers']:
-                                self.suite_ini['servers'][name].deploy(silent=True)
-                            else:
-                                raise LuaPreprocessorException("Wrong server name: " + name)
-                        elif matched2.group(1) == 'reconfigure':
-                            name, params = re.match(r"(.*)\s+with\s+(.*)", matched2.group(2)).groups()
-                            if name not in self.suite_ini['servers']:
-                                raise LuaPreprocessorException("Wrong server name: " + name)
-                            params = re.split(r'\s*=\s*|\s+|\s*,\s*', params)
-                            params = { params[i] : params[i + 1] for i in xrange(0, len(params) - 1, 2) }
-                            tmp_srv = self.suite_ini['servers'][name]
-
-                            if 'configuration' in params:
-                                tmp_srv.reconfigure(params['configuration'][1:-1], silent=True)
-                            else:
-                                tmp_srv.config = self.suite_ini['config']
-
-                            if tmp_srv.init_lua != None:
-                                var_init_lua = os.path.join(tmp_srv.vardir, tmp_srv.default_init_lua_name)
-                                if os.path.exists(var_init_lua):
-                                    os.path.remove(var_init_lua)
-                            if 'init' in params:
-                                tmp_srv.init_lua = params['init'][1:-1]
-                                var_init_lua = os.path.join(tmp_srv.vardir, tmp_srv.default_init_lua_name)
-                                shutil.copy(tmp_srv.init_lua, var_init_lua)
-                                tmp_srv.restart()
-
-                        elif matched2.group(1) == 'cleanup':
-                            name = matched2.group(2)
-                            if name in self.suite_ini['servers']:
-                                self.suite_ini['servers'][name].cleanup()
-                        else:
-                            raise LuaPreprocessorException("Wrong command for server - " + repr(matched2.group(1)))
-                    except (AttributeError, ValueError) as e:
-                        raise LuaPreprocessorException("Wrong command for server - " + repr(e.message))
-                elif matched3:
-                    try:
-                        if matched3.group(1) == 'create':
-                            namecon, name = re.match("(.*)\s+to\s+(.*)", matched3.group(2)).groups()
-                            self.suite_ini['connections'][namecon] = AdminConnection('localhost', self.suite_ini['servers'][name].port)
-                            self.suite_ini['connections'][namecon].connect()
-                        elif matched3.group(1) == 'drop':
-                            name = matched3.group(2)
-                            if name in self.suite_ini['connections']:
-                                self.suite_ini['connections'][name].disconnect()
-                                self.suite_ini['connections'].pop(name)
-                            else:
-                                raise LuaPreprocessorException("Wrong connection name: " + name)
-                        elif matched3.group(1) == 'set':
-                            name = re.split(",\s*", matched3.group(2))
-                            for _name in name:
-                                if not _name in self.suite_ini['connections']:
-                                    raise LuaPreprocessorException("Wrong connection name: " + _name)
-                            if len(name) == 1:
-                                curcon = self.suite_ini['connections'][name[0]]
-                            else:
-                                curcon = [self.suite_ini['connections'][_name] for _name in name]
-                        else:
-                            raise LuaPreprocessorException("Wrong command for connection - " + repr(matched3.group(1)))
-                    except (AttributeError, ValueError) as e:
-                        raise LuaPreprocessorException("Wrong command for connection - " + repr(e.message))
-                elif matched4:
-                    try:
-                        if matched4.group(1) == 'push':
-                            p1, p2 = re.split(r"\"\s*to\s*\"|\'\s*to\s*\'", matched4.group(2).strip())
-                            sys.stdout.push_filter(p1[1:], p2[:-1])
-                        elif matched4.group(1) == 'pop':
-                            sys.stdout.pop_filter()
-                        elif matched4.group(1) == 'clear':
-                            sys.stdout.clear_all_filters()
-                        else:
-                            raise LuaPreprocessorException("Wrong command for filters - " + repr(matched4.group(1)))
-                    except (AttributeError, ValueError) as e:
-                        raise LuaPreprocessorException("Wrong command for filters - " + repr(e.message))
-                else:
-                    raise LuaPreprocessorException("Wrong command - " + repr(line))
-                sys.stdout.write("--# " + line + '\n')
+                rescom = cmd.getvalue().replace('\n\n', '\n')
+                if rescom:
+                    result = send_command(rescom)
+                    sys.stdout.write(cmd.getvalue())
+                    sys.stdout.write(result.replace("\r\n", "\n"))
+                sys.stdout.write(line)
+                ts(line)
             elif line.find('--') == 0:
                 sys.stdout.write(line)
             else:
-                if not delimiter:
-                    if line.strip():
-                        curcon(line.strip())
-                    continue
-                cmd.write(line)
-                if cmd.getvalue().endswith(delimiter + '\n') and cmd.getvalue():
-                    if isinstance(curcon, list):
-                        for con in curcon:
-                            res = con(cmd.getvalue()[:-len(delimiter)].replace('\n\n', '\n'), silent=True)
-                    else:
-                        res = curcon(cmd.getvalue()[:-len(delimiter)].replace('\n\n', '\n'), silent=True)
-                    sys.stdout.write(cmd.getvalue()[:-1].strip() + '\n')
-                    sys.stdout.write(res.replace("\r\n", "\n"))
+                if line.strip() or cmd.getvalue():
+                    cmd.write(line)
+                delim_len = -len(ts.delimiter) if len(ts.delimiter) else None
+                if line.endswith(ts.delimiter+'\n') and cmd.getvalue().strip()[:delim_len].strip():
+                    rescom = cmd.getvalue()[:delim_len].replace('\n\n', '\n')
+                    result = send_command(rescom)
+                    sys.stdout.write(cmd.getvalue())
+                    sys.stdout.write(result.replace("\r\n", "\n"))
                     cmd.close()
                     cmd = None
-        if cmd and cmd.getvalue().strip():
-            if isinstance(curcon, list):
-                for con in curcon:
-                    res = con(cmd.getvalue()[:-len(delimiter)].replace('\n\n', '\n'), silent=True)
-            else:
-                res = curcon(cmd.getvalue()[:-len(delimiter)].replace('\n\n', '\n'), silent=True)
-            sys.stdout.write(cmd.getvalue()[:-1].strip() + '\n')
-            sys.stdout.write(res.replace("\r\n", "\n"))
-            cmd.close
-            cmd = None
-        sys.stdout.clear_all_filters()
+        ts.flush()
+
 
 class PythonTest(FuncTest):
     def execute(self, server):
         execfile(self.name, dict(locals(), **server.__dict__))
 
+
 class TarantoolConfigFile:
     """ConfigParser can't read files without sections, work it around"""
     def __init__(self, fp, section_name):

test/lib/test_suite.py

@@ -286,7 +286,7 @@ class TestSuite:
                       init_lua=self.ini["init_lua"], silent=False)
         if self.ini['core'] != 'unittest':
             self.ini['servers'] = {'default' : self.server}
-            self.ini['connections'] = {'default' : self.server.admin}
+            self.ini['connections'] = {'default' : [self.server.admin, 'default']}
             self.ini['vardir'] = self.args.vardir
             self.ini['builddir'] = self.args.builddir
         for i in self.ini['lua_libs']:

test/replication/consistent.test.lua

@@ -51,7 +51,7 @@ _select(1, 10)
 -- Master LSN:
 _print_lsn()
 
---# set connection replica 
+--# set connection replica
 -- Replica LSN:
 _print_lsn()
 
@@ -72,7 +72,7 @@ _select (11, 15)
 --# set connection default
 -- Master LSN:
 _print_lsn()
---# set connection replica 
+--# set connection replica
 -- Replica LSN:
 _print_lsn()
 
@@ -84,7 +84,7 @@ _select(11, 20)
 --# set connection default
 -- Master LSN:
 _print_lsn()
---# set connection replica 
+--# set connection replica
 -- Replica LSN:
 _print_lsn()
 
@@ -105,7 +105,7 @@ _select(21, 30)
 --# set connection default
 -- Master LSN:
 _print_lsn()
---# set connection replica 
+--# set connection replica
 -- Replica LSN:
 _print_lsn()
 
@@ -118,7 +118,7 @@ _select(21, 30)
 --# set connection default
 -- Master LSN:
 _print_lsn()
---# set connection replica 
+--# set connection replica
 -- Replica LSN:
 _print_lsn()
 
@@ -139,7 +139,7 @@ _select(31, 50)
 --# set connection default
 -- Master LSN:
 _print_lsn()
---# set connection replica 
+--# set connection replica
 -- Replica LSN:
 _print_lsn()
 
@@ -156,7 +156,7 @@ _select(41, 60)
 --# set connection default
 -- Master LSN:
 _print_lsn()
---# set connection replica 
+--# set connection replica
 -- Replica LSN:
 _print_lsn()