Refactoring: 'refactoring-python-preprocessor':

tarantool preprocessor routine was changed:
 - Now 'exec obj ...' command will translate to obj.execute(...);
 - New command 'send' and 'recv' was added. 'send obj ...' and 'recv obj
   ...' will translate to 'obj.send(...)' and 'obj.recv(...)'
   respectively;
 - New command modifiers 'silent' and 'verbose' was added. Verbose
   (silent = False) value is default.

Tests - box, box_big, box_memcached, box_replication was updated.

test/box/snapshot.test

@@ -62,7 +62,8 @@ print """
 
 exec sql "insert into t0 values (1, 'Test tuple')"
 
-info = yaml.load(admin.execute("show info\n"))["info"]
+result = exec admin silent "show info"
+info = yaml.load(result)["info"]
 
 pid = info["pid"]
 snapshot = str(info["lsn"]).zfill(20) + ".snap"

test/box_memcached/cas.result

@@ -24,24 +24,19 @@ barva2
 EXISTS
 # gets foo and verify identifier exists 
 gets foo
-
-VALUE foo 0 6 42
-barval
-END
-
 # cas success 
-cas foo 0 0 6 42
+cas foo 0 0 6 <unique_id>
 barva2
 STORED
 # cas failure (reusing the same key) 
-cas foo 0 0 6 42
+cas foo 0 0 6 <unique_id>
 barva2
 EXISTS
 # delete foo 
 delete foo
 DELETED
 # cas missing 
-cas foo 0 0 6 42
+cas foo 0 0 6 <unique_id>
 barva2
 NOT_FOUND
 # set foo1 
@@ -54,18 +49,14 @@ set foo2 0 0 1
 STORED
 # gets foo1 check 
 gets foo1
-
 VALUE foo1 0 1 44
 1
 END
-
 # gets foo2 check 
 gets foo2
-
 VALUE foo2 0 1 45
 2
 END
-
 # validate foo1 != foo2 
 pass: foo1_cas != foo2_cas
 # gets foo from memcached1 - should success 

test/box_memcached/cas.test

 # encoding: tarantool
+import sys
 from lib.memcached_connection import MemcachedConnection
 
 exec memcached "cas bad blah 0 0 0\r\n"
@@ -19,20 +20,27 @@ print """# cas fail """
 exec memcached "cas foo 0 0 6 123\r\nbarva2\r\n"
 
 print """# gets foo and verify identifier exists """
-result = server.memcached.execute("gets foo\r\n", silent=False)
-identifier = int(result.split()[4])
+sys.stdout.write("gets foo\r\n")
+result = exec memcached silent "gets foo\r\n"
+unique_id = int(result.split()[4])
 
 print """# cas success """
-exec memcached "cas foo 0 0 6 %d\r\nbarva2\r\n" % identifier
+sys.stdout.write("cas foo 0 0 6 <unique_id>\r\nbarva2\r\n")
+result = exec memcached silent "cas foo 0 0 6 %d\r\nbarva2\r\n" % unique_id
+sys.stdout.write(result)
 
 print """# cas failure (reusing the same key) """
-exec memcached "cas foo 0 0 6 %d\r\nbarva2\r\n" % identifier
+sys.stdout.write("cas foo 0 0 6 <unique_id>\r\nbarva2\r\n")
+result = exec memcached silent "cas foo 0 0 6 %d\r\nbarva2\r\n" % unique_id
+sys.stdout.write(result)
 
 print """# delete foo """
 exec memcached "delete foo\r\n"
 
 print """# cas missing """
-exec memcached "cas foo 0 0 6 %d\r\nbarva2\r\n" % identifier
+sys.stdout.write("cas foo 0 0 6 <unique_id>\r\nbarva2\r\n")
+result = exec memcached silent "cas foo 0 0 6 %d\r\nbarva2\r\n" % unique_id
+sys.stdout.write(result)
 
 print """# set foo1 """
 exec memcached "set foo1 0 0 1\r\n1\r\n"
@@ -41,11 +49,11 @@ print """# set foo2 """
 exec memcached "set foo2 0 0 1\r\n2\r\n"
 
 print """# gets foo1 check """
-result = server.memcached.execute("gets foo1\r\n", silent=False)
+result = exec memcached "gets foo1\r\n"
 foo1_cas = int(result.split()[4])
 
 print """# gets foo2 check """
-result = server.memcached.execute("gets foo2\r\n", silent=False)
+result = exec memcached "gets foo2\r\n"
 foo2_cas = int(result.split()[4])
 
 print """# validate foo1 != foo2 """
@@ -58,23 +66,23 @@ memcached1 = server.memcached
 memcached2 = MemcachedConnection('localhost', server.memcached_port)
 
 print """# gets foo from memcached1 - should success """
-result = memcached1.execute("gets foo1\r\n", silent = True)
+result = exec memcached1 silent "gets foo1\r\n"
 mem1_cas = int(result.split()[4])
 
 print """# gets foo from memcached2 - should success """
-result = memcached2.execute("gets foo1\r\n", silent = True)
+result = exec memcached2 silent "gets foo1\r\n"
 mem2_cas = int(result.split()[4])
 
 print """# send 'cas foo1' from memcached1 """
-memcached1.send("cas foo1 0 0 6 %d\r\nbarva2\r\n" % mem1_cas, silent = True)
+send memcached1 silent "cas foo1 0 0 6 %d\r\nbarva2\r\n" % mem1_cas
 print """# send 'cas foo1' from memcached2 """
-memcached2.send("cas foo1 0 0 4 %d\r\npear\r\n" % mem2_cas, silent = True)
+send memcached2 silent "cas foo1 0 0 4 %d\r\npear\r\n" % mem2_cas
 
 print """# recv reply 'cas foo1' from memcached1 """
-result = memcached1.recv(silent = True)
+result = recv memcached1 silent
 mem1_cas_result = result.split()[0]
 print """# recv reply 'cas foo1' from memcached2 """
-result = memcached2.recv(silent = True)
+result = recv memcached2 silent
 mem2_cas_result = result.split()[0]
 
 if mem1_cas_result == "STORED" and mem2_cas_result == "EXISTS":
@@ -90,13 +98,13 @@ print """# set bug15 """
 exec memcached "set bug15 0 0 1\r\n0\r\n"
 
 print """# Check out the first gets. """
-result = server.memcached.execute("gets bug15\r\n", silent=True)
+result = exec memcached silent "gets bug15\r\n"
 bug15_cas = int(result.split()[4])
 
 print """# Increment. """
 exec memcached "incr bug15 1\r\n"
 print """# Validate a changed CAS. """
-result = server.memcached.execute("gets bug15\r\n", silent=True)
+result = exec memcached silent "gets bug15\r\n"
 next_bug15_cas = int(result.split()[4])
 
 print """# validate bug15_cas != next_bug15_cas """

test/box_memcached/expirations.test

@@ -18,7 +18,7 @@ print """# expire: time - 1 second"""
 expire = time.time() - 1
 
 print """# set foo"""
-server.memcached.execute("set foo 0 %d 6\r\nfooval\r\n" % expire, silent=True)
+exec memcached silent "set foo 0 %d 6\r\nfooval\r\n" % expire
 
 print """# foo shoud expired"""
 exec memcached "get foo\r\n"
@@ -28,7 +28,7 @@ print """# expire: time + 1 second"""
 expire = time.time() + 1
 
 print """# set foo"""
-server.memcached.execute("set foo 0 %d 6\r\nfooval\r\n" % expire, silent=True)
+exec memcached silent "set foo 0 %d 6\r\nfooval\r\n" % expire
 
 print """# foo shoud be exists"""
 exec memcached "get foo\r\n"
@@ -42,7 +42,7 @@ print """# expire: time - 20 second"""
 expire = time.time() - 20
 
 print """# set boo"""
-server.memcached.execute("set boo 0 %d 6\r\nbooval\r\n" % expire, silent=True)
+exec memcached silent "set boo 0 %d 6\r\nbooval\r\n" % expire
 
 print """# foo shoud expired"""
 exec memcached "get boo\r\n"

test/box_memcached/flags.result

@@ -2,29 +2,23 @@ set foo 0 0 6
 fooval
 STORED
 gets foo
-
 VALUE foo 0 6 42
 fooval
 END
-
 success: flags (0x0) == ret_flags (0x0)
 set foo 123 0 6
 fooval
 STORED
 gets foo
-
 VALUE foo 123 6 43
 fooval
 END
-
 success: flags (0x7b) == ret_flags (0x7b)
 set foo 65535 0 6
 fooval
 STORED
 gets foo
-
 VALUE foo 65535 6 44
 fooval
 END
-
 success: flags (0xffff) == ret_flags (0xffff)

test/box_memcached/flags.test

@@ -3,7 +3,7 @@ flags_list = [ 0x0, 0x7b, 0xffff ]
 
 for flags in flags_list:
     exec memcached "set foo %d 0 6\r\nfooval\r\n" % flags
-    result = server.memcached.execute("gets foo\r\n", silent=False);
+    result = exec memcached "gets foo\r\n"
     ret_flags = int(result.split()[2])
     if flags == ret_flags:
         print "success: flags (0x%x) == ret_flags (0x%x)" % (flags, ret_flags)

test/box_memcached/flush-all.test

@@ -14,7 +14,7 @@ exec memcached "get foo\r\n"
 print """# and the other form, specifying a flush_all time... """
 expire = time.time() + 2
 print "flush_all time + 2"
-print server.memcached.execute("flush_all %d\r\n" % expire, silent=True)
+print exec memcached silent "flush_all %d\r\n" % expire
 exec memcached "get foo\r\n"
 
 exec memcached "set foo 0 0 3\r\n123\r\n"

test/box_memcached/getset.test

@@ -35,11 +35,11 @@ print """# check-and-set (cas) failure case, try to set value with incorrect cas
 exec memcached "cas moo 0 0 6 0\r\nMOOVAL\r\n"
 exec memcached "get moo\r\n"
 
-result = server.memcached.execute("gets moo\r\n", silent=True)
+result = exec memcached silent "gets moo\r\n"
 unique_id = int(result.split()[4])
 
 print """# now test that we can store moo with the correct unique id"""
-server.memcached.execute("cas moo 0 0 6 %d\r\nMOOVAL\r\n" % unique_id, silent=True)
+exec memcached silent "cas moo 0 0 6 %d\r\nMOOVAL\r\n" % unique_id
 exec memcached "get moo\r\n"
 
 exec memcached "set foo 0 0 6\r\nfooval\r\ndelete foo\r\nset foo 0 0 6\r\nfooval\r\ndelete foo\r\n"
@@ -54,11 +54,11 @@ while len < (1024 * 1028):
         exec memcached "get foo_%d\r\n" % (len)
         print "# set big data: - should fail"
         print "set foo_%d 0 0 %d\r\n<big-data>\r\n" % (len, len)
-        print server.memcached.execute("set foo_%d 0 0 %d\r\n%s\r\n" % (len, len, val), silent=True)
+        print exec memcached silent "set foo_%d 0 0 %d\r\n%s\r\n" % (len, len, val)
     else:
         print "# set big data: - should pass"
         print "set foo_%d 0 0 %d\r\n<big-data>\r\n" % (len, len)
-        print server.memcached.execute("set foo_%d 0 0 %d\r\n%s\r\n" % (len, len, val), silent=True)
+        print exec memcached silent "set foo_%d 0 0 %d\r\n%s\r\n" % (len, len, val)
     len += 1024 * 512
 
 # resore default suite config

test/box_memcached/multiversioning.result

@@ -5,8 +5,9 @@ STORED
 
 # send command 'get big' to firs memcached client 
 get big
-
 # send command 'delete big' to second client 
+delete big
+DELETED
 # Store big in lower case via first memcached client 
 set big 0 0 262144
 <big-value-upper-case>

test/box_memcached/multiversioning.test

@@ -10,21 +10,22 @@ memcached2 = MemcachedConnection('localhost', server.memcached_port)
 
 print """# Store big in lower case via first memcached client """
 print "set big 0 0 %d\r\n<big-value-lower-case>" % buf_size
-print memcached1.execute("set big 0 0 %d\r\n%s\r\n" % (buf_size, buf), silent = True)
+print exec memcached1 silent "set big 0 0 %d\r\n%s\r\n" % (buf_size, buf)
 
 print """# send command 'get big' to firs memcached client """
-memcached1.send("get big\r\n", silent = False)
+send memcached1 "get big\r\n"
 
 print """# send command 'delete big' to second client """
-memcached2.execute("delete big\r\n")
+exec memcached2 "delete big\r\n"
 
 print """# Store big in lower case via first memcached client """
 print "set big 0 0 %d\r\n<big-value-upper-case>" % buf_size
-print memcached2.execute("set big 0 0 %d\r\n%s\r\n" % (buf_size, buf_upper), silent = True)
+print exec memcached2 silent "set big 0 0 %d\r\n%s\r\n" % (buf_size, buf_upper)
 
 print """# recv reply 'get big' from firs memcached client """
-reply = memcached1.recv(silent = True).split('\r\n')[1]
-if buf == reply:
+reply = recv memcached1 silent
+reply_buf = reply.split('\r\n')[1]
+if buf == reply_buf:
     print "succes: buf == reply"
 else:
     print "fail: buf != reply"

test/box_memcached/noreply.result

@@ -32,12 +32,7 @@ VALUE noreply:foo 0 3
 534
 END
 gets noreply:foo
-
-VALUE noreply:foo 0 3 46
-534
-END
-
-cas noreply:foo 0 0 1 46 noreply
+cas noreply:foo 0 0 1 <unique_id> noreply
 6
 get noreply:foo
 VALUE noreply:foo 0 1

test/box_memcached/noreply.test

 # encoding: tarantool
+import sys
+
 print """# Test that commands can take 'noreply' parameter. """
 exec memcached "flush_all noreply\r\n"
 exec memcached "flush_all 0 noreply\r\n"
@@ -18,10 +20,12 @@ exec memcached "get noreply:foo\r\n"
 exec memcached "prepend noreply:foo 0 0 1 noreply\r\n5\r\n"
 exec memcached "get noreply:foo\r\n"
 
-result = server.memcached.execute("gets noreply:foo\r\n", silent = False);
+sys.stdout.write("gets noreply:foo\r\n")
+result = exec memcached silent "gets noreply:foo\r\n"
 unique_id = int(result.split()[4])
 
-exec memcached "cas noreply:foo 0 0 1 %d noreply\r\n6\r\n" % unique_id
+sys.stdout.write("cas noreply:foo 0 0 1 <unique_id> noreply\r\n6\r\n")
+exec memcached silent "cas noreply:foo 0 0 1 %d noreply\r\n6\r\n" % unique_id
 exec memcached "get noreply:foo\r\n"
 
 exec memcached "incr noreply:foo 3 noreply\r\n"

test/box_replication/consistent.test

@@ -7,14 +7,16 @@ ID_BEGIN = 0
 ID_STEP = 10
 
 def insert_tuples(server, begin, end, msg = "tuple"):
+    server_sql = server.sql
     for i in range(begin, end):
-        server.sql.execute("insert into t0 values (%d, '%s %d')" % (i, msg, i), silent=False)
+        exec server_sql "insert into t0 values (%d, '%s %d')" % (i, msg, i)
 
 def select_tuples(server, begin, end):
+    server_sql = server.sql
     # the last lsn is end id + 1
     server.wait_lsn(end + 1)
     for i in range(begin, end):
-        server.sql.execute("select * from t0 where k0 = %d" % i, silent=False)
+        exec server_sql "select * from t0 where k0 = %d" % i
 
 # master server
 master = server

test/box_replication/hot_standby.test

+# encoding: tarantool
 import os
 import time
 from lib.tarantool_box_server import TarantoolBoxServer
 
 # master server
 master = server
+master_sql = master.sql
 
 # hot standby server
 hot_standby = TarantoolBoxServer()
 hot_standby.deploy("box_replication/cfg/hot_standby.cfg",
                    hot_standby.find_exe(self.args.builddir),
                    os.path.join(self.args.vardir, "hot_standby"), need_init=False)
+hot_standby_sql = hot_standby.sql
 
 # replica server
 replica = TarantoolBoxServer()
 replica.deploy("box_replication/cfg/replica.cfg",
                replica.find_exe(self.args.builddir),
                os.path.join(self.args.vardir, "replica"))
+replica_sql = replica.sql
 
 # Begin tuple id
 id = 1
@@ -25,14 +29,14 @@ print """
 # Insert 10 tuples to master
 """
 for i in range(id, id + 10):
-    master.sql.execute("insert into t0 values (%d, 'the tuple %d')" % (i, i), silent=False)
+    exec master_sql "insert into t0 values (%d, 'the tuple %d')" % (i, i)
 
 
 print """
 # Select 10 tuples from master
 """
 for i in range(id, id + 10):
-    master.sql.execute("select * from t0 where k0 = %d" % i, silent=False)
+    exec master_sql "select * from t0 where k0 = %d" % i
 
 
 print """
@@ -40,7 +44,7 @@ print """
 """
 replica.wait_lsn(11)
 for i in range(id, id + 10):
-    replica.sql.execute("select * from t0 where k0 = %d" % i, silent=False)
+    exec replica_sql "select * from t0 where k0 = %d" % i
 
 
 print """
@@ -57,14 +61,14 @@ print """
 # Insert 10 tuples to hot_standby
 """
 for i in range(id, id + 10):
-    hot_standby.sql.execute("insert into t0 values (%d, 'the tuple %d')" % (i, i), silent=False)
+    exec hot_standby_sql "insert into t0 values (%d, 'the tuple %d')" % (i, i)
 
 
 print """
 # Select 10 tuples from hot_standby
 """
 for i in range(id, id + 10):
-    hot_standby.sql.execute("select * from t0 where k0 = %d" % i, silent=False)
+    exec hot_standby_sql "select * from t0 where k0 = %d" % i
 
 
 print """
@@ -72,7 +76,7 @@ print """
 """
 replica.wait_lsn(21)
 for i in range(id, id + 10):
-    replica.sql.execute("select * from t0 where k0 = %d" % i, silent=False)
+    exec replica_sql "select * from t0 where k0 = %d" % i
 
 
 # Cleanup.

test/box_replication/swap.test

@@ -8,14 +8,16 @@ ID_BEGIN = 0
 ID_STEP = 5
 
 def insert_tuples(server, begin, end, msg = "tuple"):
+    server_sql = server.sql
     for i in range(begin, end):
-        server.sql.execute("insert into t0 values (%d, '%s %d')" % (i, msg, i), silent=False)
+        exec server_sql "insert into t0 values (%d, '%s %d')" % (i, msg, i)
 
 def select_tuples(server, begin, end):
+    server_sql = server.sql
     # the last lsn is end id + 1
     server.wait_lsn(end + 1)
     for i in range(begin, end):
-        server.sql.execute("select * from t0 where k0 = %d" % i, silent=False)
+        exec server_sql "select * from t0 where k0 = %d" % i
 
 # master server
 master = server

test/lib/admin_connection.py

@@ -23,14 +23,17 @@ __author__ = "Konstantin Osipov <kostja.osipov@gmail.com>"
 
 import socket
 import yaml
+import sys
 import re
 from tarantool_connection import TarantoolConnection
 
 is_admin_re = re.compile("^\s*(show|save|exec|exit|reload|help)", re.I)
 
+ADMIN_SEPARATOR = '\n'
+
 class AdminConnection(TarantoolConnection):
     def execute_no_reconnect(self, command, silent):
-        self.socket.sendall(command)
+        self.socket.sendall(command + ADMIN_SEPARATOR)
 
         bufsiz = 4096
         res = ""
@@ -47,8 +50,8 @@ class AdminConnection(TarantoolConnection):
         yaml.load(res)
 
         if not silent:
-            print command.replace('\n', '')
-            print res[:-1]
+            sys.stdout.write(command + ADMIN_SEPARATOR)
+            sys.stdout.write(res)
 
         return res
 

test/lib/memcached_connection.py

@@ -26,11 +26,6 @@ class MemcachedCommandBuffer:
 
 class MemcachedConnection(TarantoolConnection):
 
-    def __init__(self, host, port):
-        TarantoolConnection.__init__(self, host, port)
-        self.separator = MEMCACHED_SEPARATOR + '\n'
-        self.separator_len = 2
-
     def execute_no_reconnect(self, commands, silent = True):
         self.send(commands, silent)
         return self.recv(silent)
@@ -39,7 +34,7 @@ class MemcachedConnection(TarantoolConnection):
         self.commands = commands
         self.socket.sendall(commands)
         if not silent:
-            print self.commands
+            sys.stdout.write(self.commands)
 
     def recv(self, silent = True):
         self.recv_buffer = ''
@@ -70,7 +65,7 @@ class MemcachedConnection(TarantoolConnection):
                 self.reply_unknown(cmd)
 
         if not silent:
-            print self.reply
+            sys.stdout.write(self.reply)
 
         return self.reply
 

test/lib/tarantool_box_server.py

@@ -49,7 +49,7 @@ class TarantoolBoxServer(TarantoolServer):
                               stderr = subprocess.PIPE)
 
     def get_param(self, param):
-        data = self.admin.execute("show info\n", silent = True)
+        data = self.admin.execute("show info", silent = True)
         info = yaml.load(data)["info"]
         return info[param]
 

test/lib/tarantool_connection.py

@@ -30,8 +30,6 @@ class TarantoolConnection:
     def __init__(self, host, port):
         self.host = host
         self.port = port
-        self.separator = '\n'
-        self.separator_len = 1
         self.is_connected = False
         self.stream = cStringIO.StringIO()
         self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -70,27 +68,6 @@ class TarantoolConnection:
         self.opt_reconnect()
         return self.execute_no_reconnect(command, silent)
 
-    def write(self, fragment):
-        """This is to support print >> admin, "command" syntax.
-        For every print statement, write is invoked twice: one to
-        write the command itself, and another to write \n. We should
-        accumulate all writes until we receive \n. When we receive it,
-        we execute the command, and rewind the stream."""
-
-        str = self.stream.getvalue() + fragment
-        separator_index = str.rfind(self.separator)
-        while separator_index >= 0:
-            statement = str[:separator_index + self.separator_len]
-            sys.stdout.write(statement)
-            sys.stdout.write(self.execute(statement))
-
-            str = str[separator_index + len(self.separator):]
-            separator_index = str.rfind(self.separator)
-
-        self.stream.seek(0)
-        self.stream.truncate()
-        self.stream.write(str)
-
     def __enter__(self):
         self.connect()
         return self

test/lib/tarantool_preprocessor.py

@@ -7,24 +7,116 @@ import re
 from encodings import utf_8
 import sys
 
+# tarantool operators
+TARANTOOL_OPERATORS = [ 'exec', 'send', 'recv' ]
+
+TARANTOOL_METHODS = {
+        'exec' : 'execute',
+        'send' : 'send',
+        'recv' : 'recv'
+}
 
 def tarantool_translate(readline):
     token_stream = tokenize.generate_tokens(readline)
     for token in token_stream:
-        type, name = token[:2]
-        if type == tokenize.NAME and name == "exec":
-            next_token = next(token_stream)
-            type, name = next_token[:2]
-            if type == tokenize.NAME and name in [ "sql", "admin", "memcached" ]:
-                yield (tokenize.NAME, 'print') + token[2:]
-                yield (tokenize.OP, '>>') + token[2:]
-                yield next_token
-                yield (tokenize.OP, ',') + next_token[2:]
-            else:
-                yield token
-                yield next_token
-        else:
-            yield token
+        token_buffer = [ token ]
+        # chec token type
+        if token_is_operator(token):
+            # translate tarantool operator
+            translate_command(token_buffer, token_stream)
+
+        while len(token_buffer) > 0:
+            yield token_buffer.pop(0)
+
+
+def translate_command(token_buffer, token_stream):
+    operator = token_buffer.pop(0)
+    object = next(token_stream)
+    if token_is_identifier(object):
+        # translate operator
+        translate_operator(token_buffer, operator, object)
+        translate_operands(token_buffer, operator, token_stream)
+    else:
+        token_buffer.append(operator)
+        token_buffer.append(object)
+
+
+def translate_operator(token_buffer, operator, object):
+    # operator object -> object.method
+    # put object
+    token_buffer.append(object[:2] + operator[2:])
+    # put comma
+    token_buffer.append((tokenize.OP, '.') + operator[2:])
+    # put method
+    operator_name = operator[1]
+    method_name = TARANTOOL_METHODS[operator_name]
+    token_buffer.append((tokenize.NAME, method_name) + operator[2:])
+
+
+def translate_operands(token_buffer, operator, token_stream):
+    # put open bracket
+    token_buffer.append((tokenize.OP, '(') + operator[2:])
+
+    # put all operatands
+    token = next(token_stream)
+
+    silent = False
+    if token_is_modifier(token):
+        silent = modifier_to_value(token[1])
+        token = next(token_stream)
+
+    comma_needed = False
+    while not token_is_separator(token):
+        token_buffer.append(token[:2] + operator[2:])
+        comma_needed = True
+        token = next(token_stream)
+
+    # set verbose flag
+    if comma_needed:
+        # we have operatands, put comma before silent
+        token_buffer.append((tokenize.OP, ',') + operator[2:])
+    token_buffer.append((tokenize.NAME, 'silent') + operator[2:])
+    token_buffer.append((tokenize.OP, '=') + operator[2:])
+    token_buffer.append((tokenize.NAME, '%s' % silent) + operator[2:])
+
+    # put close bracket
+    token_buffer.append((tokenize.OP, ')') + operator[2:])
+    # new line
+    token_buffer.append((tokenize.NEWLINE, '\n') + operator[2:])
+
+
+def modifier_to_value(name):
+    if name == 'silent':
+        return True
+    return False
+
+
+def token_is_modifier(token):
+    token_type, token_name = token[:2]
+    if token_type == tokenize.NAME and token_name in [ 'silent' , 'verbose' ]:
+        return True
+    return False
+
+
+def token_is_operator(token):
+    token_type, token_name = token[:2]
+    if token_type == tokenize.NAME and token_name in TARANTOOL_OPERATORS:
+        return True
+    return False
+
+
+def token_is_identifier(token):
+    token_type = token[0]
+    if token_type == tokenize.NAME:
+        return True
+    return False
+
+
+def token_is_separator(token):
+    token_type = token[0]
+    if token_type == tokenize.NEWLINE or token_type == tokenize.ENDMARKER:
+        return True
+    return False
 
 
 class TarantoolStreamReader(utf_8.StreamReader):
@@ -33,7 +125,7 @@ class TarantoolStreamReader(utf_8.StreamReader):
         try:
             data = tokenize.untokenize(tarantool_translate(self.stream.readline))
             self.stream = cStringIO.StringIO(data)
-        except Exception:
+        except Exception, e :
             self.stream.seek(0)