Skip to content
Snippets Groups Projects
Commit 74328386 authored by Alexander V. Tikhonov's avatar Alexander V. Tikhonov Committed by Kirill Yukhin
Browse files

Enable test reruns on failed fragiled tests 

In test-run implemented the new format of the fragile lists based on
JSON format set as fragile option in 'suite.ini' files per each suite:

   fragile = {
        "retries": 10,
        "tests": {
            "bitset.test.lua": {
                "issues": [ "gh-4095" ],
                "checksums": [ "050af3a99561a724013995668a4bc71c", "f34be60193cfe9221d3fe50df657e9d3" ]
            }
        }}

Added ability to check results file checksum on tests fail and
compare with the checksums of the known issues mentioned in the
fragile list.

Also added ability to set 'retries' option, which sets the number
of accepted reruns of the tests failed from 'fragile' list that
have checksums on its fails.

Closes #5050
parent bb856247
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 247 additions and 53 deletions
test/app-tap/suite.ini
+ 8
− 1
View file @ 74328386
......@@ -5,4 +5,11 @@ lua_libs = lua/require_mod.lua lua/serializer_test.lua
is_parallel = True
pretest_clean = True
use_unix_sockets_iproto = True
fragile = tarantoolctl.test.lua ; gh-5059
fragile = {
"retries": 10,
"tests": {
"tarantoolctl.test.lua": {
"issues": [ "gh-5059" ]
}
}
}
test/app/suite.ini
+ 14
− 3
View file @ 74328386
......@@ -7,6 +7,17 @@ use_unix_sockets = True
use_unix_sockets_iproto = True
is_parallel = True
pretest_clean = True
fragile = fiber_channel.test.lua ; gh-4961
socket.test.lua ; gh-4978
fiber.test.lua ; gh-4987
fragile = {
"retries": 10,
"tests": {
"fiber_channel.test.lua": {
"issues": [ "gh-4961" ]
},
"socket.test.lua": {
"issues": [ "gh-4978" ]
},
"fiber.test.lua": {
"issues": [ "gh-4987" ]
}
}
}
test/box-py/suite.ini
+ 9
− 1
View file @ 74328386
......@@ -5,4 +5,12 @@ script = box.lua
lua_libs = lua/fiber.lua lua/fifo.lua
use_unix_sockets = True
is_parallel = True
fragile = snapshot.test.py ; gh-4514
fragile = {
"retries": 10,
"tests": {
"snapshot.test.py": {
"issues": [ "gh-4514" ],
"checksums": [ "0e9db9dd391ea124bc5181089350a36e" ]
}
}
}
test/box-tap/suite.ini
+ 11
− 2
View file @ 74328386
......@@ -5,5 +5,14 @@ is_parallel = True
pretest_clean = True
use_unix_sockets_iproto = True
config = suite.cfg
fragile = cfg.test.lua ; gh-4344
key_def.test.lua ; gh-5049
fragile = {
"retries": 10,
"tests": {
"cfg.test.lua": {
"issues": [ "gh-4344" ]
},
"key_def.test.lua": {
"issues": [ "gh-5049" ]
}
}
}
test/box/suite.ini
+ 61
− 15
View file @ 74328386
......@@ -11,18 +11,64 @@ use_unix_sockets = True
use_unix_sockets_iproto = True
is_parallel = True
pretest_clean = True
fragile = bitset.test.lua ; gh-4095
func_reload.test.lua ; gh-4425
function1.test.lua ; gh-4199
push.test.lua ; gh-4882
tree_pk.test.lua ; gh-4882
alter_limits.test.lua ; gh-4926
misc.test.lua ; gh-4982
tuple.test.lua ; gh-4988
transaction.test.lua ; gh-4990
rtree_rect.test.lua ; gh-4994
sequence.test.lua ; gh-4996
on_replace.test.lua ; gh-4997
role.test.lua ; gh-4998
net.box_reconnect_after_gh-3164.test.lua ; gh-5081
net.box_disconnect_gh-3859.test.lua ; gh-5056
fragile = {
"retries": 10,
"tests": {
"bitset.test.lua": {
"issues": [ "gh-4095" ]
},
"func_reload.test.lua": {
"issues": [ "gh-4425" ],
"checksums": [ "2e8206fbddc62d12332d71b82bbcce6d" ]
},
"function1.test.lua": {
"issues": [ "gh-4199" ]
},
"push.test.lua": {
"issues": [ "gh-4882" ]
},
"tree_pk.test.lua": {
"issues": [ "gh-4882" ]
},
"alter_limits.test.lua": {
"issues": [ "gh-4926" ]
},
"misc.test.lua": {
"issues": [ "gh-4982" ]
},
"tuple.test.lua": {
"issues": [ "gh-4988" ]
},
"transaction.test.lua": {
"issues": [ "gh-4990" ]
},
"rtree_rect.test.lua": {
"issues": [ "gh-4994" ]
},
"sequence.test.lua": {
"issues": [ "gh-4996" ],
"checksums": [ "062ab51f8b36bad59ba5a137f1c48b8e" ]
},
"on_replace.test.lua": {
"issues": [ "gh-4997" ]
},
"role.test.lua": {
"issues": [ "gh-4998" ]
},
"net.box_reconnect_after_gh-3164.test.lua": {
"issues": [ "gh-5081" ]
},
"net.box_disconnect_gh-3859.test.lua": {
"issues": [ "gh-5156" ],
"checksums": [ "345f55562f152e52d79f78e2920b28a2" ]
},
"access_sysview.test.lua": {
"issues": [ "gh-5327" ],
"checksums": [ "d6136f214b81556b1858e05970a90963" ]
},
"net.box_reconnect_after.test.lua": {
"issues": [ "gh-5333" ],
"checksums": [ "cca5df51000ec0076450a6b1a8b21053" ]
}
}
}
test/engine/suite.ini
+ 13
− 2
View file @ 74328386
......@@ -10,5 +10,16 @@ config = engine.cfg
lua_libs = conflict.lua ../box/lua/utils.lua ../box/lua/push.lua
is_parallel = True
pretest_clean = True
fragile = ddl.test.lua ; gh-4353
gh-4973-concurrent-alter-fails.test.lua ; gh-5157
fragile = {
"retries": 10,
"tests": {
"ddl.test.lua": {
"issues": [ "gh-4353" ],
"checksums": [ "dd8851d80183cc75052119ba646e295d" ]
},
"gh-4973-concurrent-alter-fails.test.lua": {
"issues": [ "gh-5157" ],
"checksums": [ "4e797e63335cebe24dab15eae4aa8044" ]
}
}
}
test/replication-py/suite.ini
+ 14
− 3
View file @ 74328386
......@@ -3,6 +3,17 @@ core = tarantool
script = master.lua
description = tarantool/box, replication
is_parallel = True
fragile = init_storage.test.py ; gh-4949
conflict.test.py ; gh-4980
cluster.test.py ; gh-5109
fragile = {
"retries": 10,
"tests": {
"init_storage.test.py": {
"issues": [ "gh-4949" ]
},
"conflict.test.py": {
"issues": [ "gh-4980" ]
},
"cluster.test.py": {
"issues": [ "gh-5109" ]
}
}
}
test/replication/suite.ini
+ 47
− 11
View file @ 74328386
......@@ -11,14 +11,50 @@ use_unix_sockets_iproto = True
long_run = prune.test.lua
is_parallel = True
pretest_clean = True
fragile = errinj.test.lua ; gh-3870
long_row_timeout.test.lua ; gh-4351
gh-3160-misc-heartbeats-on-master-changes.test.lua ; gh-4940
skip_conflict_row.test.lua ; gh-4958
sync.test.lua ; gh-3835
transaction.test.lua ; gh-4312
autobootstrap.test.lua ; gh-4533
replica_rejoin.test.lua ; gh-4985
recover_missing_xlog.test.lua ; gh-4989
box_set_replication_stress.test.lua ; gh-4992 gh-4986
gh-4605-empty-password.test.lua ; gh-5030
fragile = {
"retries": 10,
"tests": {
"errinj.test.lua": {
"issues": [ "gh-3870" ],
"checksums": [ "5d3f58323aafc1a11d9b9264258f7acf" ]
},
"long_row_timeout.test.lua": {
"issues": [ "gh-4351" ]
},
"gh-3160-misc-heartbeats-on-master-changes.test.lua": {
"issues": [ "gh-4940" ]
},
"skip_conflict_row.test.lua": {
"issues": [ "gh-4958" ]
},
"sync.test.lua": {
"issues": [ "gh-3835" ]
},
"transaction.test.lua": {
"issues": [ "gh-4312", "gh-5331" ],
"checksums": [ "302cbcfde971bc8cb65b059165509d20", "a488d1f66d4d0325c28746db77cf5e17" ]
},
"autobootstrap.test.lua": {
"issues": [ "gh-4533" ]
},
"replica_rejoin.test.lua": {
"issues": [ "gh-4985" ]
},
"recover_missing_xlog.test.lua": {
"issues": [ "gh-4989" ],
"checksums": [ "e88f791d1276b1ba9b041dfd6b4187d2" ]
},
"box_set_replication_stress.test.lua": {
"issues": [ "gh-4992", "gh-4986" ],
"checksums": [ "58cd2e36248c230e96c02397af5d7dbd" ]
},
"gh-4605-empty-password.test.lua": {
"issues": [ "gh-5030" ]
},
"on_schema_init.test.lua": {
"issues": [ "gh-5291" ],
"checksums": [ "1cb01c103258e26c8a80458f6c40fd44" ]
}
}
}
>>>>>>> 0003d64fd... Enable test reruns on failed fragiled tests
test/sql/suite.ini
+ 8
− 1
View file @ 74328386
......@@ -10,4 +10,11 @@ lua_libs = lua/sql_tokenizer.lua
release_disabled = errinj.test.lua view_delayed_wal.test.lua
disabled = sql-statN-index-drop.test.lua
pretest_clean = True
fragile = func-recreate.test.lua ; gh-4384
fragile = {
"retries": 10,
"tests": {
"func-recreate.test.lua": {
"issues": [ "gh-4384" ]
}
}
}
test/vinyl/suite.ini
+ 42
− 11
View file @ 74328386
......@@ -12,14 +12,45 @@ is_parallel = True
# throttle.test.lua temporary disabled for gh-4168
disabled = upgrade.test.lua throttle.test.lua
pretest_clean = True
fragile = errinj.test.lua ; gh-4346
select_consistency.test.lua ; gh-4385
throttle.test.lua ; gh-4168
stat.test.lua ; gh-4951
misc.test.lua ; gh-4979
snapshot.test.lua ; gh-4984
write_iterator.test.lua ; gh-4572
errinj_ddl.test.lua ; gh-4993
gh-4810-dump-during-index-build.test.lua ; gh-5031
deferred_delete.test.lua ; gh-5089
gh.test.lua ; gh-5141
fragile = {
"retries": 10,
"tests": {
"tx_gap_lock.test.lua": {
"issues": [ "gh-4309" ],
"checksums": [ "99dbd33845b40f5399a657fe40abe826" ]
},
"errinj.test.lua": {
"issues": [ "gh-4346" ]
},
"select_consistency.test.lua": {
"issues": [ "gh-4385" ]
},
"throttle.test.lua": {
"issues": [ "gh-4168" ]
},
"stat.test.lua": {
"issues": [ "gh-4951" ]
},
"misc.test.lua": {
"issues": [ "gh-4979" ]
},
"snapshot.test.lua": {
"issues": [ "gh-4984" ]
},
"write_iterator.test.lua": {
"issues": [ "gh-4572" ]
},
"errinj_ddl.test.lua": {
"issues": [ "gh-4993" ]
},
"gh-4810-dump-during-index-build.test.lua": {
"issues": [ "gh-5031" ]
},
"deferred_delete.test.lua": {
"issues": [ "gh-5089" ]
},
"gh.test.lua": {
"issues": [ "gh-5141" ]
}
}
}
test/wal_off/suite.ini
+ 8
− 1
View file @ 74328386
......@@ -7,4 +7,11 @@ use_unix_sockets_iproto = True
is_parallel = True
pretest_clean = True
disabled = iterator_lt_gt.test.lua
fragile = iterator_lt_gt.test.lua ; gh-3925
fragile = {
"retries": 10,
"tests": {
"iterator_lt_gt.test.lua": {
"issues": [ "gh-3925" ]
}
}
}
test/xlog/suite.ini
+ 12
− 2
View file @ 74328386
......@@ -10,5 +10,15 @@ use_unix_sockets_iproto = True
long_run = snap_io_rate.test.lua
is_parallel = True
pretest_clean = True
fragile = checkpoint_daemon.test.lua ; gh-4952
panic_on_broken_lsn.test.lua ; gh-4991
fragile = {
"retries": 10,
"tests": {
"checkpoint_daemon.test.lua": {
"issues": [ "gh-4952" ],
"checksums": [ "7c9571a53f3025f02ab23703939a02d6" ]
},
"panic_on_broken_lsn.test.lua": {
"issues": [ "gh-4991" ]
}
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment