Skip to content
Snippets Groups Projects
Unverified Commit ed09e726 authored by Sergey Bronnikov's avatar Sergey Bronnikov Committed by Igor Munkin
Browse files

test/fuzz: collect and print Lua metrics

Fuzzing test for LuaJIT generates random Lua programs and executes them.
We want to build a fuzzing test that will produce Lua programs that will
not contain semantic errors and will trigger as much as possible
components in LuaJIT.

This proposed patch introduces metrics that gathered after running the
test. LuaJIT metrics gathered using LuaJIT getmetrics module [1]. All
gathered metrics test will output after running with a finite number of
runs or finite duration of time (options `-runs` and `-max_total_time`)
or after sending SIGUSR1 to a test process.

```
$ ./build/test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer -runs=1000

<snipped>

Done 1000 runs in 1 second(s)
Total number of samples: 1000
Total number of samples with errors: 438 (43%)
Total number of samples with recorded traces: 87 (8%)
Total number of samples with snap restores: 30 (3%)
Total number of samples with abort traces: 55 (5%)
```

1. https://www.tarantool.io/en/doc/latest/reference/tooling/luajit_getmetrics/#getmetrics-c-api

NO_CHANGELOG=testing
NO_DOC=testing
parent 20be04bd
No related branches found
No related tags found
No related merge requests found
extern "C"
{
#include <signal.h>
#include <unistd.h>
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
/* luaM_metrics */
#include <lmisclib.h>
}
#include "lua_grammar.pb.h"
......@@ -11,6 +16,77 @@ extern "C"
#include <libprotobuf-mutator/port/protobuf.h>
#include <libprotobuf-mutator/src/libfuzzer/libfuzzer_macro.h>
#define PRINT_METRIC(desc, val, total) \
std::cerr << (desc) << (val) \
<< " (" << (val) * 100 / (total) << "%)" \
<< std::endl
struct metrics {
size_t total_num;
size_t total_num_with_errors;
size_t jit_snap_restore;
size_t jit_trace_abort;
size_t jit_trace_num;
};
static struct metrics metrics;
static inline void
print_metrics(struct metrics *metrics)
{
if (metrics->total_num == 0)
return;
std::cerr << "Total number of samples: "
<< metrics->total_num << std::endl;
PRINT_METRIC("Total number of samples with errors: ",
metrics->total_num_with_errors, metrics->total_num);
PRINT_METRIC("Total number of samples with recorded traces: ",
metrics->jit_trace_num, metrics->total_num);
PRINT_METRIC("Total number of samples with snap restores: ",
metrics->jit_snap_restore, metrics->total_num);
PRINT_METRIC("Total number of samples with abort traces: ",
metrics->jit_trace_abort, metrics->total_num);
}
/* https://www.tarantool.io/en/doc/latest/reference/tooling/luajit_getmetrics/#getmetrics-c-api */
static inline void
collect_lj_metrics(struct metrics *metrics, lua_State *L)
{
struct luam_Metrics lj_metrics;
luaM_metrics(L, &lj_metrics);
if (lj_metrics.jit_snap_restore != 0)
metrics->jit_snap_restore++;
if (lj_metrics.jit_trace_abort != 0)
metrics->jit_trace_abort++;
if (lj_metrics.jit_trace_num != 0)
metrics->jit_trace_num++;
}
void
sig_handler(int signo, siginfo_t *info, void *context)
{
print_metrics(&metrics);
}
__attribute__((constructor))
static void
setup(void)
{
metrics = {};
struct sigaction act = {};
act.sa_flags = SA_SIGINFO;
act.sa_sigaction = &sig_handler;
sigaction(SIGUSR1, &act, NULL);
}
__attribute__((destructor))
static void
teardown(void)
{
print_metrics(&metrics);
}
/**
* Get an error message from the stack, and report it to std::cerr.
* Remove the message from the stack.
......@@ -18,6 +94,7 @@ extern "C"
static inline void
report_error(lua_State *L, const std::string &prefix)
{
metrics.total_num_with_errors++;
const char *verbose = ::getenv("LUA_FUZZER_VERBOSE");
if (!verbose)
return;
......@@ -67,6 +144,8 @@ DEFINE_PROTO_FUZZER(const lua_grammar::Block &message)
report_error(L, "lua_pcall()");
end:
metrics.total_num++;
collect_lj_metrics(&metrics, L);
lua_settop(L, 0);
lua_close(L);
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment