From ed09e72678726004ee3de177513bfb188aeff7c8 Mon Sep 17 00:00:00 2001
From: Sergey Bronnikov <sergeyb@tarantool.org>
Date: Sat, 29 Jul 2023 14:04:21 +0300
Subject: [PATCH] test/fuzz: collect and print Lua metrics
Fuzzing test for LuaJIT generates random Lua programs and executes them.
We want to build a fuzzing test that will produce Lua programs that will
not contain semantic errors and will trigger as much as possible
components in LuaJIT.
This proposed patch introduces metrics that gathered after running the
test. LuaJIT metrics gathered using LuaJIT getmetrics module [1]. All
gathered metrics test will output after running with a finite number of
runs or finite duration of time (options `-runs` and `-max_total_time`)
or after sending SIGUSR1 to a test process.
```
$ ./build/test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer -runs=1000
<snipped>
Done 1000 runs in 1 second(s)
Total number of samples: 1000
Total number of samples with errors: 438 (43%)
Total number of samples with recorded traces: 87 (8%)
Total number of samples with snap restores: 30 (3%)
Total number of samples with abort traces: 55 (5%)
```
1. https://www.tarantool.io/en/doc/latest/reference/tooling/luajit_getmetrics/#getmetrics-c-api
NO_CHANGELOG=testing
NO_DOC=testing
---
.../luaL_loadbuffer/luaL_loadbuffer_fuzzer.cc | 79 +++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer.cc b/test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer.cc
index f8bb4e8243..15ccaf9227 100644
--- a/test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer.cc
+++ b/test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer.cc
@@ -1,8 +1,13 @@
extern "C"
{
+#include <signal.h>
+#include <unistd.h>
+
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
+/* luaM_metrics */
+#include <lmisclib.h>
}
#include "lua_grammar.pb.h"
@@ -11,6 +16,77 @@ extern "C"
#include <libprotobuf-mutator/port/protobuf.h>
#include <libprotobuf-mutator/src/libfuzzer/libfuzzer_macro.h>
+#define PRINT_METRIC(desc, val, total) \
+ std::cerr << (desc) << (val) \
+ << " (" << (val) * 100 / (total) << "%)" \
+ << std::endl
+
+struct metrics {
+ size_t total_num;
+ size_t total_num_with_errors;
+ size_t jit_snap_restore;
+ size_t jit_trace_abort;
+ size_t jit_trace_num;
+};
+
+static struct metrics metrics;
+
+static inline void
+print_metrics(struct metrics *metrics)
+{
+ if (metrics->total_num == 0)
+ return;
+
+ std::cerr << "Total number of samples: "
+ << metrics->total_num << std::endl;
+ PRINT_METRIC("Total number of samples with errors: ",
+ metrics->total_num_with_errors, metrics->total_num);
+ PRINT_METRIC("Total number of samples with recorded traces: ",
+ metrics->jit_trace_num, metrics->total_num);
+ PRINT_METRIC("Total number of samples with snap restores: ",
+ metrics->jit_snap_restore, metrics->total_num);
+ PRINT_METRIC("Total number of samples with abort traces: ",
+ metrics->jit_trace_abort, metrics->total_num);
+}
+
+/* https://www.tarantool.io/en/doc/latest/reference/tooling/luajit_getmetrics/#getmetrics-c-api */
+static inline void
+collect_lj_metrics(struct metrics *metrics, lua_State *L)
+{
+ struct luam_Metrics lj_metrics;
+ luaM_metrics(L, &lj_metrics);
+ if (lj_metrics.jit_snap_restore != 0)
+ metrics->jit_snap_restore++;
+ if (lj_metrics.jit_trace_abort != 0)
+ metrics->jit_trace_abort++;
+ if (lj_metrics.jit_trace_num != 0)
+ metrics->jit_trace_num++;
+}
+
+void
+sig_handler(int signo, siginfo_t *info, void *context)
+{
+ print_metrics(&metrics);
+}
+
+__attribute__((constructor))
+static void
+setup(void)
+{
+ metrics = {};
+ struct sigaction act = {};
+ act.sa_flags = SA_SIGINFO;
+ act.sa_sigaction = &sig_handler;
+ sigaction(SIGUSR1, &act, NULL);
+}
+
+__attribute__((destructor))
+static void
+teardown(void)
+{
+ print_metrics(&metrics);
+}
+
/**
* Get an error message from the stack, and report it to std::cerr.
* Remove the message from the stack.
@@ -18,6 +94,7 @@ extern "C"
static inline void
report_error(lua_State *L, const std::string &prefix)
{
+ metrics.total_num_with_errors++;
const char *verbose = ::getenv("LUA_FUZZER_VERBOSE");
if (!verbose)
return;
@@ -67,6 +144,8 @@ DEFINE_PROTO_FUZZER(const lua_grammar::Block &message)
report_error(L, "lua_pcall()");
end:
+ metrics.total_num++;
+ collect_lj_metrics(&metrics, L);
lua_settop(L, 0);
lua_close(L);
}
--
GitLab