test: fix fiber stack overflow test not overflowing

test/unit/guard.cc calls stack_break_f() recursively until the stack
overflows and a signal is fired, however it relies on undefined behavior
when compares pointers to local variables. Fixed by comparing
__builtin_frame_address() instead.

One of the examples of this UB is when ASAN allocates local variables on
fake stacks, in that case the test completes without the stack overflow.

Also this patch disables ASAN for stack_break_f() to keep the array on the
fiber stack (see the corresponding comment) and marks it as volatile to
avoid optimizing it away by the compiler.

Closes tarantool/tarantool-qa#323

NO_DOC=test fix
NO_CHANGELOG=test fix

(cherry picked from commit 05b696c7)

src/trivia/util.h

@@ -408,6 +408,17 @@ strnindex(const char *const *haystack, const char *needle, uint32_t len,
 #  define PACKED
 #endif
 
+/**
+ * NO_SANITIZE_ADDRESS attribute disables AddressSanitizer for a given function.
+ * The attribute may not be supported by old compilers, but they do not support
+ * ASAN as well, so it's safe to define the attribute only if ASAN is enabled.
+ */
+#if __has_feature(address_sanitizer)
+#  define NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address))
+#else
+#  define NO_SANITIZE_ADDRESS
+#endif
+
 /** Function Attributes }}} */
 
 /** {{{ Statement Attributes */

test/unit/guard.cc

@@ -11,20 +11,28 @@ sigsegf_handler(int signo)
 	exit(0);
 }
 
-static int __attribute__((noinline))
-stack_break_f(char *ptr)
+/*
+ * ASAN is disabled for this function, because for the stack-use-after-return
+ * detection it could allocate the `block' on a fake stack, rather than placing
+ * it on a fiber stack. In that case, a lot more recursive calls will be
+ * required to overflow the stack.
+ */
+static NOINLINE NO_SANITIZE_ADDRESS int
+stack_break_f(char *frame_zero)
 {
-	char block[2048];
+	char *frame_curr = (char *)__builtin_frame_address(0);
+	volatile char block[2048];
 	/*
 	 * Make sum volatile to prevent a compiler from
 	 * optimizing away call to this function.
 	 */
 	volatile char sum = 0;
-	memset(block, 0xff, 2048);
+	memset((void *)block, 0xff, sizeof(block));
 	sum += block[(unsigned char) block[4]];
-	ptrdiff_t stack_diff = ptr > block ? ptr - block : block - ptr;
-	if (stack_diff < (ptrdiff_t)default_attr.stack_size)
-		sum += stack_break_f(ptr);
+
+	ptrdiff_t stack_diff = frame_curr - frame_zero;
+	if ((size_t)abs(stack_diff) < default_attr.stack_size)
+		sum += stack_break_f(frame_zero);
 	return sum;
 }
 
@@ -53,7 +61,7 @@ main_f(va_list ap)
 	sigaction(SIGSEGV, &sa, NULL);
 	sigaction(SIGBUS, &sa, NULL);
 
-	int res = stack_break_f((char *)&stack);
+	int res = stack_break_f((char *)__builtin_frame_address(0));
 
 	ev_break(loop(), EVBREAK_ALL);
 	free(stack.ss_sp);