Add syslog and fatal signal handler feature

I see random ENOTCONN failures in xfstest generic/013 and generic/014
in my branch, but earliest on the 2nd run - takes ~12hours to get
the issue, but then there are no further information logged.
ENOTCONN points to a daemon crash - I need backtraces and a core dump.

This adds optional handling of fatal signals to print a core dump
and optional syslog logging with these new public functions:

fuse_set_fail_signal_handlers()
    In addition to the existing fuse_set_signal_handlers(). This is not
    enabled together with fuse_set_signal_handlers(), as it is change
    in behavior and file systems might already have their own fatal
    handlers.

fuse_log_enable_syslog
    Print logs to syslog instead of stderr

fuse_log_close_syslog
    Close syslog (for now just does closelog())

Code in fuse_signals.c is also updated, to be an array of signals,
and setting signal handlers is now down with a for-loop instead
of one hand coded set_one_signal_handler() per signal.

ChangeLog.rst

@@ -1,3 +1,11 @@
+libfuse 3.17 (unreleased)
+========================
+
+* Allows to handle fatal signals and to print a backtrace.
+  New public function: fuse_set_fail_signal_handlers()
+* Allows fuse_log() messages to be send to syslog instead of stderr
+  New public functions: fuse_log_enable_syslog() and fuse_log_close_syslog()
+
 libfuse 3.16.2 (2023-10-10)
 ===========================
 

example/passthrough_hp.cc

@@ -75,6 +75,7 @@
 #include <fstream>
 #include <thread>
 #include <iomanip>
+#include <syslog.h>
 
 using namespace std;
 
@@ -1436,6 +1437,9 @@ int main(int argc, char *argv[]) {
     if (fuse_set_signal_handlers(se) != 0)
         goto err_out2;
 
+    if (fuse_set_fail_signal_handlers(se) != 0)
+        goto err_out2;
+
     // Don't apply umask, use modes exactly as specified
     umask(0);
 
@@ -1452,12 +1456,14 @@ int main(int argc, char *argv[]) {
 
     fuse_daemonize(fs.foreground);
 
+    if (!fs.foreground)
+        fuse_log_enable_syslog("passthrough-hp", LOG_PID | LOG_CONS, LOG_DAEMON);
+
     if (options.count("single"))
         ret = fuse_session_loop(se);
     else
         ret = fuse_session_loop_mt(se, loop_config);
 
-
     fuse_session_unmount(se);
 
 err_out3:
@@ -1469,6 +1475,9 @@ err_out1:
     fuse_loop_cfg_destroy(loop_config);
     fuse_opt_free_args(&args);
 
+    if (!fs.foreground)
+        fuse_log_close_syslog();
+
     return ret ? 1 : 0;
 }
 

include/fuse_common.h

@@ -954,6 +954,23 @@ ssize_t fuse_buf_copy(struct fuse_bufvec *dst, struct fuse_bufvec *src,
  */
 int fuse_set_signal_handlers(struct fuse_session *se);
 
+/**
+ * Print a stack backtrace diagnostic on critical signals ()
+ *
+ * Stores session in a global variable.	 May only be called once per
+ * process until fuse_remove_signal_handlers() is called.
+ *
+ * Once either of the POSIX signals arrives, the signal handler calls
+ * fuse_session_exit().
+ *
+ * @param se the session to exit
+ * @return 0 on success, -1 on failure
+ *
+ * See also:
+ * fuse_remove_signal_handlers()
+ */
+int fuse_set_fail_signal_handlers(struct fuse_session *se);
+
 /**
  * Restore default signal handlers
  *

include/fuse_log.h

@@ -75,6 +75,18 @@ void fuse_set_log_func(fuse_log_func_t func);
  */
 void fuse_log(enum fuse_log_level level, const char *fmt, ...);
 
+/**
+ * Switch default log handler from stderr to syslog
+ *
+ * Passed options are according to 'man 3 openlog'
+ */
+void fuse_log_enable_syslog(const char *ident, int option, int facility);
+
+/**
+ * To be called at teardown to close syslog.
+*/
+void fuse_log_close_syslog(void);
+
 #ifdef __cplusplus
 }
 #endif

lib/fuse_log.c

@@ -12,12 +12,56 @@
 
 #include <stdarg.h>
 #include <stdio.h>
+#include <stdbool.h>
+#include <syslog.h>
 
-static void default_log_func(
+#define MAX_SYSLOG_LINE_LEN 512
-		__attribute__(( unused )) enum fuse_log_level level,
+
-		const char *fmt, va_list ap)
+static bool to_syslog = false;
+
+static void default_log_func(__attribute__((unused)) enum fuse_log_level level,
+			     const char *fmt, va_list ap)
 {
-	vfprintf(stderr, fmt, ap);
+	if (to_syslog) {
+		int sys_log_level;
+
+		/*
+		 * with glibc fuse_log_level has identical values as
+		 * syslog levels, but we also support BSD - better we convert to
+		 * be sure.
+		 */
+		switch (level) {
+		case FUSE_LOG_DEBUG:
+			sys_log_level = LOG_DEBUG;
+			break;
+		case FUSE_LOG_INFO:
+			sys_log_level = LOG_INFO;
+			break;
+		case FUSE_LOG_NOTICE:
+			sys_log_level = LOG_NOTICE;
+			break;
+		case FUSE_LOG_WARNING:
+			sys_log_level = LOG_WARNING;
+			break;
+		case FUSE_LOG_ERR:
+			sys_log_level = LOG_ERR;
+			break;
+		case FUSE_LOG_CRIT:
+			sys_log_level = LOG_CRIT;
+			break;
+		case FUSE_LOG_ALERT:
+			sys_log_level = LOG_ALERT;
+			break;
+		case FUSE_LOG_EMERG:
+			sys_log_level = LOG_EMERG;
+		}
+
+		char log[MAX_SYSLOG_LINE_LEN];
+		vsnprintf(log, MAX_SYSLOG_LINE_LEN, fmt, ap);
+		syslog(sys_log_level, "%s", log);
+	} else {
+		vfprintf(stderr, fmt, ap);
+	}
 }
 
 static fuse_log_func_t log_func = default_log_func;
@@ -38,3 +82,15 @@ void fuse_log(enum fuse_log_level level, const char *fmt, ...)
 	log_func(level, fmt, ap);
 	va_end(ap);
 }
+
+void fuse_log_enable_syslog(const char *ident, int option, int facility)
+{
+	to_syslog = true;
+
+	openlog(ident, option, facility);
+}
+
+void fuse_log_close_syslog(void)
+{
+	closelog();
+}

lib/fuse_signals.c

@@ -17,35 +17,72 @@
 #include <signal.h>
 #include <stdlib.h>
 #include <execinfo.h>
+#include <errno.h>
 
+static int teardown_sigs[] = { SIGHUP, SIGINT, SIGTERM };
+static int ignore_sigs[] = { SIGPIPE};
+static int fail_sigs[] = { SIGILL, SIGTRAP, SIGABRT, SIGBUS, SIGFPE, SIGSEGV };
 static struct fuse_session *fuse_instance;
 
+#define BT_STACK_SZ (1024 * 1024)
+static void *backtrace_buffer[BT_STACK_SZ];
+
 static void dump_stack(void)
 {
+	fprintf(stderr, "%s:%d\n", __func__, __LINE__);
 #ifdef HAVE_BACKTRACE
-	const size_t backtrace_sz = 1024 * 1024;
+	char **strings;
-	void* backtrace_buffer[backtrace_sz];
 
-	int err_fd = fileno(stderr);
+	int nptrs = backtrace(backtrace_buffer, BT_STACK_SZ);
+	strings = backtrace_symbols(backtrace_buffer, nptrs);
 
-	int trace_len = backtrace(backtrace_buffer, backtrace_sz);
+	fprintf(stderr, "%s: nptrs=%d\n", __func__, nptrs);
-	backtrace_symbols_fd(backtrace_buffer, trace_len, err_fd);
+
+	if (strings == NULL) {
+		fuse_log(FUSE_LOG_ERR, "Failed to get backtrace symbols: %s\n",
+			 strerror(errno));
+		return;
+	}
+
+	for (int idx = 0; idx < nptrs; idx++)
+		fuse_log(FUSE_LOG_ERR, "%s\n", strings[idx]);
+
+	free(strings);
 #endif
 }
 
 static void exit_handler(int sig)
 {
-	if (fuse_instance) {
+	if (fuse_instance == NULL)
-		fuse_session_exit(fuse_instance);
+		return;
-		if(sig <= 0) {
+
-			fuse_log(FUSE_LOG_ERR, "assertion error: signal value <= 0\n");
+	fuse_session_exit(fuse_instance);
-			dump_stack();
+
-			abort();
+	if (sig < 0) {
-		}
+		fuse_log(FUSE_LOG_ERR,
+				"assertion error: signal value <= 0\n");
+		dump_stack();
+		abort();
 		fuse_instance->error = sig;
 	}
+
+	fuse_instance->error = sig;
 }
 
+static void exit_backtrace(int sig)
+{
+	if (fuse_instance == NULL)
+		return;
+
+	fuse_session_exit(fuse_instance);
+
+	fuse_remove_signal_handlers(fuse_instance);
+	fuse_log(FUSE_LOG_ERR, "Got signal: %d\n", sig);
+	dump_stack();
+	abort();
+}
+
+
 static void do_nothing(int sig)
 {
 	(void) sig;
@@ -74,33 +111,88 @@ static int set_one_signal_handler(int sig, void (*handler)(int), int remove)
 	return 0;
 }
 
+static int _fuse_set_signal_handlers(int signals[], int nr_signals,
+				     void (*handler)(int))
+{
+	for (int idx = 0; idx < nr_signals; idx++) {
+		int signal = signals[idx];
+
+		/*
+		 * If we used SIG_IGN instead of the do_nothing function,
+		 * then we would be unable to tell if we set SIG_IGN (and
+		 * thus should reset to SIG_DFL in fuse_remove_signal_handlers)
+		 * or if it was already set to SIG_IGN (and should be left
+		 * untouched.
+		*/
+		if (set_one_signal_handler(signal, handler, 0) == -1) {
+			fuse_log(FUSE_LOG_ERR,
+				 "Failed to install signal handler for sig %d\n",
+				 signal);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
 int fuse_set_signal_handlers(struct fuse_session *se)
 {
-	/* If we used SIG_IGN instead of the do_nothing function,
+	size_t nr_signals;
-	   then we would be unable to tell if we set SIG_IGN (and
+	int rc;
-	   thus should reset to SIG_DFL in fuse_remove_signal_handlers)
-	   or if it was already set to SIG_IGN (and should be left
-	   untouched. */
-	if (set_one_signal_handler(SIGHUP, exit_handler, 0) == -1 ||
-	    set_one_signal_handler(SIGINT, exit_handler, 0) == -1 ||
-	    set_one_signal_handler(SIGTERM, exit_handler, 0) == -1 ||
-	    set_one_signal_handler(SIGPIPE, do_nothing, 0) == -1)
-		return -1;
 
-	fuse_instance = se;
+	nr_signals = sizeof(teardown_sigs) / sizeof(teardown_sigs[0]);
+	rc = _fuse_set_signal_handlers(teardown_sigs, nr_signals, exit_handler);
+	if (rc < 0)
+		return rc;
+
+	nr_signals = sizeof(ignore_sigs) / sizeof(ignore_sigs[0]);
+	rc = _fuse_set_signal_handlers(ignore_sigs, nr_signals, do_nothing);
+	if (rc < 0)
+		return rc;
+
+	if (fuse_instance == NULL)
+		fuse_instance = se;
 	return 0;
 }
 
+int fuse_set_fail_signal_handlers(struct fuse_session *se)
+{
+	size_t nr_signals = sizeof(fail_sigs) / sizeof(fail_sigs[0]);
+
+	int rc = _fuse_set_signal_handlers(fail_sigs, nr_signals,
+					   exit_backtrace);
+	if (rc < 0)
+		return rc;
+
+	if (fuse_instance == NULL)
+		fuse_instance = se;
+
+	return 0;
+}
+
+static void _fuse_remove_signal_handlers(int signals[], int nr_signals,
+					 void (*handler)(int))
+{
+	for (int idx = 0; idx < nr_signals; idx++)
+		set_one_signal_handler(signals[idx], handler, 1);
+}
+
 void fuse_remove_signal_handlers(struct fuse_session *se)
 {
+	size_t nr_signals;
+
 	if (fuse_instance != se)
 		fuse_log(FUSE_LOG_ERR,
 			"fuse: fuse_remove_signal_handlers: unknown session\n");
 	else
 		fuse_instance = NULL;
 
-	set_one_signal_handler(SIGHUP, exit_handler, 1);
+	nr_signals = sizeof(teardown_sigs) / sizeof(teardown_sigs[0]);
-	set_one_signal_handler(SIGINT, exit_handler, 1);
+	_fuse_remove_signal_handlers(teardown_sigs, nr_signals, exit_handler);
-	set_one_signal_handler(SIGTERM, exit_handler, 1);
+
-	set_one_signal_handler(SIGPIPE, do_nothing, 1);
+	nr_signals = sizeof(ignore_sigs) / sizeof(ignore_sigs[0]);
+	_fuse_remove_signal_handlers(ignore_sigs, nr_signals, do_nothing);
+
+	nr_signals = sizeof(fail_sigs) / sizeof(fail_sigs[0]);
+	_fuse_remove_signal_handlers(fail_sigs, nr_signals, exit_backtrace);
 }

lib/fuse_versionscript

@@ -198,6 +198,9 @@ FUSE_3.17 {
 		fuse_passthrough_close;
 		fuse_session_custom_io_30;
 		fuse_session_custom_io_317;
+		fuse_set_fail_signal_handlers;
+		fuse_log_enable_syslog;
+		fuse_log_close_syslog;
 } FUSE_3.12;
 
 # Local Variables: