From dae1184302834b52cff438fbf5322cd1c9c79c06 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Wed, 10 Jul 2024 23:04:46 +0200 Subject: [PATCH] Add syslog and fatal signal handler feature I see random ENOTCONN failures in xfstest generic/013 and generic/014 in my branch, but earliest on the 2nd run - takes ~12hours to get the issue, but then there are no further information logged. ENOTCONN points to a daemon crash - I need backtraces and a core dump. This adds optional handling of fatal signals to print a core dump and optional syslog logging with these new public functions: fuse_set_fail_signal_handlers() In addition to the existing fuse_set_signal_handlers(). This is not enabled together with fuse_set_signal_handlers(), as it is change in behavior and file systems might already have their own fatal handlers. fuse_log_enable_syslog Print logs to syslog instead of stderr fuse_log_close_syslog Close syslog (for now just does closelog()) Code in fuse_signals.c is also updated, to be an array of signals, and setting signal handlers is now down with a for-loop instead of one hand coded set_one_signal_handler() per signal. --- ChangeLog.rst | 8 +++ example/passthrough_hp.cc | 11 ++- include/fuse_common.h | 17 +++++ include/fuse_log.h | 12 ++++ lib/fuse_log.c | 64 +++++++++++++++-- lib/fuse_signals.c | 146 +++++++++++++++++++++++++++++++------- lib/fuse_versionscript | 3 + 7 files changed, 229 insertions(+), 32 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index cab3e81..0ed7d78 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +libfuse 3.17 (unreleased) +======================== + +* Allows to handle fatal signals and to print a backtrace. + New public function: fuse_set_fail_signal_handlers() +* Allows fuse_log() messages to be send to syslog instead of stderr + New public functions: fuse_log_enable_syslog() and fuse_log_close_syslog() + libfuse 3.16.2 (2023-10-10) =========================== diff --git a/example/passthrough_hp.cc b/example/passthrough_hp.cc index e3c8225..708a38e 100644 --- a/example/passthrough_hp.cc +++ b/example/passthrough_hp.cc @@ -75,6 +75,7 @@ #include #include #include +#include using namespace std; @@ -1436,6 +1437,9 @@ int main(int argc, char *argv[]) { if (fuse_set_signal_handlers(se) != 0) goto err_out2; + if (fuse_set_fail_signal_handlers(se) != 0) + goto err_out2; + // Don't apply umask, use modes exactly as specified umask(0); @@ -1452,12 +1456,14 @@ int main(int argc, char *argv[]) { fuse_daemonize(fs.foreground); + if (!fs.foreground) + fuse_log_enable_syslog("passthrough-hp", LOG_PID | LOG_CONS, LOG_DAEMON); + if (options.count("single")) ret = fuse_session_loop(se); else ret = fuse_session_loop_mt(se, loop_config); - fuse_session_unmount(se); err_out3: @@ -1469,6 +1475,9 @@ err_out1: fuse_loop_cfg_destroy(loop_config); fuse_opt_free_args(&args); + if (!fs.foreground) + fuse_log_close_syslog(); + return ret ? 1 : 0; } diff --git a/include/fuse_common.h b/include/fuse_common.h index 04ecb95..6de2640 100644 --- a/include/fuse_common.h +++ b/include/fuse_common.h @@ -954,6 +954,23 @@ ssize_t fuse_buf_copy(struct fuse_bufvec *dst, struct fuse_bufvec *src, */ int fuse_set_signal_handlers(struct fuse_session *se); +/** + * Print a stack backtrace diagnostic on critical signals () + * + * Stores session in a global variable. May only be called once per + * process until fuse_remove_signal_handlers() is called. + * + * Once either of the POSIX signals arrives, the signal handler calls + * fuse_session_exit(). + * + * @param se the session to exit + * @return 0 on success, -1 on failure + * + * See also: + * fuse_remove_signal_handlers() + */ +int fuse_set_fail_signal_handlers(struct fuse_session *se); + /** * Restore default signal handlers * diff --git a/include/fuse_log.h b/include/fuse_log.h index 5e112e0..c855957 100644 --- a/include/fuse_log.h +++ b/include/fuse_log.h @@ -75,6 +75,18 @@ void fuse_set_log_func(fuse_log_func_t func); */ void fuse_log(enum fuse_log_level level, const char *fmt, ...); +/** + * Switch default log handler from stderr to syslog + * + * Passed options are according to 'man 3 openlog' + */ +void fuse_log_enable_syslog(const char *ident, int option, int facility); + +/** + * To be called at teardown to close syslog. +*/ +void fuse_log_close_syslog(void); + #ifdef __cplusplus } #endif diff --git a/lib/fuse_log.c b/lib/fuse_log.c index 0d268ab..95d6379 100644 --- a/lib/fuse_log.c +++ b/lib/fuse_log.c @@ -12,12 +12,56 @@ #include #include +#include +#include -static void default_log_func( - __attribute__(( unused )) enum fuse_log_level level, - const char *fmt, va_list ap) +#define MAX_SYSLOG_LINE_LEN 512 + +static bool to_syslog = false; + +static void default_log_func(__attribute__((unused)) enum fuse_log_level level, + const char *fmt, va_list ap) { - vfprintf(stderr, fmt, ap); + if (to_syslog) { + int sys_log_level; + + /* + * with glibc fuse_log_level has identical values as + * syslog levels, but we also support BSD - better we convert to + * be sure. + */ + switch (level) { + case FUSE_LOG_DEBUG: + sys_log_level = LOG_DEBUG; + break; + case FUSE_LOG_INFO: + sys_log_level = LOG_INFO; + break; + case FUSE_LOG_NOTICE: + sys_log_level = LOG_NOTICE; + break; + case FUSE_LOG_WARNING: + sys_log_level = LOG_WARNING; + break; + case FUSE_LOG_ERR: + sys_log_level = LOG_ERR; + break; + case FUSE_LOG_CRIT: + sys_log_level = LOG_CRIT; + break; + case FUSE_LOG_ALERT: + sys_log_level = LOG_ALERT; + break; + case FUSE_LOG_EMERG: + sys_log_level = LOG_EMERG; + } + + char log[MAX_SYSLOG_LINE_LEN]; + vsnprintf(log, MAX_SYSLOG_LINE_LEN, fmt, ap); + syslog(sys_log_level, "%s", log); + } else { + vfprintf(stderr, fmt, ap); + } } static fuse_log_func_t log_func = default_log_func; @@ -38,3 +82,15 @@ void fuse_log(enum fuse_log_level level, const char *fmt, ...) log_func(level, fmt, ap); va_end(ap); } + +void fuse_log_enable_syslog(const char *ident, int option, int facility) +{ + to_syslog = true; + + openlog(ident, option, facility); +} + +void fuse_log_close_syslog(void) +{ + closelog(); +} \ No newline at end of file diff --git a/lib/fuse_signals.c b/lib/fuse_signals.c index 4c1d172..0380c82 100644 --- a/lib/fuse_signals.c +++ b/lib/fuse_signals.c @@ -17,35 +17,72 @@ #include #include #include +#include +static int teardown_sigs[] = { SIGHUP, SIGINT, SIGTERM }; +static int ignore_sigs[] = { SIGPIPE}; +static int fail_sigs[] = { SIGILL, SIGTRAP, SIGABRT, SIGBUS, SIGFPE, SIGSEGV }; static struct fuse_session *fuse_instance; +#define BT_STACK_SZ (1024 * 1024) +static void *backtrace_buffer[BT_STACK_SZ]; + static void dump_stack(void) { + fprintf(stderr, "%s:%d\n", __func__, __LINE__); #ifdef HAVE_BACKTRACE - const size_t backtrace_sz = 1024 * 1024; - void* backtrace_buffer[backtrace_sz]; + char **strings; - int err_fd = fileno(stderr); + int nptrs = backtrace(backtrace_buffer, BT_STACK_SZ); + strings = backtrace_symbols(backtrace_buffer, nptrs); - int trace_len = backtrace(backtrace_buffer, backtrace_sz); - backtrace_symbols_fd(backtrace_buffer, trace_len, err_fd); + fprintf(stderr, "%s: nptrs=%d\n", __func__, nptrs); + + if (strings == NULL) { + fuse_log(FUSE_LOG_ERR, "Failed to get backtrace symbols: %s\n", + strerror(errno)); + return; + } + + for (int idx = 0; idx < nptrs; idx++) + fuse_log(FUSE_LOG_ERR, "%s\n", strings[idx]); + + free(strings); #endif } static void exit_handler(int sig) { - if (fuse_instance) { - fuse_session_exit(fuse_instance); - if(sig <= 0) { - fuse_log(FUSE_LOG_ERR, "assertion error: signal value <= 0\n"); - dump_stack(); - abort(); - } + if (fuse_instance == NULL) + return; + + fuse_session_exit(fuse_instance); + + if (sig < 0) { + fuse_log(FUSE_LOG_ERR, + "assertion error: signal value <= 0\n"); + dump_stack(); + abort(); fuse_instance->error = sig; } + + fuse_instance->error = sig; } +static void exit_backtrace(int sig) +{ + if (fuse_instance == NULL) + return; + + fuse_session_exit(fuse_instance); + + fuse_remove_signal_handlers(fuse_instance); + fuse_log(FUSE_LOG_ERR, "Got signal: %d\n", sig); + dump_stack(); + abort(); +} + + static void do_nothing(int sig) { (void) sig; @@ -74,33 +111,88 @@ static int set_one_signal_handler(int sig, void (*handler)(int), int remove) return 0; } +static int _fuse_set_signal_handlers(int signals[], int nr_signals, + void (*handler)(int)) +{ + for (int idx = 0; idx < nr_signals; idx++) { + int signal = signals[idx]; + + /* + * If we used SIG_IGN instead of the do_nothing function, + * then we would be unable to tell if we set SIG_IGN (and + * thus should reset to SIG_DFL in fuse_remove_signal_handlers) + * or if it was already set to SIG_IGN (and should be left + * untouched. + */ + if (set_one_signal_handler(signal, handler, 0) == -1) { + fuse_log(FUSE_LOG_ERR, + "Failed to install signal handler for sig %d\n", + signal); + return -1; + } + } + + return 0; +} + int fuse_set_signal_handlers(struct fuse_session *se) { - /* If we used SIG_IGN instead of the do_nothing function, - then we would be unable to tell if we set SIG_IGN (and - thus should reset to SIG_DFL in fuse_remove_signal_handlers) - or if it was already set to SIG_IGN (and should be left - untouched. */ - if (set_one_signal_handler(SIGHUP, exit_handler, 0) == -1 || - set_one_signal_handler(SIGINT, exit_handler, 0) == -1 || - set_one_signal_handler(SIGTERM, exit_handler, 0) == -1 || - set_one_signal_handler(SIGPIPE, do_nothing, 0) == -1) - return -1; + size_t nr_signals; + int rc; - fuse_instance = se; + nr_signals = sizeof(teardown_sigs) / sizeof(teardown_sigs[0]); + rc = _fuse_set_signal_handlers(teardown_sigs, nr_signals, exit_handler); + if (rc < 0) + return rc; + + nr_signals = sizeof(ignore_sigs) / sizeof(ignore_sigs[0]); + rc = _fuse_set_signal_handlers(ignore_sigs, nr_signals, do_nothing); + if (rc < 0) + return rc; + + if (fuse_instance == NULL) + fuse_instance = se; return 0; } +int fuse_set_fail_signal_handlers(struct fuse_session *se) +{ + size_t nr_signals = sizeof(fail_sigs) / sizeof(fail_sigs[0]); + + int rc = _fuse_set_signal_handlers(fail_sigs, nr_signals, + exit_backtrace); + if (rc < 0) + return rc; + + if (fuse_instance == NULL) + fuse_instance = se; + + return 0; +} + +static void _fuse_remove_signal_handlers(int signals[], int nr_signals, + void (*handler)(int)) +{ + for (int idx = 0; idx < nr_signals; idx++) + set_one_signal_handler(signals[idx], handler, 1); +} + void fuse_remove_signal_handlers(struct fuse_session *se) { + size_t nr_signals; + if (fuse_instance != se) fuse_log(FUSE_LOG_ERR, "fuse: fuse_remove_signal_handlers: unknown session\n"); else fuse_instance = NULL; - set_one_signal_handler(SIGHUP, exit_handler, 1); - set_one_signal_handler(SIGINT, exit_handler, 1); - set_one_signal_handler(SIGTERM, exit_handler, 1); - set_one_signal_handler(SIGPIPE, do_nothing, 1); + nr_signals = sizeof(teardown_sigs) / sizeof(teardown_sigs[0]); + _fuse_remove_signal_handlers(teardown_sigs, nr_signals, exit_handler); + + nr_signals = sizeof(ignore_sigs) / sizeof(ignore_sigs[0]); + _fuse_remove_signal_handlers(ignore_sigs, nr_signals, do_nothing); + + nr_signals = sizeof(fail_sigs) / sizeof(fail_sigs[0]); + _fuse_remove_signal_handlers(fail_sigs, nr_signals, exit_backtrace); } diff --git a/lib/fuse_versionscript b/lib/fuse_versionscript index 1058372..14cbca1 100644 --- a/lib/fuse_versionscript +++ b/lib/fuse_versionscript @@ -198,6 +198,9 @@ FUSE_3.17 { fuse_passthrough_close; fuse_session_custom_io_30; fuse_session_custom_io_317; + fuse_set_fail_signal_handlers; + fuse_log_enable_syslog; + fuse_log_close_syslog; } FUSE_3.12; # Local Variables: