From 01aa59979bc61125f599a5b8a6c911fff5daaee7 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 4 Apr 2023 17:11:28 +0900 Subject: [PATCH 1/6] journald: fix log message --- src/journal/journald-server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index a8958fcb0cf..4d1820f9b00 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -2124,7 +2124,7 @@ static int vl_method_synchronize(Varlink *link, JsonVariant *parameters, Varlink if (json_variant_elements(parameters) > 0) return varlink_error_invalid_parameter(link, parameters); - log_info("Received client request to rotate journal."); + log_info("Received client request to sync journal."); /* We don't do the main work now, but instead enqueue a deferred event loop job which will do * it. That job is scheduled at low priority, so that we return from this method call only after all From 9dfbae203e3afa500163bc46e0070c4cb2180aac Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 4 Apr 2023 16:43:44 +0900 Subject: [PATCH 2/6] sd-journal: cache results of parsing environment variables --- src/libsystemd/sd-journal/journal-file.c | 35 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 8da4ca9cadc..675ecea7b60 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -27,6 +27,7 @@ #include "journal-internal.h" #include "lookup3.h" #include "memory-util.h" +#include "missing_threads.h" #include "path-util.h" #include "prioq.h" #include "random-util.h" @@ -313,27 +314,37 @@ JournalFile* journal_file_close(JournalFile *f) { } static bool keyed_hash_requested(void) { + static thread_local int cached = -1; int r; - r = getenv_bool("SYSTEMD_JOURNAL_KEYED_HASH"); - if (r >= 0) - return r; - if (r != -ENXIO) - log_debug_errno(r, "Failed to parse $SYSTEMD_JOURNAL_KEYED_HASH environment variable, ignoring: %m"); + if (cached < 0) { + r = getenv_bool("SYSTEMD_JOURNAL_KEYED_HASH"); + if (r < 0) { + if (r != -ENXIO) + log_debug_errno(r, "Failed to parse $SYSTEMD_JOURNAL_KEYED_HASH environment variable, ignoring: %m"); + cached = true; + } else + cached = r; + } - return true; + return cached; } static bool compact_mode_requested(void) { + static thread_local int cached = -1; int r; - r = getenv_bool("SYSTEMD_JOURNAL_COMPACT"); - if (r >= 0) - return r; - if (r != -ENXIO) - log_debug_errno(r, "Failed to parse $SYSTEMD_JOURNAL_COMPACT environment variable, ignoring: %m"); + if (cached < 0) { + r = getenv_bool("SYSTEMD_JOURNAL_COMPACT"); + if (r < 0) { + if (r != -ENXIO) + log_debug_errno(r, "Failed to parse $SYSTEMD_JOURNAL_COMPACT environment variable, ignoring: %m"); + cached = true; + } else + cached = r; + } - return true; + return cached; } static int journal_file_init_header( From 83f3d73da8d132773dd91aae0fa7babb74920774 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 4 Apr 2023 14:34:32 +0900 Subject: [PATCH 3/6] compress: introduce compression_supported() helper function --- src/basic/compress.c | 10 ++++++++++ src/basic/compress.h | 3 +++ 2 files changed, 13 insertions(+) diff --git a/src/basic/compress.c b/src/basic/compress.c index 0a330ecb551..59621dc05ba 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -65,6 +65,16 @@ static const char* const compression_table[_COMPRESSION_MAX] = { DEFINE_STRING_TABLE_LOOKUP(compression, Compression); +bool compression_supported(Compression c) { + static const unsigned supported = + (1U << COMPRESSION_NONE) | + (1U << COMPRESSION_XZ) * HAVE_XZ | + (1U << COMPRESSION_LZ4) * HAVE_LZ4 | + (1U << COMPRESSION_ZSTD) * HAVE_ZSTD; + + return c >= 0 && c < _COMPRESSION_MAX && FLAGS_SET(supported, 1U << c); +} + int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t dst_alloc_size, size_t *dst_size) { #if HAVE_XZ diff --git a/src/basic/compress.h b/src/basic/compress.h index 583b105c66f..2201bca74ca 100644 --- a/src/basic/compress.h +++ b/src/basic/compress.h @@ -2,6 +2,7 @@ #pragma once #include +#include #include #include @@ -17,6 +18,8 @@ typedef enum Compression { const char* compression_to_string(Compression compression); Compression compression_from_string(const char *compression); +bool compression_supported(Compression c); + int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t dst_alloc_size, size_t *dst_size); int compress_blob_lz4(const void *src, uint64_t src_size, From 2360352ef02548723ac0c8eaf5ff6905eb9eeca5 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 4 Apr 2023 18:22:50 +0900 Subject: [PATCH 4/6] sd-journal: always use the compression algorithm specified in the header Previously, data object may be compressed with an algorithm that is not mentioned in the header. --- src/libsystemd/sd-journal/journal-file.c | 42 ++++++++++++++---------- src/libsystemd/sd-journal/journal-file.h | 12 +++++-- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 675ecea7b60..4b00129c7ff 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -1630,24 +1630,31 @@ static int journal_file_append_field( } static Compression maybe_compress_payload(JournalFile *f, uint8_t *dst, const uint8_t *src, uint64_t size, size_t *rsize) { - Compression compression = COMPRESSION_NONE; - assert(f); assert(f->header); #if HAVE_COMPRESSION - if (JOURNAL_FILE_COMPRESS(f) && size >= f->compress_threshold_bytes) { - compression = compress_blob(src, size, dst, size - 1, rsize); - if (compression > 0) - log_debug("Compressed data object %"PRIu64" -> %zu using %s", - size, *rsize, compression_to_string(compression)); - else - /* Compression didn't work, we don't really care why, let's continue without compression */ - compression = COMPRESSION_NONE; - } -#endif + Compression c; + int r; - return compression; + c = JOURNAL_FILE_COMPRESSION(f); + if (c == COMPRESSION_NONE || size < f->compress_threshold_bytes) + return COMPRESSION_NONE; + + r = compress_blob_explicit(c, src, size, dst, size - 1, rsize); + if (r < 0) { + log_debug_errno(r, "Failed to compress data object using %s, ignoring: %m", compression_to_string(c)); + /* Compression didn't work, we don't really care why, let's continue without compression */ + return COMPRESSION_NONE; + } + + assert(r == c); + log_debug("Compressed data object %"PRIu64" -> %zu using %s", size, *rsize, compression_to_string(c)); + + return c; +#else + return COMPRESSION_NONE; +#endif } static int journal_file_append_data( @@ -3967,20 +3974,21 @@ int journal_file_open( f->close_fd = true; if (DEBUG_LOGGING) { - static int last_seal = -1, last_compress = -1, last_keyed_hash = -1; + static int last_seal = -1, last_keyed_hash = -1; + static Compression last_compression = _COMPRESSION_INVALID; static uint64_t last_bytes = UINT64_MAX; if (last_seal != JOURNAL_HEADER_SEALED(f->header) || last_keyed_hash != JOURNAL_HEADER_KEYED_HASH(f->header) || - last_compress != JOURNAL_FILE_COMPRESS(f) || + last_compression != JOURNAL_FILE_COMPRESSION(f) || last_bytes != f->compress_threshold_bytes) { log_debug("Journal effective settings seal=%s keyed_hash=%s compress=%s compress_threshold_bytes=%s", yes_no(JOURNAL_HEADER_SEALED(f->header)), yes_no(JOURNAL_HEADER_KEYED_HASH(f->header)), - yes_no(JOURNAL_FILE_COMPRESS(f)), FORMAT_BYTES(f->compress_threshold_bytes)); + compression_to_string(JOURNAL_FILE_COMPRESSION(f)), FORMAT_BYTES(f->compress_threshold_bytes)); last_seal = JOURNAL_HEADER_SEALED(f->header); last_keyed_hash = JOURNAL_HEADER_KEYED_HASH(f->header); - last_compress = JOURNAL_FILE_COMPRESS(f); + last_compression = JOURNAL_FILE_COMPRESSION(f); last_bytes = f->compress_threshold_bytes; } } diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index 70d2276ced1..a4d8912aa8e 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -321,10 +321,16 @@ bool journal_file_rotate_suggested(JournalFile *f, usec_t max_file_usec, int log int journal_file_map_data_hash_table(JournalFile *f); int journal_file_map_field_hash_table(JournalFile *f); -static inline bool JOURNAL_FILE_COMPRESS(JournalFile *f) { +static inline Compression JOURNAL_FILE_COMPRESSION(JournalFile *f) { assert(f); - return JOURNAL_HEADER_COMPRESSED_XZ(f->header) || JOURNAL_HEADER_COMPRESSED_LZ4(f->header) || - JOURNAL_HEADER_COMPRESSED_ZSTD(f->header); + + if (JOURNAL_HEADER_COMPRESSED_XZ(f->header)) + return COMPRESSION_XZ; + if (JOURNAL_HEADER_COMPRESSED_LZ4(f->header)) + return COMPRESSION_LZ4; + if (JOURNAL_HEADER_COMPRESSED_ZSTD(f->header)) + return COMPRESSION_ZSTD; + return COMPRESSION_NONE; } uint64_t journal_file_hash_data(JournalFile *f, const void *data, size_t sz); From 1f06ea747b5939a1083c436dd7dae97b37bedee7 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 4 Apr 2023 14:19:51 +0900 Subject: [PATCH 5/6] sd-journal: allow to specify compression algorithm through env Fixes RHBZ#2183546 (https://bugzilla.redhat.com/show_bug.cgi?id=2183546). Previously, journal file is always compressed with the default algorithm set at compile time. So, if a newer algorithm is used, journal files cannot be read by older version of journalctl that does not support the algorithm. Co-authored-by: Colin Walters --- docs/ENVIRONMENT.md | 8 +++++ src/libsystemd/sd-journal/journal-file.c | 44 +++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 3ec5573ff95..a77992d2be7 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -513,6 +513,14 @@ SYSTEMD_HOME_DEBUG_SUFFIX=foo \ journal. Note that journal files in compact mode are limited to 4G to allow use of 32-bit offsets. Enabled by default. +* `$SYSTEMD_JOURNAL_COMPRESS` – Takes a boolean, or one of the compression + algorithms "XZ", "LZ4", and "ZSTD". If enabled, the default compression + algorithm set at compile time will be used when opening a new journal file. + If disabled, the journal file compression will be disabled. Note that the + compression mode of existing journal files are not changed. To make the + specified algorithm takes an effect immediately, you need to explicitly run + `journalctl --rotate`. + `systemd-pcrphase`, `systemd-cryptsetup`: * `$SYSTEMD_FORCE_MEASURE=1` — If set, force measuring of resources (which are diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 4b00129c7ff..a616390ad84 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -347,6 +347,48 @@ static bool compact_mode_requested(void) { return cached; } +#if HAVE_COMPRESSION +static Compression getenv_compression(void) { + Compression c; + const char *e; + int r; + + e = getenv("SYSTEMD_JOURNAL_COMPRESS"); + if (!e) + return DEFAULT_COMPRESSION; + + r = parse_boolean(e); + if (r >= 0) + return r ? DEFAULT_COMPRESSION : COMPRESSION_NONE; + + c = compression_from_string(e); + if (c < 0) { + log_debug_errno(c, "Failed to parse SYSTEMD_JOURNAL_COMPRESS value, ignoring: %s", e); + return DEFAULT_COMPRESSION; + } + + if (!compression_supported(c)) { + log_debug("Unsupported compression algorithm specified, ignoring: %s", e); + return DEFAULT_COMPRESSION; + } + + return c; +} +#endif + +static Compression compression_requested(void) { +#if HAVE_COMPRESSION + static thread_local Compression cached = _COMPRESSION_INVALID; + + if (cached < 0) + cached = getenv_compression(); + + return cached; +#else + return COMPRESSION_NONE; +#endif +} + static int journal_file_init_header( JournalFile *f, JournalFileFlags file_flags, @@ -366,7 +408,7 @@ static int journal_file_init_header( Header h = { .header_size = htole64(ALIGN64(sizeof(h))), .incompatible_flags = htole32( - FLAGS_SET(file_flags, JOURNAL_COMPRESS) * COMPRESSION_TO_HEADER_INCOMPATIBLE_FLAG(DEFAULT_COMPRESSION) | + FLAGS_SET(file_flags, JOURNAL_COMPRESS) * COMPRESSION_TO_HEADER_INCOMPATIBLE_FLAG(compression_requested()) | keyed_hash_requested() * HEADER_INCOMPATIBLE_KEYED_HASH | compact_mode_requested() * HEADER_INCOMPATIBLE_COMPACT), .compatible_flags = htole32( From d23a1c52a93206b0dbabcb4336752ccb796c11c3 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 4 Apr 2023 15:03:35 +0900 Subject: [PATCH 6/6] test: add test case that journal file is created with the requested compression algorithm --- test/units/testsuite-04.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/units/testsuite-04.sh b/test/units/testsuite-04.sh index 3f3d8bc9c89..b7cbfeb75a3 100755 --- a/test/units/testsuite-04.sh +++ b/test/units/testsuite-04.sh @@ -290,4 +290,36 @@ rm -rf "$JTMP" rm /tmp/lb1 +# https://bugzilla.redhat.com/show_bug.cgi?id=2183546 +mkdir /run/systemd/system/systemd-journald.service.d +MID=$(cat /etc/machine-id) +for c in "NONE" "XZ" "LZ4" "ZSTD"; do + cat >/run/systemd/system/systemd-journald.service.d/compress.conf <&1 | grep -q -F 'compress=${c}'; do sleep .5; done" + + # $SYSTEMD_JOURNAL_COMPRESS= also works for journal-remote + if [[ -x /usr/lib/systemd/systemd-journal-remote ]]; then + for cc in "NONE" "XZ" "LZ4" "ZSTD"; do + rm -f /tmp/foo.journal + SYSTEMD_JOURNAL_COMPRESS="${cc}" /usr/lib/systemd/systemd-journal-remote --split-mode=none -o /tmp/foo.journal --getter="journalctl -b -o export -t $ID" + SYSTEMD_LOG_LEVEL=debug journalctl --verify --quiet --file /tmp/foo.journal 2>&1 | grep -q -F "compress=${cc}" + journalctl -t "$ID" -o cat --file /tmp/foo.journal | grep -q -F "hoge with ${c}" + done + fi +done +rm /run/systemd/system/systemd-journald.service.d/compress.conf +systemctl daemon-reload +systemctl restart systemd-journald.service +journalctl --rotate + touch /testok