add Zstandard compression support

I changed the style of the hackargs variable in autogen.sh to multiline
because said line was becoming a bit long with the new --with-zstd arg
added.

A previous version of this patch has been running on my two Arch Linux
installations (with an accompanying mkinitcpio patch) for several months
over many kernel updates without any issues.
Any additional testing and/or patch review would of course be appreciated.

Signed-off-by: Torge Matthies <openglfreak@googlemail.com>

.semaphore/semaphore.yml

@@ -22,7 +22,7 @@ blocks:
       prologue:
         commands:
           - sudo apt update
-          - sudo apt --yes install docbook-xsl liblzma-dev zlib1g-dev cython linux-headers-generic libssl-dev
+          - sudo apt --yes install docbook-xsl libzstd-dev liblzma-dev zlib1g-dev cython linux-headers-generic libssl-dev
           - checkout
 
       epilogue:
@@ -42,5 +42,5 @@ blocks:
       prologue:
         commands:
           - sudo apt update
-          - sudo apt --yes install docbook-xsl liblzma-dev zlib1g-dev cython linux-headers-generic libssl-dev
+          - sudo apt --yes install docbook-xsl libzstd-dev liblzma-dev zlib1g-dev cython linux-headers-generic libssl-dev
           - checkout

.travis.yml

@@ -14,6 +14,7 @@ matrix:
 before_install:
   - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
   - sudo apt-get update -qq
+  - sudo apt-get install -qq libzstd-dev
   - sudo apt-get install -qq liblzma-dev
   - sudo apt-get install -qq zlib1g-dev
   - sudo apt-get install -qq xsltproc docbook-xsl

Makefile.am

@@ -31,6 +31,8 @@ SED_PROCESS = \
 	-e 's,@exec_prefix\@,$(exec_prefix),g' \
 	-e 's,@libdir\@,$(libdir),g' \
 	-e 's,@includedir\@,$(includedir),g' \
+	-e 's,@libzstd_CFLAGS\@,${libzstd_CFLAGS},g' \
+	-e 's,@libzstd_LIBS\@,${libzstd_LIBS},g' \
 	-e 's,@liblzma_CFLAGS\@,${liblzma_CFLAGS},g' \
 	-e 's,@liblzma_LIBS\@,${liblzma_LIBS},g' \
 	-e 's,@zlib_CFLAGS\@,${zlib_CFLAGS},g' \
@@ -90,7 +92,7 @@ libkmod_libkmod_la_DEPENDENCIES = \
 	${top_srcdir}/libkmod/libkmod.sym
 libkmod_libkmod_la_LIBADD = \
 	shared/libshared.la \
-	${liblzma_LIBS} ${zlib_LIBS} ${libcrypto_LIBS}
+	${libzstd_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libcrypto_LIBS}
 
 noinst_LTLIBRARIES += libkmod/libkmod-internal.la
 libkmod_libkmod_internal_la_SOURCES = $(libkmod_libkmod_la_SOURCES)

autogen.sh

@@ -32,7 +32,14 @@ fi
 
 cd $oldpwd
 
-hackargs="--enable-debug --enable-python --with-xz --with-zlib --with-openssl"
+hackargs="\
+--enable-debug \
+--enable-python \
+--with-zstd \
+--with-xz \
+--with-zlib \
+--with-openssl \
+"
 
 if [ "x$1" = "xc" ]; then
         shift

configure.ac

@@ -83,6 +83,17 @@ AC_ARG_WITH([rootlibdir],
         [], [with_rootlibdir=$libdir])
 AC_SUBST([rootlibdir], [$with_rootlibdir])
 
+AC_ARG_WITH([zstd],
+	AS_HELP_STRING([--with-zstd], [handle Zstandard-compressed modules @<:@default=disabled@:>@]),
+	[], [with_zstd=no])
+AS_IF([test "x$with_zstd" != "xno"], [
+	PKG_CHECK_MODULES([libzstd], [libzstd >= 1.4.4])
+	AC_DEFINE([ENABLE_ZSTD], [1], [Enable Zstandard for modules.])
+], [
+	AC_MSG_NOTICE([Zstandard support not requested])
+])
+CC_FEATURE_APPEND([with_features], [with_zstd], [ZSTD])
+
 AC_ARG_WITH([xz],
 	AS_HELP_STRING([--with-xz], [handle Xz-compressed modules @<:@default=disabled@:>@]),
 	[], [with_xz=no])
@@ -307,7 +318,7 @@ AC_MSG_RESULT([
 	tools:			${enable_tools}
 	python bindings:	${enable_python}
 	logging:		${enable_logging}
-	compression:		xz=${with_xz}  zlib=${with_zlib}
+	compression:		zstd=${with_zstd}  xz=${with_xz}  zlib=${with_zlib}
 	debug:			${enable_debug}
 	coverage:		${enable_coverage}
 	doc:			${enable_gtk_doc}

libkmod/libkmod-file.c

@@ -26,6 +26,9 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+#ifdef ENABLE_ZSTD
+#include <zstd.h>
+#endif
 #ifdef ENABLE_XZ
 #include <lzma.h>
 #endif
@@ -45,6 +48,9 @@ struct file_ops {
 };
 
 struct kmod_file {
+#ifdef ENABLE_ZSTD
+	bool zstd_used;
+#endif
 #ifdef ENABLE_XZ
 	bool xz_used;
 #endif
@@ -60,6 +66,141 @@ struct kmod_file {
 	struct kmod_elf *elf;
 };
 
+#ifdef ENABLE_ZSTD
+static int zstd_read_block(struct kmod_file *file, size_t block_size,
+			   ZSTD_inBuffer *input, size_t *input_capacity)
+{
+	ssize_t rdret;
+	int ret;
+
+	if (*input_capacity < block_size) {
+		free((void *)input->src);
+		input->src = malloc(block_size);
+		if (input->src == NULL) {
+			ret = -errno;
+			ERR(file->ctx, "zstd: %m\n");
+			return ret;
+		}
+		*input_capacity = block_size;
+	}
+
+	rdret = read(file->fd, (void *)input->src, block_size);
+	if (rdret < 0) {
+		ret = -errno;
+		ERR(file->ctx, "zstd: %m\n");
+		return ret;
+	}
+
+	input->pos = 0;
+	input->size = rdret;
+	return 0;
+}
+
+static int zstd_ensure_outbuffer_space(ZSTD_outBuffer *buffer, size_t min_free)
+{
+	uint8_t *old_buffer = buffer->dst;
+	int ret = 0;
+
+	if (buffer->size - buffer->pos >= min_free)
+		return 0;
+
+	buffer->size += min_free;
+	buffer->dst = realloc(buffer->dst, buffer->size);
+	if (buffer->dst == NULL) {
+		ret = -errno;
+		free(old_buffer);
+	}
+
+	return ret;
+}
+
+static int zstd_decompress_block(struct kmod_file *file, ZSTD_DStream *dstr,
+				 ZSTD_inBuffer *input, ZSTD_outBuffer *output,
+				 size_t *next_block_size)
+{
+	size_t out_buf_min_size = ZSTD_DStreamOutSize();
+	int ret = 0;
+
+	do {
+		ssize_t dsret;
+
+		ret = zstd_ensure_outbuffer_space(output, out_buf_min_size);
+		if (ret) {
+			ERR(file->ctx, "zstd: %s\n", strerror(-ret));
+			break;
+		}
+
+		dsret = ZSTD_decompressStream(dstr, output, input);
+		if (ZSTD_isError(dsret)) {
+			ret = -EINVAL;
+			ERR(file->ctx, "zstd: %s\n", ZSTD_getErrorName(dsret));
+			break;
+		}
+		if (dsret > 0)
+			*next_block_size = (size_t)dsret;
+	} while (input->pos < input->size
+		 || output->pos > output->size
+		 || output->size - output->pos < out_buf_min_size);
+
+	return ret;
+}
+
+static int load_zstd(struct kmod_file *file)
+{
+	ZSTD_DStream *dstr;
+	size_t next_block_size;
+	size_t zst_inb_capacity = 0;
+	ZSTD_inBuffer zst_inb = { 0 };
+	ZSTD_outBuffer zst_outb = { 0 };
+	int ret;
+
+	dstr = ZSTD_createDStream();
+	if (dstr == NULL) {
+		ret = -EINVAL;
+		ERR(file->ctx, "zstd: Failed to create decompression stream\n");
+		goto out;
+	}
+
+	next_block_size = ZSTD_initDStream(dstr);
+
+	while (true) {
+		ret = zstd_read_block(file, next_block_size, &zst_inb,
+				      &zst_inb_capacity);
+		if (ret != 0)
+			goto out;
+		if (zst_inb.size == 0) /* EOF */
+			break;
+
+		ret = zstd_decompress_block(file, dstr, &zst_inb, &zst_outb,
+					    &next_block_size);
+		if (ret != 0)
+			goto out;
+	}
+
+	ZSTD_freeDStream(dstr);
+	free((void *)zst_inb.src);
+	file->zstd_used = true;
+	file->memory = zst_outb.dst;
+	file->size = zst_outb.pos;
+	return 0;
+out:
+	if (dstr != NULL)
+		ZSTD_freeDStream(dstr);
+	free((void *)zst_inb.src);
+	free((void *)zst_outb.dst);
+	return ret;
+}
+
+static void unload_zstd(struct kmod_file *file)
+{
+	if (!file->zstd_used)
+		return;
+	free(file->memory);
+}
+
+static const char magic_zstd[] = {0x28, 0xB5, 0x2F, 0xFD};
+#endif
+
 #ifdef ENABLE_XZ
 static void xz_uncompress_belch(struct kmod_file *file, lzma_ret ret)
 {
@@ -238,6 +379,9 @@ static const struct comp_type {
 	const char *magic_bytes;
 	const struct file_ops ops;
 } comp_types[] = {
+#ifdef ENABLE_ZSTD
+	{sizeof(magic_zstd), magic_zstd, {load_zstd, unload_zstd}},
+#endif
 #ifdef ENABLE_XZ
 	{sizeof(magic_xz), magic_xz, {load_xz, unload_xz}},
 #endif

libkmod/libkmod.pc.in

@@ -7,5 +7,5 @@ Name: libkmod
 Description: Library to deal with kernel modules
 Version: @VERSION@
 Libs: -L${libdir} -lkmod
-Libs.private: @liblzma_LIBS@ @zlib_LIBS@
+Libs.private: @libzstd_LIBS@ @liblzma_LIBS@ @zlib_LIBS@
 Cflags: -I${includedir}

shared/util.c

@@ -45,6 +45,9 @@ static const struct kmod_ext {
 #endif
 #ifdef ENABLE_XZ
 	{".ko.xz", sizeof(".ko.xz") - 1},
+#endif
+#ifdef ENABLE_ZSTD
+	{".ko.zst", sizeof(".ko.zst") - 1},
 #endif
 	{ }
 };

testsuite/mkosi/mkosi.fedora

@@ -19,6 +19,7 @@ BuildPackages =
 	make
 	pkgconf-pkg-config
 	xml-common
+	libzstd-devel
 	xz-devel
 	zlib-devel
 	openssl-devel

testsuite/test-util.c

@@ -156,6 +156,9 @@ static int test_path_ends_with_kmod_ext(const struct test *t)
 #endif
 #ifdef ENABLE_XZ
 		{ "/bla.ko.xz", true },
+#endif
+#ifdef ENABLE_ZSTD
+		{ "/bla.ko.zst", true },
 #endif
 		{ "/bla.ko.x", false },
 		{ "/bla.ko.", false },