Support application-defined I/O functions for FUSE fd

The io for FUSE requests and responses can now be further customized by allowing to write custom functions for reading/writing the responses. This includes overriding the splice io.

The reason for this addition is that having a custom file descriptor is not sufficient to allow custom io. Different types of file descriptor require different mechanisms of io interaction. For example, some file descriptor communication has boundaries (SOCK_DGRAM, EOF, etc...), while other types of fd:s might be unbounded (SOCK_STREAMS, ...). For unbounded communication, you have to read the header of the FUSE request first, and then read the remaining packet data. Furthermore, the one read call does not necessarily return all the data expected, requiring further
calls in a loop.

ChangeLog.rst

@@ -1,3 +1,9 @@
+libfuse next.release.version (xxxx-xx-xx)
+=========================================
+* There is a new low-level API function `fuse_session_custom_io` that allows to implement
+  a daemon with a custom io. This can be used to create a daemon that can process incoming
+  FUSE requests to other destinations than `/dev/fuse`.
+
 libfuse 3.12.0 (2022-09-08)
 ===========================
 

example/hello_ll_uds.c

@@ -0,0 +1,358 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+  Copyright (C) 2022  Tofik Sonono <tofik.sonono@intel.com>
+
+  This program can be distributed under the terms of the GNU GPLv2.
+  See the file COPYING.
+*/
+
+/** @file
+ *
+ * minimal example filesystem using low-level API and a custom io. This custom
+ * io is implemented using UNIX domain sockets (of type SOCK_STREAM)
+ *
+ * Compile with:
+ *
+ *     gcc -Wall hello_ll_uds.c `pkg-config fuse3 --cflags --libs` -o hello_ll_uds
+ *
+ * ## Source code ##
+ * \include hello_ll.c
+ */
+
+#define FUSE_USE_VERSION 34
+
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <fuse_lowlevel.h>
+#include <fuse_kernel.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <assert.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+static const char *hello_str = "Hello World!\n";
+static const char *hello_name = "hello";
+
+static int hello_stat(fuse_ino_t ino, struct stat *stbuf)
+{
+	stbuf->st_ino = ino;
+	switch (ino) {
+	case 1:
+		stbuf->st_mode = S_IFDIR | 0755;
+		stbuf->st_nlink = 2;
+		break;
+
+	case 2:
+		stbuf->st_mode = S_IFREG | 0444;
+		stbuf->st_nlink = 1;
+		stbuf->st_size = strlen(hello_str);
+		break;
+
+	default:
+		return -1;
+	}
+	return 0;
+}
+
+static void hello_ll_getattr(fuse_req_t req, fuse_ino_t ino,
+			     struct fuse_file_info *fi)
+{
+	struct stat stbuf;
+
+	(void) fi;
+
+	memset(&stbuf, 0, sizeof(stbuf));
+	if (hello_stat(ino, &stbuf) == -1)
+		fuse_reply_err(req, ENOENT);
+	else
+		fuse_reply_attr(req, &stbuf, 1.0);
+}
+
+static void hello_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name)
+{
+	struct fuse_entry_param e;
+
+	if (parent != 1 || strcmp(name, hello_name) != 0)
+		fuse_reply_err(req, ENOENT);
+	else {
+		memset(&e, 0, sizeof(e));
+		e.ino = 2;
+		e.attr_timeout = 1.0;
+		e.entry_timeout = 1.0;
+		hello_stat(e.ino, &e.attr);
+
+		fuse_reply_entry(req, &e);
+	}
+}
+
+struct dirbuf {
+	char *p;
+	size_t size;
+};
+
+static void dirbuf_add(fuse_req_t req, struct dirbuf *b, const char *name,
+		       fuse_ino_t ino)
+{
+	struct stat stbuf;
+	size_t oldsize = b->size;
+	b->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
+	b->p = (char *) realloc(b->p, b->size);
+	memset(&stbuf, 0, sizeof(stbuf));
+	stbuf.st_ino = ino;
+	fuse_add_direntry(req, b->p + oldsize, b->size - oldsize, name, &stbuf,
+			  b->size);
+}
+
+#define min(x, y) ((x) < (y) ? (x) : (y))
+
+static int reply_buf_limited(fuse_req_t req, const char *buf, size_t bufsize,
+			     off_t off, size_t maxsize)
+{
+	if (off < bufsize)
+		return fuse_reply_buf(req, buf + off,
+				      min(bufsize - off, maxsize));
+	else
+		return fuse_reply_buf(req, NULL, 0);
+}
+
+static void hello_ll_readdir(fuse_req_t req, fuse_ino_t ino, size_t size,
+			     off_t off, struct fuse_file_info *fi)
+{
+	(void) fi;
+
+	if (ino != 1)
+		fuse_reply_err(req, ENOTDIR);
+	else {
+		struct dirbuf b;
+
+		memset(&b, 0, sizeof(b));
+		dirbuf_add(req, &b, ".", 1);
+		dirbuf_add(req, &b, "..", 1);
+		dirbuf_add(req, &b, hello_name, 2);
+		reply_buf_limited(req, b.p, b.size, off, size);
+		free(b.p);
+	}
+}
+
+static void hello_ll_open(fuse_req_t req, fuse_ino_t ino,
+			  struct fuse_file_info *fi)
+{
+	if (ino != 2)
+		fuse_reply_err(req, EISDIR);
+	else if ((fi->flags & O_ACCMODE) != O_RDONLY)
+		fuse_reply_err(req, EACCES);
+	else
+		fuse_reply_open(req, fi);
+}
+
+static void hello_ll_read(fuse_req_t req, fuse_ino_t ino, size_t size,
+			  off_t off, struct fuse_file_info *fi)
+{
+	(void) fi;
+
+	assert(ino == 2);
+	reply_buf_limited(req, hello_str, strlen(hello_str), off, size);
+}
+
+static const struct fuse_lowlevel_ops hello_ll_oper = {
+	.lookup		= hello_ll_lookup,
+	.getattr	= hello_ll_getattr,
+	.readdir	= hello_ll_readdir,
+	.open		= hello_ll_open,
+	.read		= hello_ll_read,
+};
+
+static int create_socket(const char *socket_path) {
+	struct sockaddr_un addr;
+
+	if (strnlen(socket_path, sizeof(addr.sun_path)) >=
+		sizeof(addr.sun_path)) {
+		printf("Socket path may not be longer than %lu characters\n",
+			 sizeof(addr.sun_path) - 1);
+		return -1;
+	}
+
+	if (remove(socket_path) == -1 && errno != ENOENT) {
+		printf("Could not delete previous socket file entry at %s. Error: "
+			 "%s\n", socket_path, strerror(errno));
+		return -1;
+	}
+
+	memset(&addr, 0, sizeof(struct sockaddr_un));
+	strcpy(addr.sun_path, socket_path);
+
+	int sfd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (sfd == -1) {
+		printf("Could not create socket. Error: %s\n", strerror(errno));
+		return -1;
+	}
+
+	addr.sun_family = AF_UNIX;
+	if (bind(sfd, (struct sockaddr *) &addr,
+		   sizeof(struct sockaddr_un)) == -1) {
+		printf("Could not bind socket. Error: %s\n", strerror(errno));
+		return -1;
+	}
+
+	if (listen(sfd, 1) == -1)
+		return -1;
+
+	printf("Awaiting connection on socket at %s...\n", socket_path);
+	int cfd = accept(sfd, NULL, NULL);
+	if (cfd == -1) {
+		printf("Could not accept connection. Error: %s\n",
+			 strerror(errno));
+		return -1;
+	} else {
+		printf("Accepted connection!\n");
+	}
+	return cfd;
+}
+
+static ssize_t stream_writev(int fd, struct iovec *iov, int count,
+                             void *userdata) {
+	(void)userdata;
+
+	ssize_t written = 0;
+	int cur = 0;
+	for (;;) {
+		written = writev(fd, iov+cur, count-cur);
+		if (written < 0)
+			return written;
+
+		while (cur < count && written >= iov[cur].iov_len)
+			written -= iov[cur++].iov_len;
+		if (cur == count)
+			break;
+
+		iov[cur].iov_base = (char *)iov[cur].iov_base + written;
+		iov[cur].iov_len -= written;
+	}
+	return written;
+}
+
+
+static ssize_t readall(int fd, void *buf, size_t len) {
+	size_t count = 0;
+
+	while (count < len) {
+		int i = read(fd, (char *)buf + count, len - count);
+		if (!i)
+			break;
+
+		if (i < 0)
+			return i;
+
+		count += i;
+	}
+	return count;
+}
+
+static ssize_t stream_read(int fd, void *buf, size_t buf_len, void *userdata) {
+    (void)userdata;
+
+	int res = readall(fd, buf, sizeof(struct fuse_in_header));
+	if (res == -1)
+    	return res;
+
+
+    uint32_t packet_len = ((struct fuse_in_header *)buf)->len;
+    if (packet_len > buf_len)
+    	return -1;
+
+    int prev_res = res;
+
+    res = readall(fd, (char *)buf + sizeof(struct fuse_in_header),
+                  packet_len - sizeof(struct fuse_in_header));
+
+    return  (res == -1) ? res : (res + prev_res);
+}
+
+static ssize_t stream_splice_send(int fdin, __off64_t *offin, int fdout,
+					    __off64_t *offout, size_t len,
+                                  unsigned int flags, void *userdata) {
+	(void)userdata;
+
+	size_t count = 0;
+	while (count < len) {
+		int i = splice(fdin, offin, fdout, offout, len - count, flags);
+		if (i < 1)
+			return i;
+
+		count += i;
+	}
+	return count;
+}
+
+static void fuse_cmdline_help_uds(void)
+{
+	printf("    -h   --help            print help\n"
+	       "    -V   --version         print version\n"
+	       "    -d   -o debug          enable debug output (implies -f)\n");
+}
+
+int main(int argc, char *argv[])
+{
+	struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
+	struct fuse_session *se;
+	struct fuse_cmdline_opts opts;
+	const struct fuse_custom_io io = {
+		.writev = stream_writev,
+		.read = stream_read,
+		.splice_receive = NULL,
+		.splice_send = stream_splice_send,
+	};
+	int cfd = -1;
+	int ret = -1;
+
+	if (fuse_parse_cmdline(&args, &opts) != 0)
+		return 1;
+	if (opts.show_help) {
+		printf("usage: %s [options]\n\n", argv[0]);
+		fuse_cmdline_help_uds();
+		fuse_lowlevel_help();
+		ret = 0;
+		goto err_out1;
+	} else if (opts.show_version) {
+		printf("FUSE library version %s\n", fuse_pkgversion());
+		fuse_lowlevel_version();
+		ret = 0;
+		goto err_out1;
+	}
+
+	se = fuse_session_new(&args, &hello_ll_oper,
+			      sizeof(hello_ll_oper), NULL);
+	if (se == NULL)
+	    goto err_out1;
+
+	if (fuse_set_signal_handlers(se) != 0)
+	    goto err_out2;
+
+	cfd = create_socket("/tmp/libfuse-hello-ll.sock");
+	if (cfd == -1)
+		goto err_out3;
+
+	if (fuse_session_custom_io(se, &io, cfd) != 0)
+		goto err_out3;
+
+	/* Block until ctrl+c */
+	ret = fuse_session_loop(se);
+err_out3:
+	fuse_remove_signal_handlers(se);
+err_out2:
+	fuse_session_destroy(se);
+err_out1:
+	free(opts.mountpoint);
+	fuse_opt_free_args(&args);
+
+	return ret ? 1 : 0;
+}

example/meson.build

@@ -1,7 +1,7 @@
 examples = [ 'passthrough', 'passthrough_fh',
-             'hello', 'hello_ll', 'printcap',
-             'ioctl_client', 'poll_client', 'ioctl',
-             'cuse', 'cuse_client' ]
+             'hello', 'hello_ll', 'hello_ll_uds',
+             'printcap', 'ioctl_client', 'poll_client',
+             'ioctl', 'cuse', 'cuse_client' ]
 
 if not platform.endswith('bsd') and platform != 'dragonfly'
     examples += 'passthrough_ll'

include/fuse_lowlevel.h

@@ -127,6 +127,17 @@ struct fuse_forget_data {
 	uint64_t nlookup;
 };
 
+struct fuse_custom_io {
+	ssize_t (*writev)(int fd, struct iovec *iov, int count, void *userdata);
+	ssize_t (*read)(int fd, void *buf, size_t buf_len, void *userdata);
+	ssize_t (*splice_receive)(int fdin, __off64_t *offin, int fdout,
+					  __off64_t *offout, size_t len,
+				  	  unsigned int flags, void *userdata);
+	ssize_t (*splice_send)(int fdin, __off64_t *offin, int fdout,
+				     __off64_t *offout, size_t len,
+			           unsigned int flags, void *userdata);
+};
+
 /**
  * Flags for fuse_lowlevel_notify_expire_entry()
  * 0 = invalidate entry
@@ -1994,6 +2005,36 @@ struct fuse_session *fuse_session_new(struct fuse_args *args,
 				      const struct fuse_lowlevel_ops *op,
 				      size_t op_size, void *userdata);
 
+/**
+ * Set a file descriptor for the session.
+ *
+ * This function can be used if you want to have a custom communication
+ * interface instead of using a mountpoint. In practice, this means that instead
+ * of calling fuse_session_mount() and fuse_session_unmount(), one could call
+ * fuse_session_custom_io() where fuse_session_mount() would have otherwise been
+ * called.
+ *
+ * In `io`, implementations for read and writev MUST be provided. Otherwise -1
+ * will be returned and `fd` will not be used. Implementations for `splice_send`
+ * and `splice_receive` are optional. If they are not provided splice will not
+ * be used for send or receive respectively.
+ *
+ * The provided file descriptor `fd` will be closed when fuse_session_destroy()
+ * is called.
+ *
+ * @param se session object
+ * @param io Custom io to use when retrieving/sending requests/responses
+ * @param fd file descriptor for the session
+ *
+ * @return 0  on success
+ * @return -EINVAL if `io`, `io->read` or `ìo->writev` are NULL
+ * @return -EBADF  if `fd` was smaller than 0
+ * @return -errno  if failed to allocate memory to store `io`
+ *
+ **/
+int fuse_session_custom_io(struct fuse_session *se,
+				   const struct fuse_custom_io *io, int fd);
+
 /**
  * Mount a FUSE file system.
  *

lib/fuse_i.h

@@ -45,6 +45,7 @@ struct fuse_session {
 	char *mountpoint;
 	volatile int exited;
 	int fd;
+	struct fuse_custom_io *io;
 	struct mount_opts *mo;
 	int debug;
 	int deny_others;

lib/fuse_lowlevel.c

@@ -186,8 +186,15 @@ static int fuse_send_msg(struct fuse_session *se, struct fuse_chan *ch,
 		}
 	}
 
-	ssize_t res = writev(ch ? ch->fd : se->fd,
-			     iov, count);
+	ssize_t res;
+	if (se->io != NULL)
+		/* se->io->writev is never NULL if se->io is not NULL as
+		specified by fuse_session_custom_io()*/
+		res = se->io->writev(ch ? ch->fd : se->fd, iov, count,
+					   se->userdata);
+	else
+		res = writev(ch ? ch->fd : se->fd, iov, count);
+
 	int err = errno;
 
 	if (res == -1) {
@@ -817,8 +824,14 @@ static int fuse_send_data_iov(struct fuse_session *se, struct fuse_chan *ch,
 	    (se->conn.want & FUSE_CAP_SPLICE_MOVE))
 		splice_flags |= SPLICE_F_MOVE;
 
-	res = splice(llp->pipe[0], NULL, ch ? ch->fd : se->fd,
-		     NULL, out->len, splice_flags);
+	if (se->io != NULL && se->io->splice_send != NULL) {
+		res = se->io->splice_send(llp->pipe[0], NULL,
+						  ch ? ch->fd : se->fd, NULL, out->len,
+					  	  splice_flags, se->userdata);
+	} else {
+		res = splice(llp->pipe[0], NULL, ch ? ch->fd : se->fd, NULL,
+			       out->len, splice_flags);
+	}
 	if (res == -1) {
 		res = -errno;
 		perror("fuse: splice from pipe");
@@ -2000,9 +2013,13 @@ void do_init(fuse_req_t req, fuse_ino_t nodeid, const void *inarg)
 	if (se->conn.proto_minor >= 14) {
 #ifdef HAVE_SPLICE
 #ifdef HAVE_VMSPLICE
-		se->conn.capable |= FUSE_CAP_SPLICE_WRITE | FUSE_CAP_SPLICE_MOVE;
+		if ((se->io == NULL) || (se->io->splice_send != NULL)) {
+			se->conn.capable |= FUSE_CAP_SPLICE_WRITE | FUSE_CAP_SPLICE_MOVE;
+		}
 #endif
-		se->conn.capable |= FUSE_CAP_SPLICE_READ;
+		if ((se->io == NULL) || (se->io->splice_receive != NULL)) {
+			se->conn.capable |= FUSE_CAP_SPLICE_READ;
+		}
 #endif
 	}
 	if (se->conn.proto_minor >= 18)
@@ -2755,6 +2772,8 @@ void fuse_session_destroy(struct fuse_session *se)
 	free(se->cuse_data);
 	if (se->fd != -1)
 		close(se->fd);
+	if (se->io != NULL)
+		free(se->io);
 	destroy_mount_opts(se->mo);
 	free(se);
 }
@@ -2804,8 +2823,14 @@ int fuse_session_receive_buf_int(struct fuse_session *se, struct fuse_buf *buf,
 			goto fallback;
 	}
 
-	res = splice(ch ? ch->fd : se->fd,
-		     NULL, llp->pipe[1], NULL, bufsize, 0);
+	if (se->io != NULL && se->io->splice_receive != NULL) {
+		res = se->io->splice_receive(ch ? ch->fd : se->fd, NULL,
+						     llp->pipe[1], NULL, bufsize, 0,
+						     se->userdata);
+	} else {
+		res = splice(ch ? ch->fd : se->fd, NULL, llp->pipe[1], NULL,
+				 bufsize, 0);
+	}
 	err = errno;
 
 	if (fuse_session_exited(se))
@@ -2891,7 +2916,14 @@ fallback:
 	}
 
 restart:
-	res = read(ch ? ch->fd : se->fd, buf->mem, se->bufsize);
+	if (se->io != NULL) {
+		/* se->io->read is never NULL if se->io is not NULL as
+		specified by fuse_session_custom_io()*/
+		res = se->io->read(ch ? ch->fd : se->fd, buf->mem, se->bufsize,
+					 se->userdata);
+	} else {
+		res = read(ch ? ch->fd : se->fd, buf->mem, se->bufsize);
+	}
 	err = errno;
 
 	if (fuse_session_exited(se))
@@ -3021,6 +3053,40 @@ out1:
 	return NULL;
 }
 
+int fuse_session_custom_io(struct fuse_session *se, const struct fuse_custom_io *io,
+			   int fd)
+{
+	if (fd < 0) {
+		fuse_log(FUSE_LOG_ERR, "Invalid file descriptor value %d passed to "
+			"fuse_session_custom_io()\n", fd);
+		return -EBADF;
+	}
+	if (io == NULL) {
+		fuse_log(FUSE_LOG_ERR, "No custom IO passed to "
+			"fuse_session_custom_io()\n");
+		return -EINVAL;
+	} else if (io->read == NULL || io->writev == NULL) {
+		/* If the user provides their own file descriptor, we can't
+		guarantee that the default behavior of the io operations made
+		in libfuse will function properly. Therefore, we enforce the
+		user to implement these io operations when using custom io. */
+		fuse_log(FUSE_LOG_ERR, "io passed to fuse_session_custom_io() must "
+			"implement both io->read() and io->writev\n");
+		return -EINVAL;
+	}
+
+	se->io = malloc(sizeof(struct fuse_custom_io));
+	if (se->io == NULL) {
+		fuse_log(FUSE_LOG_ERR, "Failed to allocate memory for custom io. "
+			"Error: %s\n", strerror(errno));
+		return -errno;
+	}
+
+	se->fd = fd;
+	*se->io = *io;
+	return 0;
+}
+
 int fuse_session_mount(struct fuse_session *se, const char *mountpoint)
 {
 	int fd;

lib/fuse_versionscript

@@ -39,6 +39,7 @@ FUSE_3.0 {
 		fuse_session_new;
 		fuse_main_real;
 		fuse_mount;
+		fuse_session_custom_io;
 		fuse_session_mount;
 		fuse_new;
 		fuse_opt_insert_arg;

test/meson.build

@@ -15,7 +15,7 @@ td += executable('readdir_inode', 'readdir_inode.c',
                  install: false)
 
 test_scripts = [ 'conftest.py', 'pytest.ini', 'test_examples.py',
-                 'util.py', 'test_ctests.py' ]
+                 'util.py', 'test_ctests.py', 'test_custom_io.py' ]
 td += custom_target('test_scripts', input: test_scripts,
                       output: test_scripts, build_by_default: true,
                       command: ['cp', '-fPp',

test/test_custom_io.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+if __name__ == '__main__':
+    import sys
+
+    import pytest
+    sys.exit(pytest.main([__file__] + sys.argv[1:]))
+
+import os
+import socket
+import struct
+import subprocess
+import sys
+import time
+from os.path import join as pjoin
+
+import pytest
+
+from util import base_cmdline, basename
+
+FUSE_OP_INIT = 26
+
+FUSE_MAJOR_VERSION = 7
+FUSE_MINOR_VERSION = 38
+
+fuse_in_header_fmt = '<IIQQIIII'
+fuse_out_header_fmt = '<IiQ'
+
+fuse_init_in_fmt = '<IIIII44x'
+fuse_init_out_fmt = '<IIIIHHIIHHI28x'
+
+
+def sock_recvall(sock: socket.socket, bufsize: int) -> bytes:
+    buf = bytes()
+    while len(buf) < bufsize:
+        buf += sock.recv(bufsize - len(buf))
+    return buf
+
+
+def tst_init(sock: socket.socket):
+    unique_req = 10
+    dummy_init_req_header = struct.pack(
+        fuse_in_header_fmt, struct.calcsize(fuse_in_header_fmt) +
+        struct.calcsize(fuse_init_in_fmt), FUSE_OP_INIT, unique_req, 0, 0, 0,
+        0, 0)
+    dummy_init_req_payload = struct.pack(
+        fuse_init_in_fmt, FUSE_MAJOR_VERSION, FUSE_MINOR_VERSION, 0, 0, 0)
+    dummy_init_req = dummy_init_req_header + dummy_init_req_payload
+
+    sock.sendall(dummy_init_req)
+
+    response_header = sock_recvall(sock, struct.calcsize(fuse_out_header_fmt))
+    packet_len, _, unique_res = struct.unpack(
+        fuse_out_header_fmt, response_header)
+    assert unique_res == unique_req
+
+    response_payload = sock_recvall(sock, packet_len - len(response_header))
+    response_payload = struct.unpack(fuse_init_out_fmt, response_payload)
+    assert response_payload[0] == FUSE_MAJOR_VERSION
+
+
+def test_hello_uds(output_checker):
+    cmdline = base_cmdline + [pjoin(basename, 'example', 'hello_ll_uds')]
+    print(cmdline)
+    uds_process = subprocess.Popen(cmdline, stdout=output_checker.fd,
+                                   stderr=output_checker.fd)
+    time.sleep(1)
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.settimeout(1)
+    sock.connect("/tmp/libfuse-hello-ll.sock")
+
+    tst_init(sock)
+
+    sock.close()
+    uds_process.terminate()
+    try:
+        uds_process.wait(1)
+    except subprocess.TimeoutExpired:
+        uds_process.kill()
+    os.remove("/tmp/libfuse-hello-ll.sock")