- djm@cvs.openbsd.org 2013/10/17 00:30:13

[PROTOCOL sftp-client.c sftp-client.h sftp-server.c sftp.1 sftp.c]
     fsync@openssh.com protocol extension for sftp-server
     client support to allow calling fsync() faster successful transfer
     patch mostly by imorgan AT nas.nasa.gov; bz#1798
     "fine" markus@ "grumble OK" deraadt@ "doesn't sound bad to me" millert@
This commit is contained in:
Damien Miller 2013-10-17 11:48:52 +11:00
parent 51682faa59
commit f29238e674
7 changed files with 182 additions and 53 deletions

View File

@ -23,6 +23,12 @@
- djm@cvs.openbsd.org 2013/10/16 22:58:01
[ssh.c ssh_config.5]
one I missed in previous: s/isation/ization/
- djm@cvs.openbsd.org 2013/10/17 00:30:13
[PROTOCOL sftp-client.c sftp-client.h sftp-server.c sftp.1 sftp.c]
fsync@openssh.com protocol extension for sftp-server
client support to allow calling fsync() faster successful transfer
patch mostly by imorgan AT nas.nasa.gov; bz#1798
"fine" markus@ "grumble OK" deraadt@ "doesn't sound bad to me" millert@
20131015
- (djm) OpenBSD CVS Sync

View File

@ -331,4 +331,18 @@ link(oldpath, newpath) and will respond with a SSH_FXP_STATUS message.
This extension is advertised in the SSH_FXP_VERSION hello with version
"1".
$OpenBSD: PROTOCOL,v 1.20 2013/01/08 18:49:04 markus Exp $
10. sftp: Extension request "fsync@openssh.com"
This request asks the server to call fsync(2) on an open file handle.
uint32 id
string "fsync@openssh.com"
string handle
One receiving this request, a server will call fsync(handle_fd) and will
respond with a SSH_FXP_STATUS message.
This extension is advertised in the SSH_FXP_VERSION hello with version
"1".
$OpenBSD: PROTOCOL,v 1.21 2013/10/17 00:30:13 djm Exp $

View File

@ -1,4 +1,4 @@
/* $OpenBSD: sftp-client.c,v 1.106 2013/10/11 02:52:23 djm Exp $ */
/* $OpenBSD: sftp-client.c,v 1.107 2013/10/17 00:30:13 djm Exp $ */
/*
* Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
*
@ -76,6 +76,7 @@ struct sftp_conn {
#define SFTP_EXT_STATVFS 0x00000002
#define SFTP_EXT_FSTATVFS 0x00000004
#define SFTP_EXT_HARDLINK 0x00000008
#define SFTP_EXT_FSYNC 0x00000010
u_int exts;
u_int64_t limit_kbps;
struct bwlimit bwlimit_in, bwlimit_out;
@ -388,6 +389,10 @@ do_init(int fd_in, int fd_out, u_int transfer_buflen, u_int num_requests,
strcmp(value, "1") == 0) {
ret->exts |= SFTP_EXT_HARDLINK;
known = 1;
} else if (strcmp(name, "fsync@openssh.com") == 0 &&
strcmp(value, "1") == 0) {
ret->exts |= SFTP_EXT_FSYNC;
known = 1;
}
if (known) {
debug2("Server supports extension \"%s\" revision %s",
@ -743,7 +748,7 @@ do_realpath(struct sftp_conn *conn, char *path)
if (type == SSH2_FXP_STATUS) {
u_int status = buffer_get_int(&msg);
error("Couldn't canonicalise: %s", fx2txt(status));
error("Couldn't canonicalize: %s", fx2txt(status));
buffer_free(&msg);
return NULL;
} else if (type != SSH2_FXP_NAME)
@ -869,6 +874,36 @@ do_symlink(struct sftp_conn *conn, char *oldpath, char *newpath)
return(status);
}
int
do_fsync(struct sftp_conn *conn, char *handle, u_int handle_len)
{
Buffer msg;
u_int status, id;
/* Silently return if the extension is not supported */
if ((conn->exts & SFTP_EXT_FSYNC) == 0)
return -1;
buffer_init(&msg);
/* Send fsync request */
id = conn->msg_id++;
buffer_put_char(&msg, SSH2_FXP_EXTENDED);
buffer_put_int(&msg, id);
buffer_put_cstring(&msg, "fsync@openssh.com");
buffer_put_string(&msg, handle, handle_len);
send_msg(conn, &msg);
debug3("Sent message fsync@openssh.com I:%u", id);
buffer_free(&msg);
status = get_status(conn, id);
if (status != SSH2_FX_OK)
error("Couldn't sync file: %s", fx2txt(status));
return status;
}
#ifdef notyet
char *
do_readlink(struct sftp_conn *conn, char *path)
@ -991,7 +1026,7 @@ send_read_request(struct sftp_conn *conn, u_int id, u_int64_t offset,
int
do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
Attrib *a, int preserve_flag, int resume_flag)
Attrib *a, int preserve_flag, int resume_flag, int fsync_flag)
{
Attrib junk;
Buffer msg;
@ -1251,6 +1286,12 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
error("Can't set times on \"%s\": %s",
local_path, strerror(errno));
}
if (fsync_flag) {
debug("syncing \"%s\"", local_path);
if (fsync(local_fd) == -1)
error("Couldn't sync file \"%s\": %s",
local_path, strerror(errno));
}
}
close(local_fd);
buffer_free(&msg);
@ -1261,7 +1302,8 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
static int
download_dir_internal(struct sftp_conn *conn, char *src, char *dst, int depth,
Attrib *dirattrib, int preserve_flag, int print_flag, int resume_flag)
Attrib *dirattrib, int preserve_flag, int print_flag, int resume_flag,
int fsync_flag)
{
int i, ret = 0;
SFTP_DIRENT **dir_entries;
@ -1314,11 +1356,12 @@ download_dir_internal(struct sftp_conn *conn, char *src, char *dst, int depth,
continue;
if (download_dir_internal(conn, new_src, new_dst,
depth + 1, &(dir_entries[i]->a), preserve_flag,
print_flag, resume_flag) == -1)
print_flag, resume_flag, fsync_flag) == -1)
ret = -1;
} else if (S_ISREG(dir_entries[i]->a.perm) ) {
if (do_download(conn, new_src, new_dst,
&(dir_entries[i]->a), preserve_flag, resume_flag) == -1) {
&(dir_entries[i]->a), preserve_flag,
resume_flag, fsync_flag) == -1) {
error("Download of file %s to %s failed",
new_src, new_dst);
ret = -1;
@ -1351,25 +1394,26 @@ download_dir_internal(struct sftp_conn *conn, char *src, char *dst, int depth,
int
download_dir(struct sftp_conn *conn, char *src, char *dst,
Attrib *dirattrib, int preserve_flag, int print_flag, int resume_flag)
Attrib *dirattrib, int preserve_flag, int print_flag,
int resume_flag, int fsync_flag)
{
char *src_canon;
int ret;
if ((src_canon = do_realpath(conn, src)) == NULL) {
error("Unable to canonicalise path \"%s\"", src);
error("Unable to canonicalize path \"%s\"", src);
return -1;
}
ret = download_dir_internal(conn, src_canon, dst, 0,
dirattrib, preserve_flag, print_flag, resume_flag);
dirattrib, preserve_flag, print_flag, resume_flag, fsync_flag);
free(src_canon);
return ret;
}
int
do_upload(struct sftp_conn *conn, char *local_path, char *remote_path,
int preserve_flag)
int preserve_flag, int fsync_flag)
{
int local_fd;
int status = SSH2_FX_OK;
@ -1545,6 +1589,9 @@ do_upload(struct sftp_conn *conn, char *local_path, char *remote_path,
if (preserve_flag)
do_fsetstat(conn, handle, handle_len, &a);
if (fsync_flag)
(void)do_fsync(conn, handle, handle_len);
if (do_close(conn, handle, handle_len) != SSH2_FX_OK)
status = -1;
free(handle);
@ -1554,7 +1601,7 @@ do_upload(struct sftp_conn *conn, char *local_path, char *remote_path,
static int
upload_dir_internal(struct sftp_conn *conn, char *src, char *dst, int depth,
int preserve_flag, int print_flag)
int preserve_flag, int print_flag, int fsync_flag)
{
int ret = 0, status;
DIR *dirp;
@ -1623,11 +1670,12 @@ upload_dir_internal(struct sftp_conn *conn, char *src, char *dst, int depth,
continue;
if (upload_dir_internal(conn, new_src, new_dst,
depth + 1, preserve_flag, print_flag) == -1)
depth + 1, preserve_flag, print_flag,
fsync_flag) == -1)
ret = -1;
} else if (S_ISREG(sb.st_mode)) {
if (do_upload(conn, new_src, new_dst,
preserve_flag) == -1) {
preserve_flag, fsync_flag) == -1) {
error("Uploading of file %s to %s failed!",
new_src, new_dst);
ret = -1;
@ -1646,18 +1694,19 @@ upload_dir_internal(struct sftp_conn *conn, char *src, char *dst, int depth,
int
upload_dir(struct sftp_conn *conn, char *src, char *dst, int preserve_flag,
int print_flag)
int print_flag, int fsync_flag)
{
char *dst_canon;
int ret;
if ((dst_canon = do_realpath(conn, dst)) == NULL) {
error("Unable to canonicalise path \"%s\"", dst);
error("Unable to canonicalize path \"%s\"", dst);
return -1;
}
ret = upload_dir_internal(conn, src, dst_canon, 0, preserve_flag,
print_flag);
print_flag, fsync_flag);
free(dst_canon);
return ret;
}

View File

@ -1,4 +1,4 @@
/* $OpenBSD: sftp-client.h,v 1.23 2013/10/11 02:53:45 djm Exp $ */
/* $OpenBSD: sftp-client.h,v 1.24 2013/10/17 00:30:13 djm Exp $ */
/*
* Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
@ -100,29 +100,33 @@ int do_hardlink(struct sftp_conn *, char *, char *);
/* Rename 'oldpath' to 'newpath' */
int do_symlink(struct sftp_conn *, char *, char *);
/* Call fsync() on open file 'handle' */
int do_fsync(struct sftp_conn *conn, char *, u_int);
/*
* Download 'remote_path' to 'local_path'. Preserve permissions and times
* if 'pflag' is set
*/
int do_download(struct sftp_conn *, char *, char *, Attrib *, int, int);
int do_download(struct sftp_conn *, char *, char *, Attrib *, int, int, int);
/*
* Recursively download 'remote_directory' to 'local_directory'. Preserve
* times if 'pflag' is set
*/
int download_dir(struct sftp_conn *, char *, char *, Attrib *, int, int, int);
int download_dir(struct sftp_conn *, char *, char *, Attrib *, int,
int, int, int);
/*
* Upload 'local_path' to 'remote_path'. Preserve permissions and times
* if 'pflag' is set
*/
int do_upload(struct sftp_conn *, char *, char *, int);
int do_upload(struct sftp_conn *, char *, char *, int, int);
/*
* Recursively upload 'local_directory' to 'remote_directory'. Preserve
* times if 'pflag' is set
*/
int upload_dir(struct sftp_conn *, char *, char *, int, int);
int upload_dir(struct sftp_conn *, char *, char *, int, int, int);
/* Concatenate paths, taking care of slashes. Caller must free result. */
char *path_append(char *, char *);

View File

@ -1,4 +1,4 @@
/* $OpenBSD: sftp-server.c,v 1.101 2013/10/14 23:28:23 djm Exp $ */
/* $OpenBSD: sftp-server.c,v 1.102 2013/10/17 00:30:13 djm Exp $ */
/*
* Copyright (c) 2000-2004 Markus Friedl. All rights reserved.
*
@ -112,6 +112,7 @@ static void process_extended_posix_rename(u_int32_t id);
static void process_extended_statvfs(u_int32_t id);
static void process_extended_fstatvfs(u_int32_t id);
static void process_extended_hardlink(u_int32_t id);
static void process_extended_fsync(u_int32_t id);
static void process_extended(u_int32_t id);
struct sftp_handler {
@ -152,6 +153,7 @@ struct sftp_handler extended_handlers[] = {
{ "statvfs", "statvfs@openssh.com", 0, process_extended_statvfs, 0 },
{ "fstatvfs", "fstatvfs@openssh.com", 0, process_extended_fstatvfs, 0 },
{ "hardlink", "hardlink@openssh.com", 0, process_extended_hardlink, 1 },
{ "fsync", "fsync@openssh.com", 0, process_extended_fsync, 1 },
{ NULL, NULL, 0, NULL, 0 }
};
@ -652,6 +654,9 @@ process_init(void)
/* hardlink extension */
buffer_put_cstring(&msg, "hardlink@openssh.com");
buffer_put_cstring(&msg, "1"); /* version */
/* fsync extension */
buffer_put_cstring(&msg, "fsync@openssh.com");
buffer_put_cstring(&msg, "1"); /* version */
send_msg(&msg);
buffer_free(&msg);
}
@ -1297,6 +1302,23 @@ process_extended_hardlink(u_int32_t id)
free(newpath);
}
static void
process_extended_fsync(u_int32_t id)
{
int handle, fd, ret, status = SSH2_FX_OP_UNSUPPORTED;
handle = get_handle();
debug3("request %u: fsync (handle %u)", id, handle);
verbose("fsync \"%s\"", handle_to_name(handle));
if ((fd = handle_to_fd(handle)) < 0)
status = SSH2_FX_NO_SUCH_FILE;
else if (handle_is_ok(handle, HANDLE_FILE)) {
ret = fsync(fd);
status = (ret == -1) ? errno_to_portable(errno) : SSH2_FX_OK;
}
send_status(id, status);
}
static void
process_extended(u_int32_t id)
{

29
sftp.1
View File

@ -1,4 +1,4 @@
.\" $OpenBSD: sftp.1,v 1.94 2013/08/07 06:24:51 jmc Exp $
.\" $OpenBSD: sftp.1,v 1.95 2013/10/17 00:30:13 djm Exp $
.\"
.\" Copyright (c) 2001 Damien Miller. All rights reserved.
.\"
@ -22,7 +22,7 @@
.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.\"
.Dd $Mdocdate: August 7 2013 $
.Dd $Mdocdate: October 17 2013 $
.Dt SFTP 1
.Os
.Sh NAME
@ -31,7 +31,7 @@
.Sh SYNOPSIS
.Nm sftp
.Bk -words
.Op Fl 1246aCpqrv
.Op Fl 1246aCfpqrv
.Op Fl B Ar buffer_size
.Op Fl b Ar batchfile
.Op Fl c Ar cipher
@ -164,6 +164,10 @@ per-user configuration file for
.Xr ssh 1 .
This option is directly passed to
.Xr ssh 1 .
.It Fl f
Requests that files be flushed to disk immediately after transfer.
When uploading files, this feature is only enabled if the server
implements the "fsync@openssh.com" extension.
.It Fl i Ar identity_file
Selects the file from which the identity (private key) for public key
authentication is read.
@ -348,7 +352,7 @@ extension.
Quit
.Nm sftp .
.It Xo Ic get
.Op Fl aPpr
.Op Fl afPpr
.Ar remote-path
.Op Ar local-path
.Xc
@ -376,6 +380,13 @@ the remote copy.
If the remote file contents differ from the partial local copy then the
resultant file is likely to be corrupt.
.Pp
If the
.Fl f
flag is specified, then
.Xr fsync 2
will ba called after the file transfer has completed to flush the file
to disk.
.Pp
If either the
.Fl P
or
@ -479,7 +490,7 @@ Create remote directory specified by
.It Ic progress
Toggle display of progress meter.
.It Xo Ic put
.Op Fl Ppr
.Op Fl fPpr
.Ar local-path
.Op Ar remote-path
.Xc
@ -498,6 +509,14 @@ is specified, then
.Ar remote-path
must specify a directory.
.Pp
If the
.Fl f
flag is specified, then a request will be sent to the server to call
.Xr fsync 2
after the file has been transferred.
Note that this is only supported by servers that implement
the "fsync@openssh.com" extension.
.Pp
If either the
.Fl P
or

65
sftp.c
View File

@ -1,4 +1,4 @@
/* $OpenBSD: sftp.c,v 1.155 2013/08/31 00:13:54 djm Exp $ */
/* $OpenBSD: sftp.c,v 1.156 2013/10/17 00:30:13 djm Exp $ */
/*
* Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
*
@ -94,6 +94,9 @@ int global_aflag = 0;
/* When this option is set, the file transfers will always preserve times */
int global_pflag = 0;
/* When this option is set, transfers will have fsync() called on each file */
int global_fflag = 0;
/* SIGINT received during command processing */
volatile sig_atomic_t interrupted = 0;
@ -359,7 +362,7 @@ make_absolute(char *p, char *pwd)
static int
parse_getput_flags(const char *cmd, char **argv, int argc,
int *aflag, int *pflag, int *rflag)
int *aflag, int *fflag, int *pflag, int *rflag)
{
extern int opterr, optind, optopt, optreset;
int ch;
@ -367,12 +370,15 @@ parse_getput_flags(const char *cmd, char **argv, int argc,
optind = optreset = 1;
opterr = 0;
*aflag = *rflag = *pflag = 0;
while ((ch = getopt(argc, argv, "aPpRr")) != -1) {
*aflag = *fflag = *rflag = *pflag = 0;
while ((ch = getopt(argc, argv, "afPpRr")) != -1) {
switch (ch) {
case 'a':
*aflag = 1;
break;
case 'f':
*fflag = 1;
break;
case 'p':
case 'P':
*pflag = 1;
@ -574,7 +580,7 @@ pathname_is_dir(char *pathname)
static int
process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd,
int pflag, int rflag, int resume)
int pflag, int rflag, int resume, int fflag)
{
char *abs_src = NULL;
char *abs_dst = NULL;
@ -633,11 +639,13 @@ process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd,
printf("Fetching %s to %s\n", g.gl_pathv[i], abs_dst);
if (pathname_is_dir(g.gl_pathv[i]) && (rflag || global_rflag)) {
if (download_dir(conn, g.gl_pathv[i], abs_dst, NULL,
pflag || global_pflag, 1, resume) == -1)
pflag || global_pflag, 1, resume,
fflag || global_fflag) == -1)
err = -1;
} else {
if (do_download(conn, g.gl_pathv[i], abs_dst, NULL,
pflag || global_pflag, resume) == -1)
pflag || global_pflag, resume,
fflag || global_fflag) == -1)
err = -1;
}
free(abs_dst);
@ -652,7 +660,7 @@ out:
static int
process_put(struct sftp_conn *conn, char *src, char *dst, char *pwd,
int pflag, int rflag)
int pflag, int rflag, int fflag)
{
char *tmp_dst = NULL;
char *abs_dst = NULL;
@ -719,11 +727,13 @@ process_put(struct sftp_conn *conn, char *src, char *dst, char *pwd,
printf("Uploading %s to %s\n", g.gl_pathv[i], abs_dst);
if (pathname_is_dir(g.gl_pathv[i]) && (rflag || global_rflag)) {
if (upload_dir(conn, g.gl_pathv[i], abs_dst,
pflag || global_pflag, 1) == -1)
pflag || global_pflag, 1,
fflag || global_fflag) == -1)
err = -1;
} else {
if (do_upload(conn, g.gl_pathv[i], abs_dst,
pflag || global_pflag) == -1)
pflag || global_pflag,
fflag || global_fflag) == -1)
err = -1;
}
}
@ -1176,9 +1186,9 @@ makeargv(const char *arg, int *argcp, int sloppy, char *lastquote,
}
static int
parse_args(const char **cpp, int *aflag, int *hflag, int *iflag, int *lflag,
int *pflag, int *rflag, int *sflag, unsigned long *n_arg,
char **path1, char **path2)
parse_args(const char **cpp, int *ignore_errors, int *aflag, int *fflag,
int *hflag, int *iflag, int *lflag, int *pflag, int *rflag, int *sflag,
unsigned long *n_arg, char **path1, char **path2)
{
const char *cmd, *cp = *cpp;
char *cp2, **argv;
@ -1190,9 +1200,9 @@ parse_args(const char **cpp, int *aflag, int *hflag, int *iflag, int *lflag,
cp = cp + strspn(cp, WHITESPACE);
/* Check for leading '-' (disable error processing) */
*iflag = 0;
*ignore_errors = 0;
if (*cp == '-') {
*iflag = 1;
*ignore_errors = 1;
cp++;
cp = cp + strspn(cp, WHITESPACE);
}
@ -1222,7 +1232,8 @@ parse_args(const char **cpp, int *aflag, int *hflag, int *iflag, int *lflag,
}
/* Get arguments and parse flags */
*aflag = *lflag = *pflag = *rflag = *hflag = *n_arg = 0;
*aflag = *fflag = *hflag = *iflag = *lflag = *pflag = 0;
*rflag = *sflag = 0;
*path1 = *path2 = NULL;
optidx = 1;
switch (cmdnum) {
@ -1230,7 +1241,7 @@ parse_args(const char **cpp, int *aflag, int *hflag, int *iflag, int *lflag,
case I_REGET:
case I_PUT:
if ((optidx = parse_getput_flags(cmd, argv, argc,
aflag, pflag, rflag)) == -1)
aflag, fflag, pflag, rflag)) == -1)
return -1;
/* Get first pathname (mandatory) */
if (argc - optidx < 1) {
@ -1371,8 +1382,8 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
int err_abort)
{
char *path1, *path2, *tmp;
int aflag = 0, hflag = 0, iflag = 0, lflag = 0, pflag = 0;
int rflag = 0, sflag = 0;
int ignore_errors = 0, aflag = 0, fflag = 0, hflag = 0, iflag = 0;
int lflag = 0, pflag = 0, rflag = 0, sflag = 0;
int cmdnum, i;
unsigned long n_arg = 0;
Attrib a, *aa;
@ -1381,9 +1392,9 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
glob_t g;
path1 = path2 = NULL;
cmdnum = parse_args(&cmd, &aflag, &hflag, &iflag, &lflag, &pflag,
&rflag, &sflag, &n_arg, &path1, &path2);
if (iflag != 0)
cmdnum = parse_args(&cmd, &ignore_errors, &aflag, &fflag, &hflag,
&iflag, &lflag, &pflag, &rflag, &sflag, &n_arg, &path1, &path2);
if (ignore_errors != 0)
err_abort = 0;
memset(&g, 0, sizeof(g));
@ -1402,10 +1413,11 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
/* FALLTHROUGH */
case I_GET:
err = process_get(conn, path1, path2, *pwd, pflag,
rflag, aflag);
rflag, aflag, fflag);
break;
case I_PUT:
err = process_put(conn, path1, path2, *pwd, pflag, rflag);
err = process_put(conn, path1, path2, *pwd, pflag,
rflag, fflag);
break;
case I_RENAME:
path1 = make_absolute(path1, *pwd);
@ -2231,7 +2243,7 @@ main(int argc, char **argv)
infile = stdin;
while ((ch = getopt(argc, argv,
"1246ahpqrvCc:D:i:l:o:s:S:b:B:F:P:R:")) != -1) {
"1246afhpqrvCc:D:i:l:o:s:S:b:B:F:P:R:")) != -1) {
switch (ch) {
/* Passed through to ssh(1) */
case '4':
@ -2291,6 +2303,9 @@ main(int argc, char **argv)
quiet = batchmode = 1;
addargs(&args, "-obatchmode yes");
break;
case 'f':
global_fflag = 1;
break;
case 'p':
global_pflag = 1;
break;