linux/tools/bpf/bpftool/main.c

413 lines
8.6 KiB
C
Raw Normal View History

/*
* Copyright (C) 2017-2018 Netronome Systems, Inc.
*
* This software is dual licensed under the GNU General License Version 2,
* June 1991 as shown in the file COPYING in the top-level directory of this
* source tree or the BSD 2-Clause License provided below. You have the
* option to license this software under the complete terms of either license.
*
* The BSD 2-Clause License:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* 1. Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* 2. Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <ctype.h>
#include <errno.h>
#include <getopt.h>
#include <linux/bpf.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <bpf.h>
#include "main.h"
#define BATCH_LINE_LEN_MAX 65536
#define BATCH_ARG_NB_MAX 4096
const char *bin_name;
static int last_argc;
static char **last_argv;
static int (*last_do_help)(int argc, char **argv);
json_writer_t *json_wtr;
bool pretty_output;
bool json_output;
bool show_pinned;
int bpf_flags;
tools: bpftool: show filenames of pinned objects Added support to show filenames of pinned objects. For example: root@test# ./bpftool prog 3: tracepoint name tracepoint__irq tag f677a7dd722299a3 loaded_at Oct 26/11:39 uid 0 xlated 160B not jited memlock 4096B map_ids 4 pinned /sys/fs/bpf/softirq_prog 4: tracepoint name tracepoint__irq tag ea5dc530d00b92b6 loaded_at Oct 26/11:39 uid 0 xlated 392B not jited memlock 4096B map_ids 4,6 root@test# ./bpftool --json --pretty prog [{ "id": 3, "type": "tracepoint", "name": "tracepoint__irq", "tag": "f677a7dd722299a3", "loaded_at": "Oct 26/11:39", "uid": 0, "bytes_xlated": 160, "jited": false, "bytes_memlock": 4096, "map_ids": [4 ], "pinned": ["/sys/fs/bpf/softirq_prog" ] },{ "id": 4, "type": "tracepoint", "name": "tracepoint__irq", "tag": "ea5dc530d00b92b6", "loaded_at": "Oct 26/11:39", "uid": 0, "bytes_xlated": 392, "jited": false, "bytes_memlock": 4096, "map_ids": [4,6 ], "pinned": [] } ] root@test# ./bpftool map 4: hash name start flags 0x0 key 4B value 16B max_entries 10240 memlock 1003520B pinned /sys/fs/bpf/softirq_map1 5: hash name iptr flags 0x0 key 4B value 8B max_entries 10240 memlock 921600B root@test# ./bpftool --json --pretty map [{ "id": 4, "type": "hash", "name": "start", "flags": 0, "bytes_key": 4, "bytes_value": 16, "max_entries": 10240, "bytes_memlock": 1003520, "pinned": ["/sys/fs/bpf/softirq_map1" ] },{ "id": 5, "type": "hash", "name": "iptr", "flags": 0, "bytes_key": 4, "bytes_value": 8, "max_entries": 10240, "bytes_memlock": 921600, "pinned": [] } ] Signed-off-by: Prashant Bhole <bhole_prashant_q7@lab.ntt.co.jp> Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-08 12:55:48 +08:00
struct pinned_obj_table prog_table;
struct pinned_obj_table map_table;
static void __noreturn clean_and_exit(int i)
{
if (json_output)
jsonw_destroy(&json_wtr);
exit(i);
}
void usage(void)
{
last_do_help(last_argc - 1, last_argv + 1);
clean_and_exit(-1);
}
static int do_help(int argc, char **argv)
{
if (json_output) {
jsonw_null(json_wtr);
return 0;
}
fprintf(stderr,
"Usage: %s [OPTIONS] OBJECT { COMMAND | help }\n"
" %s batch file FILE\n"
" %s version\n"
"\n"
tools/bpf: bpftool: add net support Add "bpftool net" support. Networking devices are enumerated to dump device index/name associated with xdp progs. For each networking device, tc classes and qdiscs are enumerated in order to check their bpf filters. In addition, root handle and clsact ingress/egress are also checked for bpf filters. Not all filter information is printed out. Only ifindex, kind, filter name, prog_id and tag are printed out, which are good enough to show attachment information. If the filter action is a bpf action, its bpf program id, bpf name and tag will be printed out as well. For example, $ ./bpftool net xdp [ ifindex 2 devname eth0 prog_id 198 ] tc_filters [ ifindex 2 kind qdisc_htb name prefix_matcher.o:[cls_prefix_matcher_htb] prog_id 111727 tag d08fe3b4319bc2fd act [] ifindex 2 kind qdisc_clsact_ingress name fbflow_icmp prog_id 130246 tag 3f265c7f26db62c9 act [] ifindex 2 kind qdisc_clsact_egress name prefix_matcher.o:[cls_prefix_matcher_clsact] prog_id 111726 tag 99a197826974c876 ifindex 2 kind qdisc_clsact_egress name cls_fg_dscp prog_id 108619 tag dc4630674fd72dcc act [] ifindex 2 kind qdisc_clsact_egress name fbflow_egress prog_id 130245 tag 72d2d830d6888d2c ] $ ./bpftool -jp net [{ "xdp": [{ "ifindex": 2, "devname": "eth0", "prog_id": 198 } ], "tc_filters": [{ "ifindex": 2, "kind": "qdisc_htb", "name": "prefix_matcher.o:[cls_prefix_matcher_htb]", "prog_id": 111727, "tag": "d08fe3b4319bc2fd", "act": [] },{ "ifindex": 2, "kind": "qdisc_clsact_ingress", "name": "fbflow_icmp", "prog_id": 130246, "tag": "3f265c7f26db62c9", "act": [] },{ "ifindex": 2, "kind": "qdisc_clsact_egress", "name": "prefix_matcher.o:[cls_prefix_matcher_clsact]", "prog_id": 111726, "tag": "99a197826974c876" },{ "ifindex": 2, "kind": "qdisc_clsact_egress", "name": "cls_fg_dscp", "prog_id": 108619, "tag": "dc4630674fd72dcc", "act": [] },{ "ifindex": 2, "kind": "qdisc_clsact_egress", "name": "fbflow_egress", "prog_id": 130245, "tag": "72d2d830d6888d2c" } ] } ] Signed-off-by: Yonghong Song <yhs@fb.com> Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2018-09-06 07:58:06 +08:00
" OBJECT := { prog | map | cgroup | perf | net }\n"
" " HELP_SPEC_OPTIONS "\n"
"",
bin_name, bin_name, bin_name);
return 0;
}
static int do_version(int argc, char **argv)
{
if (json_output) {
jsonw_start_object(json_wtr);
jsonw_name(json_wtr, "version");
jsonw_printf(json_wtr, "\"%s\"", BPFTOOL_VERSION);
jsonw_end_object(json_wtr);
} else {
printf("%s v%s\n", bin_name, BPFTOOL_VERSION);
}
return 0;
}
int cmd_select(const struct cmd *cmds, int argc, char **argv,
int (*help)(int argc, char **argv))
{
unsigned int i;
last_argc = argc;
last_argv = argv;
last_do_help = help;
if (argc < 1 && cmds[0].func)
return cmds[0].func(argc, argv);
for (i = 0; cmds[i].func; i++)
if (is_prefix(*argv, cmds[i].cmd))
return cmds[i].func(argc - 1, argv + 1);
help(argc - 1, argv + 1);
return -1;
}
bool is_prefix(const char *pfx, const char *str)
{
if (!pfx)
return false;
if (strlen(str) < strlen(pfx))
return false;
return !memcmp(str, pfx, strlen(pfx));
}
void fprint_hex(FILE *f, void *arg, unsigned int n, const char *sep)
{
unsigned char *data = arg;
unsigned int i;
for (i = 0; i < n; i++) {
const char *pfx = "";
if (!i)
/* nothing */;
else if (!(i % 16))
fprintf(f, "\n");
else if (!(i % 8))
fprintf(f, " ");
else
pfx = sep;
fprintf(f, "%s%02hhx", i ? pfx : "", data[i]);
}
}
/* Split command line into argument vector. */
static int make_args(char *line, char *n_argv[], int maxargs, int cmd_nb)
{
static const char ws[] = " \t\r\n";
char *cp = line;
int n_argc = 0;
while (*cp) {
/* Skip leading whitespace. */
cp += strspn(cp, ws);
if (*cp == '\0')
break;
if (n_argc >= (maxargs - 1)) {
p_err("too many arguments to command %d", cmd_nb);
return -1;
}
/* Word begins with quote. */
if (*cp == '\'' || *cp == '"') {
char quote = *cp++;
n_argv[n_argc++] = cp;
/* Find ending quote. */
cp = strchr(cp, quote);
if (!cp) {
p_err("unterminated quoted string in command %d",
cmd_nb);
return -1;
}
} else {
n_argv[n_argc++] = cp;
/* Find end of word. */
cp += strcspn(cp, ws);
if (*cp == '\0')
break;
}
/* Separate words. */
*cp++ = 0;
}
n_argv[n_argc] = NULL;
return n_argc;
}
static int do_batch(int argc, char **argv);
static const struct cmd cmds[] = {
{ "help", do_help },
{ "batch", do_batch },
{ "prog", do_prog },
{ "map", do_map },
{ "cgroup", do_cgroup },
tools/bpftool: add perf subcommand The new command "bpftool perf [show | list]" will traverse all processes under /proc, and if any fd is associated with a perf event, it will print out related perf event information. Documentation is also added. Below is an example to show the results using bcc commands. Running the following 4 bcc commands: kprobe: trace.py '__x64_sys_nanosleep' kretprobe: trace.py 'r::__x64_sys_nanosleep' tracepoint: trace.py 't:syscalls:sys_enter_nanosleep' uprobe: trace.py 'p:/home/yhs/a.out:main' The bpftool command line and result: $ bpftool perf pid 21711 fd 5: prog_id 5 kprobe func __x64_sys_write offset 0 pid 21765 fd 5: prog_id 7 kretprobe func __x64_sys_nanosleep offset 0 pid 21767 fd 5: prog_id 8 tracepoint sys_enter_nanosleep pid 21800 fd 5: prog_id 9 uprobe filename /home/yhs/a.out offset 1159 $ bpftool -j perf [{"pid":21711,"fd":5,"prog_id":5,"fd_type":"kprobe","func":"__x64_sys_write","offset":0}, \ {"pid":21765,"fd":5,"prog_id":7,"fd_type":"kretprobe","func":"__x64_sys_nanosleep","offset":0}, \ {"pid":21767,"fd":5,"prog_id":8,"fd_type":"tracepoint","tracepoint":"sys_enter_nanosleep"}, \ {"pid":21800,"fd":5,"prog_id":9,"fd_type":"uprobe","filename":"/home/yhs/a.out","offset":1159}] $ bpftool prog 5: kprobe name probe___x64_sys tag e495a0c82f2c7a8d gpl loaded_at 2018-05-15T04:46:37-0700 uid 0 xlated 200B not jited memlock 4096B map_ids 4 7: kprobe name probe___x64_sys tag f2fdee479a503abf gpl loaded_at 2018-05-15T04:48:32-0700 uid 0 xlated 200B not jited memlock 4096B map_ids 7 8: tracepoint name tracepoint__sys tag 5390badef2395fcf gpl loaded_at 2018-05-15T04:48:48-0700 uid 0 xlated 200B not jited memlock 4096B map_ids 8 9: kprobe name probe_main_1 tag 0a87bdc2e2953b6d gpl loaded_at 2018-05-15T04:49:52-0700 uid 0 xlated 200B not jited memlock 4096B map_ids 9 $ ps ax | grep "python ./trace.py" 21711 pts/0 T 0:03 python ./trace.py __x64_sys_write 21765 pts/0 S+ 0:00 python ./trace.py r::__x64_sys_nanosleep 21767 pts/2 S+ 0:00 python ./trace.py t:syscalls:sys_enter_nanosleep 21800 pts/3 S+ 0:00 python ./trace.py p:/home/yhs/a.out:main 22374 pts/1 S+ 0:00 grep --color=auto python ./trace.py Reviewed-by: Jakub Kicinski <jakub.kicinski@netronome.com> Signed-off-by: Yonghong Song <yhs@fb.com> Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2018-05-25 02:21:58 +08:00
{ "perf", do_perf },
tools/bpf: bpftool: add net support Add "bpftool net" support. Networking devices are enumerated to dump device index/name associated with xdp progs. For each networking device, tc classes and qdiscs are enumerated in order to check their bpf filters. In addition, root handle and clsact ingress/egress are also checked for bpf filters. Not all filter information is printed out. Only ifindex, kind, filter name, prog_id and tag are printed out, which are good enough to show attachment information. If the filter action is a bpf action, its bpf program id, bpf name and tag will be printed out as well. For example, $ ./bpftool net xdp [ ifindex 2 devname eth0 prog_id 198 ] tc_filters [ ifindex 2 kind qdisc_htb name prefix_matcher.o:[cls_prefix_matcher_htb] prog_id 111727 tag d08fe3b4319bc2fd act [] ifindex 2 kind qdisc_clsact_ingress name fbflow_icmp prog_id 130246 tag 3f265c7f26db62c9 act [] ifindex 2 kind qdisc_clsact_egress name prefix_matcher.o:[cls_prefix_matcher_clsact] prog_id 111726 tag 99a197826974c876 ifindex 2 kind qdisc_clsact_egress name cls_fg_dscp prog_id 108619 tag dc4630674fd72dcc act [] ifindex 2 kind qdisc_clsact_egress name fbflow_egress prog_id 130245 tag 72d2d830d6888d2c ] $ ./bpftool -jp net [{ "xdp": [{ "ifindex": 2, "devname": "eth0", "prog_id": 198 } ], "tc_filters": [{ "ifindex": 2, "kind": "qdisc_htb", "name": "prefix_matcher.o:[cls_prefix_matcher_htb]", "prog_id": 111727, "tag": "d08fe3b4319bc2fd", "act": [] },{ "ifindex": 2, "kind": "qdisc_clsact_ingress", "name": "fbflow_icmp", "prog_id": 130246, "tag": "3f265c7f26db62c9", "act": [] },{ "ifindex": 2, "kind": "qdisc_clsact_egress", "name": "prefix_matcher.o:[cls_prefix_matcher_clsact]", "prog_id": 111726, "tag": "99a197826974c876" },{ "ifindex": 2, "kind": "qdisc_clsact_egress", "name": "cls_fg_dscp", "prog_id": 108619, "tag": "dc4630674fd72dcc", "act": [] },{ "ifindex": 2, "kind": "qdisc_clsact_egress", "name": "fbflow_egress", "prog_id": 130245, "tag": "72d2d830d6888d2c" } ] } ] Signed-off-by: Yonghong Song <yhs@fb.com> Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2018-09-06 07:58:06 +08:00
{ "net", do_net },
{ "version", do_version },
{ 0 }
};
static int do_batch(int argc, char **argv)
{
char buf[BATCH_LINE_LEN_MAX], contline[BATCH_LINE_LEN_MAX];
char *n_argv[BATCH_ARG_NB_MAX];
unsigned int lines = 0;
int n_argc;
FILE *fp;
char *cp;
int err;
int i;
if (argc < 2) {
p_err("too few parameters for batch");
return -1;
} else if (!is_prefix(*argv, "file")) {
p_err("expected 'file', got: %s", *argv);
return -1;
} else if (argc > 2) {
p_err("too many parameters for batch");
return -1;
}
NEXT_ARG();
if (!strcmp(*argv, "-"))
fp = stdin;
else
fp = fopen(*argv, "r");
if (!fp) {
p_err("Can't open file (%s): %s", *argv, strerror(errno));
return -1;
}
if (json_output)
jsonw_start_array(json_wtr);
while (fgets(buf, sizeof(buf), fp)) {
cp = strchr(buf, '#');
if (cp)
*cp = '\0';
if (strlen(buf) == sizeof(buf) - 1) {
errno = E2BIG;
break;
}
/* Append continuation lines if any (coming after a line ending
* with '\' in the batch file).
*/
while ((cp = strstr(buf, "\\\n")) != NULL) {
if (!fgets(contline, sizeof(contline), fp) ||
strlen(contline) == 0) {
p_err("missing continuation line on command %d",
lines);
err = -1;
goto err_close;
}
cp = strchr(contline, '#');
if (cp)
*cp = '\0';
if (strlen(buf) + strlen(contline) + 1 > sizeof(buf)) {
p_err("command %d is too long", lines);
err = -1;
goto err_close;
}
buf[strlen(buf) - 2] = '\0';
strcat(buf, contline);
}
n_argc = make_args(buf, n_argv, BATCH_ARG_NB_MAX, lines);
if (!n_argc)
continue;
if (n_argc < 0)
goto err_close;
if (json_output) {
jsonw_start_object(json_wtr);
jsonw_name(json_wtr, "command");
jsonw_start_array(json_wtr);
for (i = 0; i < n_argc; i++)
jsonw_string(json_wtr, n_argv[i]);
jsonw_end_array(json_wtr);
jsonw_name(json_wtr, "output");
}
err = cmd_select(cmds, n_argc, n_argv, do_help);
if (json_output)
jsonw_end_object(json_wtr);
if (err)
goto err_close;
lines++;
}
if (errno && errno != ENOENT) {
p_err("reading batch file failed: %s", strerror(errno));
err = -1;
} else {
if (!json_output)
printf("processed %d commands\n", lines);
err = 0;
}
err_close:
if (fp != stdin)
fclose(fp);
if (json_output)
jsonw_end_array(json_wtr);
return err;
}
int main(int argc, char **argv)
{
static const struct option options[] = {
{ "json", no_argument, NULL, 'j' },
{ "help", no_argument, NULL, 'h' },
{ "pretty", no_argument, NULL, 'p' },
{ "version", no_argument, NULL, 'V' },
{ "bpffs", no_argument, NULL, 'f' },
{ "mapcompat", no_argument, NULL, 'm' },
{ 0 }
};
int opt, ret;
last_do_help = do_help;
pretty_output = false;
json_output = false;
show_pinned = false;
bin_name = argv[0];
tools: bpftool: show filenames of pinned objects Added support to show filenames of pinned objects. For example: root@test# ./bpftool prog 3: tracepoint name tracepoint__irq tag f677a7dd722299a3 loaded_at Oct 26/11:39 uid 0 xlated 160B not jited memlock 4096B map_ids 4 pinned /sys/fs/bpf/softirq_prog 4: tracepoint name tracepoint__irq tag ea5dc530d00b92b6 loaded_at Oct 26/11:39 uid 0 xlated 392B not jited memlock 4096B map_ids 4,6 root@test# ./bpftool --json --pretty prog [{ "id": 3, "type": "tracepoint", "name": "tracepoint__irq", "tag": "f677a7dd722299a3", "loaded_at": "Oct 26/11:39", "uid": 0, "bytes_xlated": 160, "jited": false, "bytes_memlock": 4096, "map_ids": [4 ], "pinned": ["/sys/fs/bpf/softirq_prog" ] },{ "id": 4, "type": "tracepoint", "name": "tracepoint__irq", "tag": "ea5dc530d00b92b6", "loaded_at": "Oct 26/11:39", "uid": 0, "bytes_xlated": 392, "jited": false, "bytes_memlock": 4096, "map_ids": [4,6 ], "pinned": [] } ] root@test# ./bpftool map 4: hash name start flags 0x0 key 4B value 16B max_entries 10240 memlock 1003520B pinned /sys/fs/bpf/softirq_map1 5: hash name iptr flags 0x0 key 4B value 8B max_entries 10240 memlock 921600B root@test# ./bpftool --json --pretty map [{ "id": 4, "type": "hash", "name": "start", "flags": 0, "bytes_key": 4, "bytes_value": 16, "max_entries": 10240, "bytes_memlock": 1003520, "pinned": ["/sys/fs/bpf/softirq_map1" ] },{ "id": 5, "type": "hash", "name": "iptr", "flags": 0, "bytes_key": 4, "bytes_value": 8, "max_entries": 10240, "bytes_memlock": 921600, "pinned": [] } ] Signed-off-by: Prashant Bhole <bhole_prashant_q7@lab.ntt.co.jp> Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-08 12:55:48 +08:00
hash_init(prog_table.table);
hash_init(map_table.table);
opterr = 0;
while ((opt = getopt_long(argc, argv, "Vhpjfm",
options, NULL)) >= 0) {
switch (opt) {
case 'V':
return do_version(argc, argv);
case 'h':
return do_help(argc, argv);
case 'p':
pretty_output = true;
/* fall through */
case 'j':
if (!json_output) {
json_wtr = jsonw_new(stdout);
if (!json_wtr) {
p_err("failed to create JSON writer");
return -1;
}
json_output = true;
}
jsonw_pretty(json_wtr, pretty_output);
break;
case 'f':
show_pinned = true;
break;
case 'm':
bpf_flags = MAPS_RELAX_COMPAT;
break;
default:
p_err("unrecognized option '%s'", argv[optind - 1]);
if (json_output)
clean_and_exit(-1);
else
usage();
}
}
argc -= optind;
argv += optind;
if (argc < 0)
usage();
ret = cmd_select(cmds, argc, argv, do_help);
if (json_output)
jsonw_destroy(&json_wtr);
if (show_pinned) {
delete_pinned_obj_table(&prog_table);
delete_pinned_obj_table(&map_table);
}
tools: bpftool: show filenames of pinned objects Added support to show filenames of pinned objects. For example: root@test# ./bpftool prog 3: tracepoint name tracepoint__irq tag f677a7dd722299a3 loaded_at Oct 26/11:39 uid 0 xlated 160B not jited memlock 4096B map_ids 4 pinned /sys/fs/bpf/softirq_prog 4: tracepoint name tracepoint__irq tag ea5dc530d00b92b6 loaded_at Oct 26/11:39 uid 0 xlated 392B not jited memlock 4096B map_ids 4,6 root@test# ./bpftool --json --pretty prog [{ "id": 3, "type": "tracepoint", "name": "tracepoint__irq", "tag": "f677a7dd722299a3", "loaded_at": "Oct 26/11:39", "uid": 0, "bytes_xlated": 160, "jited": false, "bytes_memlock": 4096, "map_ids": [4 ], "pinned": ["/sys/fs/bpf/softirq_prog" ] },{ "id": 4, "type": "tracepoint", "name": "tracepoint__irq", "tag": "ea5dc530d00b92b6", "loaded_at": "Oct 26/11:39", "uid": 0, "bytes_xlated": 392, "jited": false, "bytes_memlock": 4096, "map_ids": [4,6 ], "pinned": [] } ] root@test# ./bpftool map 4: hash name start flags 0x0 key 4B value 16B max_entries 10240 memlock 1003520B pinned /sys/fs/bpf/softirq_map1 5: hash name iptr flags 0x0 key 4B value 8B max_entries 10240 memlock 921600B root@test# ./bpftool --json --pretty map [{ "id": 4, "type": "hash", "name": "start", "flags": 0, "bytes_key": 4, "bytes_value": 16, "max_entries": 10240, "bytes_memlock": 1003520, "pinned": ["/sys/fs/bpf/softirq_map1" ] },{ "id": 5, "type": "hash", "name": "iptr", "flags": 0, "bytes_key": 4, "bytes_value": 8, "max_entries": 10240, "bytes_memlock": 921600, "pinned": [] } ] Signed-off-by: Prashant Bhole <bhole_prashant_q7@lab.ntt.co.jp> Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-08 12:55:48 +08:00
return ret;
}