mirror of
https://github.com/git/git.git
synced 2024-11-27 12:03:55 +08:00
Merge branch 'it/fetch-pack-many-refs'
When "git fetch" encounters repositories with too many references, the command line of "fetch-pack" that is run by a helper e.g. remote-curl, may fail to hold all of them. Now such an internal invocation can feed the references through the standard input of "fetch-pack". By Ivan Todoroski * it/fetch-pack-many-refs: remote-curl: main test case for the OS command line overflow fetch-pack: test cases for the new --stdin option remote-curl: send the refs to fetch-pack on stdin fetch-pack: new --stdin option to read refs from stdin
This commit is contained in:
commit
77cab8af4a
@ -32,6 +32,16 @@ OPTIONS
|
||||
--all::
|
||||
Fetch all remote refs.
|
||||
|
||||
--stdin::
|
||||
Take the list of refs from stdin, one per line. If there
|
||||
are refs specified on the command line in addition to this
|
||||
option, then the refs from stdin are processed after those
|
||||
on the command line.
|
||||
+
|
||||
If '--stateless-rpc' is specified together with this option then
|
||||
the list of refs must be in packet format (pkt-line). Each ref must
|
||||
be in a separate packet, and the list must end with a flush packet.
|
||||
|
||||
-q::
|
||||
--quiet::
|
||||
Pass '-q' flag to 'git unpack-objects'; this makes the
|
||||
|
@ -23,7 +23,9 @@ static struct fetch_pack_args args = {
|
||||
};
|
||||
|
||||
static const char fetch_pack_usage[] =
|
||||
"git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
|
||||
"git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
|
||||
"[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
|
||||
"[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
|
||||
|
||||
#define COMPLETE (1U << 0)
|
||||
#define COMMON (1U << 1)
|
||||
@ -942,6 +944,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
|
||||
args.fetch_all = 1;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp("--stdin", arg)) {
|
||||
args.stdin_refs = 1;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp("-v", arg)) {
|
||||
args.verbose = 1;
|
||||
continue;
|
||||
@ -973,6 +979,40 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
|
||||
if (!dest)
|
||||
usage(fetch_pack_usage);
|
||||
|
||||
if (args.stdin_refs) {
|
||||
/*
|
||||
* Copy refs from cmdline to new growable list, then
|
||||
* append the refs from the standard input.
|
||||
*/
|
||||
int alloc_heads = nr_heads;
|
||||
int size = nr_heads * sizeof(*heads);
|
||||
heads = memcpy(xmalloc(size), heads, size);
|
||||
if (args.stateless_rpc) {
|
||||
/* in stateless RPC mode we use pkt-line to read
|
||||
* from stdin, until we get a flush packet
|
||||
*/
|
||||
static char line[1000];
|
||||
for (;;) {
|
||||
int n = packet_read_line(0, line, sizeof(line));
|
||||
if (!n)
|
||||
break;
|
||||
if (line[n-1] == '\n')
|
||||
n--;
|
||||
ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
|
||||
heads[nr_heads++] = xmemdupz(line, n);
|
||||
}
|
||||
}
|
||||
else {
|
||||
/* read from stdin one ref per line, until EOF */
|
||||
struct strbuf line = STRBUF_INIT;
|
||||
while (strbuf_getline(&line, stdin, '\n') != EOF) {
|
||||
ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
|
||||
heads[nr_heads++] = strbuf_detach(&line, NULL);
|
||||
}
|
||||
strbuf_release(&line);
|
||||
}
|
||||
}
|
||||
|
||||
if (args.stateless_rpc) {
|
||||
conn = NULL;
|
||||
fd[0] = 0;
|
||||
|
@ -10,6 +10,7 @@ struct fetch_pack_args {
|
||||
lock_pack:1,
|
||||
use_thin_pack:1,
|
||||
fetch_all:1,
|
||||
stdin_refs:1,
|
||||
verbose:1,
|
||||
no_progress:1,
|
||||
include_tag:1,
|
||||
|
@ -290,6 +290,7 @@ static void output_refs(struct ref *refs)
|
||||
struct rpc_state {
|
||||
const char *service_name;
|
||||
const char **argv;
|
||||
struct strbuf *stdin_preamble;
|
||||
char *service_url;
|
||||
char *hdr_content_type;
|
||||
char *hdr_accept;
|
||||
@ -535,6 +536,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
|
||||
{
|
||||
const char *svc = rpc->service_name;
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
struct strbuf *preamble = rpc->stdin_preamble;
|
||||
struct child_process client;
|
||||
int err = 0;
|
||||
|
||||
@ -545,6 +547,8 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
|
||||
client.argv = rpc->argv;
|
||||
if (start_command(&client))
|
||||
exit(1);
|
||||
if (preamble)
|
||||
write_or_die(client.in, preamble->buf, preamble->len);
|
||||
if (heads)
|
||||
write_or_die(client.in, heads->buf, heads->len);
|
||||
|
||||
@ -626,13 +630,14 @@ static int fetch_git(struct discovery *heads,
|
||||
int nr_heads, struct ref **to_fetch)
|
||||
{
|
||||
struct rpc_state rpc;
|
||||
struct strbuf preamble = STRBUF_INIT;
|
||||
char *depth_arg = NULL;
|
||||
const char **argv;
|
||||
int argc = 0, i, err;
|
||||
const char *argv[15];
|
||||
|
||||
argv = xmalloc((15 + nr_heads) * sizeof(char*));
|
||||
argv[argc++] = "fetch-pack";
|
||||
argv[argc++] = "--stateless-rpc";
|
||||
argv[argc++] = "--stdin";
|
||||
argv[argc++] = "--lock-pack";
|
||||
if (options.followtags)
|
||||
argv[argc++] = "--include-tag";
|
||||
@ -651,24 +656,27 @@ static int fetch_git(struct discovery *heads,
|
||||
argv[argc++] = depth_arg;
|
||||
}
|
||||
argv[argc++] = url;
|
||||
argv[argc++] = NULL;
|
||||
|
||||
for (i = 0; i < nr_heads; i++) {
|
||||
struct ref *ref = to_fetch[i];
|
||||
if (!ref->name || !*ref->name)
|
||||
die("cannot fetch by sha1 over smart http");
|
||||
argv[argc++] = ref->name;
|
||||
packet_buf_write(&preamble, "%s\n", ref->name);
|
||||
}
|
||||
argv[argc++] = NULL;
|
||||
packet_buf_flush(&preamble);
|
||||
|
||||
memset(&rpc, 0, sizeof(rpc));
|
||||
rpc.service_name = "git-upload-pack",
|
||||
rpc.argv = argv;
|
||||
rpc.stdin_preamble = &preamble;
|
||||
rpc.gzip_request = 1;
|
||||
|
||||
err = rpc_service(&rpc, heads);
|
||||
if (rpc.result.len)
|
||||
safe_write(1, rpc.result.buf, rpc.result.len);
|
||||
strbuf_release(&rpc.result);
|
||||
free(argv);
|
||||
strbuf_release(&preamble);
|
||||
free(depth_arg);
|
||||
return err;
|
||||
}
|
||||
|
@ -326,4 +326,70 @@ EOF
|
||||
test_cmp count7.expected count7.actual
|
||||
'
|
||||
|
||||
test_expect_success 'setup tests for the --stdin parameter' '
|
||||
for head in C D E F
|
||||
do
|
||||
add $head
|
||||
done &&
|
||||
for head in A B C D E F
|
||||
do
|
||||
git tag $head $head
|
||||
done &&
|
||||
cat >input <<-\EOF
|
||||
refs/heads/C
|
||||
refs/heads/A
|
||||
refs/heads/D
|
||||
refs/tags/C
|
||||
refs/heads/B
|
||||
refs/tags/A
|
||||
refs/heads/E
|
||||
refs/tags/B
|
||||
refs/tags/E
|
||||
refs/tags/D
|
||||
EOF
|
||||
sort <input >expect &&
|
||||
(
|
||||
echo refs/heads/E &&
|
||||
echo refs/tags/E &&
|
||||
cat input
|
||||
) >input.dup
|
||||
'
|
||||
|
||||
test_expect_success 'fetch refs from cmdline' '
|
||||
(
|
||||
cd client &&
|
||||
git fetch-pack --no-progress .. $(cat ../input)
|
||||
) >output &&
|
||||
cut -d " " -f 2 <output | sort >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success 'fetch refs from stdin' '
|
||||
(
|
||||
cd client &&
|
||||
git fetch-pack --stdin --no-progress .. <../input
|
||||
) >output &&
|
||||
cut -d " " -f 2 <output | sort >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success 'fetch mixed refs from cmdline and stdin' '
|
||||
(
|
||||
cd client &&
|
||||
tail -n +5 ../input |
|
||||
git fetch-pack --stdin --no-progress .. $(head -n 4 ../input)
|
||||
) >output &&
|
||||
cut -d " " -f 2 <output | sort >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success 'test duplicate refs from stdin' '
|
||||
(
|
||||
cd client &&
|
||||
test_must_fail git fetch-pack --stdin --no-progress .. <../input.dup
|
||||
) >output &&
|
||||
cut -d " " -f 2 <output | sort >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_done
|
||||
|
@ -109,5 +109,36 @@ test_expect_success 'follow redirects (302)' '
|
||||
git clone $HTTPD_URL/smart-redir-temp/repo.git --quiet repo-t
|
||||
'
|
||||
|
||||
test -n "$GIT_TEST_LONG" && test_set_prereq EXPENSIVE
|
||||
|
||||
test_expect_success EXPENSIVE 'create 50,000 tags in the repo' '
|
||||
(
|
||||
cd "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
|
||||
for i in `seq 50000`
|
||||
do
|
||||
echo "commit refs/heads/too-many-refs"
|
||||
echo "mark :$i"
|
||||
echo "committer git <git@example.com> $i +0000"
|
||||
echo "data 0"
|
||||
echo "M 644 inline bla.txt"
|
||||
echo "data 4"
|
||||
echo "bla"
|
||||
# make every commit dangling by always
|
||||
# rewinding the branch after each commit
|
||||
echo "reset refs/heads/too-many-refs"
|
||||
echo "from :1"
|
||||
done | git fast-import --export-marks=marks &&
|
||||
|
||||
# now assign tags to all the dangling commits we created above
|
||||
tag=$(perl -e "print \"bla\" x 30") &&
|
||||
sed -e "s/^:\(.\+\) \(.\+\)$/\2 refs\/tags\/$tag-\1/" <marks >>packed-refs
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success EXPENSIVE 'clone the 50,000 tag repo to check OS command line overflow' '
|
||||
git clone $HTTPD_URL/smart/repo.git too-many-refs 2>err &&
|
||||
test_line_count = 0 err
|
||||
'
|
||||
|
||||
stop_httpd
|
||||
test_done
|
||||
|
Loading…
Reference in New Issue
Block a user