Merge branch 'it/fetch-pack-many-refs'

When "git fetch" encounters repositories with too many references, the
command line of "fetch-pack" that is run by a helper e.g. remote-curl,
may fail to hold all of them. Now such an internal invocation can feed
the references through the standard input of "fetch-pack".

By Ivan Todoroski
* it/fetch-pack-many-refs:
  remote-curl: main test case for the OS command line overflow
  fetch-pack: test cases for the new --stdin option
  remote-curl: send the refs to fetch-pack on stdin
  fetch-pack: new --stdin option to read refs from stdin
This commit is contained in:
Junio C Hamano 2012-04-24 14:40:51 -07:00
commit 77cab8af4a
6 changed files with 162 additions and 6 deletions

View File

@ -32,6 +32,16 @@ OPTIONS
--all::
Fetch all remote refs.
--stdin::
Take the list of refs from stdin, one per line. If there
are refs specified on the command line in addition to this
option, then the refs from stdin are processed after those
on the command line.
+
If '--stateless-rpc' is specified together with this option then
the list of refs must be in packet format (pkt-line). Each ref must
be in a separate packet, and the list must end with a flush packet.
-q::
--quiet::
Pass '-q' flag to 'git unpack-objects'; this makes the

View File

@ -23,7 +23,9 @@ static struct fetch_pack_args args = {
};
static const char fetch_pack_usage[] =
"git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
"git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
"[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
"[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
#define COMPLETE (1U << 0)
#define COMMON (1U << 1)
@ -942,6 +944,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
args.fetch_all = 1;
continue;
}
if (!strcmp("--stdin", arg)) {
args.stdin_refs = 1;
continue;
}
if (!strcmp("-v", arg)) {
args.verbose = 1;
continue;
@ -973,6 +979,40 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
if (!dest)
usage(fetch_pack_usage);
if (args.stdin_refs) {
/*
* Copy refs from cmdline to new growable list, then
* append the refs from the standard input.
*/
int alloc_heads = nr_heads;
int size = nr_heads * sizeof(*heads);
heads = memcpy(xmalloc(size), heads, size);
if (args.stateless_rpc) {
/* in stateless RPC mode we use pkt-line to read
* from stdin, until we get a flush packet
*/
static char line[1000];
for (;;) {
int n = packet_read_line(0, line, sizeof(line));
if (!n)
break;
if (line[n-1] == '\n')
n--;
ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
heads[nr_heads++] = xmemdupz(line, n);
}
}
else {
/* read from stdin one ref per line, until EOF */
struct strbuf line = STRBUF_INIT;
while (strbuf_getline(&line, stdin, '\n') != EOF) {
ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
heads[nr_heads++] = strbuf_detach(&line, NULL);
}
strbuf_release(&line);
}
}
if (args.stateless_rpc) {
conn = NULL;
fd[0] = 0;

View File

@ -10,6 +10,7 @@ struct fetch_pack_args {
lock_pack:1,
use_thin_pack:1,
fetch_all:1,
stdin_refs:1,
verbose:1,
no_progress:1,
include_tag:1,

View File

@ -290,6 +290,7 @@ static void output_refs(struct ref *refs)
struct rpc_state {
const char *service_name;
const char **argv;
struct strbuf *stdin_preamble;
char *service_url;
char *hdr_content_type;
char *hdr_accept;
@ -535,6 +536,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
{
const char *svc = rpc->service_name;
struct strbuf buf = STRBUF_INIT;
struct strbuf *preamble = rpc->stdin_preamble;
struct child_process client;
int err = 0;
@ -545,6 +547,8 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
client.argv = rpc->argv;
if (start_command(&client))
exit(1);
if (preamble)
write_or_die(client.in, preamble->buf, preamble->len);
if (heads)
write_or_die(client.in, heads->buf, heads->len);
@ -626,13 +630,14 @@ static int fetch_git(struct discovery *heads,
int nr_heads, struct ref **to_fetch)
{
struct rpc_state rpc;
struct strbuf preamble = STRBUF_INIT;
char *depth_arg = NULL;
const char **argv;
int argc = 0, i, err;
const char *argv[15];
argv = xmalloc((15 + nr_heads) * sizeof(char*));
argv[argc++] = "fetch-pack";
argv[argc++] = "--stateless-rpc";
argv[argc++] = "--stdin";
argv[argc++] = "--lock-pack";
if (options.followtags)
argv[argc++] = "--include-tag";
@ -651,24 +656,27 @@ static int fetch_git(struct discovery *heads,
argv[argc++] = depth_arg;
}
argv[argc++] = url;
argv[argc++] = NULL;
for (i = 0; i < nr_heads; i++) {
struct ref *ref = to_fetch[i];
if (!ref->name || !*ref->name)
die("cannot fetch by sha1 over smart http");
argv[argc++] = ref->name;
packet_buf_write(&preamble, "%s\n", ref->name);
}
argv[argc++] = NULL;
packet_buf_flush(&preamble);
memset(&rpc, 0, sizeof(rpc));
rpc.service_name = "git-upload-pack",
rpc.argv = argv;
rpc.stdin_preamble = &preamble;
rpc.gzip_request = 1;
err = rpc_service(&rpc, heads);
if (rpc.result.len)
safe_write(1, rpc.result.buf, rpc.result.len);
strbuf_release(&rpc.result);
free(argv);
strbuf_release(&preamble);
free(depth_arg);
return err;
}

View File

@ -326,4 +326,70 @@ EOF
test_cmp count7.expected count7.actual
'
test_expect_success 'setup tests for the --stdin parameter' '
for head in C D E F
do
add $head
done &&
for head in A B C D E F
do
git tag $head $head
done &&
cat >input <<-\EOF
refs/heads/C
refs/heads/A
refs/heads/D
refs/tags/C
refs/heads/B
refs/tags/A
refs/heads/E
refs/tags/B
refs/tags/E
refs/tags/D
EOF
sort <input >expect &&
(
echo refs/heads/E &&
echo refs/tags/E &&
cat input
) >input.dup
'
test_expect_success 'fetch refs from cmdline' '
(
cd client &&
git fetch-pack --no-progress .. $(cat ../input)
) >output &&
cut -d " " -f 2 <output | sort >actual &&
test_cmp expect actual
'
test_expect_success 'fetch refs from stdin' '
(
cd client &&
git fetch-pack --stdin --no-progress .. <../input
) >output &&
cut -d " " -f 2 <output | sort >actual &&
test_cmp expect actual
'
test_expect_success 'fetch mixed refs from cmdline and stdin' '
(
cd client &&
tail -n +5 ../input |
git fetch-pack --stdin --no-progress .. $(head -n 4 ../input)
) >output &&
cut -d " " -f 2 <output | sort >actual &&
test_cmp expect actual
'
test_expect_success 'test duplicate refs from stdin' '
(
cd client &&
test_must_fail git fetch-pack --stdin --no-progress .. <../input.dup
) >output &&
cut -d " " -f 2 <output | sort >actual &&
test_cmp expect actual
'
test_done

View File

@ -109,5 +109,36 @@ test_expect_success 'follow redirects (302)' '
git clone $HTTPD_URL/smart-redir-temp/repo.git --quiet repo-t
'
test -n "$GIT_TEST_LONG" && test_set_prereq EXPENSIVE
test_expect_success EXPENSIVE 'create 50,000 tags in the repo' '
(
cd "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
for i in `seq 50000`
do
echo "commit refs/heads/too-many-refs"
echo "mark :$i"
echo "committer git <git@example.com> $i +0000"
echo "data 0"
echo "M 644 inline bla.txt"
echo "data 4"
echo "bla"
# make every commit dangling by always
# rewinding the branch after each commit
echo "reset refs/heads/too-many-refs"
echo "from :1"
done | git fast-import --export-marks=marks &&
# now assign tags to all the dangling commits we created above
tag=$(perl -e "print \"bla\" x 30") &&
sed -e "s/^:\(.\+\) \(.\+\)$/\2 refs\/tags\/$tag-\1/" <marks >>packed-refs
)
'
test_expect_success EXPENSIVE 'clone the 50,000 tag repo to check OS command line overflow' '
git clone $HTTPD_URL/smart/repo.git too-many-refs 2>err &&
test_line_count = 0 err
'
stop_httpd
test_done