Merge branch 'mh/fetch-filter-refs'

Code simplification and clarification.

* mh/fetch-filter-refs:
  test-string-list.c: Fix some sparse warnings
  fetch-pack: eliminate spurious error messages
  cmd_fetch_pack(): simplify computation of return value
  fetch-pack: report missing refs even if no existing refs were received
  cmd_fetch_pack(): return early if finish_connect() fails
  filter_refs(): simplify logic
  filter_refs(): build refs list as we go
  filter_refs(): delete matched refs from sought list
  fetch_pack(): update sought->nr to reflect number of unique entries
  filter_refs(): do not check the same sought_pos twice
  Change fetch_pack() and friends to take string_list arguments
  fetch_pack(): reindent function decl and defn
  Rename static function fetch_pack() to http_fetch_pack()
  t5500: add tests of fetch-pack --all --depth=N $URL $REF
  t5500: add tests of error output for missing refs

builtin/fetch-pack.c

@@ -525,72 +525,59 @@ static void mark_recent_complete_commits(unsigned long cutoff)
 	}
 }
 
-static void filter_refs(struct ref **refs, int nr_match, char **match)
+static int non_matching_ref(struct string_list_item *item, void *unused)
+{
+	if (item->util) {
+		item->util = NULL;
+		return 0;
+	}
+	else
+		return 1;
+}
+
+static void filter_refs(struct ref **refs, struct string_list *sought)
 {
-	struct ref **return_refs;
 	struct ref *newlist = NULL;
 	struct ref **newtail = &newlist;
 	struct ref *ref, *next;
-	struct ref *fastarray[32];
+	int sought_pos;
-	int match_pos;
 
-	if (nr_match && !args.fetch_all) {
+	sought_pos = 0;
-		if (ARRAY_SIZE(fastarray) < nr_match)
-			return_refs = xcalloc(nr_match, sizeof(struct ref *));
-		else {
-			return_refs = fastarray;
-			memset(return_refs, 0, sizeof(struct ref *) * nr_match);
-		}
-	}
-	else
-		return_refs = NULL;
-
-	match_pos = 0;
 	for (ref = *refs; ref; ref = next) {
+		int keep = 0;
 		next = ref->next;
 		if (!memcmp(ref->name, "refs/", 5) &&
 		    check_refname_format(ref->name + 5, 0))
 			; /* trash */
-		else if (args.fetch_all &&
+		else {
-			 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
+			while (sought_pos < sought->nr) {
+				int cmp = strcmp(ref->name, sought->items[sought_pos].string);
+				if (cmp < 0)
+					break; /* definitely do not have it */
+				else if (cmp == 0) {
+					keep = 1; /* definitely have it */
+					sought->items[sought_pos++].util = "matched";
+					break;
+				}
+				else
+					sought_pos++; /* might have it; keep looking */
+			}
+		}
+
+		if (! keep && args.fetch_all &&
+		    (!args.depth || prefixcmp(ref->name, "refs/tags/")))
+			keep = 1;
+
+		if (keep) {
 			*newtail = ref;
 			ref->next = NULL;
 			newtail = &ref->next;
-			continue;
+		} else {
+			free(ref);
 		}
-		else {
-			int cmp = -1;
-			while (match_pos < nr_match) {
-				cmp = strcmp(ref->name, match[match_pos]);
-				if (cmp < 0) /* definitely do not have it */
-					break;
-				else if (cmp == 0) { /* definitely have it */
-					match[match_pos][0] = '\0';
-					return_refs[match_pos] = ref;
-					break;
-				}
-				else /* might have it; keep looking */
-					match_pos++;
-			}
-			if (!cmp)
-				continue; /* we will link it later */
-		}
-		free(ref);
 	}
 
-	if (!args.fetch_all) {
+	filter_string_list(sought, 0, non_matching_ref, NULL);
-		int i;
-		for (i = 0; i < nr_match; i++) {
-			ref = return_refs[i];
-			if (ref) {
-				*newtail = ref;
-				ref->next = NULL;
-				newtail = &ref->next;
-			}
-		}
-		if (return_refs != fastarray)
-			free(return_refs);
-	}
 	*refs = newlist;
 }
 
@@ -599,7 +586,7 @@ static void mark_alternate_complete(const struct ref *ref, void *unused)
 	mark_complete(NULL, ref->old_sha1, 0, NULL);
 }
 
-static int everything_local(struct ref **refs, int nr_match, char **match)
+static int everything_local(struct ref **refs, struct string_list *sought)
 {
 	struct ref *ref;
 	int retval;
@@ -650,7 +637,7 @@ static int everything_local(struct ref **refs, int nr_match, char **match)
 		}
 	}
 
-	filter_refs(refs, nr_match, match);
+	filter_refs(refs, sought);
 
 	for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 		const unsigned char *remote = ref->old_sha1;
@@ -781,8 +768,7 @@ static int get_pack(int xd[2], char **pack_lockfile)
 
 static struct ref *do_fetch_pack(int fd[2],
 		const struct ref *orig_ref,
-		int nr_match,
+		struct string_list *sought,
-		char **match,
 		char **pack_lockfile)
 {
 	struct ref *ref = copy_ref_list(orig_ref);
@@ -839,7 +825,7 @@ static struct ref *do_fetch_pack(int fd[2],
 				agent_len, agent_feature);
 	}
 
-	if (everything_local(&ref, nr_match, match)) {
+	if (everything_local(&ref, sought)) {
 		packet_flush(fd[1]);
 		goto all_done;
 	}
@@ -859,19 +845,6 @@ static struct ref *do_fetch_pack(int fd[2],
 	return ref;
 }
 
-static int remove_duplicates(int nr_heads, char **heads)
-{
-	int src, dst;
-
-	if (!nr_heads)
-		return 0;
-
-	for (src = dst = 1; src < nr_heads; src++)
-		if (strcmp(heads[src], heads[dst-1]))
-			heads[dst++] = heads[src];
-	return dst;
-}
-
 static int fetch_pack_config(const char *var, const char *value, void *cb)
 {
 	if (strcmp(var, "fetch.unpacklimit") == 0) {
@@ -922,8 +895,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	int i, ret;
 	struct ref *ref = NULL;
 	const char *dest = NULL;
-	int alloc_heads = 0, nr_heads = 0;
+	struct string_list sought = STRING_LIST_INIT_DUP;
-	char **heads = NULL;
 	int fd[2];
 	char *pack_lockfile = NULL;
 	char **pack_lockfile_ptr = NULL;
@@ -1000,9 +972,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	 * Copy refs from cmdline to growable list, then append any
 	 * refs from the standard input:
 	 */
-	ALLOC_GROW(heads, argc - i, alloc_heads);
 	for (; i < argc; i++)
-		heads[nr_heads++] = xstrdup(argv[i]);
+		string_list_append(&sought, xstrdup(argv[i]));
 	if (args.stdin_refs) {
 		if (args.stateless_rpc) {
 			/* in stateless RPC mode we use pkt-line to read
@@ -1015,17 +986,14 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 					break;
 				if (line[n-1] == '\n')
 					n--;
-				ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+				string_list_append(&sought, xmemdupz(line, n));
-				heads[nr_heads++] = xmemdupz(line, n);
 			}
 		}
 		else {
 			/* read from stdin one ref per line, until EOF */
 			struct strbuf line = STRBUF_INIT;
-			while (strbuf_getline(&line, stdin, '\n') != EOF) {
+			while (strbuf_getline(&line, stdin, '\n') != EOF)
-				ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+				string_list_append(&sought, strbuf_detach(&line, NULL));
-				heads[nr_heads++] = strbuf_detach(&line, NULL);
-			}
 			strbuf_release(&line);
 		}
 	}
@@ -1042,7 +1010,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	get_remote_heads(fd[0], &ref, 0, NULL);
 
 	ref = fetch_pack(&args, fd, conn, ref, dest,
-		nr_heads, heads, pack_lockfile_ptr);
+			 &sought, pack_lockfile_ptr);
 	if (pack_lockfile) {
 		printf("lock %s\n", pack_lockfile);
 		fflush(stdout);
@@ -1050,21 +1018,18 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	close(fd[0]);
 	close(fd[1]);
 	if (finish_connect(conn))
-		ref = NULL;
+		return 1;
-	ret = !ref;
 
-	if (!ret && nr_heads) {
+	ret = !ref || sought.nr;
-		/* If the heads to pull were given, we should have
+
-		 * consumed all of them by matching the remote.
+	/*
-		 * Otherwise, 'git fetch remote no-such-ref' would
+	 * If the heads to pull were given, we should have consumed
-		 * silently succeed without issuing an error.
+	 * all of them by matching the remote.  Otherwise, 'git fetch
-		 */
+	 * remote no-such-ref' would silently succeed without issuing
-		for (i = 0; i < nr_heads; i++)
+	 * an error.
-			if (heads[i] && heads[i][0]) {
+	 */
-				error("no such remote ref %s", heads[i]);
+	for (i = 0; i < sought.nr; i++)
-				ret = 1;
+		error("no such remote ref %s", sought.items[i].string);
-			}
-	}
 	while (ref) {
 		printf("%s %s\n",
 		       sha1_to_hex(ref->old_sha1), ref->name);
@@ -1074,18 +1039,12 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	return ret;
 }
 
-static int compare_heads(const void *a, const void *b)
-{
-	return strcmp(*(const char **)a, *(const char **)b);
-}
-
 struct ref *fetch_pack(struct fetch_pack_args *my_args,
 		       int fd[], struct child_process *conn,
 		       const struct ref *ref,
-		const char *dest,
+		       const char *dest,
-		int nr_heads,
+		       struct string_list *sought,
-		char **heads,
+		       char **pack_lockfile)
-		char **pack_lockfile)
 {
 	struct stat st;
 	struct ref *ref_cpy;
@@ -1098,16 +1057,16 @@ struct ref *fetch_pack(struct fetch_pack_args *my_args,
 			st.st_mtime = 0;
 	}
 
-	if (heads && nr_heads) {
+	if (sought->nr) {
-		qsort(heads, nr_heads, sizeof(*heads), compare_heads);
+		sort_string_list(sought);
-		nr_heads = remove_duplicates(nr_heads, heads);
+		string_list_remove_duplicates(sought, 0);
 	}
 
 	if (!ref) {
 		packet_flush(fd[1]);
 		die("no matching remote head");
 	}
-	ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
+	ref_cpy = do_fetch_pack(fd, ref, sought, pack_lockfile);
 
 	if (args.depth > 0) {
 		struct cache_time mtime;

fetch-pack.h

@@ -1,6 +1,8 @@
 #ifndef FETCH_PACK_H
 #define FETCH_PACK_H
 
+#include "string-list.h"
+
 struct fetch_pack_args {
 	const char *uploadpack;
 	int unpacklimit;
@@ -17,12 +19,18 @@ struct fetch_pack_args {
 		stateless_rpc:1;
 };
 
+/*
+ * sought contains the full names of remote references that should be
+ * updated from.  On return, the names that were found on the remote
+ * will have been removed from the list.  The util members of the
+ * string_list_items are used internally; they must be NULL on entry
+ * (and will be NULL on exit).
+ */
 struct ref *fetch_pack(struct fetch_pack_args *args,
-		int fd[], struct child_process *conn,
+		       int fd[], struct child_process *conn,
-		const struct ref *ref,
+		       const struct ref *ref,
-		const char *dest,
+		       const char *dest,
-		int nr_heads,
+		       struct string_list *sought,
-		char **heads,
+		       char **pack_lockfile);
-		char **pack_lockfile);
 
 #endif

http-walker.c

@@ -396,7 +396,7 @@ static int fetch_indices(struct walker *walker, struct alt_base *repo)
 	return ret;
 }
 
-static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
+static int http_fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 {
 	struct packed_git *target;
 	int ret;
@@ -524,7 +524,7 @@ static int fetch(struct walker *walker, unsigned char *sha1)
 	if (!fetch_object(walker, altbase, sha1))
 		return 0;
 	while (altbase) {
-		if (!fetch_pack(walker, altbase, sha1))
+		if (!http_fetch_pack(walker, altbase, sha1))
 			return 0;
 		fetch_alternates(walker, data->alt->base);
 		altbase = altbase->next;

t/t5500-fetch-pack.sh

@@ -391,10 +391,55 @@ test_expect_success 'fetch mixed refs from cmdline and stdin' '
 test_expect_success 'test duplicate refs from stdin' '
 	(
 	cd client &&
-	test_must_fail git fetch-pack --stdin --no-progress .. <../input.dup
+	git fetch-pack --stdin --no-progress .. <../input.dup
 	) >output &&
 	cut -d " " -f 2 <output | sort >actual &&
 	test_cmp expect actual
 '
 
+test_expect_success 'set up tests of missing reference' '
+	cat >expect-error <<-\EOF
+	error: no such remote ref refs/heads/xyzzy
+	EOF
+'
+
+test_expect_success 'test lonely missing ref' '
+	(
+		cd client &&
+		test_must_fail git fetch-pack --no-progress .. refs/heads/xyzzy
+	) >/dev/null 2>error-m &&
+	test_cmp expect-error error-m
+'
+
+test_expect_success 'test missing ref after existing' '
+	(
+		cd client &&
+		test_must_fail git fetch-pack --no-progress .. refs/heads/A refs/heads/xyzzy
+	) >/dev/null 2>error-em &&
+	test_cmp expect-error error-em
+'
+
+test_expect_success 'test missing ref before existing' '
+	(
+		cd client &&
+		test_must_fail git fetch-pack --no-progress .. refs/heads/xyzzy refs/heads/A
+	) >/dev/null 2>error-me &&
+	test_cmp expect-error error-me
+'
+
+test_expect_success 'test --all, --depth, and explicit head' '
+	(
+		cd client &&
+		git fetch-pack --no-progress --all --depth=1 .. refs/heads/A
+	) >out-adh 2>error-adh
+'
+
+test_expect_success 'test --all, --depth, and explicit tag' '
+	git tag OLDTAG refs/heads/B~5 &&
+	(
+		cd client &&
+		git fetch-pack --no-progress --all --depth=1 .. refs/tags/OLDTAG
+	) >out-adt 2>error-adt
+'
+
 test_done

test-string-list.c

@@ -7,7 +7,7 @@
  * list (as opposed to "", which indicates a string list containing a
  * single empty string).  list->strdup_strings must be set.
  */
-void parse_string_list(struct string_list *list, const char *arg)
+static void parse_string_list(struct string_list *list, const char *arg)
 {
 	if (!strcmp(arg, "-"))
 		return;
@@ -15,14 +15,14 @@ void parse_string_list(struct string_list *list, const char *arg)
 	(void)string_list_split(list, arg, ':', -1);
 }
 
-void write_list(const struct string_list *list)
+static void write_list(const struct string_list *list)
 {
 	int i;
 	for (i = 0; i < list->nr; i++)
 		printf("[%d]: \"%s\"\n", i, list->items[i].string);
 }
 
-void write_list_compact(const struct string_list *list)
+static void write_list_compact(const struct string_list *list)
 {
 	int i;
 	if (!list->nr)
@@ -35,7 +35,7 @@ void write_list_compact(const struct string_list *list)
 	}
 }
 
-int prefix_cb(struct string_list_item *item, void *cb_data)
+static int prefix_cb(struct string_list_item *item, void *cb_data)
 {
 	const char *prefix = (const char *)cb_data;
 	return !prefixcmp(item->string, prefix);

transport.c

@@ -518,8 +518,7 @@ static int fetch_refs_via_pack(struct transport *transport,
 			       int nr_heads, struct ref **to_fetch)
 {
 	struct git_transport_data *data = transport->data;
-	char **heads = xmalloc(nr_heads * sizeof(*heads));
+	struct string_list sought = STRING_LIST_INIT_DUP;
-	char **origh = xmalloc(nr_heads * sizeof(*origh));
 	const struct ref *refs;
 	char *dest = xstrdup(transport->url);
 	struct fetch_pack_args args;
@@ -538,7 +537,7 @@ static int fetch_refs_via_pack(struct transport *transport,
 	args.depth = data->options.depth;
 
 	for (i = 0; i < nr_heads; i++)
-		origh[i] = heads[i] = xstrdup(to_fetch[i]->name);
+		string_list_append(&sought, to_fetch[i]->name);
 
 	if (!data->got_remote_heads) {
 		connect_setup(transport, 0, 0);
@@ -548,7 +547,7 @@ static int fetch_refs_via_pack(struct transport *transport,
 
 	refs = fetch_pack(&args, data->fd, data->conn,
 			  refs_tmp ? refs_tmp : transport->remote_refs,
-			  dest, nr_heads, heads, &transport->pack_lockfile);
+			  dest, &sought, &transport->pack_lockfile);
 	close(data->fd[0]);
 	close(data->fd[1]);
 	if (finish_connect(data->conn))
@@ -558,10 +557,7 @@ static int fetch_refs_via_pack(struct transport *transport,
 
 	free_refs(refs_tmp);
 
-	for (i = 0; i < nr_heads; i++)
+	string_list_clear(&sought, 0);
-		free(origh[i]);
-	free(origh);
-	free(heads);
 	free(dest);
 	return (refs ? 0 : -1);
 }