mirrors/git
0
0
Fork 0
mirror of https://github.com/git/git.git synced 2024-11-23 01:46:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ps/reftable-exclude'

Browse Source 
The reftable backend learned to more efficiently handle exclude
patterns while enumerating the refs.

* ps/reftable-exclude:
  refs/reftable: wire up support for exclude patterns
  reftable/reader: make table iterator reseekable
  t/unit-tests: introduce reftable library
  Makefile: stop listing test library objects twice
  builtin/receive-pack: fix exclude patterns when announcing refs
  refs: properly apply exclude patterns to namespaced refs
This commit is contained in:
Junio C Hamano 2024-09-25 10:37:11 -07:00
commit 52f57e94bd
16 changed files with 594 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -1356,6 +1356,7 @@ UNIT_TEST_PROGRAMS += t-reftable-basics
UNIT_TEST_PROGRAMS += t-reftable-block
UNIT_TEST_PROGRAMS += t-reftable-merged
UNIT_TEST_PROGRAMS += t-reftable-pq
UNIT_TEST_PROGRAMS += t-reftable-reader
UNIT_TEST_PROGRAMS += t-reftable-readwrite
UNIT_TEST_PROGRAMS += t-reftable-record
UNIT_TEST_PROGRAMS += t-reftable-stack
@ -1365,9 +1366,9 @@ UNIT_TEST_PROGRAMS += t-strcmp-offset
UNIT_TEST_PROGRAMS += t-trailer
UNIT_TEST_PROGRAMS += t-urlmatch-normalization
UNIT_TEST_PROGS = $(patsubst %,$(UNIT_TEST_BIN)/%$X,$(UNIT_TEST_PROGRAMS))
UNIT_TEST_OBJS = $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(UNIT_TEST_PROGRAMS))
UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o
UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/lib-oid.o
UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/lib-reftable.o
# xdiff and reftable libs may in turn depend on what is in libgit.a
GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(LIB_FILE)
@ -2725,6 +2726,7 @@ OBJECTS += $(FUZZ_OBJS)
OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS)
OBJECTS += $(UNIT_TEST_OBJS)
OBJECTS += $(CLAR_TEST_OBJS)
OBJECTS += $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(UNIT_TEST_PROGRAMS))
ifndef NO_CURL
OBJECTS += http.o http-walker.o remote-curl.o
@ -3864,9 +3866,7 @@ $(FUZZ_PROGRAMS): %: %.o oss-fuzz/dummy-cmd-main.o $(GITLIBS) GIT-LDFLAGS
-Wl,--allow-multiple-definition \
$(filter %.o,$^) $(filter %.a,$^) $(LIBS) $(LIB_FUZZING_ENGINE)
$(UNIT_TEST_PROGS): $(UNIT_TEST_BIN)/%$X: $(UNIT_TEST_DIR)/%.o \
$(UNIT_TEST_DIR)/test-lib.o \
$(UNIT_TEST_DIR)/lib-oid.o \
$(UNIT_TEST_PROGS): $(UNIT_TEST_BIN)/%$X: $(UNIT_TEST_DIR)/%.o $(UNIT_TEST_OBJS) \
$(GITLIBS) GIT-LDFLAGS
$(call mkdir_p_parent_template)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \

18
builtin/receive-pack.c
View File

@ -340,12 +340,26 @@ static void show_one_alternate_ref(const struct object_id *oid,
static void write_head_info(void)
{
static struct oidset seen = OIDSET_INIT;
struct strvec excludes_vector = STRVEC_INIT;
const char **exclude_patterns;
/*
* We need access to the reference names both with and without their
* namespace and thus cannot use `refs_for_each_namespaced_ref()`. We
* thus have to adapt exclude patterns to carry the namespace prefix
* ourselves.
*/
exclude_patterns = get_namespaced_exclude_patterns(
hidden_refs_to_excludes(&hidden_refs),
get_git_namespace(), &excludes_vector);
refs_for_each_fullref_in(get_main_ref_store(the_repository), "",
hidden_refs_to_excludes(&hidden_refs),
show_ref_cb, &seen);
exclude_patterns, show_ref_cb, &seen);
for_each_alternate_ref(show_one_alternate_ref, &seen);
oidset_clear(&seen);
strvec_clear(&excludes_vector);
if (!sent_capabilities)
show_ref("capabilities^{}", null_oid());

35
refs.c
View File

@ -1518,6 +1518,19 @@ const char **hidden_refs_to_excludes(const struct strvec *hide_refs)
return hide_refs->v;
}
const char **get_namespaced_exclude_patterns(const char **exclude_patterns,
const char *namespace,
struct strvec *out)
{
if (!namespace || !*namespace || !exclude_patterns || !*exclude_patterns)
return exclude_patterns;
for (size_t i = 0; exclude_patterns[i]; i++)
strvec_pushf(out, "%s%s", namespace, exclude_patterns[i]);
return out->v;
}
const char *find_descendant_ref(const char *dirname,
const struct string_list *extras,
const struct string_list *skip)
@ -1635,11 +1648,19 @@ int refs_for_each_namespaced_ref(struct ref_store *refs,
const char **exclude_patterns,
each_ref_fn fn, void *cb_data)
{
struct strbuf buf = STRBUF_INIT;
struct strvec namespaced_exclude_patterns = STRVEC_INIT;
struct strbuf prefix = STRBUF_INIT;
int ret;
strbuf_addf(&buf, "%srefs/", get_git_namespace());
ret = do_for_each_ref(refs, buf.buf, exclude_patterns, fn, 0, 0, cb_data);
strbuf_release(&buf);
exclude_patterns = get_namespaced_exclude_patterns(exclude_patterns,
get_git_namespace(),
&namespaced_exclude_patterns);
strbuf_addf(&prefix, "%srefs/", get_git_namespace());
ret = do_for_each_ref(refs, prefix.buf, exclude_patterns, fn, 0, 0, cb_data);
strvec_clear(&namespaced_exclude_patterns);
strbuf_release(&prefix);
return ret;
}
@ -1720,6 +1741,7 @@ int refs_for_each_fullref_in_prefixes(struct ref_store *ref_store,
const char **exclude_patterns,
each_ref_fn fn, void *cb_data)
{
struct strvec namespaced_exclude_patterns = STRVEC_INIT;
struct string_list prefixes = STRING_LIST_INIT_DUP;
struct string_list_item *prefix;
struct strbuf buf = STRBUF_INIT;
@ -1731,6 +1753,10 @@ int refs_for_each_fullref_in_prefixes(struct ref_store *ref_store,
strbuf_addstr(&buf, namespace);
namespace_len = buf.len;
exclude_patterns = get_namespaced_exclude_patterns(exclude_patterns,
namespace,
&namespaced_exclude_patterns);
for_each_string_list_item(prefix, &prefixes) {
strbuf_addstr(&buf, prefix->string);
ret = refs_for_each_fullref_in(ref_store, buf.buf,
@ -1740,6 +1766,7 @@ int refs_for_each_fullref_in_prefixes(struct ref_store *ref_store,
strbuf_setlen(&buf, namespace_len);
}
strvec_clear(&namespaced_exclude_patterns);
string_list_clear(&prefixes, 0);
strbuf_release(&buf);
return ret;

9
refs.h
View File

@ -861,6 +861,15 @@ int ref_is_hidden(const char *, const char *, const struct strvec *);
*/
const char **hidden_refs_to_excludes(const struct strvec *hide_refs);
/*
* Prefix all exclude patterns with the namespace, if any. This is required
* because exclude patterns apply to the stripped reference name, not the full
* reference name with the namespace.
*/
const char **get_namespaced_exclude_patterns(const char **exclude_patterns,
const char *namespace,
struct strvec *out);
/* Is this a per-worktree ref living in the refs/ namespace? */
int is_per_worktree_ref(const char *refname);

133
refs/reftable-backend.c
View File

@ -22,6 +22,7 @@
#include "../repo-settings.h"
#include "../setup.h"
#include "../strmap.h"
#include "../trace2.h"
#include "parse.h"
#include "refs-internal.h"
@ -451,10 +452,81 @@ struct reftable_ref_iterator {
const char *prefix;
size_t prefix_len;
char **exclude_patterns;
size_t exclude_patterns_index;
size_t exclude_patterns_strlen;
unsigned int flags;
int err;
};
/*
* Handle exclude patterns. Returns either `1`, which tells the caller that the
* current reference shall not be shown. Or `0`, which indicates that it should
* be shown.
*/
static int should_exclude_current_ref(struct reftable_ref_iterator *iter)
{
while (iter->exclude_patterns[iter->exclude_patterns_index]) {
const char *pattern = iter->exclude_patterns[iter->exclude_patterns_index];
char *ref_after_pattern;
int cmp;
/*
* Lazily cache the pattern length so that we don't have to
* recompute it every time this function is called.
*/
if (!iter->exclude_patterns_strlen)
iter->exclude_patterns_strlen = strlen(pattern);
/*
* When the reference name is lexicographically bigger than the
* current exclude pattern we know that it won't ever match any
* of the following references, either. We thus advance to the
* next pattern and re-check whether it matches.
*
* Otherwise, if it's smaller, then we do not have a match and
* thus want to show the current reference.
*/
cmp = strncmp(iter->ref.refname, pattern,
iter->exclude_patterns_strlen);
if (cmp > 0) {
iter->exclude_patterns_index++;
iter->exclude_patterns_strlen = 0;
continue;
}
if (cmp < 0)
return 0;
/*
* The reference shares a prefix with the exclude pattern and
* shall thus be omitted. We skip all references that match the
* pattern by seeking to the first reference after the block of
* matches.
*
* This is done by appending the highest possible character to
* the pattern. Consequently, all references that have the
* pattern as prefix and whose suffix starts with anything in
* the range [0x00, 0xfe] are skipped. And given that 0xff is a
* non-printable character that shouldn't ever be in a ref name,
* we'd not yield any such record, either.
*
* Note that the seeked-to reference may also be excluded. This
* is not handled here though, but the caller is expected to
* loop and re-verify the next reference for us.
*/
ref_after_pattern = xstrfmt("%s%c", pattern, 0xff);
iter->err = reftable_iterator_seek_ref(&iter->iter, ref_after_pattern);
iter->exclude_patterns_index++;
iter->exclude_patterns_strlen = 0;
trace2_counter_add(TRACE2_COUNTER_ID_REFTABLE_RESEEKS, 1);
free(ref_after_pattern);
return 1;
}
return 0;
}
static int reftable_ref_iterator_advance(struct ref_iterator *ref_iterator)
{
struct reftable_ref_iterator *iter =
@ -485,6 +557,9 @@ static int reftable_ref_iterator_advance(struct ref_iterator *ref_iterator)
break;
}
if (iter->exclude_patterns && should_exclude_current_ref(iter))
continue;
if (iter->flags & DO_FOR_EACH_PER_WORKTREE_ONLY &&
parse_worktree_ref(iter->ref.refname, NULL, NULL, NULL) !=
REF_WORKTREE_CURRENT)
@ -574,6 +649,11 @@ static int reftable_ref_iterator_abort(struct ref_iterator *ref_iterator)
(struct reftable_ref_iterator *)ref_iterator;
reftable_ref_record_release(&iter->ref);
reftable_iterator_destroy(&iter->iter);
if (iter->exclude_patterns) {
for (size_t i = 0; iter->exclude_patterns[i]; i++)
free(iter->exclude_patterns[i]);
free(iter->exclude_patterns);
}
free(iter);
return ITER_DONE;
}
@ -584,9 +664,53 @@ static struct ref_iterator_vtable reftable_ref_iterator_vtable = {
.abort = reftable_ref_iterator_abort
};
static int qsort_strcmp(const void *va, const void *vb)
{
const char *a = *(const char **)va;
const char *b = *(const char **)vb;
return strcmp(a, b);
}
static char **filter_exclude_patterns(const char **exclude_patterns)
{
size_t filtered_size = 0, filtered_alloc = 0;
char **filtered = NULL;
if (!exclude_patterns)
return NULL;
for (size_t i = 0; ; i++) {
const char *exclude_pattern = exclude_patterns[i];
int has_glob = 0;
if (!exclude_pattern)
break;
for (const char *p = exclude_pattern; *p; p++) {
has_glob = is_glob_special(*p);
if (has_glob)
break;
}
if (has_glob)
continue;
ALLOC_GROW(filtered, filtered_size + 1, filtered_alloc);
filtered[filtered_size++] = xstrdup(exclude_pattern);
}
if (filtered_size) {
QSORT(filtered, filtered_size, qsort_strcmp);
ALLOC_GROW(filtered, filtered_size + 1, filtered_alloc);
filtered[filtered_size++] = NULL;
}
return filtered;
}
static struct reftable_ref_iterator *ref_iterator_for_stack(struct reftable_ref_store *refs,
struct reftable_stack *stack,
const char *prefix,
const char **exclude_patterns,
int flags)
{
struct reftable_ref_iterator *iter;
@ -599,6 +723,7 @@ static struct reftable_ref_iterator *ref_iterator_for_stack(struct reftable_ref_
iter->base.oid = &iter->oid;
iter->flags = flags;
iter->refs = refs;
iter->exclude_patterns = filter_exclude_patterns(exclude_patterns);
ret = refs->err;
if (ret)
@ -620,7 +745,7 @@ done:
static struct ref_iterator *reftable_be_iterator_begin(struct ref_store *ref_store,
const char *prefix,
const char **exclude_patterns UNUSED,
const char **exclude_patterns,
unsigned int flags)
{
struct reftable_ref_iterator *main_iter, *worktree_iter;
@ -631,7 +756,8 @@ static struct ref_iterator *reftable_be_iterator_begin(struct ref_store *ref_sto
required_flags |= REF_STORE_ODB;
refs = reftable_be_downcast(ref_store, required_flags, "ref_iterator_begin");
main_iter = ref_iterator_for_stack(refs, refs->main_stack, prefix, flags);
main_iter = ref_iterator_for_stack(refs, refs->main_stack, prefix,
exclude_patterns, flags);
/*
* The worktree stack is only set when we're in an actual worktree
@ -645,7 +771,8 @@ static struct ref_iterator *reftable_be_iterator_begin(struct ref_store *ref_sto
* Otherwise we merge both the common and the per-worktree refs into a
* single iterator.
*/
worktree_iter = ref_iterator_for_stack(refs, refs->worktree_stack, prefix, flags);
worktree_iter = ref_iterator_for_stack(refs, refs->worktree_stack, prefix,
exclude_patterns, flags);
return merge_ref_iterator_begin(&worktree_iter->base, &main_iter->base,
ref_iterator_select, NULL);
}

1
reftable/reader.c
View File

@ -328,6 +328,7 @@ static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ)
ti->typ = block_reader_type(&ti->br);
ti->block_off = off;
block_iter_seek_start(&ti->bi, &ti->br);
ti->is_finished = 0;
return 0;
}

49
t/t1419-exclude-refs.sh
View File

@ -8,12 +8,6 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
TEST_PASSES_SANITIZE_LEAK=true
. ./test-lib.sh
if test_have_prereq !REFFILES
then
skip_all='skipping `git for-each-ref --exclude` tests; need files backend'
test_done
fi
for_each_ref__exclude () {
GIT_TRACE2_PERF=1 test-tool ref-store main \
for-each-ref--exclude "$@" >actual.raw
@ -28,7 +22,14 @@ assert_jumps () {
local nr="$1"
local trace="$2"
grep -q "name:jumps_made value:$nr$" $trace
case "$GIT_DEFAULT_REF_FORMAT" in
files)
grep -q "name:jumps_made value:$nr$" $trace;;
reftable)
grep -q "name:reseeks_made value:$nr$" $trace;;
*)
BUG "unhandled ref format $GIT_DEFAULT_REF_FORMAT";;
esac
}
assert_no_jumps () {
@ -89,7 +90,14 @@ test_expect_success 'adjacent, non-overlapping excluded regions' '
for_each_ref refs/heads/foo refs/heads/quux >expect &&
test_cmp expect actual &&
assert_jumps 1 perf
case "$GIT_DEFAULT_REF_FORMAT" in
files)
assert_jumps 1 perf;;
reftable)
assert_jumps 2 perf;;
*)
BUG "unhandled ref format $GIT_DEFAULT_REF_FORMAT";;
esac
'
test_expect_success 'overlapping excluded regions' '
@ -106,7 +114,30 @@ test_expect_success 'several overlapping excluded regions' '
for_each_ref refs/heads/quux >expect &&
test_cmp expect actual &&
assert_jumps 1 perf
case "$GIT_DEFAULT_REF_FORMAT" in
files)
assert_jumps 1 perf;;
reftable)
assert_jumps 3 perf;;
*)
BUG "unhandled ref format $GIT_DEFAULT_REF_FORMAT";;
esac
'
test_expect_success 'unordered excludes' '
for_each_ref__exclude refs/heads \
refs/heads/foo refs/heads/baz >actual 2>perf &&
for_each_ref refs/heads/bar refs/heads/quux >expect &&
test_cmp expect actual &&
case "$GIT_DEFAULT_REF_FORMAT" in
files)
assert_jumps 1 perf;;
reftable)
assert_jumps 2 perf;;
*)
BUG "unhandled ref format $GIT_DEFAULT_REF_FORMAT";;
esac
'
test_expect_success 'non-matching excluded section' '

9
t/t5509-fetch-push-namespaces.sh
View File

@ -96,6 +96,7 @@ test_expect_success 'hide namespaced refs with transfer.hideRefs' '
'
test_expect_success 'check that transfer.hideRefs does not match unstripped refs' '
git -C pushee pack-refs --all &&
GIT_NAMESPACE=namespace \
git -C pushee -c transfer.hideRefs=refs/namespaces/namespace/refs/tags \
ls-remote "ext::git %s ." >actual &&
@ -123,6 +124,14 @@ test_expect_success 'try to update a ref that is not hidden' '
git -C original push pushee-namespaced main
'
test_expect_success 'git-receive-pack(1) with transfer.hideRefs does not match unstripped refs during advertisement' '
git -C pushee update-ref refs/namespaces/namespace/refs/heads/foo/1 refs/namespaces/namespace/refs/heads/main &&
git -C pushee pack-refs --all &&
test_config -C pushee transfer.hideRefs refs/namespaces/namespace/refs/heads/foo &&
GIT_TRACE_PACKET="$(pwd)/trace" git -C original push pushee-namespaced main &&
test_grep refs/heads/foo/1 trace
'
test_expect_success 'try to update a hidden full ref' '
test_config -C pushee transfer.hideRefs "^refs/namespaces/namespace/refs/heads/main" &&
test_must_fail git -C original push pushee-namespaced main

93
t/unit-tests/lib-reftable.c Normal file
View File

@ -0,0 +1,93 @@
#include "lib-reftable.h"
#include "test-lib.h"
#include "reftable/constants.h"
#include "reftable/writer.h"
void t_reftable_set_hash(uint8_t *p, int i, uint32_t id)
{
memset(p, (uint8_t)i, hash_size(id));
}
static ssize_t strbuf_writer_write(void *b, const void *data, size_t sz)
{
strbuf_add(b, data, sz);
return sz;
}
static int strbuf_writer_flush(void *arg UNUSED)
{
return 0;
}
struct reftable_writer *t_reftable_strbuf_writer(struct strbuf *buf,
struct reftable_write_options *opts)
{
return reftable_new_writer(&strbuf_writer_write,
&strbuf_writer_flush,
buf, opts);
}
void t_reftable_write_to_buf(struct strbuf *buf,
struct reftable_ref_record *refs,
size_t nrefs,
struct reftable_log_record *logs,
size_t nlogs,
struct reftable_write_options *_opts)
{
struct reftable_write_options opts = { 0 };
const struct reftable_stats *stats;
struct reftable_writer *writer;
uint64_t min = 0xffffffff;
uint64_t max = 0;
int ret;
if (_opts)
opts = *_opts;
for (size_t i = 0; i < nrefs; i++) {
uint64_t ui = refs[i].update_index;
if (ui > max)
max = ui;
if (ui < min)
min = ui;
}
for (size_t i = 0; i < nlogs; i++) {
uint64_t ui = logs[i].update_index;
if (ui > max)
max = ui;
if (ui < min)
min = ui;
}
writer = t_reftable_strbuf_writer(buf, &opts);
reftable_writer_set_limits(writer, min, max);
if (nrefs) {
ret = reftable_writer_add_refs(writer, refs, nrefs);
check_int(ret, ==, 0);
}
if (nlogs) {
ret = reftable_writer_add_logs(writer, logs, nlogs);
check_int(ret, ==, 0);
}
ret = reftable_writer_close(writer);
check_int(ret, ==, 0);
stats = reftable_writer_stats(writer);
for (size_t i = 0; i < stats->ref_stats.blocks; i++) {
size_t off = i * (opts.block_size ? opts.block_size
: DEFAULT_BLOCK_SIZE);
if (!off)
off = header_size(opts.hash_id == GIT_SHA256_FORMAT_ID ? 2 : 1);
check_char(buf->buf[off], ==, 'r');
}
if (nrefs)
check_int(stats->ref_stats.blocks, >, 0);
if (nlogs)
check_int(stats->log_stats.blocks, >, 0);
reftable_writer_free(writer);
}

20
t/unit-tests/lib-reftable.h Normal file
View File

@ -0,0 +1,20 @@
#ifndef LIB_REFTABLE_H
#define LIB_REFTABLE_H
#include "git-compat-util.h"
#include "strbuf.h"
#include "reftable/reftable-writer.h"
void t_reftable_set_hash(uint8_t *p, int i, uint32_t id);
struct reftable_writer *t_reftable_strbuf_writer(struct strbuf *buf,
struct reftable_write_options *opts);
void t_reftable_write_to_buf(struct strbuf *buf,
struct reftable_ref_record *refs,
size_t nrecords,
struct reftable_log_record *logs,
size_t nlogs,
struct reftable_write_options *opts);
#endif

163
t/unit-tests/t-reftable-merged.c
View File

@ -7,6 +7,7 @@ https://developers.google.com/open-source/licenses/bsd
*/
#include "test-lib.h"
#include "lib-reftable.h"
#include "reftable/blocksource.h"
#include "reftable/constants.h"
#include "reftable/merged.h"
@ -15,77 +16,6 @@ https://developers.google.com/open-source/licenses/bsd
#include "reftable/reftable-merged.h"
#include "reftable/reftable-writer.h"
static ssize_t strbuf_add_void(void *b, const void *data, const size_t sz)
{
strbuf_add(b, data, sz);
return sz;
}
static int noop_flush(void *arg UNUSED)
{
return 0;
}
static void write_test_table(struct strbuf *buf,
struct reftable_ref_record refs[], const size_t n)
{
uint64_t min = 0xffffffff;
uint64_t max = 0;
size_t i;
int err;
struct reftable_write_options opts = {
.block_size = 256,
};
struct reftable_writer *w = NULL;
for (i = 0; i < n; i++) {
uint64_t ui = refs[i].update_index;
if (ui > max)
max = ui;
if (ui < min)
min = ui;
}
w = reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts);
reftable_writer_set_limits(w, min, max);
for (i = 0; i < n; i++) {
uint64_t before = refs[i].update_index;
int n = reftable_writer_add_ref(w, &refs[i]);
check_int(n, ==, 0);
check_int(before, ==, refs[i].update_index);
}
err = reftable_writer_close(w);
check(!err);
reftable_writer_free(w);
}
static void write_test_log_table(struct strbuf *buf, struct reftable_log_record logs[],
const size_t n, const uint64_t update_index)
{
int err;
struct reftable_write_options opts = {
.block_size = 256,
.exact_log_message = 1,
};
struct reftable_writer *w = NULL;
w = reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts);
reftable_writer_set_limits(w, update_index, update_index);
for (size_t i = 0; i < n; i++) {
int err = reftable_writer_add_log(w, &logs[i]);
check(!err);
}
err = reftable_writer_close(w);
check(!err);
reftable_writer_free(w);
}
static struct reftable_merged_table *
merged_table_from_records(struct reftable_ref_record **refs,
struct reftable_block_source **source,
@ -93,13 +23,16 @@ merged_table_from_records(struct reftable_ref_record **refs,
struct strbuf *buf, const size_t n)
{
struct reftable_merged_table *mt = NULL;
struct reftable_write_options opts = {
.block_size = 256,
};
int err;
REFTABLE_CALLOC_ARRAY(*readers, n);
REFTABLE_CALLOC_ARRAY(*source, n);
for (size_t i = 0; i < n; i++) {
write_test_table(&buf[i], refs[i], sizes[i]);
t_reftable_write_to_buf(&buf[i], refs[i], sizes[i], NULL, 0, &opts);
block_source_from_strbuf(&(*source)[i], &buf[i]);
err = reftable_reader_new(&(*readers)[i], &(*source)[i],
@ -261,6 +194,81 @@ static void t_merged_refs(void)
reftable_free(bs);
}
static void t_merged_seek_multiple_times(void)
{
struct reftable_ref_record r1[] = {
{
.refname = (char *) "a",
.update_index = 1,
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 1 },
},
{
.refname = (char *) "c",
.update_index = 1,
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 2 },
}
};
struct reftable_ref_record r2[] = {
{
.refname = (char *) "b",
.update_index = 2,
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 3 },
},
{
.refname = (char *) "d",
.update_index = 2,
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 4 },
},
};
struct reftable_ref_record *refs[] = {
r1, r2,
};
size_t sizes[] = {
ARRAY_SIZE(r1), ARRAY_SIZE(r2),
};
struct strbuf bufs[] = {
STRBUF_INIT, STRBUF_INIT,
};
struct reftable_block_source *sources = NULL;
struct reftable_reader **readers = NULL;
struct reftable_ref_record rec = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_merged_table *mt;
mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2);
merged_table_init_iter(mt, &it, BLOCK_TYPE_REF);
for (size_t i = 0; i < 5; i++) {
int err = reftable_iterator_seek_ref(&it, "c");
check(!err);
err = reftable_iterator_next_ref(&it, &rec);
check(!err);
err = reftable_ref_record_equal(&rec, &r1[1], GIT_SHA1_RAWSZ);
check(err == 1);
err = reftable_iterator_next_ref(&it, &rec);
check(!err);
err = reftable_ref_record_equal(&rec, &r2[1], GIT_SHA1_RAWSZ);
check(err == 1);
err = reftable_iterator_next_ref(&it, &rec);
check(err > 0);
}
for (size_t i = 0; i < ARRAY_SIZE(bufs); i++)
strbuf_release(&bufs[i]);
readers_destroy(readers, ARRAY_SIZE(refs));
reftable_ref_record_release(&rec);
reftable_iterator_destroy(&it);
reftable_merged_table_free(mt);
reftable_free(sources);
}
static struct reftable_merged_table *
merged_table_from_log_records(struct reftable_log_record **logs,
struct reftable_block_source **source,
@ -268,13 +276,17 @@ merged_table_from_log_records(struct reftable_log_record **logs,
struct strbuf *buf, const size_t n)
{
struct reftable_merged_table *mt = NULL;
struct reftable_write_options opts = {
.block_size = 256,
.exact_log_message = 1,
};
int err;
REFTABLE_CALLOC_ARRAY(*readers, n);
REFTABLE_CALLOC_ARRAY(*source, n);
for (size_t i = 0; i < n; i++) {
write_test_log_table(&buf[i], logs[i], sizes[i], i + 1);
t_reftable_write_to_buf(&buf[i], NULL, 0, logs[i], sizes[i], &opts);
block_source_from_strbuf(&(*source)[i], &buf[i]);
err = reftable_reader_new(&(*readers)[i], &(*source)[i],
@ -402,9 +414,7 @@ static void t_default_write_opts(void)
{
struct reftable_write_options opts = { 0 };
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
struct reftable_ref_record rec = {
.refname = (char *) "master",
.update_index = 1,
@ -448,6 +458,7 @@ int cmd_main(int argc UNUSED, const char *argv[] UNUSED)
TEST(t_default_write_opts(), "merged table with default write opts");
TEST(t_merged_logs(), "merged table with multiple log updates for same ref");
TEST(t_merged_refs(), "merged table with multiple updates to same ref");
TEST(t_merged_seek_multiple_times(), "merged table can seek multiple times");
TEST(t_merged_single_record(), "ref ocurring in only one record can be fetched");
return test_done();

96
t/unit-tests/t-reftable-reader.c Normal file
View File

@ -0,0 +1,96 @@
#include "test-lib.h"
#include "lib-reftable.h"
#include "reftable/blocksource.h"
#include "reftable/reader.h"
static int t_reader_seek_once(void)
{
struct reftable_ref_record records[] = {
{
.refname = (char *) "refs/heads/main",
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 42 },
},
};
struct reftable_block_source source = { 0 };
struct reftable_ref_record ref = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_reader *reader;
struct strbuf buf = STRBUF_INIT;
int ret;
t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL);
block_source_from_strbuf(&source, &buf);
ret = reftable_reader_new(&reader, &source, "name");
check(!ret);
reftable_reader_init_ref_iterator(reader, &it);
ret = reftable_iterator_seek_ref(&it, "");
check(!ret);
ret = reftable_iterator_next_ref(&it, &ref);
check(!ret);
ret = reftable_ref_record_equal(&ref, &records[0], GIT_SHA1_RAWSZ);
check_int(ret, ==, 1);
ret = reftable_iterator_next_ref(&it, &ref);
check_int(ret, ==, 1);
reftable_ref_record_release(&ref);
reftable_iterator_destroy(&it);
reftable_reader_decref(reader);
strbuf_release(&buf);
return 0;
}
static int t_reader_reseek(void)
{
struct reftable_ref_record records[] = {
{
.refname = (char *) "refs/heads/main",
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 42 },
},
};
struct reftable_block_source source = { 0 };
struct reftable_ref_record ref = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_reader *reader;
struct strbuf buf = STRBUF_INIT;
int ret;
t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL);
block_source_from_strbuf(&source, &buf);
ret = reftable_reader_new(&reader, &source, "name");
check(!ret);
reftable_reader_init_ref_iterator(reader, &it);
for (size_t i = 0; i < 5; i++) {
ret = reftable_iterator_seek_ref(&it, "");
check(!ret);
ret = reftable_iterator_next_ref(&it, &ref);
check(!ret);
ret = reftable_ref_record_equal(&ref, &records[0], GIT_SHA1_RAWSZ);
check_int(ret, ==, 1);
ret = reftable_iterator_next_ref(&it, &ref);
check_int(ret, ==, 1);
}
reftable_ref_record_release(&ref);
reftable_iterator_destroy(&it);
reftable_reader_decref(reader);
strbuf_release(&buf);
return 0;
}
int cmd_main(int argc UNUSED, const char *argv[] UNUSED)
{
TEST(t_reader_seek_once(), "reader can seek once");
TEST(t_reader_reseek(), "reader can reseek multiple times");
return test_done();
}

130
t/unit-tests/t-reftable-readwrite.c
View File

@ -7,6 +7,7 @@ https://developers.google.com/open-source/licenses/bsd
*/
#include "test-lib.h"
#include "lib-reftable.h"
#include "reftable/basics.h"
#include "reftable/blocksource.h"
#include "reftable/reader.h"
@ -15,22 +16,6 @@ https://developers.google.com/open-source/licenses/bsd
static const int update_index = 5;
static void set_test_hash(uint8_t *p, int i)
{
memset(p, (uint8_t)i, hash_size(GIT_SHA1_FORMAT_ID));
}
static ssize_t strbuf_add_void(void *b, const void *data, size_t sz)
{
strbuf_add(b, data, sz);
return sz;
}
static int noop_flush(void *arg UNUSED)
{
return 0;
}
static void t_buffer(void)
{
struct strbuf buf = STRBUF_INIT;
@ -62,61 +47,34 @@ static void write_table(char ***names, struct strbuf *buf, int N,
.block_size = block_size,
.hash_id = hash_id,
};
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts);
struct reftable_ref_record ref = { 0 };
int i = 0, n;
struct reftable_log_record log = { 0 };
const struct reftable_stats *stats = NULL;
struct reftable_ref_record *refs;
struct reftable_log_record *logs;
int i;
REFTABLE_CALLOC_ARRAY(*names, N + 1);
REFTABLE_CALLOC_ARRAY(refs, N);
REFTABLE_CALLOC_ARRAY(logs, N);
reftable_writer_set_limits(w, update_index, update_index);
for (i = 0; i < N; i++) {
char name[100];
int n;
snprintf(name, sizeof(name), "refs/heads/branch%02d", i);
ref.refname = name;
ref.update_index = update_index;
ref.value_type = REFTABLE_REF_VAL1;
set_test_hash(ref.value.val1, i);
(*names)[i] = xstrdup(name);
n = reftable_writer_add_ref(w, &ref);
check_int(n, ==, 0);
refs[i].refname = (*names)[i] = xstrfmt("refs/heads/branch%02d", i);
refs[i].update_index = update_index;
refs[i].value_type = REFTABLE_REF_VAL1;
t_reftable_set_hash(refs[i].value.val1, i, GIT_SHA1_FORMAT_ID);
}
for (i = 0; i < N; i++) {
char name[100];
int n;
snprintf(name, sizeof(name), "refs/heads/branch%02d", i);
log.refname = name;
log.update_index = update_index;
log.value_type = REFTABLE_LOG_UPDATE;
set_test_hash(log.value.update.new_hash, i);
log.value.update.message = (char *) "message";
n = reftable_writer_add_log(w, &log);
check_int(n, ==, 0);
logs[i].refname = (*names)[i];
logs[i].update_index = update_index;
logs[i].value_type = REFTABLE_LOG_UPDATE;
t_reftable_set_hash(logs[i].value.update.new_hash, i,
GIT_SHA1_FORMAT_ID);
logs[i].value.update.message = (char *) "message";
}
n = reftable_writer_close(w);
check_int(n, ==, 0);
t_reftable_write_to_buf(buf, refs, N, logs, N, &opts);
stats = reftable_writer_stats(w);
for (i = 0; i < stats->ref_stats.blocks; i++) {
int off = i * opts.block_size;
if (!off)
off = header_size((hash_id == GIT_SHA256_FORMAT_ID) ? 2 : 1);
check_char(buf->buf[off], ==, 'r');
}
check_int(stats->log_stats.blocks, >, 0);
reftable_writer_free(w);
free(refs);
free(logs);
}
static void t_log_buffer_size(void)
@ -138,8 +96,7 @@ static void t_log_buffer_size(void)
.time = 0x5e430672,
.message = (char *) "commit: 9\n",
} } };
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
/* This tests buffer extension for log compression. Must use a random
hash, to ensure that the compressed part is larger than the original.
@ -181,8 +138,7 @@ static void t_log_overflow(void)
},
},
};
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
memset(msg, 'x', sizeof(msg) - 1);
reftable_writer_set_limits(w, update_index, update_index);
@ -208,8 +164,7 @@ static void t_log_write_read(void)
struct reftable_reader *reader;
struct reftable_block_source source = { 0 };
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
const struct reftable_stats *stats = NULL;
reftable_writer_set_limits(w, 0, N);
for (i = 0; i < N; i++) {
@ -229,8 +184,10 @@ static void t_log_write_read(void)
log.refname = names[i];
log.update_index = i;
log.value_type = REFTABLE_LOG_UPDATE;
set_test_hash(log.value.update.old_hash, i);
set_test_hash(log.value.update.new_hash, i + 1);
t_reftable_set_hash(log.value.update.old_hash, i,
GIT_SHA1_FORMAT_ID);
t_reftable_set_hash(log.value.update.new_hash, i + 1,
GIT_SHA1_FORMAT_ID);
err = reftable_writer_add_log(w, &log);
check(!err);
@ -297,8 +254,7 @@ static void t_log_zlib_corruption(void)
struct reftable_reader *reader;
struct reftable_block_source source = { 0 };
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
const struct reftable_stats *stats = NULL;
char message[100] = { 0 };
int err, i, n;
@ -528,15 +484,12 @@ static void t_table_refs_for(int indexed)
int err;
struct reftable_reader *reader;
struct reftable_block_source source = { 0 };
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
struct reftable_iterator it = { 0 };
int j;
set_test_hash(want_hash, 4);
t_reftable_set_hash(want_hash, 4, GIT_SHA1_FORMAT_ID);
for (i = 0; i < N; i++) {
uint8_t hash[GIT_SHA1_RAWSZ];
@ -552,8 +505,10 @@ static void t_table_refs_for(int indexed)
ref.refname = name;
ref.value_type = REFTABLE_REF_VAL2;
set_test_hash(ref.value.val2.value, i / 4);
set_test_hash(ref.value.val2.target_value, 3 + i / 4);
t_reftable_set_hash(ref.value.val2.value, i / 4,
GIT_SHA1_FORMAT_ID);
t_reftable_set_hash(ref.value.val2.target_value, 3 + i / 4,
GIT_SHA1_FORMAT_ID);
/* 80 bytes / entry, so 3 entries per block. Yields 17
*/
@ -618,8 +573,7 @@ static void t_write_empty_table(void)
{
struct reftable_write_options opts = { 0 };
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
struct reftable_block_source source = { 0 };
struct reftable_reader *rd = NULL;
struct reftable_ref_record rec = { 0 };
@ -657,8 +611,7 @@ static void t_write_object_id_min_length(void)
.block_size = 75,
};
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
struct reftable_ref_record ref = {
.update_index = 1,
.value_type = REFTABLE_REF_VAL1,
@ -692,8 +645,7 @@ static void t_write_object_id_length(void)
.block_size = 75,
};
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
struct reftable_ref_record ref = {
.update_index = 1,
.value_type = REFTABLE_REF_VAL1,
@ -726,8 +678,7 @@ static void t_write_empty_key(void)
{
struct reftable_write_options opts = { 0 };
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
struct reftable_ref_record ref = {
.refname = (char *) "",
.update_index = 1,
@ -749,8 +700,7 @@ static void t_write_key_order(void)
{
struct reftable_write_options opts = { 0 };
struct strbuf buf = STRBUF_INIT;
struct reftable_writer *w =
reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts);
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
struct reftable_ref_record refs[2] = {
{
.refname = (char *) "b",
@ -798,7 +748,7 @@ static void t_write_multiple_indices(void)
struct reftable_reader *reader;
int err, i;
writer = reftable_new_writer(&strbuf_add_void, &noop_flush, &writer_buf, &opts);
writer = t_reftable_strbuf_writer(&writer_buf, &opts);
reftable_writer_set_limits(writer, 1, 1);
for (i = 0; i < 100; i++) {
struct reftable_ref_record ref = {
@ -876,7 +826,7 @@ static void t_write_multi_level_index(void)
struct reftable_reader *reader;
int err;
writer = reftable_new_writer(&strbuf_add_void, &noop_flush, &writer_buf, &opts);
writer = t_reftable_strbuf_writer(&writer_buf, &opts);
reftable_writer_set_limits(writer, 1, 1);
for (size_t i = 0; i < 200; i++) {
struct reftable_ref_record ref = {

25
t/unit-tests/t-reftable-stack.c
View File

@ -7,17 +7,13 @@ https://developers.google.com/open-source/licenses/bsd
*/
#include "test-lib.h"
#include "lib-reftable.h"
#include "reftable/merged.h"
#include "reftable/reader.h"
#include "reftable/reftable-error.h"
#include "reftable/stack.h"
#include <dirent.h>
static void set_test_hash(uint8_t *p, int i)
{
memset(p, (uint8_t)i, hash_size(GIT_SHA1_FORMAT_ID));
}
static void clear_dir(const char *dirname)
{
struct strbuf path = STRBUF_INIT;
@ -125,7 +121,7 @@ static void write_n_ref_tables(struct reftable_stack *st,
strbuf_reset(&buf);
strbuf_addf(&buf, "refs/heads/branch-%04"PRIuMAX, (uintmax_t)i);
ref.refname = buf.buf;
set_test_hash(ref.value.val1, i);
t_reftable_set_hash(ref.value.val1, i, GIT_SHA1_FORMAT_ID);
err = reftable_stack_add(st, &write_test_ref, &ref);
check(!err);
@ -470,13 +466,13 @@ static void t_reftable_stack_add(void)
refs[i].refname = xstrdup(buf);
refs[i].update_index = i + 1;
refs[i].value_type = REFTABLE_REF_VAL1;
set_test_hash(refs[i].value.val1, i);
t_reftable_set_hash(refs[i].value.val1, i, GIT_SHA1_FORMAT_ID);
logs[i].refname = xstrdup(buf);
logs[i].update_index = N + i + 1;
logs[i].value_type = REFTABLE_LOG_UPDATE;
logs[i].value.update.email = xstrdup("identity@invalid");
set_test_hash(logs[i].value.update.new_hash, i);
t_reftable_set_hash(logs[i].value.update.new_hash, i, GIT_SHA1_FORMAT_ID);
}
for (i = 0; i < N; i++) {
@ -562,14 +558,14 @@ static void t_reftable_stack_iterator(void)
refs[i].refname = xstrfmt("branch%02"PRIuMAX, (uintmax_t)i);
refs[i].update_index = i + 1;
refs[i].value_type = REFTABLE_REF_VAL1;
set_test_hash(refs[i].value.val1, i);
t_reftable_set_hash(refs[i].value.val1, i, GIT_SHA1_FORMAT_ID);
logs[i].refname = xstrfmt("branch%02"PRIuMAX, (uintmax_t)i);
logs[i].update_index = i + 1;
logs[i].value_type = REFTABLE_LOG_UPDATE;
logs[i].value.update.email = xstrdup("johndoe@invalid");
logs[i].value.update.message = xstrdup("commit\n");
set_test_hash(logs[i].value.update.new_hash, i);
t_reftable_set_hash(logs[i].value.update.new_hash, i, GIT_SHA1_FORMAT_ID);
}
for (i = 0; i < N; i++) {
@ -704,7 +700,8 @@ static void t_reftable_stack_tombstone(void)
refs[i].update_index = i + 1;
if (i % 2 == 0) {
refs[i].value_type = REFTABLE_REF_VAL1;
set_test_hash(refs[i].value.val1, i);
t_reftable_set_hash(refs[i].value.val1, i,
GIT_SHA1_FORMAT_ID);
}
logs[i].refname = xstrdup(buf);
@ -712,7 +709,8 @@ static void t_reftable_stack_tombstone(void)
logs[i].update_index = 42;
if (i % 2 == 0) {
logs[i].value_type = REFTABLE_LOG_UPDATE;
set_test_hash(logs[i].value.update.new_hash, i);
t_reftable_set_hash(logs[i].value.update.new_hash, i,
GIT_SHA1_FORMAT_ID);
logs[i].value.update.email =
xstrdup("identity@invalid");
}
@ -844,7 +842,8 @@ static void t_reflog_expire(void)
logs[i].value_type = REFTABLE_LOG_UPDATE;
logs[i].value.update.time = i;
logs[i].value.update.email = xstrdup("identity@invalid");
set_test_hash(logs[i].value.update.new_hash, i);
t_reftable_set_hash(logs[i].value.update.new_hash, i,
GIT_SHA1_FORMAT_ID);
}
for (i = 1; i <= N; i++) {

1
trace2.h
View File

@ -554,6 +554,7 @@ enum trace2_counter_id {
TRACE2_COUNTER_ID_TEST2, /* emits summary and thread events */
TRACE2_COUNTER_ID_PACKED_REFS_JUMPS, /* counts number of jumps */
TRACE2_COUNTER_ID_REFTABLE_RESEEKS, /* counts number of re-seeks */
/* counts number of fsyncs */
TRACE2_COUNTER_ID_FSYNC_WRITEOUT_ONLY,

5
trace2/tr2_ctr.c
View File

@ -31,6 +31,11 @@ static struct tr2_counter_metadata tr2_counter_metadata[TRACE2_NUMBER_OF_COUNTER
.name = "jumps_made",
.want_per_thread_events = 0,
},
[TRACE2_COUNTER_ID_REFTABLE_RESEEKS] = {
.category = "reftable",
.name = "reseeks_made",
.want_per_thread_events = 0,
},
[TRACE2_COUNTER_ID_FSYNC_WRITEOUT_ONLY] = {
.category = "fsync",
.name = "writeout-only",