mirrors/git
0
0
Fork 0
mirror of https://github.com/git/git.git synced 2024-11-27 20:14:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'np/pack'

Browse Source 
* np/pack: (27 commits)
  document --index-version for index-pack and pack-objects
  pack-objects: remove obsolete comments
  pack-objects: better check_object() performances
  add get_size_from_delta()
  pack-objects: make in_pack_header_size a variable of its own
  pack-objects: get rid of create_final_object_list()
  pack-objects: get rid of reuse_cached_pack
  pack-objects: clean up list sorting
  pack-objects: rework check_delta_limit usage
  pack-objects: equal objects in size should delta against newer objects
  pack-objects: optimize preferred base handling a bit
  clean up add_object_entry()
  tests for various pack index features
  use test-genrandom in tests instead of /dev/urandom
  simple random data generator for tests
  validate reused pack data with CRC when possible
  allow forcing index v2 and 64-bit offset treshold
  pack-redundant.c: learn about index v2
  show-index.c: learn about index v2
  sha1_file.c: learn about index version 2
  ...
This commit is contained in:
Junio C Hamano 2007-04-21 17:20:50 -07:00
commit 99ebd06c18
21 changed files with 1016 additions and 504 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -149,6 +149,7 @@ test-chmtime
test-date
test-delta
test-dump-cache-tree
test-genrandom
test-match-trees
common-cmds.h
*.tar.gz

5
Documentation/git-index-pack.txt
View File

@ -68,6 +68,11 @@ OPTIONS
message can later be searched for within all .keep files to
locate any which have outlived their usefulness.
--index-version=<version>[,<offset>]::
This is intended to be used by the test suite only. It allows
to force the version for the generated pack index, and to force
64-bit index entries on objects located above the given offset.
Note
----

5
Documentation/git-pack-objects.txt
View File

@ -138,6 +138,11 @@ base-name::
length, this option typically shrinks the resulting
packfile by 3-5 per-cent.
--index-version=<version>[,<offset>]::
This is intended to be used by the test suite only. It allows
to force the version for the generated pack index, and to force
64-bit index entries on objects located above the given offset.
Author
------

7
Makefile
View File

@ -933,7 +933,7 @@ endif
export NO_SVN_TESTS
test: all test-chmtime$X
test: all test-chmtime$X test-genrandom$X
$(MAKE) -C t/ all
test-date$X: test-date.c date.o ctype.o
@ -954,6 +954,9 @@ test-match-trees$X: test-match-trees.o $(GITLIBS)
test-chmtime$X: test-chmtime.c
$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $<
test-genrandom$X: test-genrandom.c
$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $<
check-sha1:: test-sha1$X
./test-sha1.sh
@ -1042,7 +1045,7 @@ dist-doc:
clean:
rm -f *.o mozilla-sha1/*.o arm/*.o ppc/*.o compat/*.o xdiff/*.o \
test-chmtime$X $(LIB_FILE) $(XDIFF_LIB)
test-chmtime$X test-genrandom$X $(LIB_FILE) $(XDIFF_LIB)
rm -f $(ALL_PROGRAMS) $(BUILT_INS) git$X
rm -f *.spec *.pyc *.pyo */*.pyc */*.pyo common-cmds.h TAGS tags
rm -rf autom4te.cache

2
builtin-count-objects.c
View File

@ -111,7 +111,7 @@ int cmd_count_objects(int ac, const char **av, const char *prefix)
for (p = packed_git; p; p = p->next) {
if (!p->pack_local)
continue;
packed += num_packed_objects(p);
packed += p->num_objects;
num_pack++;
}
printf("count: %lu\n", loose);

2
builtin-fsck.c
View File

@ -661,7 +661,7 @@ int cmd_fsck(int argc, char **argv, const char *prefix)
verify_pack(p, 0);
for (p = packed_git; p; p = p->next) {
uint32_t i, num = num_packed_objects(p);
uint32_t i, num = p->num_objects;
for (i = 0; i < num; i++)
fsck_sha1(nth_packed_object_sha1(p, i));
}

821
builtin-pack-objects.c
View File

File diff suppressed because it is too large Load Diff

19
builtin-unpack-objects.c
View File

@ -13,7 +13,8 @@ static const char unpack_usage[] = "git-unpack-objects [-n] [-q] [-r] < pack-fil
/* We always read in 4kB chunks. */
static unsigned char buffer[4096];
static unsigned long offset, len, consumed_bytes;
static unsigned int offset, len;
static off_t consumed_bytes;
static SHA_CTX ctx;
/*
@ -49,6 +50,10 @@ static void use(int bytes)
die("used more bytes than were available");
len -= bytes;
offset += bytes;
/* make sure off_t is sufficiently large not to wrap */
if (consumed_bytes > consumed_bytes + bytes)
die("pack too large for current definition of off_t");
consumed_bytes += bytes;
}
@ -88,17 +93,17 @@ static void *get_data(unsigned long size)
struct delta_info {
unsigned char base_sha1[20];
unsigned long base_offset;
unsigned nr;
off_t base_offset;
unsigned long size;
void *delta;
unsigned nr;
struct delta_info *next;
};
static struct delta_info *delta_list;
static void add_delta_to_list(unsigned nr, unsigned const char *base_sha1,
unsigned long base_offset,
off_t base_offset,
void *delta, unsigned long size)
{
struct delta_info *info = xmalloc(sizeof(*info));
@ -113,7 +118,7 @@ static void add_delta_to_list(unsigned nr, unsigned const char *base_sha1,
}
struct obj_info {
unsigned long offset;
off_t offset;
unsigned char sha1[20];
};
@ -200,7 +205,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
} else {
unsigned base_found = 0;
unsigned char *pack, c;
unsigned long base_offset;
off_t base_offset;
unsigned lo, mid, hi;
pack = fill(1);
@ -209,7 +214,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
base_offset = c & 127;
while (c & 128) {
base_offset += 1;
if (!base_offset || base_offset & ~(~0UL >> 7))
if (!base_offset || MSB(base_offset, 7))
die("offset value overflow for delta base object");
pack = fill(1);
c = *pack;

9
cache.h
View File

@ -376,11 +376,12 @@ struct pack_window {
extern struct packed_git {
struct packed_git *next;
struct pack_window *windows;
const void *index_data;
off_t index_size;
off_t pack_size;
time_t mtime;
const void *index_data;
size_t index_size;
uint32_t num_objects;
int index_version;
time_t mtime;
int pack_fd;
int pack_local;
unsigned char sha1[20];
@ -431,11 +432,11 @@ extern void pack_report(void);
extern unsigned char* use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
extern void unuse_pack(struct pack_window **);
extern struct packed_git *add_packed_git(const char *, int, int);
extern uint32_t num_packed_objects(const struct packed_git *p);
extern const unsigned char *nth_packed_object_sha1(const struct packed_git *, uint32_t);
extern off_t find_pack_entry_one(const unsigned char *, struct packed_git *);
extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *);
extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);
extern const char *packed_object_info_detail(struct packed_git *, off_t, unsigned long *, unsigned long *, unsigned int *, unsigned char *);
/* Dumb servers support */

14
csum-file.c
View File

@ -49,6 +49,8 @@ int sha1close(struct sha1file *f, unsigned char *result, int update)
int sha1write(struct sha1file *f, void *buf, unsigned int count)
{
if (f->do_crc)
f->crc32 = crc32(f->crc32, buf, count);
while (count) {
unsigned offset = f->offset;
unsigned left = sizeof(f->buffer) - offset;
@ -91,6 +93,7 @@ struct sha1file *sha1create(const char *fmt, ...)
f->fd = fd;
f->error = 0;
f->offset = 0;
f->do_crc = 0;
SHA1_Init(&f->ctx);
return f;
}
@ -111,6 +114,7 @@ struct sha1file *sha1fd(int fd, const char *name)
f->fd = fd;
f->error = 0;
f->offset = 0;
f->do_crc = 0;
SHA1_Init(&f->ctx);
return f;
}
@ -143,4 +147,14 @@ int sha1write_compressed(struct sha1file *f, void *in, unsigned int size)
return size;
}
void crc32_begin(struct sha1file *f)
{
f->crc32 = crc32(0, Z_NULL, 0);
f->do_crc = 1;
}
uint32_t crc32_end(struct sha1file *f)
{
f->do_crc = 0;
return f->crc32;
}

4
csum-file.h
View File

@ -7,6 +7,8 @@ struct sha1file {
unsigned int offset, namelen;
SHA_CTX ctx;
char name[PATH_MAX];
int do_crc;
uint32_t crc32;
unsigned char buffer[8192];
};
@ -15,5 +17,7 @@ extern struct sha1file *sha1create(const char *fmt, ...) __attribute__((format (
extern int sha1close(struct sha1file *, unsigned char *, int);
extern int sha1write(struct sha1file *, void *, unsigned int);
extern int sha1write_compressed(struct sha1file *, void *, unsigned int);
extern void crc32_begin(struct sha1file *);
extern uint32_t crc32_end(struct sha1file *);
#endif

8
git-compat-util.h
View File

@ -13,6 +13,14 @@
#define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
#ifdef __GNUC__
#define TYPEOF(x) (__typeof__(x))
#else
#define TYPEOF(x)
#endif
#define MSB(x, bits) ((x) & TYPEOF(x)(~0ULL << (sizeof(x) * 8 - (bits))))
#if !defined(__APPLE__) && !defined(__FreeBSD__)
#define _XOPEN_SOURCE 600 /* glibc2 and AIX 5.3L need 500, OpenBSD needs 600 for S_ISLNK() */
#define _XOPEN_SOURCE_EXTENDED 1 /* AIX 5.3L needs this */

110
index-pack.c
View File

@ -12,9 +12,10 @@ static const char index_pack_usage[] =
struct object_entry
{
unsigned long offset;
off_t offset;
unsigned long size;
unsigned int hdr_size;
uint32_t crc32;
enum object_type type;
enum object_type real_type;
unsigned char sha1[20];
@ -22,7 +23,7 @@ struct object_entry
union delta_base {
unsigned char sha1[20];
unsigned long offset;
off_t offset;
};
/*
@ -83,8 +84,10 @@ static unsigned display_progress(unsigned n, unsigned total, unsigned last_pc)
/* We always read in 4kB chunks. */
static unsigned char input_buffer[4096];
static unsigned long input_offset, input_len, consumed_bytes;
static unsigned int input_offset, input_len;
static off_t consumed_bytes;
static SHA_CTX input_ctx;
static uint32_t input_crc32;
static int input_fd, output_fd, pack_fd;
/* Discard current buffer used content. */
@ -127,8 +130,13 @@ static void use(int bytes)
{
if (bytes > input_len)
die("used more bytes than were available");
input_crc32 = crc32(input_crc32, input_buffer + input_offset, bytes);
input_len -= bytes;
input_offset += bytes;
/* make sure off_t is sufficiently large not to wrap */
if (consumed_bytes > consumed_bytes + bytes)
die("pack too large for current definition of off_t");
consumed_bytes += bytes;
}
@ -216,10 +224,13 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size)
static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_base)
{
unsigned char *p, c;
unsigned long size, base_offset;
unsigned long size;
off_t base_offset;
unsigned shift;
void *data;
obj->offset = consumed_bytes;
input_crc32 = crc32(0, Z_NULL, 0);
p = fill(1);
c = *p;
@ -249,7 +260,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
base_offset = c & 127;
while (c & 128) {
base_offset += 1;
if (!base_offset || base_offset & ~(~0UL >> 7))
if (!base_offset || MSB(base_offset, 7))
bad_object(obj->offset, "offset value overflow for delta base object");
p = fill(1);
c = *p;
@ -270,7 +281,9 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
}
obj->hdr_size = consumed_bytes - obj->offset;
return unpack_entry_data(obj->offset, obj->size);
data = unpack_entry_data(obj->offset, obj->size);
obj->crc32 = input_crc32;
return data;
}
static void *get_data_from_pack(struct object_entry *obj)
@ -515,7 +528,7 @@ static void parse_pack_objects(unsigned char *sha1)
fputc('\n', stderr);
}
static int write_compressed(int fd, void *in, unsigned int size)
static int write_compressed(int fd, void *in, unsigned int size, uint32_t *obj_crc)
{
z_stream stream;
unsigned long maxsize;
@ -536,6 +549,7 @@ static int write_compressed(int fd, void *in, unsigned int size)
size = stream.total_out;
write_or_die(fd, out, size);
*obj_crc = crc32(*obj_crc, out, size);
free(out);
return size;
}
@ -556,8 +570,10 @@ static void append_obj_to_pack(const unsigned char *sha1, void *buf,
}
header[n++] = c;
write_or_die(output_fd, header, n);
obj[0].crc32 = crc32(0, Z_NULL, 0);
obj[0].crc32 = crc32(obj[0].crc32, header, n);
obj[1].offset = obj[0].offset + n;
obj[1].offset += write_compressed(output_fd, buf, size);
obj[1].offset += write_compressed(output_fd, buf, size, &obj[0].crc32);
hashcpy(obj->sha1, sha1);
}
@ -655,6 +671,9 @@ static void readjust_pack_header_and_sha1(unsigned char *sha1)
write_or_die(output_fd, sha1, 20);
}
static uint32_t index_default_version = 1;
static uint32_t index_off32_limit = 0x7fffffff;
static int sha1_compare(const void *_a, const void *_b)
{
struct object_entry *a = *(struct object_entry **)_a;
@ -670,9 +689,10 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
{
struct sha1file *f;
struct object_entry **sorted_by_sha, **list, **last;
unsigned int array[256];
uint32_t array[256];
int i, fd;
SHA_CTX ctx;
uint32_t index_version;
if (nr_objects) {
sorted_by_sha =
@ -683,7 +703,6 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
sorted_by_sha[i] = &objects[i];
qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
sha1_compare);
}
else
sorted_by_sha = list = last = NULL;
@ -702,6 +721,17 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
die("unable to create %s: %s", index_name, strerror(errno));
f = sha1fd(fd, index_name);
/* if last object's offset is >= 2^31 we should use index V2 */
index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version;
/* index versions 2 and above need a header */
if (index_version >= 2) {
struct pack_idx_header hdr;
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(index_version);
sha1write(f, &hdr, sizeof(hdr));
}
/*
* Write the first-level table (the list is sorted,
* but we use a 256-entry lookup to be able to avoid
@ -718,24 +748,61 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
array[i] = htonl(next - sorted_by_sha);
list = next;
}
sha1write(f, array, 256 * sizeof(int));
sha1write(f, array, 256 * 4);
/* recompute the SHA1 hash of sorted object names.
* currently pack-objects does not do this, but that
* can be fixed.
*/
/* compute the SHA1 hash of sorted object names. */
SHA1_Init(&ctx);
/*
* Write the actual SHA1 entries..
*/
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
unsigned int offset = htonl(obj->offset);
sha1write(f, &offset, 4);
if (index_version < 2) {
uint32_t offset = htonl(obj->offset);
sha1write(f, &offset, 4);
}
sha1write(f, obj->sha1, 20);
SHA1_Update(&ctx, obj->sha1, 20);
}
if (index_version >= 2) {
unsigned int nr_large_offset = 0;
/* write the crc32 table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
uint32_t crc32_val = htonl(obj->crc32);
sha1write(f, &crc32_val, 4);
}
/* write the 32-bit offset table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
uint32_t offset = (obj->offset <= index_off32_limit) ?
obj->offset : (0x80000000 | nr_large_offset++);
offset = htonl(offset);
sha1write(f, &offset, 4);
}
/* write the large offset table */
list = sorted_by_sha;
while (nr_large_offset) {
struct object_entry *obj = *list++;
uint64_t offset = obj->offset;
if (offset > index_off32_limit) {
uint32_t split[2];
split[0] = htonl(offset >> 32);
split[1] = htonl(offset & 0xffffffff);
sha1write(f, split, 8);
nr_large_offset--;
}
}
}
sha1write(f, sha1, 20);
sha1close(f, NULL, 1);
free(sorted_by_sha);
@ -865,6 +932,15 @@ int main(int argc, char **argv)
if (index_name || (i+1) >= argc)
usage(index_pack_usage);
index_name = argv[++i];
} else if (!prefixcmp(arg, "--index-version=")) {
char *c;
index_default_version = strtoul(arg + 16, &c, 10);
if (index_default_version > 2)
die("bad %s", arg);
if (*c == ',')
index_off32_limit = strtoul(c+1, &c, 0);
if (*c || index_off32_limit & 0x80000000)
die("bad %s", arg);
} else
usage(index_pack_usage);
continue;

4
pack-check.c
View File

@ -40,7 +40,7 @@ static int verify_packfile(struct packed_git *p,
* have verified that nr_objects matches between idx and pack,
* we do not do scan-streaming check on the pack file.
*/
nr_objects = num_packed_objects(p);
nr_objects = p->num_objects;
for (i = 0, err = 0; i < nr_objects; i++) {
const unsigned char *sha1;
void *data;
@ -79,7 +79,7 @@ static void show_pack_info(struct packed_git *p)
{
uint32_t nr_objects, i, chain_histogram[MAX_CHAIN];
nr_objects = num_packed_objects(p);
nr_objects = p->num_objects;
memset(chain_histogram, 0, sizeof(chain_histogram));
for (i = 0; i < nr_objects; i++) {

47
pack-redundant.c
View File

@ -247,16 +247,19 @@ static struct pack_list * pack_list_difference(const struct pack_list *A,
static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
{
int p1_off, p2_off;
unsigned long p1_off = 0, p2_off = 0, p1_step, p2_step;
const unsigned char *p1_base, *p2_base;
struct llist_item *p1_hint = NULL, *p2_hint = NULL;
p1_off = p2_off = 256 * 4 + 4;
p1_base = p1->pack->index_data;
p2_base = p2->pack->index_data;
p1_base += 256 * 4 + ((p1->pack->index_version < 2) ? 4 : 8);
p2_base += 256 * 4 + ((p2->pack->index_version < 2) ? 4 : 8);
p1_step = (p1->pack->index_version < 2) ? 24 : 20;
p2_step = (p2->pack->index_version < 2) ? 24 : 20;
while (p1_off <= p1->pack->index_size - 3 * 20 &&
p2_off <= p2->pack->index_size - 3 * 20)
while (p1_off < p1->pack->num_objects * p1_step &&
p2_off < p2->pack->num_objects * p2_step)
{
int cmp = hashcmp(p1_base + p1_off, p2_base + p2_off);
/* cmp ~ p1 - p2 */
@ -265,14 +268,14 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
p1_base + p1_off, p1_hint);
p2_hint = llist_sorted_remove(p2->unique_objects,
p1_base + p1_off, p2_hint);
p1_off+=24;
p2_off+=24;
p1_off += p1_step;
p2_off += p2_step;
continue;
}
if (cmp < 0) { /* p1 has the object, p2 doesn't */
p1_off+=24;
p1_off += p1_step;
} else { /* p2 has the object, p1 doesn't */
p2_off+=24;
p2_off += p2_step;
}
}
}
@ -352,28 +355,31 @@ static int is_superset(struct pack_list *pl, struct llist *list)
static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
{
size_t ret = 0;
int p1_off, p2_off;
unsigned long p1_off = 0, p2_off = 0, p1_step, p2_step;
const unsigned char *p1_base, *p2_base;
p1_off = p2_off = 256 * 4 + 4;
p1_base = p1->index_data;
p2_base = p2->index_data;
p1_base += 256 * 4 + ((p1->index_version < 2) ? 4 : 8);
p2_base += 256 * 4 + ((p2->index_version < 2) ? 4 : 8);
p1_step = (p1->index_version < 2) ? 24 : 20;
p2_step = (p2->index_version < 2) ? 24 : 20;
while (p1_off <= p1->index_size - 3 * 20 &&
p2_off <= p2->index_size - 3 * 20)
while (p1_off < p1->num_objects * p1_step &&
p2_off < p2->num_objects * p2_step)
{
int cmp = hashcmp(p1_base + p1_off, p2_base + p2_off);
/* cmp ~ p1 - p2 */
if (cmp == 0) {
ret++;
p1_off+=24;
p2_off+=24;
p1_off += p1_step;
p2_off += p2_step;
continue;
}
if (cmp < 0) { /* p1 has the object, p2 doesn't */
p1_off+=24;
p1_off += p1_step;
} else { /* p2 has the object, p1 doesn't */
p2_off+=24;
p2_off += p2_step;
}
}
return ret;
@ -535,7 +541,7 @@ static void scan_alt_odb_packs(void)
static struct pack_list * add_pack(struct packed_git *p)
{
struct pack_list l;
size_t off;
unsigned long off = 0, step;
const unsigned char *base;
if (!p->pack_local && !(alt_odb || verbose))
@ -544,11 +550,12 @@ static struct pack_list * add_pack(struct packed_git *p)
l.pack = p;
llist_init(&l.all_objects);
off = 256 * 4 + 4;
base = p->index_data;
while (off <= p->index_size - 3 * 20) {
base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
step = (p->index_version < 2) ? 24 : 20;
while (off < p->num_objects * step) {
llist_insert_back(l.all_objects, base + off);
off += 24;
off += step;
}
/* this list will be pruned in cmp_two_packs later */
l.unique_objects = llist_copy(l.all_objects);

210
sha1_file.c
View File

@ -437,7 +437,7 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
void *idx_map;
struct pack_idx_header *hdr;
size_t idx_size;
uint32_t nr, i, *index;
uint32_t version, nr, i, *index;
int fd = open(path, O_RDONLY);
struct stat st;
@ -455,21 +455,23 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
close(fd);
/* a future index format would start with this, as older git
* binaries would fail the non-monotonic index check below.
* give a nicer warning to the user if we can.
*/
hdr = idx_map;
if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
munmap(idx_map, idx_size);
return error("index file %s is a newer version"
" and is not supported by this binary"
" (try upgrading GIT to a newer version)",
path);
}
version = ntohl(hdr->idx_version);
if (version < 2 || version > 2) {
munmap(idx_map, idx_size);
return error("index file %s is version %d"
" and is not supported by this binary"
" (try upgrading GIT to a newer version)",
path, version);
}
} else
version = 1;
nr = 0;
index = idx_map;
if (version > 1)
index += 2; /* skip index header */
for (i = 0; i < 256; i++) {
uint32_t n = ntohl(index[i]);
if (n < nr) {
@ -479,21 +481,51 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
nr = n;
}
/*
* Total size:
* - 256 index entries 4 bytes each
* - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
* - 20-byte SHA1 of the packfile
* - 20-byte SHA1 file checksum
*/
if (idx_size != 4*256 + nr * 24 + 20 + 20) {
munmap(idx_map, idx_size);
return error("wrong index file size in %s", path);
if (version == 1) {
/*
* Total size:
* - 256 index entries 4 bytes each
* - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
* - 20-byte SHA1 of the packfile
* - 20-byte SHA1 file checksum
*/
if (idx_size != 4*256 + nr * 24 + 20 + 20) {
munmap(idx_map, idx_size);
return error("wrong index file size in %s", path);
}
} else if (version == 2) {
/*
* Minimum size:
* - 8 bytes of header
* - 256 index entries 4 bytes each
* - 20-byte sha1 entry * nr
* - 4-byte crc entry * nr
* - 4-byte offset entry * nr
* - 20-byte SHA1 of the packfile
* - 20-byte SHA1 file checksum
* And after the 4-byte offset table might be a
* variable sized table containing 8-byte entries
* for offsets larger than 2^31.
*/
unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
if (idx_size < min_size || idx_size > min_size + (nr - 1)*8) {
munmap(idx_map, idx_size);
return error("wrong index file size in %s", path);
}
if (idx_size != min_size) {
/* make sure we can deal with large pack offsets */
off_t x = 0x7fffffffUL, y = 0xffffffffUL;
if (x > (x + 1) || y > (y + 1)) {
munmap(idx_map, idx_size);
return error("pack too large for current definition of off_t in %s", path);
}
}
}
p->index_version = 1;
p->index_version = version;
p->index_data = idx_map;
p->index_size = idx_size;
p->num_objects = nr;
return 0;
}
@ -605,11 +637,11 @@ static int open_packed_git_1(struct packed_git *p)
p->pack_name, ntohl(hdr.hdr_version));
/* Verify the pack matches its index. */
if (num_packed_objects(p) != ntohl(hdr.hdr_entries))
if (p->num_objects != ntohl(hdr.hdr_entries))
return error("packfile %s claims to have %u objects"
" while index size indicates %u objects",
p->pack_name, ntohl(hdr.hdr_entries),
num_packed_objects(p));
" while index indicates %u objects",
p->pack_name, ntohl(hdr.hdr_entries),
p->num_objects);
if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
return error("end of packfile %s is unavailable", p->pack_name);
if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
@ -1128,6 +1160,43 @@ static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type
return unpack_sha1_rest(&stream, hdr, *size, sha1);
}
unsigned long get_size_from_delta(struct packed_git *p,
struct pack_window **w_curs,
off_t curpos)
{
const unsigned char *data;
unsigned char delta_head[20], *in;
z_stream stream;
int st;
memset(&stream, 0, sizeof(stream));
stream.next_out = delta_head;
stream.avail_out = sizeof(delta_head);
inflateInit(&stream);
do {
in = use_pack(p, w_curs, curpos, &stream.avail_in);
stream.next_in = in;
st = inflate(&stream, Z_FINISH);
curpos += stream.next_in - in;
} while ((st == Z_OK || st == Z_BUF_ERROR) &&
stream.total_out < sizeof(delta_head));
inflateEnd(&stream);
if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head))
die("delta data unpack-initial failed");
/* Examine the initial part of the delta to figure out
* the result size.
*/
data = delta_head;
/* ignore base size */
get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
/* Read the result size */
return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
}
static off_t get_delta_base(struct packed_git *p,
struct pack_window **w_curs,
off_t *curpos,
@ -1149,7 +1218,7 @@ static off_t get_delta_base(struct packed_git *p,
base_offset = c & 127;
while (c & 128) {
base_offset += 1;
if (!base_offset || base_offset & ~(~0UL >> 7))
if (!base_offset || MSB(base_offset, 7))
die("offset value overflow for delta base object");
c = base_info[used++];
base_offset = (base_offset << 7) + (c & 127);
@ -1191,40 +1260,8 @@ static int packed_delta_info(struct packed_git *p,
* based on a base with a wrong size. This saves tons of
* inflate() calls.
*/
if (sizep) {
const unsigned char *data;
unsigned char delta_head[20], *in;
z_stream stream;
int st;
memset(&stream, 0, sizeof(stream));
stream.next_out = delta_head;
stream.avail_out = sizeof(delta_head);
inflateInit(&stream);
do {
in = use_pack(p, w_curs, curpos, &stream.avail_in);
stream.next_in = in;
st = inflate(&stream, Z_FINISH);
curpos += stream.next_in - in;
} while ((st == Z_OK || st == Z_BUF_ERROR)
&& stream.total_out < sizeof(delta_head));
inflateEnd(&stream);
if ((st != Z_STREAM_END) &&
stream.total_out != sizeof(delta_head))
die("delta data unpack-initial failed");
/* Examine the initial part of the delta to figure out
* the result size.
*/
data = delta_head;
/* ignore base size */
get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
/* Read the result size */
*sizep = get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
}
if (sizep)
*sizep = get_size_from_delta(p, w_curs, curpos);
return type;
}
@ -1526,37 +1563,60 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
return data;
}
uint32_t num_packed_objects(const struct packed_git *p)
{
/* See check_packed_git_idx() */
return (uint32_t)((p->index_size - 20 - 20 - 4*256) / 24);
}
const unsigned char *nth_packed_object_sha1(const struct packed_git *p,
uint32_t n)
{
const unsigned char *index = p->index_data;
index += 4 * 256;
if (num_packed_objects(p) <= n)
if (n >= p->num_objects)
return NULL;
return index + 24 * n + 4;
index += 4 * 256;
if (p->index_version == 1) {
return index + 24 * n + 4;
} else {
index += 8;
return index + 20 * n;
}
}
static off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
{
const unsigned char *index = p->index_data;
index += 4 * 256;
if (p->index_version == 1) {
return ntohl(*((uint32_t *)(index + 24 * n)));
} else {
uint32_t off;
index += 8 + p->num_objects * (20 + 4);
off = ntohl(*((uint32_t *)(index + 4 * n)));
if (!(off & 0x80000000))
return off;
index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
ntohl(*((uint32_t *)(index + 4)));
}
}
off_t find_pack_entry_one(const unsigned char *sha1,
struct packed_git *p)
{
const uint32_t *level1_ofs = p->index_data;
int hi = ntohl(level1_ofs[*sha1]);
int lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
const unsigned char *index = p->index_data;
unsigned hi, lo;
if (p->index_version > 1) {
level1_ofs += 2;
index += 8;
}
index += 4 * 256;
hi = ntohl(level1_ofs[*sha1]);
lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
do {
int mi = (lo + hi) / 2;
int cmp = hashcmp(index + 24 * mi + 4, sha1);
unsigned mi = (lo + hi) / 2;
unsigned x = (p->index_version > 1) ? (mi * 20) : (mi * 24 + 4);
int cmp = hashcmp(index + x, sha1);
if (!cmp)
return ntohl(*((uint32_t *)((char *)index + (24 * mi))));
return nth_packed_object_offset(p, mi);
if (cmp > 0)
hi = mi;
else

2
sha1_name.c
View File

@ -76,7 +76,7 @@ static int find_short_packed_object(int len, const unsigned char *match, unsigne
prepare_packed_git();
for (p = packed_git; p && found < 2; p = p->next) {
uint32_t num = num_packed_objects(p);
uint32_t num = p->num_objects;
uint32_t first = 0, last = num;
while (first < last) {
uint32_t mid = (first + last) / 2;

68
show-index.c
View File

@ -1,14 +1,26 @@
#include "cache.h"
#include "pack.h"
int main(int argc, char **argv)
{
int i;
unsigned nr;
unsigned int entry[6];
unsigned int version;
static unsigned int top_index[256];
if (fread(top_index, sizeof(top_index), 1, stdin) != 1)
die("unable to read index");
if (fread(top_index, 2 * 4, 1, stdin) != 1)
die("unable to read header");
if (top_index[0] == htonl(PACK_IDX_SIGNATURE)) {
version = ntohl(top_index[1]);
if (version < 2 || version > 2)
die("unknown index version");
if (fread(top_index, 256 * 4, 1, stdin) != 1)
die("unable to read index");
} else {
version = 1;
if (fread(&top_index[2], 254 * 4, 1, stdin) != 1)
die("unable to read index");
}
nr = 0;
for (i = 0; i < 256; i++) {
unsigned n = ntohl(top_index[i]);
@ -16,13 +28,51 @@ int main(int argc, char **argv)
die("corrupt index file");
nr = n;
}
for (i = 0; i < nr; i++) {
unsigned offset;
if (version == 1) {
for (i = 0; i < nr; i++) {
unsigned int offset, entry[6];
if (fread(entry, 24, 1, stdin) != 1)
die("unable to read entry %u/%u", i, nr);
offset = ntohl(entry[0]);
printf("%u %s\n", offset, sha1_to_hex((void *)(entry+1)));
if (fread(entry, 4 + 20, 1, stdin) != 1)
die("unable to read entry %u/%u", i, nr);
offset = ntohl(entry[0]);
printf("%u %s\n", offset, sha1_to_hex((void *)(entry+1)));
}
} else {
unsigned off64_nr = 0;
struct {
unsigned char sha1[20];
uint32_t crc;
uint32_t off;
} *entries = xmalloc(nr * sizeof(entries[0]));
for (i = 0; i < nr; i++)
if (fread(entries[i].sha1, 20, 1, stdin) != 1)
die("unable to read sha1 %u/%u", i, nr);
for (i = 0; i < nr; i++)
if (fread(&entries[i].crc, 4, 1, stdin) != 1)
die("unable to read crc %u/%u", i, nr);
for (i = 0; i < nr; i++)
if (fread(&entries[i].off, 4, 1, stdin) != 1)
die("unable to read 32b offset %u/%u", i, nr);
for (i = 0; i < nr; i++) {
uint64_t offset;
uint32_t off = ntohl(entries[i].off);
if (!(off & 0x80000000)) {
offset = off;
} else {
uint32_t off64[2];
if ((off & 0x7fffffff) != off64_nr)
die("inconsistent 64b offset index");
if (fread(off64, 8, 1, stdin) != 1)
die("unable to read 64b offset %u", off64_nr);
offset = (((uint64_t)ntohl(off64[0])) << 32) |
ntohl(off64[1]);
off64_nr++;
}
printf("%llu %s (%08x)\n", (unsigned long long) offset,
sha1_to_hex(entries[i].sha1),
ntohl(entries[i].crc));
}
free(entries);
}
return 0;
}

2
t/t5301-sliding-window.sh
View File

@ -12,7 +12,7 @@ test_expect_success \
for i in a b c
do
echo $i >$i &&
dd if=/dev/urandom bs=32k count=1 >>$i &&
test-genrandom "$i" 32768 >>$i &&
git-update-index --add $i || return 1
done &&
echo d >d && cat c >>d && git-update-index --add d &&

146
t/t5302-pack-index.sh Executable file
View File

@ -0,0 +1,146 @@
#!/bin/sh
#
# Copyright (c) 2007 Nicolas Pitre
#
test_description='pack index with 64-bit offsets and object CRC'
. ./test-lib.sh
test_expect_success \
'setup' \
'rm -rf .git
git-init &&
for i in `seq -w 100`
do
echo $i >file_$i &&
test-genrandom "$i" 8192 >>file_$i &&
git-update-index --add file_$i || return 1
done &&
echo 101 >file_101 && tail -c 8192 file_100 >>file_101 &&
git-update-index --add file_101 &&
tree=`git-write-tree` &&
commit=`git-commit-tree $tree </dev/null` && {
echo $tree &&
git-ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\) .*/\\1/"
} >obj-list &&
git-update-ref HEAD $commit'
test_expect_success \
'pack-objects with index version 1' \
'pack1=$(git-pack-objects --index-version=1 test-1 <obj-list) &&
git-verify-pack -v "test-1-${pack1}.pack"'
test_expect_success \
'pack-objects with index version 2' \
'pack2=$(git-pack-objects --index-version=2 test-2 <obj-list) &&
git-verify-pack -v "test-2-${pack2}.pack"'
test_expect_success \
'both packs should be identical' \
'cmp "test-1-${pack1}.pack" "test-2-${pack2}.pack"'
test_expect_failure \
'index v1 and index v2 should be different' \
'cmp "test-1-${pack1}.idx" "test-2-${pack2}.idx"'
test_expect_success \
'index-pack with index version 1' \
'git-index-pack --index-version=1 -o 1.idx "test-1-${pack1}.pack"'
test_expect_success \
'index-pack with index version 2' \
'git-index-pack --index-version=2 -o 2.idx "test-1-${pack1}.pack"'
test_expect_success \
'index-pack results should match pack-objects ones' \
'cmp "test-1-${pack1}.idx" "1.idx" &&
cmp "test-2-${pack2}.idx" "2.idx"'
test_expect_success \
'index v2: force some 64-bit offsets with pack-objects' \
'pack3=$(git-pack-objects --index-version=2,0x40000 test-3 <obj-list) &&
git-verify-pack -v "test-3-${pack3}.pack"'
test_expect_failure \
'64-bit offsets: should be different from previous index v2 results' \
'cmp "test-2-${pack2}.idx" "test-3-${pack3}.idx"'
test_expect_success \
'index v2: force some 64-bit offsets with index-pack' \
'git-index-pack --index-version=2,0x40000 -o 3.idx "test-1-${pack1}.pack"'
test_expect_success \
'64-bit offsets: index-pack result should match pack-objects one' \
'cmp "test-3-${pack3}.idx" "3.idx"'
test_expect_success \
'[index v1] 1) stream pack to repository' \
'git-index-pack --index-version=1 --stdin < "test-1-${pack1}.pack" &&
git-prune-packed &&
git-count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
cmp "test-1-${pack1}.idx" ".git/objects/pack/pack-${pack1}.idx"'
test_expect_success \
'[index v1] 2) create a stealth corruption in a delta base reference' \
'# this test assumes a delta smaller than 16 bytes at the end of the pack
git-show-index <1.idx | sort -n | tail -n 1 | (
read delta_offs delta_sha1 &&
git-cat-file blob "$delta_sha1" > blob_1 &&
chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($delta_offs + 1)) \
if=".git/objects/pack/pack-${pack1}.idx" skip=$((256 * 4 + 4)) \
bs=1 count=20 conv=notrunc &&
git-cat-file blob "$delta_sha1" > blob_2 )'
test_expect_failure \
'[index v1] 3) corrupted delta happily returned wrong data' \
'cmp blob_1 blob_2'
test_expect_failure \
'[index v1] 4) confirm that the pack is actually corrupted' \
'git-fsck --full $commit'
test_expect_success \
'[index v1] 5) pack-objects happily reuses corrupted data' \
'pack4=$(git-pack-objects test-4 <obj-list) &&
test -f "test-4-${pack1}.pack"'
test_expect_failure \
'[index v1] 6) newly created pack is BAD !' \
'git-verify-pack -v "test-4-${pack1}.pack"'
test_expect_success \
'[index v2] 1) stream pack to repository' \
'rm -f .git/objects/pack/* &&
git-index-pack --index-version=2,0x40000 --stdin < "test-1-${pack1}.pack" &&
git-prune-packed &&
git-count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
cmp "test-3-${pack1}.idx" ".git/objects/pack/pack-${pack1}.idx"'
test_expect_success \
'[index v2] 2) create a stealth corruption in a delta base reference' \
'# this test assumes a delta smaller than 16 bytes at the end of the pack
git-show-index <1.idx | sort -n | tail -n 1 | (
read delta_offs delta_sha1 delta_crc &&
git-cat-file blob "$delta_sha1" > blob_3 &&
chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($delta_offs + 1)) \
if=".git/objects/pack/pack-${pack1}.idx" skip=$((8 + 256 * 4)) \
bs=1 count=20 conv=notrunc &&
git-cat-file blob "$delta_sha1" > blob_4 )'
test_expect_failure \
'[index v2] 3) corrupted delta happily returned wrong data' \
'cmp blob_3 blob_4'
test_expect_failure \
'[index v2] 4) confirm that the pack is actually corrupted' \
'git-fsck --full $commit'
test_expect_failure \
'[index v2] 5) pack-objects refuses to reuse corrupted data' \
'git-pack-objects test-5 <obj-list'
test_done

34
test-genrandom.c Normal file
View File

@ -0,0 +1,34 @@
/*
* Simple random data generator used to create reproducible test files.
* This is inspired from POSIX.1-2001 implementation example for rand().
* Copyright (C) 2007 by Nicolas Pitre, licensed under the GPL version 2.
*/
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char *argv[])
{
unsigned long count, next = 0;
unsigned char *c;
if (argc < 2 || argc > 3) {
fprintf( stderr, "Usage: %s <seed_string> [<size>]", argv[0]);
return 1;
}
c = (unsigned char *) argv[1];
do {
next = next * 11 + *c;
} while (*c++);
count = (argc == 3) ? strtoul(argv[2], NULL, 0) : -1L;
while (count--) {
next = next * 1103515245 + 12345;
if (putchar((next >> 16) & 0xff) == EOF)
return -1;
}
return 0;
}