mirrors/git
0
0
Fork 0
mirror of https://github.com/git/git.git synced 2024-11-23 09:56:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

Unify write_index_file functions

Browse Source 
This patch unifies the write_index_file functions in
builtin-pack-objects.c and index-pack.c.  As the name
"index" is overloaded in git, move in the direction of
using "idx" and "pack idx" when refering to the pack index.
There should be no change in functionality.

Signed-off-by: Geert Bosch <bosch@gnat.com>
Acked-by: Nicolas Pitre <nico@cam.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
This commit is contained in:
Geert Bosch 2007-06-01 15:18:05 -04:00 committed by Junio C Hamano
parent 4bc708347e
commit aa7e44bf57
4 changed files with 245 additions and 341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

222
builtin-pack-objects.c
View File

@ -23,10 +23,9 @@ git-pack-objects [{ -q | --progress | --all-progress }] [--max-pack-size=N] \n\
[--stdout | base-name] [<ref-list | <object-list]";
struct object_entry {
unsigned char sha1[20];
uint32_t crc32; /* crc of raw pack data for this object */
off_t offset; /* offset into the final pack file */
struct pack_idx_entry idx;
unsigned long size; /* uncompressed size */
unsigned int hash; /* name hint hash */
unsigned int depth; /* delta depth */
struct packed_git *in_pack; /* already in pack */
@ -66,7 +65,6 @@ static int allow_ofs_delta;
static const char *pack_tmp_name, *idx_tmp_name;
static char tmpname[PATH_MAX];
static const char *base_name;
static unsigned char pack_file_sha1[20];
static int progress = 1;
static int window = 10;
static uint32_t pack_size_limit;
@ -246,11 +244,11 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
{
unsigned long othersize, delta_size;
enum object_type type;
void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize);
void *otherbuf = read_sha1_file(entry->delta->idx.sha1, &type, &othersize);
void *delta_buf;
if (!otherbuf)
die("unable to read %s", sha1_to_hex(entry->delta->sha1));
die("unable to read %s", sha1_to_hex(entry->delta->idx.sha1));
delta_buf = diff_delta(otherbuf, othersize,
buf, size, &delta_size, 0);
if (!delta_buf || delta_size != entry->delta_size)
@ -379,11 +377,11 @@ static unsigned long write_object(struct sha1file *f,
/* yes if unlimited packfile */
!pack_size_limit ? 1 :
/* no if base written to previous pack */
entry->delta->offset == (off_t)-1 ? 0 :
entry->delta->idx.offset == (off_t)-1 ? 0 :
/* otherwise double-check written to this
* pack, like we do below
*/
entry->delta->offset ? 1 : 0;
entry->delta->idx.offset ? 1 : 0;
if (!pack_to_stdout)
crc32_begin(f);
@ -411,22 +409,22 @@ static unsigned long write_object(struct sha1file *f,
unsigned long maxsize;
void *out;
if (!usable_delta) {
buf = read_sha1_file(entry->sha1, &obj_type, &size);
buf = read_sha1_file(entry->idx.sha1, &obj_type, &size);
if (!buf)
die("unable to read %s", sha1_to_hex(entry->sha1));
die("unable to read %s", sha1_to_hex(entry->idx.sha1));
} else if (entry->delta_data) {
size = entry->delta_size;
buf = entry->delta_data;
entry->delta_data = NULL;
obj_type = (allow_ofs_delta && entry->delta->offset) ?
obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA;
} else {
buf = read_sha1_file(entry->sha1, &type, &size);
buf = read_sha1_file(entry->idx.sha1, &type, &size);
if (!buf)
die("unable to read %s", sha1_to_hex(entry->sha1));
die("unable to read %s", sha1_to_hex(entry->idx.sha1));
buf = delta_against(buf, size, entry);
size = entry->delta_size;
obj_type = (allow_ofs_delta && entry->delta->offset) ?
obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA;
}
/* compress the data to store and put compressed length in datalen */
@ -456,7 +454,7 @@ static unsigned long write_object(struct sha1file *f,
* encoding of the relative offset for the delta
* base from this object's position in the pack.
*/
off_t ofs = entry->offset - entry->delta->offset;
off_t ofs = entry->idx.offset - entry->delta->idx.offset;
unsigned pos = sizeof(dheader) - 1;
dheader[pos] = ofs & 127;
while (ofs >>= 7)
@ -480,7 +478,7 @@ static unsigned long write_object(struct sha1file *f,
return 0;
}
sha1write(f, header, hdrlen);
sha1write(f, entry->delta->sha1, 20);
sha1write(f, entry->delta->idx.sha1, 20);
hdrlen += 20;
} else {
if (limit && hdrlen + datalen + 20 >= limit) {
@ -501,7 +499,7 @@ static unsigned long write_object(struct sha1file *f,
off_t offset;
if (entry->delta) {
obj_type = (allow_ofs_delta && entry->delta->offset) ?
obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA;
reused_delta++;
}
@ -511,11 +509,11 @@ static unsigned long write_object(struct sha1file *f,
datalen = revidx[1].offset - offset;
if (!pack_to_stdout && p->index_version > 1 &&
check_pack_crc(p, &w_curs, offset, datalen, revidx->nr))
die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1));
offset += entry->in_pack_header_size;
datalen -= entry->in_pack_header_size;
if (obj_type == OBJ_OFS_DELTA) {
off_t ofs = entry->offset - entry->delta->offset;
off_t ofs = entry->idx.offset - entry->delta->idx.offset;
unsigned pos = sizeof(dheader) - 1;
dheader[pos] = ofs & 127;
while (ofs >>= 7)
@ -529,7 +527,7 @@ static unsigned long write_object(struct sha1file *f,
if (limit && hdrlen + 20 + datalen + 20 >= limit)
return 0;
sha1write(f, header, hdrlen);
sha1write(f, entry->delta->sha1, 20);
sha1write(f, entry->delta->idx.sha1, 20);
hdrlen += 20;
} else {
if (limit && hdrlen + datalen + 20 >= limit)
@ -539,7 +537,7 @@ static unsigned long write_object(struct sha1file *f,
if (!pack_to_stdout && p->index_version == 1 &&
check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1));
copy_pack_data(f, p, &w_curs, offset, datalen);
unuse_pack(&w_curs);
reused++;
@ -548,7 +546,7 @@ static unsigned long write_object(struct sha1file *f,
written_delta++;
written++;
if (!pack_to_stdout)
entry->crc32 = crc32_end(f);
entry->idx.crc32 = crc32_end(f);
return hdrlen + datalen;
}
@ -559,7 +557,7 @@ static off_t write_one(struct sha1file *f,
unsigned long size;
/* offset is non zero if object is written already. */
if (e->offset || e->preferred_base)
if (e->idx.offset || e->preferred_base)
return offset;
/* if we are deltified, write out base object first. */
@ -569,10 +567,10 @@ static off_t write_one(struct sha1file *f,
return 0;
}
e->offset = offset;
e->idx.offset = offset;
size = write_object(f, e, offset);
if (!size) {
e->offset = 0;
e->idx.offset = 0;
return 0;
}
written_list[nr_written++] = e;
@ -589,8 +587,7 @@ static int open_object_dir_tmp(const char *path)
return mkstemp(tmpname);
}
/* forward declarations for write_pack_file */
static void write_index_file(off_t last_obj_offset, unsigned char *sha1);
/* forward declaration for write_pack_file */
static int adjust_perm(const char *path, mode_t mode);
static void write_pack_file(void)
@ -607,6 +604,8 @@ static void write_pack_file(void)
written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
do {
unsigned char sha1[20];
if (pack_to_stdout) {
f = sha1fd(1, "<stdout>");
} else {
@ -638,23 +637,23 @@ static void write_pack_file(void)
* If so, rewrite it like in fast-import
*/
if (pack_to_stdout || nr_written == nr_remaining) {
sha1close(f, pack_file_sha1, 1);
sha1close(f, sha1, 1);
} else {
sha1close(f, pack_file_sha1, 0);
fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written);
sha1close(f, sha1, 0);
fixup_pack_header_footer(f->fd, sha1, pack_tmp_name, nr_written);
close(f->fd);
}
if (!pack_to_stdout) {
unsigned char object_list_sha1[20];
mode_t mode = umask(0);
umask(mode);
mode = 0444 & ~mode;
write_index_file(last_obj_offset, object_list_sha1);
idx_tmp_name = write_idx_file(NULL,
(struct pack_idx_entry **) written_list, nr_written, sha1);
snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
base_name, sha1_to_hex(object_list_sha1));
base_name, sha1_to_hex(sha1));
if (adjust_perm(pack_tmp_name, mode))
die("unable to make temporary pack file readable: %s",
strerror(errno));
@ -662,19 +661,19 @@ static void write_pack_file(void)
die("unable to rename temporary pack file: %s",
strerror(errno));
snprintf(tmpname, sizeof(tmpname), "%s-%s.idx",
base_name, sha1_to_hex(object_list_sha1));
base_name, sha1_to_hex(sha1));
if (adjust_perm(idx_tmp_name, mode))
die("unable to make temporary index file readable: %s",
strerror(errno));
if (rename(idx_tmp_name, tmpname))
die("unable to rename temporary index file: %s",
strerror(errno));
puts(sha1_to_hex(object_list_sha1));
puts(sha1_to_hex(sha1));
}
/* mark written objects as written to previous pack */
for (j = 0; j < nr_written; j++) {
written_list[j]->offset = (off_t)-1;
written_list[j]->idx.offset = (off_t)-1;
}
nr_remaining -= nr_written;
} while (nr_remaining && i < nr_objects);
@ -692,129 +691,12 @@ static void write_pack_file(void)
*/
for (j = 0; i < nr_objects; i++) {
struct object_entry *e = objects + i;
j += !e->offset && !e->preferred_base;
j += !e->idx.offset && !e->preferred_base;
}
if (j)
die("wrote %u objects as expected but %u unwritten", written, j);
}
static int sha1_sort(const void *_a, const void *_b)
{
const struct object_entry *a = *(struct object_entry **)_a;
const struct object_entry *b = *(struct object_entry **)_b;
return hashcmp(a->sha1, b->sha1);
}
static uint32_t index_default_version = 1;
static uint32_t index_off32_limit = 0x7fffffff;
static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
{
struct sha1file *f;
struct object_entry **sorted_by_sha, **list, **last;
uint32_t array[256];
uint32_t i, index_version;
SHA_CTX ctx;
int fd = open_object_dir_tmp("tmp_idx_XXXXXX");
if (fd < 0)
die("unable to create %s: %s\n", tmpname, strerror(errno));
idx_tmp_name = xstrdup(tmpname);
f = sha1fd(fd, idx_tmp_name);
if (nr_written) {
sorted_by_sha = written_list;
qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
list = sorted_by_sha;
last = sorted_by_sha + nr_written;
} else
sorted_by_sha = list = last = NULL;
/* if last object's offset is >= 2^31 we should use index V2 */
index_version = (last_obj_offset >> 31) ? 2 : index_default_version;
/* index versions 2 and above need a header */
if (index_version >= 2) {
struct pack_idx_header hdr;
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(index_version);
sha1write(f, &hdr, sizeof(hdr));
}
/*
* Write the first-level table (the list is sorted,
* but we use a 256-entry lookup to be able to avoid
* having to do eight extra binary search iterations).
*/
for (i = 0; i < 256; i++) {
struct object_entry **next = list;
while (next < last) {
struct object_entry *entry = *next;
if (entry->sha1[0] != i)
break;
next++;
}
array[i] = htonl(next - sorted_by_sha);
list = next;
}
sha1write(f, array, 256 * 4);
/* Compute the SHA1 hash of sorted object names. */
SHA1_Init(&ctx);
/* Write the actual SHA1 entries. */
list = sorted_by_sha;
for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
if (index_version < 2) {
uint32_t offset = htonl(entry->offset);
sha1write(f, &offset, 4);
}
sha1write(f, entry->sha1, 20);
SHA1_Update(&ctx, entry->sha1, 20);
}
if (index_version >= 2) {
unsigned int nr_large_offset = 0;
/* write the crc32 table */
list = sorted_by_sha;
for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
uint32_t crc32_val = htonl(entry->crc32);
sha1write(f, &crc32_val, 4);
}
/* write the 32-bit offset table */
list = sorted_by_sha;
for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
uint32_t offset = (entry->offset <= index_off32_limit) ?
entry->offset : (0x80000000 | nr_large_offset++);
offset = htonl(offset);
sha1write(f, &offset, 4);
}
/* write the large offset table */
list = sorted_by_sha;
while (nr_large_offset) {
struct object_entry *entry = *list++;
uint64_t offset = entry->offset;
if (offset > index_off32_limit) {
uint32_t split[2];
split[0] = htonl(offset >> 32);
split[1] = htonl(offset & 0xffffffff);
sha1write(f, split, 8);
nr_large_offset--;
}
}
}
sha1write(f, pack_file_sha1, 20);
sha1close(f, NULL, 1);
SHA1_Final(sha1, &ctx);
}
static int locate_object_entry_hash(const unsigned char *sha1)
{
int i;
@ -822,7 +704,7 @@ static int locate_object_entry_hash(const unsigned char *sha1)
memcpy(&ui, sha1, sizeof(unsigned int));
i = ui % object_ix_hashsz;
while (0 < object_ix[i]) {
if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1))
if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1))
return i;
if (++i == object_ix_hashsz)
i = 0;
@ -854,7 +736,7 @@ static void rehash_objects(void)
object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
int ix = locate_object_entry_hash(oe->sha1);
int ix = locate_object_entry_hash(oe->idx.sha1);
if (0 <= ix)
continue;
ix = -1 - ix;
@ -948,7 +830,7 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
entry = objects + nr_objects++;
memset(entry, 0, sizeof(*entry));
hashcpy(entry->sha1, sha1);
hashcpy(entry->idx.sha1, sha1);
entry->hash = hash;
if (type)
entry->type = type;
@ -1268,13 +1150,13 @@ static void check_object(struct object_entry *entry)
ofs += 1;
if (!ofs || MSB(ofs, 7))
die("delta base offset overflow in pack for %s",
sha1_to_hex(entry->sha1));
sha1_to_hex(entry->idx.sha1));
c = buf[used_0++];
ofs = (ofs << 7) + (c & 127);
}
if (ofs >= entry->in_pack_offset)
die("delta base offset out of bound for %s",
sha1_to_hex(entry->sha1));
sha1_to_hex(entry->idx.sha1));
ofs = entry->in_pack_offset - ofs;
if (!no_reuse_delta && !entry->preferred_base)
base_ref = find_packed_object_name(p, ofs);
@ -1321,10 +1203,10 @@ static void check_object(struct object_entry *entry)
unuse_pack(&w_curs);
}
entry->type = sha1_object_info(entry->sha1, &entry->size);
entry->type = sha1_object_info(entry->idx.sha1, &entry->size);
if (entry->type < 0)
die("unable to get type of object %s",
sha1_to_hex(entry->sha1));
sha1_to_hex(entry->idx.sha1));
}
static int pack_offset_sort(const void *_a, const void *_b)
@ -1334,7 +1216,7 @@ static int pack_offset_sort(const void *_a, const void *_b)
/* avoid filesystem trashing with loose objects */
if (!a->in_pack && !b->in_pack)
return hashcmp(a->sha1, b->sha1);
return hashcmp(a->idx.sha1, b->idx.sha1);
if (a->in_pack < b->in_pack)
return -1;
@ -1463,16 +1345,16 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
/* Load data if not already done */
if (!trg->data) {
trg->data = read_sha1_file(trg_entry->sha1, &type, &sz);
trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
if (sz != trg_size)
die("object %s inconsistent object length (%lu vs %lu)",
sha1_to_hex(trg_entry->sha1), sz, trg_size);
sha1_to_hex(trg_entry->idx.sha1), sz, trg_size);
}
if (!src->data) {
src->data = read_sha1_file(src_entry->sha1, &type, &sz);
src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
if (sz != src_size)
die("object %s inconsistent object length (%lu vs %lu)",
sha1_to_hex(src_entry->sha1), sz, src_size);
sha1_to_hex(src_entry->idx.sha1), sz, src_size);
}
if (!src->index) {
src->index = create_delta_index(src->data, src_size);
@ -1869,12 +1751,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
}
if (!prefixcmp(arg, "--index-version=")) {
char *c;
index_default_version = strtoul(arg + 16, &c, 10);
if (index_default_version > 2)
pack_idx_default_version = strtoul(arg + 16, &c, 10);
if (pack_idx_default_version > 2)
die("bad %s", arg);
if (*c == ',')
index_off32_limit = strtoul(c+1, &c, 0);
if (*c || index_off32_limit & 0x80000000)
pack_idx_off32_limit = strtoul(c+1, &c, 0);
if (*c || pack_idx_off32_limit & 0x80000000)
die("bad %s", arg);
continue;
}

208
index-pack.c
View File

@ -13,13 +13,11 @@ static const char index_pack_usage[] =
struct object_entry
{
off_t offset;
struct pack_idx_entry idx;
unsigned long size;
unsigned int hdr_size;
uint32_t crc32;
enum object_type type;
enum object_type real_type;
unsigned char sha1[20];
};
union delta_base {
@ -197,7 +195,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
unsigned shift;
void *data;
obj->offset = consumed_bytes;
obj->idx.offset = consumed_bytes;
input_crc32 = crc32(0, Z_NULL, 0);
p = fill(1);
@ -229,15 +227,15 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
while (c & 128) {
base_offset += 1;
if (!base_offset || MSB(base_offset, 7))
bad_object(obj->offset, "offset value overflow for delta base object");
bad_object(obj->idx.offset, "offset value overflow for delta base object");
p = fill(1);
c = *p;
use(1);
base_offset = (base_offset << 7) + (c & 127);
}
delta_base->offset = obj->offset - base_offset;
if (delta_base->offset >= obj->offset)
bad_object(obj->offset, "delta base offset is out of bound");
delta_base->offset = obj->idx.offset - base_offset;
if (delta_base->offset >= obj->idx.offset)
bad_object(obj->idx.offset, "delta base offset is out of bound");
break;
case OBJ_COMMIT:
case OBJ_TREE:
@ -245,19 +243,19 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
case OBJ_TAG:
break;
default:
bad_object(obj->offset, "unknown object type %d", obj->type);
bad_object(obj->idx.offset, "unknown object type %d", obj->type);
}
obj->hdr_size = consumed_bytes - obj->offset;
obj->hdr_size = consumed_bytes - obj->idx.offset;
data = unpack_entry_data(obj->offset, obj->size);
obj->crc32 = input_crc32;
data = unpack_entry_data(obj->idx.offset, obj->size);
obj->idx.crc32 = input_crc32;
return data;
}
static void *get_data_from_pack(struct object_entry *obj)
{
unsigned long from = obj[0].offset + obj[0].hdr_size;
unsigned long len = obj[1].offset - from;
unsigned long from = obj[0].idx.offset + obj[0].hdr_size;
unsigned long len = obj[1].idx.offset - from;
unsigned long rdy = 0;
unsigned char *src, *data;
z_stream stream;
@ -360,11 +358,11 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
&result_size);
free(delta_data);
if (!result)
bad_object(delta_obj->offset, "failed to apply delta");
sha1_object(result, result_size, type, delta_obj->sha1);
bad_object(delta_obj->idx.offset, "failed to apply delta");
sha1_object(result, result_size, type, delta_obj->idx.sha1);
nr_resolved_deltas++;
hashcpy(delta_base.sha1, delta_obj->sha1);
hashcpy(delta_base.sha1, delta_obj->idx.sha1);
if (!find_delta_children(&delta_base, &first, &last)) {
for (j = first; j <= last; j++) {
struct object_entry *child = objects + deltas[j].obj_no;
@ -374,7 +372,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
}
memset(&delta_base, 0, sizeof(delta_base));
delta_base.offset = delta_obj->offset;
delta_base.offset = delta_obj->idx.offset;
if (!find_delta_children(&delta_base, &first, &last)) {
for (j = first; j <= last; j++) {
struct object_entry *child = objects + deltas[j].obj_no;
@ -418,12 +416,12 @@ static void parse_pack_objects(unsigned char *sha1)
delta->obj_no = i;
delta++;
} else
sha1_object(data, obj->size, obj->type, obj->sha1);
sha1_object(data, obj->size, obj->type, obj->idx.sha1);
free(data);
if (verbose)
display_progress(&progress, i+1);
}
objects[i].offset = consumed_bytes;
objects[i].idx.offset = consumed_bytes;
if (verbose)
stop_progress(&progress);
@ -465,10 +463,10 @@ static void parse_pack_objects(unsigned char *sha1)
if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
continue;
hashcpy(base.sha1, obj->sha1);
hashcpy(base.sha1, obj->idx.sha1);
ref = !find_delta_children(&base, &ref_first, &ref_last);
memset(&base, 0, sizeof(base));
base.offset = obj->offset;
base.offset = obj->idx.offset;
ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
if (!ref && !ofs)
continue;
@ -535,11 +533,11 @@ static void append_obj_to_pack(const unsigned char *sha1, void *buf,
}
header[n++] = c;
write_or_die(output_fd, header, n);
obj[0].crc32 = crc32(0, Z_NULL, 0);
obj[0].crc32 = crc32(obj[0].crc32, header, n);
obj[1].offset = obj[0].offset + n;
obj[1].offset += write_compressed(output_fd, buf, size, &obj[0].crc32);
hashcpy(obj->sha1, sha1);
obj[0].idx.crc32 = crc32(0, Z_NULL, 0);
obj[0].idx.crc32 = crc32(obj[0].idx.crc32, header, n);
obj[1].idx.offset = obj[0].idx.offset + n;
obj[1].idx.offset += write_compressed(output_fd, buf, size, &obj[0].idx.crc32);
hashcpy(obj->idx.sha1, sha1);
}
static int delta_pos_compare(const void *_a, const void *_b)
@ -602,145 +600,6 @@ static void fix_unresolved_deltas(int nr_unresolved)
free(sorted_by_pos);
}
static uint32_t index_default_version = 1;
static uint32_t index_off32_limit = 0x7fffffff;
static int sha1_compare(const void *_a, const void *_b)
{
struct object_entry *a = *(struct object_entry **)_a;
struct object_entry *b = *(struct object_entry **)_b;
return hashcmp(a->sha1, b->sha1);
}
/*
* On entry *sha1 contains the pack content SHA1 hash, on exit it is
* the SHA1 hash of sorted object names.
*/
static const char *write_index_file(const char *index_name, unsigned char *sha1)
{
struct sha1file *f;
struct object_entry **sorted_by_sha, **list, **last;
uint32_t array[256];
int i, fd;
SHA_CTX ctx;
uint32_t index_version;
if (nr_objects) {
sorted_by_sha =
xcalloc(nr_objects, sizeof(struct object_entry *));
list = sorted_by_sha;
last = sorted_by_sha + nr_objects;
for (i = 0; i < nr_objects; ++i)
sorted_by_sha[i] = &objects[i];
qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
sha1_compare);
}
else
sorted_by_sha = list = last = NULL;
if (!index_name) {
static char tmpfile[PATH_MAX];
snprintf(tmpfile, sizeof(tmpfile),
"%s/tmp_idx_XXXXXX", get_object_directory());
fd = mkstemp(tmpfile);
index_name = xstrdup(tmpfile);
} else {
unlink(index_name);
fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
}
if (fd < 0)
die("unable to create %s: %s", index_name, strerror(errno));
f = sha1fd(fd, index_name);
/* if last object's offset is >= 2^31 we should use index V2 */
index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version;
/* index versions 2 and above need a header */
if (index_version >= 2) {
struct pack_idx_header hdr;
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(index_version);
sha1write(f, &hdr, sizeof(hdr));
}
/*
* Write the first-level table (the list is sorted,
* but we use a 256-entry lookup to be able to avoid
* having to do eight extra binary search iterations).
*/
for (i = 0; i < 256; i++) {
struct object_entry **next = list;
while (next < last) {
struct object_entry *obj = *next;
if (obj->sha1[0] != i)
break;
next++;
}
array[i] = htonl(next - sorted_by_sha);
list = next;
}
sha1write(f, array, 256 * 4);
/* compute the SHA1 hash of sorted object names. */
SHA1_Init(&ctx);
/*
* Write the actual SHA1 entries..
*/
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
if (index_version < 2) {
uint32_t offset = htonl(obj->offset);
sha1write(f, &offset, 4);
}
sha1write(f, obj->sha1, 20);
SHA1_Update(&ctx, obj->sha1, 20);
}
if (index_version >= 2) {
unsigned int nr_large_offset = 0;
/* write the crc32 table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
uint32_t crc32_val = htonl(obj->crc32);
sha1write(f, &crc32_val, 4);
}
/* write the 32-bit offset table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
uint32_t offset = (obj->offset <= index_off32_limit) ?
obj->offset : (0x80000000 | nr_large_offset++);
offset = htonl(offset);
sha1write(f, &offset, 4);
}
/* write the large offset table */
list = sorted_by_sha;
while (nr_large_offset) {
struct object_entry *obj = *list++;
uint64_t offset = obj->offset;
if (offset > index_off32_limit) {
uint32_t split[2];
split[0] = htonl(offset >> 32);
split[1] = htonl(offset & 0xffffffff);
sha1write(f, split, 8);
nr_large_offset--;
}
}
}
sha1write(f, sha1, 20);
sha1close(f, NULL, 1);
free(sorted_by_sha);
SHA1_Final(sha1, &ctx);
return index_name;
}
static void final(const char *final_pack_name, const char *curr_pack_name,
const char *final_index_name, const char *curr_index_name,
const char *keep_name, const char *keep_msg,
@ -830,6 +689,7 @@ int main(int argc, char **argv)
const char *curr_index, *index_name = NULL;
const char *keep_name = NULL, *keep_msg = NULL;
char *index_name_buf = NULL, *keep_name_buf = NULL;
struct pack_idx_entry **idx_objects;
unsigned char sha1[20];
for (i = 1; i < argc; i++) {
@ -865,12 +725,12 @@ int main(int argc, char **argv)
index_name = argv[++i];
} else if (!prefixcmp(arg, "--index-version=")) {
char *c;
index_default_version = strtoul(arg + 16, &c, 10);
if (index_default_version > 2)
pack_idx_default_version = strtoul(arg + 16, &c, 10);
if (pack_idx_default_version > 2)
die("bad %s", arg);
if (*c == ',')
index_off32_limit = strtoul(c+1, &c, 0);
if (*c || index_off32_limit & 0x80000000)
pack_idx_off32_limit = strtoul(c+1, &c, 0);
if (*c || pack_idx_off32_limit & 0x80000000)
die("bad %s", arg);
} else
usage(index_pack_usage);
@ -940,7 +800,13 @@ int main(int argc, char **argv)
nr_deltas - nr_resolved_deltas);
}
free(deltas);
curr_index = write_index_file(index_name, sha1);
idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *));
for (i = 0; i < nr_objects; i++)
idx_objects[i] = &objects[i].idx;
curr_index = write_idx_file(index_name, idx_objects, nr_objects, sha1);
free(idx_objects);
final(pack_name, curr_pack,
index_name, curr_index,
keep_name, keep_msg,

142
pack-write.c
View File

@ -1,5 +1,147 @@
#include "cache.h"
#include "pack.h"
#include "csum-file.h"
uint32_t pack_idx_default_version = 1;
uint32_t pack_idx_off32_limit = 0x7fffffff;
static int sha1_compare(const void *_a, const void *_b)
{
struct pack_idx_entry *a = *(struct pack_idx_entry **)_a;
struct pack_idx_entry *b = *(struct pack_idx_entry **)_b;
return hashcmp(a->sha1, b->sha1);
}
/*
* On entry *sha1 contains the pack content SHA1 hash, on exit it is
* the SHA1 hash of sorted object names. The objects array passed in
* will be sorted by SHA1 on exit.
*/
const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1)
{
struct sha1file *f;
struct pack_idx_entry **sorted_by_sha, **list, **last;
off_t last_obj_offset = 0;
uint32_t array[256];
int i, fd;
SHA_CTX ctx;
uint32_t index_version;
if (nr_objects) {
sorted_by_sha = objects;
list = sorted_by_sha;
last = sorted_by_sha + nr_objects;
for (i = 0; i < nr_objects; ++i) {
if (objects[i]->offset > last_obj_offset)
last_obj_offset = objects[i]->offset;
}
qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
sha1_compare);
}
else
sorted_by_sha = list = last = NULL;
if (!index_name) {
static char tmpfile[PATH_MAX];
snprintf(tmpfile, sizeof(tmpfile),
"%s/tmp_idx_XXXXXX", get_object_directory());
fd = mkstemp(tmpfile);
index_name = xstrdup(tmpfile);
} else {
unlink(index_name);
fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
}
if (fd < 0)
die("unable to create %s: %s", index_name, strerror(errno));
f = sha1fd(fd, index_name);
/* if last object's offset is >= 2^31 we should use index V2 */
index_version = (last_obj_offset >> 31) ? 2 : pack_idx_default_version;
/* index versions 2 and above need a header */
if (index_version >= 2) {
struct pack_idx_header hdr;
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(index_version);
sha1write(f, &hdr, sizeof(hdr));
}
/*
* Write the first-level table (the list is sorted,
* but we use a 256-entry lookup to be able to avoid
* having to do eight extra binary search iterations).
*/
for (i = 0; i < 256; i++) {
struct pack_idx_entry **next = list;
while (next < last) {
struct pack_idx_entry *obj = *next;
if (obj->sha1[0] != i)
break;
next++;
}
array[i] = htonl(next - sorted_by_sha);
list = next;
}
sha1write(f, array, 256 * 4);
/* compute the SHA1 hash of sorted object names. */
SHA1_Init(&ctx);
/*
* Write the actual SHA1 entries..
*/
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct pack_idx_entry *obj = *list++;
if (index_version < 2) {
uint32_t offset = htonl(obj->offset);
sha1write(f, &offset, 4);
}
sha1write(f, obj->sha1, 20);
SHA1_Update(&ctx, obj->sha1, 20);
}
if (index_version >= 2) {
unsigned int nr_large_offset = 0;
/* write the crc32 table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct pack_idx_entry *obj = *list++;
uint32_t crc32_val = htonl(obj->crc32);
sha1write(f, &crc32_val, 4);
}
/* write the 32-bit offset table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct pack_idx_entry *obj = *list++;
uint32_t offset = (obj->offset <= pack_idx_off32_limit) ?
obj->offset : (0x80000000 | nr_large_offset++);
offset = htonl(offset);
sha1write(f, &offset, 4);
}
/* write the large offset table */
list = sorted_by_sha;
while (nr_large_offset) {
struct pack_idx_entry *obj = *list++;
uint64_t offset = obj->offset;
if (offset > pack_idx_off32_limit) {
uint32_t split[2];
split[0] = htonl(offset >> 32);
split[1] = htonl(offset & 0xffffffff);
sha1write(f, split, 8);
nr_large_offset--;
}
}
}
sha1write(f, sha1, 20);
sha1close(f, NULL, 1);
SHA1_Final(sha1, &ctx);
return index_name;
}
void fixup_pack_header_footer(int pack_fd,
unsigned char *pack_file_sha1,

14
pack.h
View File

@ -34,6 +34,10 @@ struct pack_header {
*/
#define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */
/* These may be overridden by command-line parameters */
extern uint32_t pack_idx_default_version;
extern uint32_t pack_idx_off32_limit;
/*
* Packed object index header
*/
@ -42,6 +46,16 @@ struct pack_idx_header {
uint32_t idx_version;
};
/*
* Common part of object structure used for write_idx_file
*/
struct pack_idx_entry {
unsigned char sha1[20];
uint32_t crc32;
off_t offset;
};
extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
extern int verify_pack(struct packed_git *, int);
extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t);