summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--builtin/pack-objects.c3
-rw-r--r--midx-write.c57
-rw-r--r--pack-bitmap-write.c65
-rw-r--r--pack-bitmap.h4
-rwxr-xr-xt/t5334-incremental-multi-pack-index.sh87
5 files changed, 179 insertions, 37 deletions
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index 58a9b16126..a7e4bb7904 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -1397,7 +1397,8 @@ static void write_pack_file(void)
if (write_bitmap_index) {
bitmap_writer_init(&bitmap_writer,
- the_repository, &to_pack);
+ the_repository, &to_pack,
+ NULL);
bitmap_writer_set_checksum(&bitmap_writer, hash);
bitmap_writer_build_type_index(&bitmap_writer,
written_list);
diff --git a/midx-write.c b/midx-write.c
index 48d6558253..0897cbd829 100644
--- a/midx-write.c
+++ b/midx-write.c
@@ -647,16 +647,22 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx)
return pack_order;
}
-static void write_midx_reverse_index(char *midx_name, unsigned char *midx_hash,
- struct write_midx_context *ctx)
+static void write_midx_reverse_index(struct write_midx_context *ctx,
+ const char *object_dir,
+ unsigned char *midx_hash)
{
struct strbuf buf = STRBUF_INIT;
char *tmp_file;
trace2_region_enter("midx", "write_midx_reverse_index", ctx->repo);
- strbuf_addf(&buf, "%s-%s.rev", midx_name, hash_to_hex_algop(midx_hash,
- ctx->repo->hash_algo));
+ if (ctx->incremental)
+ get_split_midx_filename_ext(ctx->repo->hash_algo, &buf,
+ object_dir, midx_hash,
+ MIDX_EXT_REV);
+ else
+ get_midx_filename_ext(ctx->repo->hash_algo, &buf, object_dir,
+ midx_hash, MIDX_EXT_REV);
tmp_file = write_rev_file_order(ctx->repo->hash_algo, NULL, ctx->pack_order,
ctx->entries_nr, midx_hash, WRITE_REV);
@@ -829,22 +835,29 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr
return cb.commits;
}
-static int write_midx_bitmap(struct repository *r, const char *midx_name,
+static int write_midx_bitmap(struct write_midx_context *ctx,
+ const char *object_dir,
const unsigned char *midx_hash,
struct packing_data *pdata,
struct commit **commits,
uint32_t commits_nr,
- uint32_t *pack_order,
unsigned flags)
{
int ret, i;
uint16_t options = 0;
struct bitmap_writer writer;
struct pack_idx_entry **index;
- char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name,
- hash_to_hex_algop(midx_hash, r->hash_algo));
+ struct strbuf bitmap_name = STRBUF_INIT;
+
+ trace2_region_enter("midx", "write_midx_bitmap", ctx->repo);
- trace2_region_enter("midx", "write_midx_bitmap", r);
+ if (ctx->incremental)
+ get_split_midx_filename_ext(ctx->repo->hash_algo, &bitmap_name,
+ object_dir, midx_hash,
+ MIDX_EXT_BITMAP);
+ else
+ get_midx_filename_ext(ctx->repo->hash_algo, &bitmap_name,
+ object_dir, midx_hash, MIDX_EXT_BITMAP);
if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
options |= BITMAP_OPT_HASH_CACHE;
@@ -861,7 +874,8 @@ static int write_midx_bitmap(struct repository *r, const char *midx_name,
for (i = 0; i < pdata->nr_objects; i++)
index[i] = &pdata->objects[i].idx;
- bitmap_writer_init(&writer, r, pdata);
+ bitmap_writer_init(&writer, ctx->repo, pdata,
+ ctx->incremental ? ctx->base_midx : NULL);
bitmap_writer_show_progress(&writer, flags & MIDX_PROGRESS);
bitmap_writer_build_type_index(&writer, index);
@@ -879,7 +893,7 @@ static int write_midx_bitmap(struct repository *r, const char *midx_name,
* bitmap_writer_finish().
*/
for (i = 0; i < pdata->nr_objects; i++)
- index[pack_order[i]] = &pdata->objects[i].idx;
+ index[ctx->pack_order[i]] = &pdata->objects[i].idx;
bitmap_writer_select_commits(&writer, commits, commits_nr);
ret = bitmap_writer_build(&writer);
@@ -887,14 +901,14 @@ static int write_midx_bitmap(struct repository *r, const char *midx_name,
goto cleanup;
bitmap_writer_set_checksum(&writer, midx_hash);
- bitmap_writer_finish(&writer, index, bitmap_name, options);
+ bitmap_writer_finish(&writer, index, bitmap_name.buf, options);
cleanup:
free(index);
- free(bitmap_name);
+ strbuf_release(&bitmap_name);
bitmap_writer_free(&writer);
- trace2_region_leave("midx", "write_midx_bitmap", r);
+ trace2_region_leave("midx", "write_midx_bitmap", ctx->repo);
return ret;
}
@@ -1077,8 +1091,6 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
ctx.repo = r;
ctx.incremental = !!(flags & MIDX_WRITE_INCREMENTAL);
- if (ctx.incremental && (flags & MIDX_WRITE_BITMAP))
- die(_("cannot write incremental MIDX with bitmap"));
if (ctx.incremental)
strbuf_addf(&midx_name,
@@ -1119,6 +1131,13 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
if (ctx.incremental) {
struct multi_pack_index *m = ctx.base_midx;
while (m) {
+ if (flags & MIDX_WRITE_BITMAP && load_midx_revindex(m)) {
+ error(_("could not load reverse index for MIDX %s"),
+ hash_to_hex_algop(get_midx_checksum(m),
+ m->repo->hash_algo));
+ result = 1;
+ goto cleanup;
+ }
ctx.num_multi_pack_indexes_before++;
m = m->base_midx;
}
@@ -1387,7 +1406,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
if (flags & MIDX_WRITE_REV_INDEX &&
git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0))
- write_midx_reverse_index(midx_name.buf, midx_hash, &ctx);
+ write_midx_reverse_index(&ctx, object_dir, midx_hash);
if (flags & MIDX_WRITE_BITMAP) {
struct packing_data pdata;
@@ -1410,8 +1429,8 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
FREE_AND_NULL(ctx.entries);
ctx.entries_nr = 0;
- if (write_midx_bitmap(r, midx_name.buf, midx_hash, &pdata,
- commits, commits_nr, ctx.pack_order,
+ if (write_midx_bitmap(&ctx, object_dir,
+ midx_hash, &pdata, commits, commits_nr,
flags) < 0) {
error(_("could not write multi-pack bitmap"));
result = 1;
diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c
index 34e86d4994..8a30853d2e 100644
--- a/pack-bitmap-write.c
+++ b/pack-bitmap-write.c
@@ -26,6 +26,8 @@
#include "alloc.h"
#include "refs.h"
#include "strmap.h"
+#include "midx.h"
+#include "pack-revindex.h"
struct bitmapped_commit {
struct commit *commit;
@@ -43,7 +45,8 @@ static inline int bitmap_writer_nr_selected_commits(struct bitmap_writer *writer
}
void bitmap_writer_init(struct bitmap_writer *writer, struct repository *r,
- struct packing_data *pdata)
+ struct packing_data *pdata,
+ struct multi_pack_index *midx)
{
memset(writer, 0, sizeof(struct bitmap_writer));
if (writer->bitmaps)
@@ -51,6 +54,7 @@ void bitmap_writer_init(struct bitmap_writer *writer, struct repository *r,
writer->bitmaps = kh_init_oid_map();
writer->pseudo_merge_commits = kh_init_oid_map();
writer->to_pack = pdata;
+ writer->midx = midx;
string_list_init_dup(&writer->pseudo_merge_groups);
@@ -113,6 +117,11 @@ void bitmap_writer_build_type_index(struct bitmap_writer *writer,
struct pack_idx_entry **index)
{
uint32_t i;
+ uint32_t base_objects = 0;
+
+ if (writer->midx)
+ base_objects = writer->midx->num_objects +
+ writer->midx->num_objects_in_base;
writer->commits = ewah_new();
writer->trees = ewah_new();
@@ -142,19 +151,19 @@ void bitmap_writer_build_type_index(struct bitmap_writer *writer,
switch (real_type) {
case OBJ_COMMIT:
- ewah_set(writer->commits, i);
+ ewah_set(writer->commits, i + base_objects);
break;
case OBJ_TREE:
- ewah_set(writer->trees, i);
+ ewah_set(writer->trees, i + base_objects);
break;
case OBJ_BLOB:
- ewah_set(writer->blobs, i);
+ ewah_set(writer->blobs, i + base_objects);
break;
case OBJ_TAG:
- ewah_set(writer->tags, i);
+ ewah_set(writer->tags, i + base_objects);
break;
default:
@@ -207,19 +216,37 @@ void bitmap_writer_push_commit(struct bitmap_writer *writer,
static uint32_t find_object_pos(struct bitmap_writer *writer,
const struct object_id *oid, int *found)
{
- struct object_entry *entry = packlist_find(writer->to_pack, oid);
+ struct object_entry *entry;
+
+ entry = packlist_find(writer->to_pack, oid);
+ if (entry) {
+ uint32_t base_objects = 0;
+ if (writer->midx)
+ base_objects = writer->midx->num_objects +
+ writer->midx->num_objects_in_base;
- if (!entry) {
if (found)
- *found = 0;
- warning("Failed to write bitmap index. Packfile doesn't have full closure "
- "(object %s is missing)", oid_to_hex(oid));
- return 0;
+ *found = 1;
+ return oe_in_pack_pos(writer->to_pack, entry) + base_objects;
+ } else if (writer->midx) {
+ uint32_t at, pos;
+
+ if (!bsearch_midx(oid, writer->midx, &at))
+ goto missing;
+ if (midx_to_pack_pos(writer->midx, at, &pos) < 0)
+ goto missing;
+
+ if (found)
+ *found = 1;
+ return pos;
}
+missing:
if (found)
- *found = 1;
- return oe_in_pack_pos(writer->to_pack, entry);
+ *found = 0;
+ warning("Failed to write bitmap index. Packfile doesn't have full closure "
+ "(object %s is missing)", oid_to_hex(oid));
+ return 0;
}
static void compute_xor_offsets(struct bitmap_writer *writer)
@@ -586,7 +613,7 @@ int bitmap_writer_build(struct bitmap_writer *writer)
struct prio_queue queue = { compare_commits_by_gen_then_commit_date };
struct prio_queue tree_queue = { NULL };
struct bitmap_index *old_bitmap;
- uint32_t *mapping;
+ uint32_t *mapping = NULL;
int closed = 1; /* until proven otherwise */
if (writer->show_progress)
@@ -1021,7 +1048,7 @@ void bitmap_writer_finish(struct bitmap_writer *writer,
struct strbuf tmp_file = STRBUF_INIT;
struct hashfile *f;
off_t *offsets = NULL;
- uint32_t i;
+ uint32_t i, base_objects;
struct bitmap_disk_header header;
@@ -1047,6 +1074,12 @@ void bitmap_writer_finish(struct bitmap_writer *writer,
if (options & BITMAP_OPT_LOOKUP_TABLE)
CALLOC_ARRAY(offsets, writer->to_pack->nr_objects);
+ if (writer->midx)
+ base_objects = writer->midx->num_objects +
+ writer->midx->num_objects_in_base;
+ else
+ base_objects = 0;
+
for (i = 0; i < bitmap_writer_nr_selected_commits(writer); i++) {
struct bitmapped_commit *stored = &writer->selected[i];
int commit_pos = oid_pos(&stored->commit->object.oid, index,
@@ -1055,7 +1088,7 @@ void bitmap_writer_finish(struct bitmap_writer *writer,
if (commit_pos < 0)
BUG(_("trying to write commit not in index"));
- stored->commit_pos = commit_pos;
+ stored->commit_pos = commit_pos + base_objects;
}
write_selected_commits_v1(writer, f, offsets);
diff --git a/pack-bitmap.h b/pack-bitmap.h
index d7f4b8b8e9..dd0951088f 100644
--- a/pack-bitmap.h
+++ b/pack-bitmap.h
@@ -111,6 +111,7 @@ struct bitmap_writer {
kh_oid_map_t *bitmaps;
struct packing_data *to_pack;
+ struct multi_pack_index *midx; /* if appending to a MIDX chain */
struct bitmapped_commit *selected;
unsigned int selected_nr, selected_alloc;
@@ -125,7 +126,8 @@ struct bitmap_writer {
};
void bitmap_writer_init(struct bitmap_writer *writer, struct repository *r,
- struct packing_data *pdata);
+ struct packing_data *pdata,
+ struct multi_pack_index *midx);
void bitmap_writer_show_progress(struct bitmap_writer *writer, int show);
void bitmap_writer_set_checksum(struct bitmap_writer *writer,
const unsigned char *sha1);
diff --git a/t/t5334-incremental-multi-pack-index.sh b/t/t5334-incremental-multi-pack-index.sh
index 26257e5660..d30d7253d6 100755
--- a/t/t5334-incremental-multi-pack-index.sh
+++ b/t/t5334-incremental-multi-pack-index.sh
@@ -44,4 +44,91 @@ test_expect_success 'convert incremental to non-incremental' '
compare_results_with_midx 'non-incremental MIDX conversion'
+write_midx_layer () {
+ n=1
+ if test -f $midx_chain
+ then
+ n="$(($(wc -l <$midx_chain) + 1))"
+ fi
+
+ for i in 1 2
+ do
+ test_commit $n.$i &&
+ git repack -d || return 1
+ done &&
+ git multi-pack-index write --bitmap --incremental
+}
+
+test_expect_success 'write initial MIDX layer' '
+ git repack -ad &&
+ write_midx_layer
+'
+
+test_expect_success 'read bitmap from first MIDX layer' '
+ git rev-list --test-bitmap 1.2
+'
+
+test_expect_success 'write another MIDX layer' '
+ write_midx_layer
+'
+
+test_expect_success 'midx verify with multiple layers' '
+ test_path_is_file "$midx_chain" &&
+ test_line_count = 2 "$midx_chain" &&
+
+ git multi-pack-index verify
+'
+
+test_expect_success 'read bitmap from second MIDX layer' '
+ git rev-list --test-bitmap 2.2
+'
+
+test_expect_success 'read earlier bitmap from second MIDX layer' '
+ git rev-list --test-bitmap 1.2
+'
+
+test_expect_success 'show object from first pack' '
+ git cat-file -p 1.1
+'
+
+test_expect_success 'show object from second pack' '
+ git cat-file -p 2.2
+'
+
+for reuse in false single multi
+do
+ test_expect_success "full clone (pack.allowPackReuse=$reuse)" '
+ rm -fr clone.git &&
+
+ git config pack.allowPackReuse $reuse &&
+ git clone --no-local --bare . clone.git
+ '
+done
+
+test_expect_success 'relink existing MIDX layer' '
+ rm -fr "$midxdir" &&
+
+ GIT_TEST_MIDX_WRITE_REV=1 git multi-pack-index write --bitmap &&
+
+ midx_hash="$(test-tool read-midx --checksum $objdir)" &&
+
+ test_path_is_file "$packdir/multi-pack-index" &&
+ test_path_is_file "$packdir/multi-pack-index-$midx_hash.bitmap" &&
+ test_path_is_file "$packdir/multi-pack-index-$midx_hash.rev" &&
+
+ test_commit another &&
+ git repack -d &&
+ git multi-pack-index write --bitmap --incremental &&
+
+ test_path_is_missing "$packdir/multi-pack-index" &&
+ test_path_is_missing "$packdir/multi-pack-index-$midx_hash.bitmap" &&
+ test_path_is_missing "$packdir/multi-pack-index-$midx_hash.rev" &&
+
+ test_path_is_file "$midxdir/multi-pack-index-$midx_hash.midx" &&
+ test_path_is_file "$midxdir/multi-pack-index-$midx_hash.bitmap" &&
+ test_path_is_file "$midxdir/multi-pack-index-$midx_hash.rev" &&
+ test_line_count = 2 "$midx_chain"
+
+'
+
test_done
generated by cgit v1.2.3 (git 2.46.0) at 2025-12-30 12:06:21 +0000