diff options
Diffstat (limited to 'commit-graph.c')
| -rw-r--r-- | commit-graph.c | 1055 |
1 files changed, 667 insertions, 388 deletions
diff --git a/commit-graph.c b/commit-graph.c index c11b59f28b..e0d92b816f 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -1,15 +1,22 @@ +#define USE_THE_REPOSITORY_VARIABLE +#define DISABLE_SIGN_COMPARE_WARNINGS + #include "git-compat-util.h" #include "config.h" +#include "csum-file.h" +#include "environment.h" +#include "gettext.h" +#include "hex.h" #include "lockfile.h" -#include "pack.h" #include "packfile.h" #include "commit.h" #include "object.h" #include "refs.h" -#include "revision.h" #include "hash-lookup.h" #include "commit-graph.h" -#include "object-store.h" +#include "odb.h" +#include "oid-array.h" +#include "path.h" #include "alloc.h" #include "hashmap.h" #include "replace-object.h" @@ -19,6 +26,7 @@ #include "shallow.h" #include "json-writer.h" #include "trace2.h" +#include "tree.h" #include "chunk-format.h" void git_test_write_commit_graph_or_die(void) @@ -30,7 +38,7 @@ void git_test_write_commit_graph_or_die(void) if (git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags = COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - if (write_commit_graph_reachable(the_repository->objects->odb, + if (write_commit_graph_reachable(the_repository->objects->sources, flags, NULL)) die("failed to write commit-graph under GIT_TEST_COMMIT_GRAPH"); } @@ -116,11 +124,19 @@ timestamp_t commit_graph_generation(const struct commit *c) struct commit_graph_data *data = commit_graph_data_slab_peek(&commit_graph_data_slab, c); - if (!data) - return GENERATION_NUMBER_INFINITY; - else if (data->graph_pos == COMMIT_NOT_FROM_GRAPH) - return GENERATION_NUMBER_INFINITY; + if (data && data->generation) + return data->generation; + + return GENERATION_NUMBER_INFINITY; +} +static timestamp_t commit_graph_generation_from_graph(const struct commit *c) +{ + struct commit_graph_data *data = + commit_graph_data_slab_peek(&commit_graph_data_slab, c); + + if (!data || data->graph_pos == COMMIT_NOT_FROM_GRAPH) + return GENERATION_NUMBER_INFINITY; return data->generation; } @@ -176,21 +192,21 @@ static int commit_gen_cmp(const void *va, const void *vb) return 0; } -char *get_commit_graph_filename(struct object_directory *obj_dir) +char *get_commit_graph_filename(struct odb_source *source) { - return xstrfmt("%s/info/commit-graph", obj_dir->path); + return xstrfmt("%s/info/commit-graph", source->path); } -static char *get_split_graph_filename(struct object_directory *odb, +static char *get_split_graph_filename(struct odb_source *source, const char *oid_hex) { - return xstrfmt("%s/info/commit-graphs/graph-%s.graph", odb->path, + return xstrfmt("%s/info/commit-graphs/graph-%s.graph", source->path, oid_hex); } -char *get_commit_graph_chain_filename(struct object_directory *odb) +char *get_commit_graph_chain_filename(struct odb_source *source) { - return xstrfmt("%s/info/commit-graphs/commit-graph-chain", odb->path); + return xstrfmt("%s/info/commit-graphs/commit-graph-chain", source->path); } static struct commit_graph *alloc_commit_graph(void) @@ -200,16 +216,14 @@ static struct commit_graph *alloc_commit_graph(void) return g; } -extern int read_replace_refs; - static int commit_graph_compatible(struct repository *r) { if (!r->gitdir) return 0; - if (read_replace_refs) { + if (replace_refs_enabled(r)) { prepare_replace_object(r); - if (hashmap_get_size(&r->objects->replace_map->map)) + if (oidmap_get_size(&r->objects->replace_map)) return 0; } @@ -237,7 +251,7 @@ int open_commit_graph(const char *graph_file, int *fd, struct stat *st) struct commit_graph *load_commit_graph_one_fd_st(struct repository *r, int fd, struct stat *st, - struct object_directory *odb) + struct odb_source *source) { void *graph_map; size_t graph_size; @@ -256,38 +270,32 @@ struct commit_graph *load_commit_graph_one_fd_st(struct repository *r, ret = parse_commit_graph(&r->settings, graph_map, graph_size); if (ret) - ret->odb = odb; + ret->odb_source = source; else munmap(graph_map, graph_size); return ret; } -static int verify_commit_graph_lite(struct commit_graph *g) +static int graph_read_oid_fanout(const unsigned char *chunk_start, + size_t chunk_size, void *data) { - /* - * Basic validation shared between parse_commit_graph() - * which'll be called every time the graph is used, and the - * much more expensive verify_commit_graph() used by - * "commit-graph verify". - * - * There should only be very basic checks here to ensure that - * we don't e.g. segfault in fill_commit_in_graph(), but - * because this is a very hot codepath nothing that e.g. loops - * over g->num_commits, or runs a checksum on the commit-graph - * itself. - */ - if (!g->chunk_oid_fanout) { - error("commit-graph is missing the OID Fanout chunk"); - return 1; - } - if (!g->chunk_oid_lookup) { - error("commit-graph is missing the OID Lookup chunk"); - return 1; - } - if (!g->chunk_commit_data) { - error("commit-graph is missing the Commit Data chunk"); - return 1; + struct commit_graph *g = data; + int i; + + if (chunk_size != 256 * sizeof(uint32_t)) + return error(_("commit-graph oid fanout chunk is wrong size")); + g->chunk_oid_fanout = (const uint32_t *)chunk_start; + g->num_commits = ntohl(g->chunk_oid_fanout[255]); + + for (i = 0; i < 255; i++) { + uint32_t oid_fanout1 = ntohl(g->chunk_oid_fanout[i]); + uint32_t oid_fanout2 = ntohl(g->chunk_oid_fanout[i + 1]); + + if (oid_fanout1 > oid_fanout2) { + error(_("commit-graph fanout values out of order")); + return 1; + } } return 0; @@ -298,7 +306,40 @@ static int graph_read_oid_lookup(const unsigned char *chunk_start, { struct commit_graph *g = data; g->chunk_oid_lookup = chunk_start; - g->num_commits = chunk_size / g->hash_len; + if (chunk_size / g->hash_len != g->num_commits) + return error(_("commit-graph OID lookup chunk is the wrong size")); + return 0; +} + +static int graph_read_commit_data(const unsigned char *chunk_start, + size_t chunk_size, void *data) +{ + struct commit_graph *g = data; + if (chunk_size / GRAPH_DATA_WIDTH != g->num_commits) + return error(_("commit-graph commit data chunk is wrong size")); + g->chunk_commit_data = chunk_start; + return 0; +} + +static int graph_read_generation_data(const unsigned char *chunk_start, + size_t chunk_size, void *data) +{ + struct commit_graph *g = data; + if (chunk_size / sizeof(uint32_t) != g->num_commits) + return error(_("commit-graph generations chunk is wrong size")); + g->chunk_generation_data = chunk_start; + return 0; +} + +static int graph_read_bloom_index(const unsigned char *chunk_start, + size_t chunk_size, void *data) +{ + struct commit_graph *g = data; + if (chunk_size / 4 != g->num_commits) { + warning(_("commit-graph changed-path index chunk is too small")); + return -1; + } + g->chunk_bloom_indexes = chunk_start; return 0; } @@ -306,15 +347,20 @@ static int graph_read_bloom_data(const unsigned char *chunk_start, size_t chunk_size, void *data) { struct commit_graph *g = data; - uint32_t hash_version; - g->chunk_bloom_data = chunk_start; - hash_version = get_be32(chunk_start); - if (hash_version != 1) - return 0; + if (chunk_size < BLOOMDATA_CHUNK_HEADER_SIZE) { + warning(_("ignoring too-small changed-path chunk" + " (%"PRIuMAX" < %"PRIuMAX") in commit-graph file"), + (uintmax_t)chunk_size, + (uintmax_t)BLOOMDATA_CHUNK_HEADER_SIZE); + return -1; + } + + g->chunk_bloom_data = chunk_start; + g->chunk_bloom_data_size = chunk_size; g->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings)); - g->bloom_filter_settings->hash_version = hash_version; + g->bloom_filter_settings->hash_version = get_be32(chunk_start); g->bloom_filter_settings->num_hashes = get_be32(chunk_start + 4); g->bloom_filter_settings->bits_per_entry = get_be32(chunk_start + 8); g->bloom_filter_settings->max_changed_paths = DEFAULT_BLOOM_MAX_CHANGES; @@ -379,29 +425,41 @@ struct commit_graph *parse_commit_graph(struct repo_settings *s, cf = init_chunkfile(NULL); if (read_table_of_contents(cf, graph->data, graph_size, - GRAPH_HEADER_SIZE, graph->num_chunks)) + GRAPH_HEADER_SIZE, graph->num_chunks, 1)) + goto free_and_return; + + if (read_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, graph_read_oid_fanout, graph)) { + error(_("commit-graph required OID fanout chunk missing or corrupted")); + goto free_and_return; + } + if (read_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, graph_read_oid_lookup, graph)) { + error(_("commit-graph required OID lookup chunk missing or corrupted")); + goto free_and_return; + } + if (read_chunk(cf, GRAPH_CHUNKID_DATA, graph_read_commit_data, graph)) { + error(_("commit-graph required commit data chunk missing or corrupted")); goto free_and_return; + } - pair_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, - (const unsigned char **)&graph->chunk_oid_fanout); - read_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, graph_read_oid_lookup, graph); - pair_chunk(cf, GRAPH_CHUNKID_DATA, &graph->chunk_commit_data); - pair_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, &graph->chunk_extra_edges); - pair_chunk(cf, GRAPH_CHUNKID_BASE, &graph->chunk_base_graphs); + pair_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, &graph->chunk_extra_edges, + &graph->chunk_extra_edges_size); + pair_chunk(cf, GRAPH_CHUNKID_BASE, &graph->chunk_base_graphs, + &graph->chunk_base_graphs_size); if (s->commit_graph_generation_version >= 2) { - pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA, - &graph->chunk_generation_data); + read_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA, + graph_read_generation_data, graph); pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW, - &graph->chunk_generation_data_overflow); + &graph->chunk_generation_data_overflow, + &graph->chunk_generation_data_overflow_size); if (graph->chunk_generation_data) graph->read_generation_data = 1; } - if (s->commit_graph_read_changed_paths) { - pair_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES, - &graph->chunk_bloom_indexes); + if (s->commit_graph_changed_paths_version) { + read_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES, + graph_read_bloom_index, graph); read_chunk(cf, GRAPH_CHUNKID_BLOOMDATA, graph_read_bloom_data, graph); } @@ -415,10 +473,8 @@ struct commit_graph *parse_commit_graph(struct repo_settings *s, FREE_AND_NULL(graph->bloom_filter_settings); } - oidread(&graph->oid, graph->data + graph->data_len - graph->hash_len); - - if (verify_commit_graph_lite(graph)) - goto free_and_return; + oidread(&graph->oid, graph->data + graph->data_len - graph->hash_len, + the_repository->hash_algo); free_chunkfile(cf); return graph; @@ -432,7 +488,7 @@ free_and_return: static struct commit_graph *load_commit_graph_one(struct repository *r, const char *graph_file, - struct object_directory *odb) + struct odb_source *source) { struct stat st; @@ -443,7 +499,7 @@ static struct commit_graph *load_commit_graph_one(struct repository *r, if (!open_ok) return NULL; - g = load_commit_graph_one_fd_st(r, fd, &st, odb); + g = load_commit_graph_one_fd_st(r, fd, &st, source); if (g) g->filename = xstrdup(graph_file); @@ -452,15 +508,65 @@ static struct commit_graph *load_commit_graph_one(struct repository *r, } static struct commit_graph *load_commit_graph_v1(struct repository *r, - struct object_directory *odb) + struct odb_source *source) { - char *graph_name = get_commit_graph_filename(odb); - struct commit_graph *g = load_commit_graph_one(r, graph_name, odb); + char *graph_name = get_commit_graph_filename(source); + struct commit_graph *g = load_commit_graph_one(r, graph_name, source); free(graph_name); return g; } +/* + * returns 1 if and only if all graphs in the chain have + * corrected commit dates stored in the generation_data chunk. + */ +static int validate_mixed_generation_chain(struct commit_graph *g) +{ + int read_generation_data = 1; + struct commit_graph *p = g; + + while (read_generation_data && p) { + read_generation_data = p->read_generation_data; + p = p->base_graph; + } + + if (read_generation_data) + return 1; + + while (g) { + g->read_generation_data = 0; + g = g->base_graph; + } + + return 0; +} + +static void validate_mixed_bloom_settings(struct commit_graph *g) +{ + struct bloom_filter_settings *settings = NULL; + for (; g; g = g->base_graph) { + if (!g->bloom_filter_settings) + continue; + if (!settings) { + settings = g->bloom_filter_settings; + continue; + } + + if (g->bloom_filter_settings->bits_per_entry != settings->bits_per_entry || + g->bloom_filter_settings->num_hashes != settings->num_hashes || + g->bloom_filter_settings->hash_version != settings->hash_version) { + g->chunk_bloom_indexes = NULL; + g->chunk_bloom_data = NULL; + FREE_AND_NULL(g->bloom_filter_settings); + + warning(_("disabling Bloom filters for commit-graph " + "layer '%s' due to incompatible settings"), + oid_to_hex(&g->oid)); + } + } +} + static int add_graph_to_chain(struct commit_graph *g, struct commit_graph *chain, struct object_id *oids, @@ -473,12 +579,18 @@ static int add_graph_to_chain(struct commit_graph *g, return 0; } + if (g->chunk_base_graphs_size / g->hash_len < n) { + warning(_("commit-graph base graphs chunk is too small")); + return 0; + } + while (n) { n--; if (!cur_g || !oideq(&oids[n], &cur_g->oid) || - !hasheq(oids[n].hash, g->chunk_base_graphs + g->hash_len * n)) { + !hasheq(oids[n].hash, g->chunk_base_graphs + st_mult(g->hash_len, n), + the_repository->hash_algo)) { warning(_("commit-graph chain does not match")); return 0; } @@ -486,45 +598,62 @@ static int add_graph_to_chain(struct commit_graph *g, cur_g = cur_g->base_graph; } + if (chain) { + if (unsigned_add_overflows(chain->num_commits, + chain->num_commits_in_base)) { + warning(_("commit count in base graph too high: %"PRIuMAX), + (uintmax_t)chain->num_commits_in_base); + return 0; + } + g->num_commits_in_base = chain->num_commits + chain->num_commits_in_base; + } + g->base_graph = chain; - if (chain) - g->num_commits_in_base = chain->num_commits + chain->num_commits_in_base; + return 1; +} +int open_commit_graph_chain(const char *chain_file, + int *fd, struct stat *st) +{ + *fd = git_open(chain_file); + if (*fd < 0) + return 0; + if (fstat(*fd, st)) { + close(*fd); + return 0; + } + if (st->st_size < the_hash_algo->hexsz) { + close(*fd); + if (!st->st_size) { + /* treat empty files the same as missing */ + errno = ENOENT; + } else { + warning(_("commit-graph chain file too small")); + errno = EINVAL; + } + return 0; + } return 1; } -static struct commit_graph *load_commit_graph_chain(struct repository *r, - struct object_directory *odb) +struct commit_graph *load_commit_graph_chain_fd_st(struct repository *r, + int fd, struct stat *st, + int *incomplete_chain) { struct commit_graph *graph_chain = NULL; struct strbuf line = STRBUF_INIT; - struct stat st; struct object_id *oids; int i = 0, valid = 1, count; - char *chain_name = get_commit_graph_chain_filename(odb); - FILE *fp; - int stat_res; - - fp = fopen(chain_name, "r"); - stat_res = stat(chain_name, &st); - free(chain_name); + FILE *fp = xfdopen(fd, "r"); - if (!fp) - return NULL; - if (stat_res || - st.st_size <= the_hash_algo->hexsz) { - fclose(fp); - return NULL; - } - - count = st.st_size / (the_hash_algo->hexsz + 1); + count = st->st_size / (the_hash_algo->hexsz + 1); CALLOC_ARRAY(oids, count); - prepare_alt_odb(r); + odb_prepare_alternates(r->objects); for (i = 0; i < count; i++) { - struct object_directory *odb; + struct odb_source *source; if (strbuf_getline_lf(&line, fp) == EOF) break; @@ -537,9 +666,9 @@ static struct commit_graph *load_commit_graph_chain(struct repository *r, } valid = 0; - for (odb = r->objects->odb; odb; odb = odb->next) { - char *graph_name = get_split_graph_filename(odb, line.buf); - struct commit_graph *g = load_commit_graph_one(r, graph_name, odb); + for (source = r->objects->sources; source; source = source->next) { + char *graph_name = get_split_graph_filename(source, line.buf); + struct commit_graph *g = load_commit_graph_one(r, graph_name, source); free(graph_name); @@ -547,6 +676,8 @@ static struct commit_graph *load_commit_graph_chain(struct repository *r, if (add_graph_to_chain(g, graph_chain, oids, i)) { graph_chain = g; valid = 1; + } else { + free_commit_graph(g); } break; @@ -559,59 +690,54 @@ static struct commit_graph *load_commit_graph_chain(struct repository *r, } } + validate_mixed_generation_chain(graph_chain); + validate_mixed_bloom_settings(graph_chain); + free(oids); fclose(fp); strbuf_release(&line); + *incomplete_chain = !valid; return graph_chain; } -/* - * returns 1 if and only if all graphs in the chain have - * corrected commit dates stored in the generation_data chunk. - */ -static int validate_mixed_generation_chain(struct commit_graph *g) +static struct commit_graph *load_commit_graph_chain(struct repository *r, + struct odb_source *source) { - int read_generation_data = 1; - struct commit_graph *p = g; - - while (read_generation_data && p) { - read_generation_data = p->read_generation_data; - p = p->base_graph; - } - - if (read_generation_data) - return 1; + char *chain_file = get_commit_graph_chain_filename(source); + struct stat st; + int fd; + struct commit_graph *g = NULL; - while (g) { - g->read_generation_data = 0; - g = g->base_graph; + if (open_commit_graph_chain(chain_file, &fd, &st)) { + int incomplete; + /* ownership of fd is taken over by load function */ + g = load_commit_graph_chain_fd_st(r, fd, &st, &incomplete); } - return 0; + free(chain_file); + return g; } struct commit_graph *read_commit_graph_one(struct repository *r, - struct object_directory *odb) + struct odb_source *source) { - struct commit_graph *g = load_commit_graph_v1(r, odb); + struct commit_graph *g = load_commit_graph_v1(r, source); if (!g) - g = load_commit_graph_chain(r, odb); - - validate_mixed_generation_chain(g); + g = load_commit_graph_chain(r, source); return g; } static void prepare_commit_graph_one(struct repository *r, - struct object_directory *odb) + struct odb_source *source) { if (r->objects->commit_graph) return; - r->objects->commit_graph = read_commit_graph_one(r, odb); + r->objects->commit_graph = read_commit_graph_one(r, source); } /* @@ -622,7 +748,7 @@ static void prepare_commit_graph_one(struct repository *r, */ static int prepare_commit_graph(struct repository *r) { - struct object_directory *odb; + struct odb_source *source; /* * Early return if there is no git dir or if the commit graph is @@ -653,11 +779,11 @@ static int prepare_commit_graph(struct repository *r) if (!commit_graph_compatible(r)) return 0; - prepare_alt_odb(r); - for (odb = r->objects->odb; - !r->objects->commit_graph && odb; - odb = odb->next) - prepare_commit_graph_one(r, odb); + odb_prepare_alternates(r->objects); + for (source = r->objects->sources; + !r->objects->commit_graph && source; + source = source->next) + prepare_commit_graph_one(r, source); return !!r->objects->commit_graph; } @@ -704,19 +830,14 @@ struct bloom_filter_settings *get_bloom_filter_settings(struct repository *r) return NULL; } -static void close_commit_graph_one(struct commit_graph *g) +void close_commit_graph(struct object_database *o) { - if (!g) + if (!o->commit_graph) return; clear_commit_graph_data_slab(&commit_graph_data_slab); - close_commit_graph_one(g->base_graph); - free_commit_graph(g); -} - -void close_commit_graph(struct raw_object_store *o) -{ - close_commit_graph_one(o->commit_graph); + deinit_bloom_filters(); + free_commit_graph(o->commit_graph); o->commit_graph = NULL; } @@ -743,7 +864,8 @@ static void load_oid_from_graph(struct commit_graph *g, lex_index = pos - g->num_commits_in_base; - oidread(oid, g->chunk_oid_lookup + g->hash_len * lex_index); + oidread(oid, g->chunk_oid_lookup + st_mult(g->hash_len, lex_index), + the_repository->hash_algo); } static struct commit_list **insert_parent_or_die(struct repository *r, @@ -779,7 +901,7 @@ static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, die(_("invalid commit position. commit-graph is likely corrupt")); lex_index = pos - g->num_commits_in_base; - commit_data = g->chunk_commit_data + GRAPH_DATA_WIDTH * lex_index; + commit_data = g->chunk_commit_data + st_mult(GRAPH_DATA_WIDTH, lex_index); graph_data = commit_graph_data_at(item); graph_data->graph_pos = pos; @@ -789,14 +911,17 @@ static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, item->date = (timestamp_t)((date_high << 32) | date_low); if (g->read_generation_data) { - offset = (timestamp_t)get_be32(g->chunk_generation_data + sizeof(uint32_t) * lex_index); + offset = (timestamp_t)get_be32(g->chunk_generation_data + st_mult(sizeof(uint32_t), lex_index)); if (offset & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW) { if (!g->chunk_generation_data_overflow) die(_("commit-graph requires overflow generation data but has none")); offset_pos = offset ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW; - graph_data->generation = item->date + get_be64(g->chunk_generation_data_overflow + 8 * offset_pos); + if (g->chunk_generation_data_overflow_size / sizeof(uint64_t) <= offset_pos) + die(_("commit-graph overflow generation data is too small")); + graph_data->generation = item->date + + get_be64(g->chunk_generation_data_overflow + sizeof(uint64_t) * offset_pos); } else graph_data->generation = item->date + offset; } else @@ -816,7 +941,7 @@ static int fill_commit_in_graph(struct repository *r, struct commit_graph *g, uint32_t pos) { uint32_t edge_value; - uint32_t *parent_data_ptr; + uint32_t parent_data_pos; struct commit_list **pptr; const unsigned char *commit_data; uint32_t lex_index; @@ -827,7 +952,7 @@ static int fill_commit_in_graph(struct repository *r, fill_commit_graph_info(item, g, pos); lex_index = pos - g->num_commits_in_base; - commit_data = g->chunk_commit_data + (g->hash_len + 16) * lex_index; + commit_data = g->chunk_commit_data + st_mult(g->hash_len + 16, lex_index); item->object.parsed = 1; @@ -848,14 +973,21 @@ static int fill_commit_in_graph(struct repository *r, return 1; } - parent_data_ptr = (uint32_t*)(g->chunk_extra_edges + - 4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK)); + parent_data_pos = edge_value & GRAPH_EDGE_LAST_MASK; do { - edge_value = get_be32(parent_data_ptr); + if (g->chunk_extra_edges_size / sizeof(uint32_t) <= parent_data_pos) { + error(_("commit-graph extra-edges pointer out of bounds")); + free_commit_list(item->parents); + item->parents = NULL; + item->object.parsed = 0; + return 0; + } + edge_value = get_be32(g->chunk_extra_edges + + sizeof(uint32_t) * parent_data_pos); pptr = insert_parent_or_die(r, g, edge_value & GRAPH_EDGE_LAST_MASK, pptr); - parent_data_ptr++; + parent_data_pos++; } while (!(edge_value & GRAPH_LAST_EDGE)); return 1; @@ -898,14 +1030,18 @@ int repo_find_commit_pos_in_graph(struct repository *r, struct commit *c, struct commit *lookup_commit_in_graph(struct repository *repo, const struct object_id *id) { + static int commit_graph_paranoia = -1; struct commit *commit; uint32_t pos; + if (commit_graph_paranoia == -1) + commit_graph_paranoia = git_env_bool(GIT_COMMIT_GRAPH_PARANOIA, 0); + if (!prepare_commit_graph(repo)) return NULL; if (!search_commit_pos_in_graph(id, repo->objects->commit_graph, &pos)) return NULL; - if (!has_object(repo, id, 0)) + if (commit_graph_paranoia && !odb_has_object(repo->objects, id, 0)) return NULL; commit = lookup_commit(repo, id); @@ -969,9 +1105,9 @@ static struct tree *load_tree_for_commit(struct repository *r, g = g->base_graph; commit_data = g->chunk_commit_data + - GRAPH_DATA_WIDTH * (graph_pos - g->num_commits_in_base); + st_mult(GRAPH_DATA_WIDTH, graph_pos - g->num_commits_in_base); - oidread(&oid, commit_data); + oidread(&oid, commit_data, the_repository->hash_algo); set_commit_tree(c, lookup_tree(r, &oid)); return c->maybe_tree; @@ -1002,7 +1138,7 @@ struct packed_commit_list { struct write_commit_graph_context { struct repository *r; - struct object_directory *odb; + struct odb_source *odb_source; char *graph_name; struct oid_array oids; struct packed_commit_list commits; @@ -1039,6 +1175,7 @@ struct write_commit_graph_context { int count_bloom_filter_not_computed; int count_bloom_filter_trunc_empty; int count_bloom_filter_trunc_large; + int count_bloom_filter_upgraded; }; static int write_graph_chunk_fanout(struct hashfile *f, @@ -1397,6 +1534,7 @@ static void close_reachable(struct write_commit_graph_context *ctx) if (ctx->report_progress) ctx->progress = start_delayed_progress( + the_repository, _("Loading known commits in commit graph"), ctx->oids.nr); for (i = 0; i < ctx->oids.nr; i++) { @@ -1414,6 +1552,7 @@ static void close_reachable(struct write_commit_graph_context *ctx) */ if (ctx->report_progress) ctx->progress = start_delayed_progress( + the_repository, _("Expanding reachable commits in commit graph"), 0); for (i = 0; i < ctx->oids.nr; i++) { @@ -1434,6 +1573,7 @@ static void close_reachable(struct write_commit_graph_context *ctx) if (ctx->report_progress) ctx->progress = start_delayed_progress( + the_repository, _("Clearing commit marks in commit graph"), ctx->oids.nr); for (i = 0; i < ctx->oids.nr; i++) { @@ -1446,24 +1586,52 @@ static void close_reachable(struct write_commit_graph_context *ctx) stop_progress(&ctx->progress); } -static void compute_topological_levels(struct write_commit_graph_context *ctx) +struct compute_generation_info { + struct repository *r; + struct packed_commit_list *commits; + struct progress *progress; + int progress_cnt; + + timestamp_t (*get_generation)(struct commit *c, void *data); + void (*set_generation)(struct commit *c, timestamp_t gen, void *data); + void *data; +}; + +static timestamp_t compute_generation_from_max(struct commit *c, + timestamp_t max_gen, + int generation_version) +{ + switch (generation_version) { + case 1: /* topological levels */ + if (max_gen > GENERATION_NUMBER_V1_MAX - 1) + max_gen = GENERATION_NUMBER_V1_MAX - 1; + return max_gen + 1; + + case 2: /* corrected commit date */ + if (c->date && c->date > max_gen) + max_gen = c->date - 1; + return max_gen + 1; + + default: + BUG("attempting unimplemented version"); + } +} + +static void compute_reachable_generation_numbers( + struct compute_generation_info *info, + int generation_version) { int i; struct commit_list *list = NULL; - if (ctx->report_progress) - ctx->progress = start_delayed_progress( - _("Computing commit graph topological levels"), - ctx->commits.nr); - for (i = 0; i < ctx->commits.nr; i++) { - struct commit *c = ctx->commits.list[i]; - uint32_t level; - - repo_parse_commit(ctx->r, c); - level = *topo_level_slab_at(ctx->topo_levels, c); + for (i = 0; i < info->commits->nr; i++) { + struct commit *c = info->commits->list[i]; + timestamp_t gen; + repo_parse_commit(info->r, c); + gen = info->get_generation(c, info->data); + display_progress(info->progress, ++info->progress_cnt); - display_progress(ctx->progress, i + 1); - if (level != GENERATION_NUMBER_ZERO) + if (gen != GENERATION_NUMBER_ZERO && gen != GENERATION_NUMBER_INFINITY) continue; commit_list_insert(c, &list); @@ -1471,41 +1639,94 @@ static void compute_topological_levels(struct write_commit_graph_context *ctx) struct commit *current = list->item; struct commit_list *parent; int all_parents_computed = 1; - uint32_t max_level = 0; + uint32_t max_gen = 0; for (parent = current->parents; parent; parent = parent->next) { - repo_parse_commit(ctx->r, parent->item); - level = *topo_level_slab_at(ctx->topo_levels, parent->item); + repo_parse_commit(info->r, parent->item); + gen = info->get_generation(parent->item, info->data); - if (level == GENERATION_NUMBER_ZERO) { + if (gen == GENERATION_NUMBER_ZERO) { all_parents_computed = 0; commit_list_insert(parent->item, &list); break; } - if (level > max_level) - max_level = level; + if (gen > max_gen) + max_gen = gen; } if (all_parents_computed) { pop_commit(&list); - - if (max_level > GENERATION_NUMBER_V1_MAX - 1) - max_level = GENERATION_NUMBER_V1_MAX - 1; - *topo_level_slab_at(ctx->topo_levels, current) = max_level + 1; + gen = compute_generation_from_max( + current, max_gen, + generation_version); + info->set_generation(current, gen, info->data); } } } +} + +static timestamp_t get_topo_level(struct commit *c, void *data) +{ + struct write_commit_graph_context *ctx = data; + return *topo_level_slab_at(ctx->topo_levels, c); +} + +static void set_topo_level(struct commit *c, timestamp_t t, void *data) +{ + struct write_commit_graph_context *ctx = data; + *topo_level_slab_at(ctx->topo_levels, c) = (uint32_t)t; +} + +static void compute_topological_levels(struct write_commit_graph_context *ctx) +{ + struct compute_generation_info info = { + .r = ctx->r, + .commits = &ctx->commits, + .get_generation = get_topo_level, + .set_generation = set_topo_level, + .data = ctx, + }; + + if (ctx->report_progress) + info.progress = ctx->progress + = start_delayed_progress( + the_repository, + _("Computing commit graph topological levels"), + ctx->commits.nr); + + compute_reachable_generation_numbers(&info, 1); + stop_progress(&ctx->progress); } +static timestamp_t get_generation_from_graph_data(struct commit *c, + void *data UNUSED) +{ + return commit_graph_data_at(c)->generation; +} + +static void set_generation_v2(struct commit *c, timestamp_t t, + void *data UNUSED) +{ + struct commit_graph_data *g = commit_graph_data_at(c); + g->generation = t; +} + static void compute_generation_numbers(struct write_commit_graph_context *ctx) { int i; - struct commit_list *list = NULL; + struct compute_generation_info info = { + .r = ctx->r, + .commits = &ctx->commits, + .get_generation = get_generation_from_graph_data, + .set_generation = set_generation_v2, + }; if (ctx->report_progress) - ctx->progress = start_delayed_progress( + info.progress = ctx->progress + = start_delayed_progress( + the_repository, _("Computing commit graph generation numbers"), ctx->commits.nr); @@ -1517,47 +1738,7 @@ static void compute_generation_numbers(struct write_commit_graph_context *ctx) } } - for (i = 0; i < ctx->commits.nr; i++) { - struct commit *c = ctx->commits.list[i]; - timestamp_t corrected_commit_date; - - repo_parse_commit(ctx->r, c); - corrected_commit_date = commit_graph_data_at(c)->generation; - - display_progress(ctx->progress, i + 1); - if (corrected_commit_date != GENERATION_NUMBER_ZERO) - continue; - - commit_list_insert(c, &list); - while (list) { - struct commit *current = list->item; - struct commit_list *parent; - int all_parents_computed = 1; - timestamp_t max_corrected_commit_date = 0; - - for (parent = current->parents; parent; parent = parent->next) { - repo_parse_commit(ctx->r, parent->item); - corrected_commit_date = commit_graph_data_at(parent->item)->generation; - - if (corrected_commit_date == GENERATION_NUMBER_ZERO) { - all_parents_computed = 0; - commit_list_insert(parent->item, &list); - break; - } - - if (corrected_commit_date > max_corrected_commit_date) - max_corrected_commit_date = corrected_commit_date; - } - - if (all_parents_computed) { - pop_commit(&list); - - if (current->date && current->date > max_corrected_commit_date) - max_corrected_commit_date = current->date - 1; - commit_graph_data_at(current)->generation = max_corrected_commit_date + 1; - } - } - } + compute_reachable_generation_numbers(&info, 2); for (i = 0; i < ctx->commits.nr; i++) { struct commit *c = ctx->commits.list[i]; @@ -1568,6 +1749,35 @@ static void compute_generation_numbers(struct write_commit_graph_context *ctx) stop_progress(&ctx->progress); } +static void set_generation_in_graph_data(struct commit *c, timestamp_t t, + void *data UNUSED) +{ + commit_graph_data_at(c)->generation = t; +} + +/* + * After this method, all commits reachable from those in the given + * list will have non-zero, non-infinite generation numbers. + */ +void ensure_generations_valid(struct repository *r, + struct commit **commits, size_t nr) +{ + int generation_version = get_configured_generation_version(r); + struct packed_commit_list list = { + .list = commits, + .alloc = nr, + .nr = nr, + }; + struct compute_generation_info info = { + .r = r, + .commits = &list, + .get_generation = get_generation_from_graph_data, + .set_generation = set_generation_in_graph_data, + }; + + compute_reachable_generation_numbers(&info, generation_version); +} + static void trace2_bloom_filter_write_statistics(struct write_commit_graph_context *ctx) { trace2_data_intmax("commit-graph", ctx->r, "filter-computed", @@ -1578,6 +1788,8 @@ static void trace2_bloom_filter_write_statistics(struct write_commit_graph_conte ctx->count_bloom_filter_trunc_empty); trace2_data_intmax("commit-graph", ctx->r, "filter-trunc-large", ctx->count_bloom_filter_trunc_large); + trace2_data_intmax("commit-graph", ctx->r, "filter-upgraded", + ctx->count_bloom_filter_upgraded); } static void compute_bloom_filters(struct write_commit_graph_context *ctx) @@ -1591,6 +1803,7 @@ static void compute_bloom_filters(struct write_commit_graph_context *ctx) if (ctx->report_progress) progress = start_delayed_progress( + the_repository, _("Computing commit changed paths Bloom filters"), ctx->commits.nr); @@ -1619,6 +1832,8 @@ static void compute_bloom_filters(struct write_commit_graph_context *ctx) ctx->count_bloom_filter_trunc_empty++; if (computed & BLOOM_TRUNC_LARGE) ctx->count_bloom_filter_trunc_large++; + } else if (computed & BLOOM_UPGRADED) { + ctx->count_bloom_filter_upgraded++; } else if (computed & BLOOM_NOT_COMPUTED) ctx->count_bloom_filter_not_computed++; ctx->total_bloom_filter_data_size += filter @@ -1639,15 +1854,16 @@ struct refs_cb_data { }; static int add_ref_to_set(const char *refname UNUSED, + const char *referent UNUSED, const struct object_id *oid, int flags UNUSED, void *cb_data) { struct object_id peeled; struct refs_cb_data *data = (struct refs_cb_data *)cb_data; - if (!peel_iterated_oid(oid, &peeled)) + if (!peel_iterated_oid(the_repository, oid, &peeled)) oid = &peeled; - if (oid_object_info(the_repository, oid, NULL) == OBJ_COMMIT) + if (odb_read_object_info(the_repository->objects, oid, NULL) == OBJ_COMMIT) oidset_insert(data->commits, oid); display_progress(data->progress, oidset_size(data->commits)); @@ -1655,7 +1871,7 @@ static int add_ref_to_set(const char *refname UNUSED, return 0; } -int write_commit_graph_reachable(struct object_directory *odb, +int write_commit_graph_reachable(struct odb_source *source, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts) { @@ -1667,13 +1883,15 @@ int write_commit_graph_reachable(struct object_directory *odb, data.commits = &commits; if (flags & COMMIT_GRAPH_WRITE_PROGRESS) data.progress = start_delayed_progress( + the_repository, _("Collecting referenced commits"), 0); - for_each_ref(add_ref_to_set, &data); + refs_for_each_ref(get_main_ref_store(the_repository), add_ref_to_set, + &data); stop_progress(&data.progress); - result = write_commit_graph(odb, NULL, &commits, + result = write_commit_graph(source, NULL, &commits, flags, opts); oidset_clear(&commits); @@ -1689,7 +1907,7 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, int dirlen; int ret = 0; - strbuf_addf(&packname, "%s/pack/", ctx->odb->path); + strbuf_addf(&packname, "%s/pack/", ctx->odb_source->path); dirlen = packname.len; if (ctx->report_progress) { strbuf_addf(&progress_title, @@ -1697,20 +1915,23 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, "Finding commits for commit graph in %"PRIuMAX" packs", pack_indexes->nr), (uintmax_t)pack_indexes->nr); - ctx->progress = start_delayed_progress(progress_title.buf, 0); + ctx->progress = start_delayed_progress(the_repository, + progress_title.buf, 0); ctx->progress_done = 0; } for (i = 0; i < pack_indexes->nr; i++) { struct packed_git *p; strbuf_setlen(&packname, dirlen); strbuf_addstr(&packname, pack_indexes->items[i].string); - p = add_packed_git(packname.buf, packname.len, 1); + p = add_packed_git(ctx->r, packname.buf, packname.len, 1); if (!p) { ret = error(_("error adding pack %s"), packname.buf); goto cleanup; } if (open_pack_index(p)) { ret = error(_("error opening index for %s"), packname.buf); + close_pack(p); + free(p); goto cleanup; } for_each_object_in_pack(p, add_packed_commits, ctx, @@ -1748,9 +1969,10 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) { if (ctx->report_progress) ctx->progress = start_delayed_progress( + the_repository, _("Finding commits for commit graph among packed objects"), ctx->approx_nr_objects); - for_each_packed_object(add_packed_commits, ctx, + for_each_packed_object(ctx->r, add_packed_commits, ctx, FOR_EACH_OBJECT_PACK_ORDER); if (ctx->progress_done < ctx->approx_nr_objects) display_progress(ctx->progress, ctx->approx_nr_objects); @@ -1766,6 +1988,7 @@ static void copy_oids_to_commits(struct write_commit_graph_context *ctx) ctx->num_extra_edges = 0; if (ctx->report_progress) ctx->progress = start_delayed_progress( + the_repository, _("Finding extra edges in commit graph"), ctx->oids.nr); oid_array_sort(&ctx->oids); @@ -1825,8 +2048,8 @@ static int write_graph_chunk_base(struct hashfile *f, static int write_commit_graph_file(struct write_commit_graph_context *ctx) { uint32_t i; - int fd; struct hashfile *f; + struct tempfile *graph_layer; /* when ctx->split is non-zero */ struct lock_file lk = LOCK_INIT; const unsigned hashsz = the_hash_algo->rawsz; struct strbuf progress_title = STRBUF_INIT; @@ -1838,79 +2061,79 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) strbuf_addf(&tmp_file, "%s/info/commit-graphs/tmp_graph_XXXXXX", - ctx->odb->path); + ctx->odb_source->path); ctx->graph_name = strbuf_detach(&tmp_file, NULL); } else { - ctx->graph_name = get_commit_graph_filename(ctx->odb); + ctx->graph_name = get_commit_graph_filename(ctx->odb_source); } - if (safe_create_leading_directories(ctx->graph_name)) { - UNLEAK(ctx->graph_name); + if (safe_create_leading_directories(the_repository, ctx->graph_name)) { error(_("unable to create leading directories of %s"), ctx->graph_name); return -1; } if (ctx->split) { - char *lock_name = get_commit_graph_chain_filename(ctx->odb); + char *lock_name = get_commit_graph_chain_filename(ctx->odb_source); hold_lock_file_for_update_mode(&lk, lock_name, LOCK_DIE_ON_ERROR, 0444); free(lock_name); - fd = git_mkstemp_mode(ctx->graph_name, 0444); - if (fd < 0) { + graph_layer = mks_tempfile_m(ctx->graph_name, 0444); + if (!graph_layer) { error(_("unable to create temporary graph layer")); return -1; } - if (adjust_shared_perm(ctx->graph_name)) { + if (adjust_shared_perm(the_repository, get_tempfile_path(graph_layer))) { error(_("unable to adjust shared permissions for '%s'"), - ctx->graph_name); + get_tempfile_path(graph_layer)); return -1; } - f = hashfd(fd, ctx->graph_name); + f = hashfd(the_repository->hash_algo, + get_tempfile_fd(graph_layer), get_tempfile_path(graph_layer)); } else { hold_lock_file_for_update_mode(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR, 0444); - fd = get_lock_file_fd(&lk); - f = hashfd(fd, get_lock_file_path(&lk)); + f = hashfd(the_repository->hash_algo, + get_lock_file_fd(&lk), get_lock_file_path(&lk)); } cf = init_chunkfile(f); add_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, GRAPH_FANOUT_SIZE, write_graph_chunk_fanout); - add_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, hashsz * ctx->commits.nr, + add_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, st_mult(hashsz, ctx->commits.nr), write_graph_chunk_oids); - add_chunk(cf, GRAPH_CHUNKID_DATA, (hashsz + 16) * ctx->commits.nr, + add_chunk(cf, GRAPH_CHUNKID_DATA, st_mult(hashsz + 16, ctx->commits.nr), write_graph_chunk_data); if (ctx->write_generation_data) add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA, - sizeof(uint32_t) * ctx->commits.nr, + st_mult(sizeof(uint32_t), ctx->commits.nr), write_graph_chunk_generation_data); if (ctx->num_generation_data_overflows) add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW, - sizeof(timestamp_t) * ctx->num_generation_data_overflows, + st_mult(sizeof(timestamp_t), ctx->num_generation_data_overflows), write_graph_chunk_generation_data_overflow); if (ctx->num_extra_edges) add_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, - 4 * ctx->num_extra_edges, + st_mult(4, ctx->num_extra_edges), write_graph_chunk_extra_edges); if (ctx->changed_paths) { add_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES, - sizeof(uint32_t) * ctx->commits.nr, + st_mult(sizeof(uint32_t), ctx->commits.nr), write_graph_chunk_bloom_indexes); add_chunk(cf, GRAPH_CHUNKID_BLOOMDATA, - sizeof(uint32_t) * 3 - + ctx->total_bloom_filter_data_size, + st_add(sizeof(uint32_t) * 3, + ctx->total_bloom_filter_data_size), write_graph_chunk_bloom_data); } if (ctx->num_commit_graphs_after > 1) add_chunk(cf, GRAPH_CHUNKID_BASE, - hashsz * (ctx->num_commit_graphs_after - 1), + st_mult(hashsz, ctx->num_commit_graphs_after - 1), write_graph_chunk_base); hashwrite_be32(f, GRAPH_SIGNATURE); @@ -1927,8 +2150,9 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) get_num_chunks(cf)), get_num_chunks(cf)); ctx->progress = start_delayed_progress( + the_repository, progress_title.buf, - get_num_chunks(cf) * ctx->commits.nr); + st_mult(get_num_chunks(cf), ctx->commits.nr)); } write_chunkfile(cf, ctx); @@ -1938,7 +2162,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) if (ctx->split && ctx->base_graph_name && ctx->num_commit_graphs_after > 1) { char *new_base_hash = xstrdup(oid_to_hex(&ctx->new_base_graph->oid)); - char *new_base_name = get_split_graph_filename(ctx->new_base_graph->odb, new_base_hash); + char *new_base_name = get_split_graph_filename(ctx->new_base_graph->odb_source, new_base_hash); free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2]); free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2]); @@ -1956,8 +2180,6 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) char *final_graph_name; int result; - close(fd); - if (!chainf) { error(_("unable to open commit-graph chain file")); return -1; @@ -1980,17 +2202,19 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) } } } else { - char *graph_name = get_commit_graph_filename(ctx->odb); + char *graph_name = get_commit_graph_filename(ctx->odb_source); unlink(graph_name); free(graph_name); } + free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]); ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1] = xstrdup(hash_to_hex(file_hash)); - final_graph_name = get_split_graph_filename(ctx->odb, + final_graph_name = get_split_graph_filename(ctx->odb_source, ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]); + free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1]); ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1] = final_graph_name; - result = rename(ctx->graph_name, final_graph_name); + result = rename_tempfile(&graph_layer, final_graph_name); for (i = 0; i < ctx->num_commit_graphs_after; i++) fprintf(get_lock_file_fp(&lk), "%s\n", ctx->commit_graph_hash_after[i]); @@ -2034,11 +2258,16 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx) if (flags != COMMIT_GRAPH_SPLIT_MERGE_PROHIBITED && flags != COMMIT_GRAPH_SPLIT_REPLACE) { - while (g && (g->num_commits <= size_mult * num_commits || + while (g && (g->num_commits <= st_mult(size_mult, num_commits) || (max_commits && num_commits > max_commits))) { - if (g->odb != ctx->odb) + if (g->odb_source != ctx->odb_source) break; + if (unsigned_add_overflows(num_commits, g->num_commits)) + die(_("cannot merge graphs with %"PRIuMAX", " + "%"PRIuMAX" commits"), + (uintmax_t)num_commits, + (uintmax_t)g->num_commits); num_commits += g->num_commits; g = g->base_graph; @@ -2053,10 +2282,10 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx) "should be 1 with --split=replace"); if (ctx->num_commit_graphs_after == 2) { - char *old_graph_name = get_commit_graph_filename(g->odb); + char *old_graph_name = get_commit_graph_filename(g->odb_source); if (!strcmp(g->filename, old_graph_name) && - g->odb != ctx->odb) { + g->odb_source != ctx->odb_source) { ctx->num_commit_graphs_after = 1; ctx->new_base_graph = NULL; } @@ -2096,6 +2325,11 @@ static void merge_commit_graph(struct write_commit_graph_context *ctx, uint32_t i; uint32_t offset = g->num_commits_in_base; + if (unsigned_add_overflows(ctx->commits.nr, g->num_commits)) + die(_("cannot merge graph %s, too many commits: %"PRIuMAX), + oid_to_hex(&g->oid), + (uintmax_t)st_add(ctx->commits.nr, g->num_commits)); + ALLOC_GROW(ctx->commits.list, ctx->commits.nr + g->num_commits, ctx->commits.alloc); for (i = 0; i < g->num_commits; i++) { @@ -2129,6 +2363,7 @@ static void sort_and_scan_merged_commits(struct write_commit_graph_context *ctx) if (ctx->report_progress) ctx->progress = start_delayed_progress( + the_repository, _("Scanning merged commits"), ctx->commits.nr); @@ -2173,7 +2408,8 @@ static void merge_commit_graphs(struct write_commit_graph_context *ctx) current_graph_number--; if (ctx->report_progress) - ctx->progress = start_delayed_progress(_("Merging commit-graph"), 0); + ctx->progress = start_delayed_progress(the_repository, + _("Merging commit-graph"), 0); merge_commit_graph(ctx, g); stop_progress(&ctx->progress); @@ -2221,13 +2457,13 @@ static void expire_commit_graphs(struct write_commit_graph_context *ctx) if (ctx->opts && ctx->opts->expire_time) expire_time = ctx->opts->expire_time; if (!ctx->split) { - char *chain_file_name = get_commit_graph_chain_filename(ctx->odb); + char *chain_file_name = get_commit_graph_chain_filename(ctx->odb_source); unlink(chain_file_name); free(chain_file_name); ctx->num_commit_graphs_after = 0; } - strbuf_addstr(&path, ctx->odb->path); + strbuf_addstr(&path, ctx->odb_source->path); strbuf_addstr(&path, "/info/commit-graphs"); dir = opendir(path.buf); @@ -2269,14 +2505,24 @@ out: strbuf_release(&path); } -int write_commit_graph(struct object_directory *odb, +int write_commit_graph(struct odb_source *source, const struct string_list *const pack_indexes, struct oidset *commits, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts) { struct repository *r = the_repository; - struct write_commit_graph_context *ctx; + struct write_commit_graph_context ctx = { + .r = r, + .odb_source = source, + .append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0, + .report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0, + .split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0, + .opts = opts, + .total_bloom_filter_data_size = 0, + .write_generation_data = (get_configured_generation_version(r) == 2), + .num_generation_data_overflows = 0, + }; uint32_t i; int res = 0; int replace = 0; @@ -2290,32 +2536,29 @@ int write_commit_graph(struct object_directory *odb, } if (!commit_graph_compatible(r)) return 0; + if (r->settings.commit_graph_changed_paths_version < -1 + || r->settings.commit_graph_changed_paths_version > 2) { + warning(_("attempting to write a commit-graph, but " + "'commitGraph.changedPathsVersion' (%d) is not supported"), + r->settings.commit_graph_changed_paths_version); + return 0; + } - CALLOC_ARRAY(ctx, 1); - ctx->r = r; - ctx->odb = odb; - ctx->append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0; - ctx->report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0; - ctx->split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0; - ctx->opts = opts; - ctx->total_bloom_filter_data_size = 0; - ctx->write_generation_data = (get_configured_generation_version(r) == 2); - ctx->num_generation_data_overflows = 0; - + bloom_settings.hash_version = r->settings.commit_graph_changed_paths_version; bloom_settings.bits_per_entry = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY", bloom_settings.bits_per_entry); bloom_settings.num_hashes = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_NUM_HASHES", bloom_settings.num_hashes); bloom_settings.max_changed_paths = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS", bloom_settings.max_changed_paths); - ctx->bloom_settings = &bloom_settings; + ctx.bloom_settings = &bloom_settings; init_topo_level_slab(&topo_levels); - ctx->topo_levels = &topo_levels; + ctx.topo_levels = &topo_levels; - prepare_commit_graph(ctx->r); - if (ctx->r->objects->commit_graph) { - struct commit_graph *g = ctx->r->objects->commit_graph; + prepare_commit_graph(ctx.r); + if (ctx.r->objects->commit_graph) { + struct commit_graph *g = ctx.r->objects->commit_graph; while (g) { g->topo_levels = &topo_levels; @@ -2324,127 +2567,144 @@ int write_commit_graph(struct object_directory *odb, } if (flags & COMMIT_GRAPH_WRITE_BLOOM_FILTERS) - ctx->changed_paths = 1; + ctx.changed_paths = 1; if (!(flags & COMMIT_GRAPH_NO_WRITE_BLOOM_FILTERS)) { struct commit_graph *g; - g = ctx->r->objects->commit_graph; + g = ctx.r->objects->commit_graph; /* We have changed-paths already. Keep them in the next graph */ - if (g && g->chunk_bloom_data) { - ctx->changed_paths = 1; - ctx->bloom_settings = g->bloom_filter_settings; + if (g && g->bloom_filter_settings) { + ctx.changed_paths = 1; + + /* don't propagate the hash_version unless unspecified */ + if (bloom_settings.hash_version == -1) + bloom_settings.hash_version = g->bloom_filter_settings->hash_version; + bloom_settings.bits_per_entry = g->bloom_filter_settings->bits_per_entry; + bloom_settings.num_hashes = g->bloom_filter_settings->num_hashes; + bloom_settings.max_changed_paths = g->bloom_filter_settings->max_changed_paths; } } - if (ctx->split) { - struct commit_graph *g = ctx->r->objects->commit_graph; + bloom_settings.hash_version = bloom_settings.hash_version == 2 ? 2 : 1; + + if (ctx.split) { + struct commit_graph *g = ctx.r->objects->commit_graph; while (g) { - ctx->num_commit_graphs_before++; + ctx.num_commit_graphs_before++; g = g->base_graph; } - if (ctx->num_commit_graphs_before) { - ALLOC_ARRAY(ctx->commit_graph_filenames_before, ctx->num_commit_graphs_before); - i = ctx->num_commit_graphs_before; - g = ctx->r->objects->commit_graph; + if (ctx.num_commit_graphs_before) { + ALLOC_ARRAY(ctx.commit_graph_filenames_before, ctx.num_commit_graphs_before); + i = ctx.num_commit_graphs_before; + g = ctx.r->objects->commit_graph; while (g) { - ctx->commit_graph_filenames_before[--i] = xstrdup(g->filename); + ctx.commit_graph_filenames_before[--i] = xstrdup(g->filename); g = g->base_graph; } } - if (ctx->opts) - replace = ctx->opts->split_flags & COMMIT_GRAPH_SPLIT_REPLACE; + if (ctx.opts) + replace = ctx.opts->split_flags & COMMIT_GRAPH_SPLIT_REPLACE; } - ctx->approx_nr_objects = approximate_object_count(); + ctx.approx_nr_objects = repo_approximate_object_count(the_repository); - if (ctx->append && ctx->r->objects->commit_graph) { - struct commit_graph *g = ctx->r->objects->commit_graph; + if (ctx.append && ctx.r->objects->commit_graph) { + struct commit_graph *g = ctx.r->objects->commit_graph; for (i = 0; i < g->num_commits; i++) { struct object_id oid; - oidread(&oid, g->chunk_oid_lookup + g->hash_len * i); - oid_array_append(&ctx->oids, &oid); + oidread(&oid, g->chunk_oid_lookup + st_mult(g->hash_len, i), + the_repository->hash_algo); + oid_array_append(&ctx.oids, &oid); } } if (pack_indexes) { - ctx->order_by_pack = 1; - if ((res = fill_oids_from_packs(ctx, pack_indexes))) + ctx.order_by_pack = 1; + if ((res = fill_oids_from_packs(&ctx, pack_indexes))) goto cleanup; } if (commits) { - if ((res = fill_oids_from_commits(ctx, commits))) + if ((res = fill_oids_from_commits(&ctx, commits))) goto cleanup; } if (!pack_indexes && !commits) { - ctx->order_by_pack = 1; - fill_oids_from_all_packs(ctx); + ctx.order_by_pack = 1; + fill_oids_from_all_packs(&ctx); } - close_reachable(ctx); + close_reachable(&ctx); - copy_oids_to_commits(ctx); + copy_oids_to_commits(&ctx); - if (ctx->commits.nr >= GRAPH_EDGE_LAST_MASK) { + if (ctx.commits.nr >= GRAPH_EDGE_LAST_MASK) { error(_("too many commits to write graph")); res = -1; goto cleanup; } - if (!ctx->commits.nr && !replace) + if (!ctx.commits.nr && !replace) goto cleanup; - if (ctx->split) { - split_graph_merge_strategy(ctx); + if (ctx.split) { + split_graph_merge_strategy(&ctx); if (!replace) - merge_commit_graphs(ctx); + merge_commit_graphs(&ctx); } else - ctx->num_commit_graphs_after = 1; + ctx.num_commit_graphs_after = 1; - ctx->trust_generation_numbers = validate_mixed_generation_chain(ctx->r->objects->commit_graph); + ctx.trust_generation_numbers = validate_mixed_generation_chain(ctx.r->objects->commit_graph); - compute_topological_levels(ctx); - if (ctx->write_generation_data) - compute_generation_numbers(ctx); + compute_topological_levels(&ctx); + if (ctx.write_generation_data) + compute_generation_numbers(&ctx); - if (ctx->changed_paths) - compute_bloom_filters(ctx); + if (ctx.changed_paths) + compute_bloom_filters(&ctx); - res = write_commit_graph_file(ctx); + res = write_commit_graph_file(&ctx); - if (ctx->split) - mark_commit_graphs(ctx); + if (ctx.changed_paths) + deinit_bloom_filters(); - expire_commit_graphs(ctx); + if (ctx.split) + mark_commit_graphs(&ctx); + + expire_commit_graphs(&ctx); cleanup: - free(ctx->graph_name); - free(ctx->commits.list); - oid_array_clear(&ctx->oids); + free(ctx.graph_name); + free(ctx.base_graph_name); + free(ctx.commits.list); + oid_array_clear(&ctx.oids); clear_topo_level_slab(&topo_levels); - if (ctx->commit_graph_filenames_after) { - for (i = 0; i < ctx->num_commit_graphs_after; i++) { - free(ctx->commit_graph_filenames_after[i]); - free(ctx->commit_graph_hash_after[i]); + if (ctx.r->objects->commit_graph) { + struct commit_graph *g = ctx.r->objects->commit_graph; + + while (g) { + g->topo_levels = NULL; + g = g->base_graph; } + } - for (i = 0; i < ctx->num_commit_graphs_before; i++) - free(ctx->commit_graph_filenames_before[i]); + for (i = 0; i < ctx.num_commit_graphs_before; i++) + free(ctx.commit_graph_filenames_before[i]); + free(ctx.commit_graph_filenames_before); - free(ctx->commit_graph_filenames_after); - free(ctx->commit_graph_filenames_before); - free(ctx->commit_graph_hash_after); + for (i = 0; i < ctx.num_commit_graphs_after; i++) { + free(ctx.commit_graph_filenames_after[i]); + free(ctx.commit_graph_hash_after[i]); } - - free(ctx); + free(ctx.commit_graph_filenames_after); + free(ctx.commit_graph_hash_after); return res; } @@ -2464,30 +2724,21 @@ static void graph_report(const char *fmt, ...) va_end(ap); } -#define GENERATION_ZERO_EXISTS 1 -#define GENERATION_NUMBER_EXISTS 2 - static int commit_graph_checksum_valid(struct commit_graph *g) { - return hashfile_checksum_valid(g->data, g->data_len); + return hashfile_checksum_valid(the_repository->hash_algo, + g->data, g->data_len); } -int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags) +static int verify_one_commit_graph(struct repository *r, + struct commit_graph *g, + struct progress *progress, + uint64_t *seen) { uint32_t i, cur_fanout_pos = 0; struct object_id prev_oid, cur_oid; - int generation_zero = 0; - struct progress *progress = NULL; - int local_error = 0; - - if (!g) { - graph_report("no commit-graph file loaded"); - return 1; - } - - verify_commit_graph_error = verify_commit_graph_lite(g); - if (verify_commit_graph_error) - return verify_commit_graph_error; + struct commit *seen_gen_zero = NULL; + struct commit *seen_gen_non_zero = NULL; if (!commit_graph_checksum_valid(g)) { graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt")); @@ -2497,7 +2748,8 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags) for (i = 0; i < g->num_commits; i++) { struct commit *graph_commit; - oidread(&cur_oid, g->chunk_oid_lookup + g->hash_len * i); + oidread(&cur_oid, g->chunk_oid_lookup + st_mult(g->hash_len, i), + the_repository->hash_algo); if (i && oidcmp(&prev_oid, &cur_oid) >= 0) graph_report(_("commit-graph has incorrect OID order: %s then %s"), @@ -2534,22 +2786,19 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags) if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH) return verify_commit_graph_error; - if (flags & COMMIT_GRAPH_WRITE_PROGRESS) - progress = start_progress(_("Verifying commits in commit graph"), - g->num_commits); - for (i = 0; i < g->num_commits; i++) { struct commit *graph_commit, *odb_commit; struct commit_list *graph_parents, *odb_parents; timestamp_t max_generation = 0; timestamp_t generation; - display_progress(progress, i + 1); - oidread(&cur_oid, g->chunk_oid_lookup + g->hash_len * i); + display_progress(progress, ++(*seen)); + oidread(&cur_oid, g->chunk_oid_lookup + st_mult(g->hash_len, i), + the_repository->hash_algo); graph_commit = lookup_commit(r, &cur_oid); odb_commit = (struct commit *)create_object(r, &cur_oid, alloc_commit_node(r)); - if (parse_commit_internal(odb_commit, 0, 0)) { + if (repo_parse_commit_internal(r, odb_commit, 0, 0)) { graph_report(_("failed to parse commit %s from object database for commit-graph"), oid_to_hex(&cur_oid)); continue; @@ -2581,7 +2830,7 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags) oid_to_hex(&graph_parents->item->object.oid), oid_to_hex(&odb_parents->item->object.oid)); - generation = commit_graph_generation(graph_parents->item); + generation = commit_graph_generation_from_graph(graph_parents->item); if (generation > max_generation) max_generation = generation; @@ -2593,16 +2842,12 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags) graph_report(_("commit-graph parent list for commit %s terminates early"), oid_to_hex(&cur_oid)); - if (!commit_graph_generation(graph_commit)) { - if (generation_zero == GENERATION_NUMBER_EXISTS) - graph_report(_("commit-graph has generation number zero for commit %s, but non-zero elsewhere"), - oid_to_hex(&cur_oid)); - generation_zero = GENERATION_ZERO_EXISTS; - } else if (generation_zero == GENERATION_ZERO_EXISTS) - graph_report(_("commit-graph has non-zero generation number for commit %s, but zero elsewhere"), - oid_to_hex(&cur_oid)); + if (commit_graph_generation_from_graph(graph_commit)) + seen_gen_non_zero = graph_commit; + else + seen_gen_zero = graph_commit; - if (generation_zero == GENERATION_ZERO_EXISTS) + if (seen_gen_zero) continue; /* @@ -2627,27 +2872,61 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags) graph_commit->date, odb_commit->date); } - stop_progress(&progress); - local_error = verify_commit_graph_error; + if (seen_gen_zero && seen_gen_non_zero) + graph_report(_("commit-graph has both zero and non-zero " + "generations (e.g., commits '%s' and '%s')"), + oid_to_hex(&seen_gen_zero->object.oid), + oid_to_hex(&seen_gen_non_zero->object.oid)); + + return verify_commit_graph_error; +} + +int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags) +{ + struct progress *progress = NULL; + int local_error = 0; + uint64_t seen = 0; + + if (!g) { + graph_report("no commit-graph file loaded"); + return 1; + } + + if (flags & COMMIT_GRAPH_WRITE_PROGRESS) { + uint64_t total = g->num_commits; + if (!(flags & COMMIT_GRAPH_VERIFY_SHALLOW)) + total += g->num_commits_in_base; - if (!(flags & COMMIT_GRAPH_VERIFY_SHALLOW) && g->base_graph) - local_error |= verify_commit_graph(r, g->base_graph, flags); + progress = start_progress(the_repository, + _("Verifying commits in commit graph"), + total); + } + + for (; g; g = g->base_graph) { + local_error |= verify_one_commit_graph(r, g, progress, &seen); + if (flags & COMMIT_GRAPH_VERIFY_SHALLOW) + break; + } + + stop_progress(&progress); return local_error; } void free_commit_graph(struct commit_graph *g) { - if (!g) - return; - if (g->data) { - munmap((void *)g->data, g->data_len); - g->data = NULL; + while (g) { + struct commit_graph *next = g->base_graph; + + if (g->data) + munmap((void *)g->data, g->data_len); + free(g->filename); + free(g->bloom_filter_settings); + free(g); + + g = next; } - free(g->filename); - free(g->bloom_filter_settings); - free(g); } void disable_commit_graph(struct repository *r) |