diff options
Diffstat (limited to 'packfile.c')
| -rw-r--r-- | packfile.c | 479 |
1 files changed, 276 insertions, 203 deletions
diff --git a/packfile.c b/packfile.c index 70c7208f02..9cc11b6dc5 100644 --- a/packfile.c +++ b/packfile.c @@ -19,7 +19,7 @@ #include "tree-walk.h" #include "tree.h" #include "object-file.h" -#include "object-store.h" +#include "odb.h" #include "midx.h" #include "commit-graph.h" #include "pack-revindex.h" @@ -47,6 +47,89 @@ static size_t pack_mapped; #define SZ_FMT PRIuMAX static inline uintmax_t sz_fmt(size_t s) { return s; } +void packfile_list_clear(struct packfile_list *list) +{ + struct packfile_list_entry *e, *next; + + for (e = list->head; e; e = next) { + next = e->next; + free(e); + } + + list->head = list->tail = NULL; +} + +static struct packfile_list_entry *packfile_list_remove_internal(struct packfile_list *list, + struct packed_git *pack) +{ + struct packfile_list_entry *e, *prev; + + for (e = list->head, prev = NULL; e; prev = e, e = e->next) { + if (e->pack != pack) + continue; + + if (prev) + prev->next = e->next; + if (list->head == e) + list->head = e->next; + if (list->tail == e) + list->tail = prev; + + return e; + } + + return NULL; +} + +void packfile_list_remove(struct packfile_list *list, struct packed_git *pack) +{ + free(packfile_list_remove_internal(list, pack)); +} + +void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack) +{ + struct packfile_list_entry *entry; + + entry = packfile_list_remove_internal(list, pack); + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->pack = pack; + } + entry->next = list->head; + + list->head = entry; + if (!list->tail) + list->tail = entry; +} + +void packfile_list_append(struct packfile_list *list, struct packed_git *pack) +{ + struct packfile_list_entry *entry; + + entry = packfile_list_remove_internal(list, pack); + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->pack = pack; + } + entry->next = NULL; + + if (list->tail) { + list->tail->next = entry; + list->tail = entry; + } else { + list->head = list->tail = entry; + } +} + +struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs, + const struct object_id *oid) +{ + for (; packs; packs = packs->next) + if (find_pack_entry_one(oid, packs->pack)) + return packs->pack; + return NULL; +} + void pack_report(struct repository *repo) { fprintf(stderr, @@ -273,13 +356,14 @@ static void scan_windows(struct packed_git *p, static int unuse_one_window(struct packed_git *current) { - struct packed_git *p, *lru_p = NULL; + struct packfile_list_entry *e; + struct packed_git *lru_p = NULL; struct pack_window *lru_w = NULL, *lru_l = NULL; if (current) scan_windows(current, &lru_p, &lru_w, &lru_l); - for (p = current->repo->objects->packed_git; p; p = p->next) - scan_windows(p, &lru_p, &lru_w, &lru_l); + for (e = current->repo->objects->packfiles->packs.head; e; e = e->next) + scan_windows(e->pack, &lru_p, &lru_w, &lru_l); if (lru_p) { munmap(lru_w->base, lru_w->len); pack_mapped -= lru_w->len; @@ -359,19 +443,16 @@ void close_pack(struct packed_git *p) oidset_clear(&p->bad_objects); } -void close_object_store(struct raw_object_store *o) +void close_object_store(struct object_database *o) { - struct packed_git *p; + struct odb_source *source; - for (p = o->packed_git; p; p = p->next) - if (p->do_not_close) - BUG("want to close pack marked 'do-not-close'"); - else - close_pack(p); + packfile_store_close(o->packfiles); - if (o->multi_pack_index) { - close_midx(o->multi_pack_index); - o->multi_pack_index = NULL; + for (source = o->sources; source; source = source->next) { + if (source->midx) + close_midx(source->midx); + source->midx = NULL; } close_commit_graph(o); @@ -462,14 +543,15 @@ static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struc static int close_one_pack(struct repository *r) { - struct packed_git *p, *lru_p = NULL; + struct packfile_list_entry *e; + struct packed_git *lru_p = NULL; struct pack_window *mru_w = NULL; int accept_windows_inuse = 1; - for (p = r->objects->packed_git; p; p = p->next) { - if (p->pack_fd == -1) + for (e = r->objects->packfiles->packs.head; e; e = e->next) { + if (e->pack->pack_fd == -1) continue; - find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse); + find_lru_pack(e->pack, &lru_p, &mru_w, &accept_windows_inuse); } if (lru_p) @@ -782,16 +864,56 @@ struct packed_git *add_packed_git(struct repository *r, const char *path, return p; } -void install_packed_git(struct repository *r, struct packed_git *pack) +void packfile_store_add_pack(struct packfile_store *store, + struct packed_git *pack) { if (pack->pack_fd != -1) pack_open_fds++; - pack->next = r->objects->packed_git; - r->objects->packed_git = pack; + packfile_list_append(&store->packs, pack); + strmap_put(&store->packs_by_path, pack->pack_name, pack); +} + +struct packed_git *packfile_store_load_pack(struct packfile_store *store, + const char *idx_path, int local) +{ + struct strbuf key = STRBUF_INIT; + struct packed_git *p; + + /* + * We're being called with the path to the index file, but `pack_map` + * holds the path to the packfile itself. + */ + strbuf_addstr(&key, idx_path); + strbuf_strip_suffix(&key, ".idx"); + strbuf_addstr(&key, ".pack"); + + p = strmap_get(&store->packs_by_path, key.buf); + if (!p) { + p = add_packed_git(store->odb->repo, idx_path, + strlen(idx_path), local); + if (p) + packfile_store_add_pack(store, p); + } + + strbuf_release(&key); + return p; +} - hashmap_entry_init(&pack->packmap_ent, strhash(pack->pack_name)); - hashmap_add(&r->objects->pack_map, &pack->packmap_ent); +int packfile_store_freshen_object(struct packfile_store *store, + const struct object_id *oid) +{ + struct pack_entry e; + if (!find_pack_entry(store->odb->repo, oid, &e)) + return 0; + if (e.p->is_cruft) + return 0; + if (e.p->freshened) + return 1; + if (utime(e.p->pack_name, NULL)) + return 0; + e.p->freshened = 1; + return 1; } void (*report_garbage)(unsigned seen_bits, const char *path); @@ -893,23 +1015,14 @@ static void prepare_pack(const char *full_name, size_t full_name_len, const char *file_name, void *_data) { struct prepare_pack_data *data = (struct prepare_pack_data *)_data; - struct packed_git *p; size_t base_len = full_name_len; if (strip_suffix_mem(full_name, &base_len, ".idx") && !(data->m && midx_contains_pack(data->m, file_name))) { - struct hashmap_entry hent; - char *pack_name = xstrfmt("%.*s.pack", (int)base_len, full_name); - unsigned int hash = strhash(pack_name); - hashmap_entry_init(&hent, hash); - - /* Don't reopen a pack we already have. */ - if (!hashmap_get(&data->r->objects->pack_map, &hent, pack_name)) { - p = add_packed_git(data->r, full_name, full_name_len, data->local); - if (p) - install_packed_git(data->r, p); - } - free(pack_name); + char *trimmed_path = xstrndup(full_name, full_name_len); + packfile_store_load_pack(data->r->objects->packfiles, + trimmed_path, data->local); + free(trimmed_path); } if (!report_garbage) @@ -933,60 +1046,26 @@ static void prepare_pack(const char *full_name, size_t full_name_len, report_garbage(PACKDIR_FILE_GARBAGE, full_name); } -static void prepare_packed_git_one(struct repository *r, char *objdir, int local) +static void prepare_packed_git_one(struct odb_source *source) { - struct prepare_pack_data data; struct string_list garbage = STRING_LIST_INIT_DUP; + struct prepare_pack_data data = { + .m = source->midx, + .r = source->odb->repo, + .garbage = &garbage, + .local = source->local, + }; - data.m = r->objects->multi_pack_index; - - /* look for the multi-pack-index for this object directory */ - while (data.m && strcmp(data.m->object_dir, objdir)) - data.m = data.m->next; - - data.r = r; - data.garbage = &garbage; - data.local = local; - - for_each_file_in_pack_dir(objdir, prepare_pack, &data); + for_each_file_in_pack_dir(source->path, prepare_pack, &data); report_pack_garbage(data.garbage); string_list_clear(data.garbage, 0); } -static void prepare_packed_git(struct repository *r); -/* - * Give a fast, rough count of the number of objects in the repository. This - * ignores loose objects completely. If you have a lot of them, then either - * you should repack because your performance will be awful, or they are - * all unreachable objects about to be pruned, in which case they're not really - * interesting as a measure of repo size in the first place. - */ -unsigned long repo_approximate_object_count(struct repository *r) -{ - if (!r->objects->approximate_object_count_valid) { - unsigned long count; - struct multi_pack_index *m; - struct packed_git *p; - - prepare_packed_git(r); - count = 0; - for (m = get_multi_pack_index(r); m; m = m->next) - count += m->num_objects; - for (p = r->objects->packed_git; p; p = p->next) { - if (open_pack_index(p)) - continue; - count += p->num_objects; - } - r->objects->approximate_object_count = count; - r->objects->approximate_object_count_valid = 1; - } - return r->objects->approximate_object_count; -} - -DEFINE_LIST_SORT(static, sort_packs, struct packed_git, next); +DEFINE_LIST_SORT(static, sort_packs, struct packfile_list_entry, next); -static int sort_pack(const struct packed_git *a, const struct packed_git *b) +static int sort_pack(const struct packfile_list_entry *a, + const struct packfile_list_entry *b) { int st; @@ -996,7 +1075,7 @@ static int sort_pack(const struct packed_git *a, const struct packed_git *b) * remote ones could be on a network mounted filesystem. * Favor local ones for these reasons. */ - st = a->pack_local - b->pack_local; + st = a->pack->pack_local - b->pack->pack_local; if (st) return -st; @@ -1005,112 +1084,86 @@ static int sort_pack(const struct packed_git *a, const struct packed_git *b) * and more recent objects tend to get accessed more * often. */ - if (a->mtime < b->mtime) + if (a->pack->mtime < b->pack->mtime) return 1; - else if (a->mtime == b->mtime) + else if (a->pack->mtime == b->pack->mtime) return 0; return -1; } -static void rearrange_packed_git(struct repository *r) -{ - sort_packs(&r->objects->packed_git, sort_pack); -} - -static void prepare_packed_git_mru(struct repository *r) -{ - struct packed_git *p; - - INIT_LIST_HEAD(&r->objects->packed_git_mru); - - for (p = r->objects->packed_git; p; p = p->next) - list_add_tail(&p->mru, &r->objects->packed_git_mru); -} - -static void prepare_packed_git(struct repository *r) +void packfile_store_prepare(struct packfile_store *store) { - struct object_directory *odb; + struct odb_source *source; - if (r->objects->packed_git_initialized) + if (store->initialized) return; - prepare_alt_odb(r); - for (odb = r->objects->odb; odb; odb = odb->next) { - int local = (odb == r->objects->odb); - prepare_multi_pack_index_one(r, odb->path, local); - prepare_packed_git_one(r, odb->path, local); + odb_prepare_alternates(store->odb); + for (source = store->odb->sources; source; source = source->next) { + prepare_multi_pack_index_one(source); + prepare_packed_git_one(source); } - rearrange_packed_git(r); - - prepare_packed_git_mru(r); - r->objects->packed_git_initialized = 1; -} - -void reprepare_packed_git(struct repository *r) -{ - struct object_directory *odb; - - obj_read_lock(); - /* - * Reprepare alt odbs, in case the alternates file was modified - * during the course of this process. This only _adds_ odbs to - * the linked list, so existing odbs will continue to exist for - * the lifetime of the process. - */ - r->objects->loaded_alternates = 0; - prepare_alt_odb(r); - - for (odb = r->objects->odb; odb; odb = odb->next) - odb_clear_loose_cache(odb); - - r->objects->approximate_object_count_valid = 0; - r->objects->packed_git_initialized = 0; - prepare_packed_git(r); - obj_read_unlock(); -} + sort_packs(&store->packs.head, sort_pack); + for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) + if (!e->next) + store->packs.tail = e; -struct packed_git *get_packed_git(struct repository *r) -{ - prepare_packed_git(r); - return r->objects->packed_git; + store->initialized = true; } -struct multi_pack_index *get_multi_pack_index(struct repository *r) +void packfile_store_reprepare(struct packfile_store *store) { - prepare_packed_git(r); - return r->objects->multi_pack_index; + store->initialized = false; + packfile_store_prepare(store); } -struct multi_pack_index *get_local_multi_pack_index(struct repository *r) +struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *store) { - struct multi_pack_index *m = get_multi_pack_index(r); + packfile_store_prepare(store); - /* no need to iterate; we always put the local one first (if any) */ - if (m && m->local) - return m; + for (struct odb_source *source = store->odb->sources; source; source = source->next) { + struct multi_pack_index *m = source->midx; + if (!m) + continue; + for (uint32_t i = 0; i < m->num_packs + m->num_packs_in_base; i++) + prepare_midx_pack(m, i); + } - return NULL; + return store->packs.head; } -struct packed_git *get_all_packs(struct repository *r) +/* + * Give a fast, rough count of the number of objects in the repository. This + * ignores loose objects completely. If you have a lot of them, then either + * you should repack because your performance will be awful, or they are + * all unreachable objects about to be pruned, in which case they're not really + * interesting as a measure of repo size in the first place. + */ +unsigned long repo_approximate_object_count(struct repository *r) { - struct multi_pack_index *m; + if (!r->objects->approximate_object_count_valid) { + struct odb_source *source; + unsigned long count = 0; + struct packed_git *p; - prepare_packed_git(r); - for (m = r->objects->multi_pack_index; m; m = m->next) { - uint32_t i; - for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) - prepare_midx_pack(r, m, i); - } + odb_prepare_alternates(r->objects); - return r->objects->packed_git; -} + for (source = r->objects->sources; source; source = source->next) { + struct multi_pack_index *m = get_multi_pack_index(source); + if (m) + count += m->num_objects + m->num_objects_in_base; + } -struct list_head *get_packed_git_mru(struct repository *r) -{ - prepare_packed_git(r); - return &r->objects->packed_git_mru; + repo_for_each_pack(r, p) { + if (p->multi_pack_index || open_pack_index(p)) + continue; + count += p->num_objects; + } + r->objects->approximate_object_count = count; + r->objects->approximate_object_count_valid = 1; + } + return r->objects->approximate_object_count; } unsigned long unpack_object_header_buffer(const unsigned char *buf, @@ -1165,7 +1218,7 @@ unsigned long get_size_from_delta(struct packed_git *p, * * Other worrying sections could be the call to close_pack_fd(), * which can close packs even with in-use windows, and to - * reprepare_packed_git(). Regarding the former, mmap doc says: + * odb_reprepare(). Regarding the former, mmap doc says: * "closing the file descriptor does not unmap the region". And * for the latter, it won't re-open already available packs. */ @@ -1227,11 +1280,11 @@ void mark_bad_packed_object(struct packed_git *p, const struct object_id *oid) const struct packed_git *has_packed_and_bad(struct repository *r, const struct object_id *oid) { - struct packed_git *p; + struct packfile_list_entry *e; - for (p = r->objects->packed_git; p; p = p->next) - if (oidset_contains(&p->bad_objects, oid)) - return p; + for (e = r->objects->packfiles->packs.head; e; e = e->next) + if (oidset_contains(&e->pack->bad_objects, oid)) + return e->pack; return NULL; } @@ -1321,7 +1374,7 @@ static int retry_bad_packed_offset(struct repository *r, return OBJ_BAD; nth_packed_object_id(&oid, p, pack_pos_to_index(p, pos)); mark_bad_packed_object(p, &oid); - type = oid_object_info(r, &oid, NULL); + type = odb_read_object_info(r->objects, &oid, NULL); if (type <= OBJ_NONE) return OBJ_BAD; return type; @@ -1849,7 +1902,8 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, oi.typep = &type; oi.sizep = &base_size; oi.contentp = &base; - if (oid_object_info_extended(r, &base_oid, &oi, 0) < 0) + if (odb_read_object_info_extended(r->objects, &base_oid, + &oi, 0) < 0) base = NULL; external_base = base; @@ -2038,19 +2092,6 @@ int is_pack_valid(struct packed_git *p) return !open_packed_git(p); } -struct packed_git *find_oid_pack(const struct object_id *oid, - struct packed_git *packs) -{ - struct packed_git *p; - - for (p = packs; p; p = p->next) { - if (find_pack_entry_one(oid, p)) - return p; - } - return NULL; - -} - static int fill_pack_entry(const struct object_id *oid, struct pack_entry *e, struct packed_git *p) @@ -2081,22 +2122,23 @@ static int fill_pack_entry(const struct object_id *oid, int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e) { - struct list_head *pos; - struct multi_pack_index *m; + struct packfile_list_entry *l; - prepare_packed_git(r); - if (!r->objects->packed_git && !r->objects->multi_pack_index) - return 0; + packfile_store_prepare(r->objects->packfiles); - for (m = r->objects->multi_pack_index; m; m = m->next) { - if (fill_midx_entry(r, oid, e, m)) + for (struct odb_source *source = r->objects->sources; source; source = source->next) + if (source->midx && fill_midx_entry(source->midx, oid, e)) return 1; - } - list_for_each(pos, &r->objects->packed_git_mru) { - struct packed_git *p = list_entry(pos, struct packed_git, mru); + if (!r->objects->packfiles->packs.head) + return 0; + + for (l = r->objects->packfiles->packs.head; l; l = l->next) { + struct packed_git *p = l->pack; + if (!p->multi_pack_index && fill_pack_entry(oid, e, p)) { - list_move(&p->mru, &r->objects->packed_git_mru); + if (!r->objects->packfiles->skip_mru_updates) + packfile_list_prepend(&r->objects->packfiles->packs, p); return 1; } } @@ -2106,19 +2148,19 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa static void maybe_invalidate_kept_pack_cache(struct repository *r, unsigned flags) { - if (!r->objects->kept_pack_cache.packs) + if (!r->objects->packfiles->kept_cache.packs) return; - if (r->objects->kept_pack_cache.flags == flags) + if (r->objects->packfiles->kept_cache.flags == flags) return; - FREE_AND_NULL(r->objects->kept_pack_cache.packs); - r->objects->kept_pack_cache.flags = 0; + FREE_AND_NULL(r->objects->packfiles->kept_cache.packs); + r->objects->packfiles->kept_cache.flags = 0; } struct packed_git **kept_pack_cache(struct repository *r, unsigned flags) { maybe_invalidate_kept_pack_cache(r, flags); - if (!r->objects->kept_pack_cache.packs) { + if (!r->objects->packfiles->kept_cache.packs) { struct packed_git **packs = NULL; size_t nr = 0, alloc = 0; struct packed_git *p; @@ -2131,7 +2173,7 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags) * covers, one kept and one not kept, but the midx returns only * the non-kept version. */ - for (p = get_all_packs(r); p; p = p->next) { + repo_for_each_pack(r, p) { if ((p->pack_keep && (flags & ON_DISK_KEEP_PACKS)) || (p->pack_keep_in_core && (flags & IN_CORE_KEEP_PACKS))) { ALLOC_GROW(packs, nr + 1, alloc); @@ -2141,11 +2183,11 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags) ALLOC_GROW(packs, nr + 1, alloc); packs[nr] = NULL; - r->objects->kept_pack_cache.packs = packs; - r->objects->kept_pack_cache.flags = flags; + r->objects->packfiles->kept_cache.packs = packs; + r->objects->packfiles->kept_cache.flags = flags; } - return r->objects->kept_pack_cache.packs; + return r->objects->packfiles->kept_cache.packs; } int find_kept_pack_entry(struct repository *r, @@ -2228,7 +2270,8 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, int r = 0; int pack_errors = 0; - for (p = get_all_packs(repo); p; p = p->next) { + repo->objects->packfiles->skip_mru_updates = true; + repo_for_each_pack(repo, p) { if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) continue; if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) && @@ -2248,6 +2291,8 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, if (r) break; } + repo->objects->packfiles->skip_mru_updates = false; + return r ? r : pack_errors; } @@ -2342,3 +2387,31 @@ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *l *len = hdr - out; return 0; } + +struct packfile_store *packfile_store_new(struct object_database *odb) +{ + struct packfile_store *store; + CALLOC_ARRAY(store, 1); + store->odb = odb; + strmap_init(&store->packs_by_path); + return store; +} + +void packfile_store_free(struct packfile_store *store) +{ + for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) + free(e->pack); + packfile_list_clear(&store->packs); + + strmap_clear(&store->packs_by_path, 0); + free(store); +} + +void packfile_store_close(struct packfile_store *store) +{ + for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) { + if (e->pack->do_not_close) + BUG("want to close pack marked 'do-not-close'"); + close_pack(e->pack); + } +} |
