summaryrefslogtreecommitdiff
path: root/packfile.c
diff options
context:
space:
mode:
Diffstat (limited to 'packfile.c')
-rw-r--r--packfile.c479
1 files changed, 276 insertions, 203 deletions
diff --git a/packfile.c b/packfile.c
index 70c7208f02..9cc11b6dc5 100644
--- a/packfile.c
+++ b/packfile.c
@@ -19,7 +19,7 @@
#include "tree-walk.h"
#include "tree.h"
#include "object-file.h"
-#include "object-store.h"
+#include "odb.h"
#include "midx.h"
#include "commit-graph.h"
#include "pack-revindex.h"
@@ -47,6 +47,89 @@ static size_t pack_mapped;
#define SZ_FMT PRIuMAX
static inline uintmax_t sz_fmt(size_t s) { return s; }
+void packfile_list_clear(struct packfile_list *list)
+{
+ struct packfile_list_entry *e, *next;
+
+ for (e = list->head; e; e = next) {
+ next = e->next;
+ free(e);
+ }
+
+ list->head = list->tail = NULL;
+}
+
+static struct packfile_list_entry *packfile_list_remove_internal(struct packfile_list *list,
+ struct packed_git *pack)
+{
+ struct packfile_list_entry *e, *prev;
+
+ for (e = list->head, prev = NULL; e; prev = e, e = e->next) {
+ if (e->pack != pack)
+ continue;
+
+ if (prev)
+ prev->next = e->next;
+ if (list->head == e)
+ list->head = e->next;
+ if (list->tail == e)
+ list->tail = prev;
+
+ return e;
+ }
+
+ return NULL;
+}
+
+void packfile_list_remove(struct packfile_list *list, struct packed_git *pack)
+{
+ free(packfile_list_remove_internal(list, pack));
+}
+
+void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack)
+{
+ struct packfile_list_entry *entry;
+
+ entry = packfile_list_remove_internal(list, pack);
+ if (!entry) {
+ entry = xmalloc(sizeof(*entry));
+ entry->pack = pack;
+ }
+ entry->next = list->head;
+
+ list->head = entry;
+ if (!list->tail)
+ list->tail = entry;
+}
+
+void packfile_list_append(struct packfile_list *list, struct packed_git *pack)
+{
+ struct packfile_list_entry *entry;
+
+ entry = packfile_list_remove_internal(list, pack);
+ if (!entry) {
+ entry = xmalloc(sizeof(*entry));
+ entry->pack = pack;
+ }
+ entry->next = NULL;
+
+ if (list->tail) {
+ list->tail->next = entry;
+ list->tail = entry;
+ } else {
+ list->head = list->tail = entry;
+ }
+}
+
+struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs,
+ const struct object_id *oid)
+{
+ for (; packs; packs = packs->next)
+ if (find_pack_entry_one(oid, packs->pack))
+ return packs->pack;
+ return NULL;
+}
+
void pack_report(struct repository *repo)
{
fprintf(stderr,
@@ -273,13 +356,14 @@ static void scan_windows(struct packed_git *p,
static int unuse_one_window(struct packed_git *current)
{
- struct packed_git *p, *lru_p = NULL;
+ struct packfile_list_entry *e;
+ struct packed_git *lru_p = NULL;
struct pack_window *lru_w = NULL, *lru_l = NULL;
if (current)
scan_windows(current, &lru_p, &lru_w, &lru_l);
- for (p = current->repo->objects->packed_git; p; p = p->next)
- scan_windows(p, &lru_p, &lru_w, &lru_l);
+ for (e = current->repo->objects->packfiles->packs.head; e; e = e->next)
+ scan_windows(e->pack, &lru_p, &lru_w, &lru_l);
if (lru_p) {
munmap(lru_w->base, lru_w->len);
pack_mapped -= lru_w->len;
@@ -359,19 +443,16 @@ void close_pack(struct packed_git *p)
oidset_clear(&p->bad_objects);
}
-void close_object_store(struct raw_object_store *o)
+void close_object_store(struct object_database *o)
{
- struct packed_git *p;
+ struct odb_source *source;
- for (p = o->packed_git; p; p = p->next)
- if (p->do_not_close)
- BUG("want to close pack marked 'do-not-close'");
- else
- close_pack(p);
+ packfile_store_close(o->packfiles);
- if (o->multi_pack_index) {
- close_midx(o->multi_pack_index);
- o->multi_pack_index = NULL;
+ for (source = o->sources; source; source = source->next) {
+ if (source->midx)
+ close_midx(source->midx);
+ source->midx = NULL;
}
close_commit_graph(o);
@@ -462,14 +543,15 @@ static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struc
static int close_one_pack(struct repository *r)
{
- struct packed_git *p, *lru_p = NULL;
+ struct packfile_list_entry *e;
+ struct packed_git *lru_p = NULL;
struct pack_window *mru_w = NULL;
int accept_windows_inuse = 1;
- for (p = r->objects->packed_git; p; p = p->next) {
- if (p->pack_fd == -1)
+ for (e = r->objects->packfiles->packs.head; e; e = e->next) {
+ if (e->pack->pack_fd == -1)
continue;
- find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
+ find_lru_pack(e->pack, &lru_p, &mru_w, &accept_windows_inuse);
}
if (lru_p)
@@ -782,16 +864,56 @@ struct packed_git *add_packed_git(struct repository *r, const char *path,
return p;
}
-void install_packed_git(struct repository *r, struct packed_git *pack)
+void packfile_store_add_pack(struct packfile_store *store,
+ struct packed_git *pack)
{
if (pack->pack_fd != -1)
pack_open_fds++;
- pack->next = r->objects->packed_git;
- r->objects->packed_git = pack;
+ packfile_list_append(&store->packs, pack);
+ strmap_put(&store->packs_by_path, pack->pack_name, pack);
+}
+
+struct packed_git *packfile_store_load_pack(struct packfile_store *store,
+ const char *idx_path, int local)
+{
+ struct strbuf key = STRBUF_INIT;
+ struct packed_git *p;
+
+ /*
+ * We're being called with the path to the index file, but `pack_map`
+ * holds the path to the packfile itself.
+ */
+ strbuf_addstr(&key, idx_path);
+ strbuf_strip_suffix(&key, ".idx");
+ strbuf_addstr(&key, ".pack");
+
+ p = strmap_get(&store->packs_by_path, key.buf);
+ if (!p) {
+ p = add_packed_git(store->odb->repo, idx_path,
+ strlen(idx_path), local);
+ if (p)
+ packfile_store_add_pack(store, p);
+ }
+
+ strbuf_release(&key);
+ return p;
+}
- hashmap_entry_init(&pack->packmap_ent, strhash(pack->pack_name));
- hashmap_add(&r->objects->pack_map, &pack->packmap_ent);
+int packfile_store_freshen_object(struct packfile_store *store,
+ const struct object_id *oid)
+{
+ struct pack_entry e;
+ if (!find_pack_entry(store->odb->repo, oid, &e))
+ return 0;
+ if (e.p->is_cruft)
+ return 0;
+ if (e.p->freshened)
+ return 1;
+ if (utime(e.p->pack_name, NULL))
+ return 0;
+ e.p->freshened = 1;
+ return 1;
}
void (*report_garbage)(unsigned seen_bits, const char *path);
@@ -893,23 +1015,14 @@ static void prepare_pack(const char *full_name, size_t full_name_len,
const char *file_name, void *_data)
{
struct prepare_pack_data *data = (struct prepare_pack_data *)_data;
- struct packed_git *p;
size_t base_len = full_name_len;
if (strip_suffix_mem(full_name, &base_len, ".idx") &&
!(data->m && midx_contains_pack(data->m, file_name))) {
- struct hashmap_entry hent;
- char *pack_name = xstrfmt("%.*s.pack", (int)base_len, full_name);
- unsigned int hash = strhash(pack_name);
- hashmap_entry_init(&hent, hash);
-
- /* Don't reopen a pack we already have. */
- if (!hashmap_get(&data->r->objects->pack_map, &hent, pack_name)) {
- p = add_packed_git(data->r, full_name, full_name_len, data->local);
- if (p)
- install_packed_git(data->r, p);
- }
- free(pack_name);
+ char *trimmed_path = xstrndup(full_name, full_name_len);
+ packfile_store_load_pack(data->r->objects->packfiles,
+ trimmed_path, data->local);
+ free(trimmed_path);
}
if (!report_garbage)
@@ -933,60 +1046,26 @@ static void prepare_pack(const char *full_name, size_t full_name_len,
report_garbage(PACKDIR_FILE_GARBAGE, full_name);
}
-static void prepare_packed_git_one(struct repository *r, char *objdir, int local)
+static void prepare_packed_git_one(struct odb_source *source)
{
- struct prepare_pack_data data;
struct string_list garbage = STRING_LIST_INIT_DUP;
+ struct prepare_pack_data data = {
+ .m = source->midx,
+ .r = source->odb->repo,
+ .garbage = &garbage,
+ .local = source->local,
+ };
- data.m = r->objects->multi_pack_index;
-
- /* look for the multi-pack-index for this object directory */
- while (data.m && strcmp(data.m->object_dir, objdir))
- data.m = data.m->next;
-
- data.r = r;
- data.garbage = &garbage;
- data.local = local;
-
- for_each_file_in_pack_dir(objdir, prepare_pack, &data);
+ for_each_file_in_pack_dir(source->path, prepare_pack, &data);
report_pack_garbage(data.garbage);
string_list_clear(data.garbage, 0);
}
-static void prepare_packed_git(struct repository *r);
-/*
- * Give a fast, rough count of the number of objects in the repository. This
- * ignores loose objects completely. If you have a lot of them, then either
- * you should repack because your performance will be awful, or they are
- * all unreachable objects about to be pruned, in which case they're not really
- * interesting as a measure of repo size in the first place.
- */
-unsigned long repo_approximate_object_count(struct repository *r)
-{
- if (!r->objects->approximate_object_count_valid) {
- unsigned long count;
- struct multi_pack_index *m;
- struct packed_git *p;
-
- prepare_packed_git(r);
- count = 0;
- for (m = get_multi_pack_index(r); m; m = m->next)
- count += m->num_objects;
- for (p = r->objects->packed_git; p; p = p->next) {
- if (open_pack_index(p))
- continue;
- count += p->num_objects;
- }
- r->objects->approximate_object_count = count;
- r->objects->approximate_object_count_valid = 1;
- }
- return r->objects->approximate_object_count;
-}
-
-DEFINE_LIST_SORT(static, sort_packs, struct packed_git, next);
+DEFINE_LIST_SORT(static, sort_packs, struct packfile_list_entry, next);
-static int sort_pack(const struct packed_git *a, const struct packed_git *b)
+static int sort_pack(const struct packfile_list_entry *a,
+ const struct packfile_list_entry *b)
{
int st;
@@ -996,7 +1075,7 @@ static int sort_pack(const struct packed_git *a, const struct packed_git *b)
* remote ones could be on a network mounted filesystem.
* Favor local ones for these reasons.
*/
- st = a->pack_local - b->pack_local;
+ st = a->pack->pack_local - b->pack->pack_local;
if (st)
return -st;
@@ -1005,112 +1084,86 @@ static int sort_pack(const struct packed_git *a, const struct packed_git *b)
* and more recent objects tend to get accessed more
* often.
*/
- if (a->mtime < b->mtime)
+ if (a->pack->mtime < b->pack->mtime)
return 1;
- else if (a->mtime == b->mtime)
+ else if (a->pack->mtime == b->pack->mtime)
return 0;
return -1;
}
-static void rearrange_packed_git(struct repository *r)
-{
- sort_packs(&r->objects->packed_git, sort_pack);
-}
-
-static void prepare_packed_git_mru(struct repository *r)
-{
- struct packed_git *p;
-
- INIT_LIST_HEAD(&r->objects->packed_git_mru);
-
- for (p = r->objects->packed_git; p; p = p->next)
- list_add_tail(&p->mru, &r->objects->packed_git_mru);
-}
-
-static void prepare_packed_git(struct repository *r)
+void packfile_store_prepare(struct packfile_store *store)
{
- struct object_directory *odb;
+ struct odb_source *source;
- if (r->objects->packed_git_initialized)
+ if (store->initialized)
return;
- prepare_alt_odb(r);
- for (odb = r->objects->odb; odb; odb = odb->next) {
- int local = (odb == r->objects->odb);
- prepare_multi_pack_index_one(r, odb->path, local);
- prepare_packed_git_one(r, odb->path, local);
+ odb_prepare_alternates(store->odb);
+ for (source = store->odb->sources; source; source = source->next) {
+ prepare_multi_pack_index_one(source);
+ prepare_packed_git_one(source);
}
- rearrange_packed_git(r);
-
- prepare_packed_git_mru(r);
- r->objects->packed_git_initialized = 1;
-}
-
-void reprepare_packed_git(struct repository *r)
-{
- struct object_directory *odb;
-
- obj_read_lock();
- /*
- * Reprepare alt odbs, in case the alternates file was modified
- * during the course of this process. This only _adds_ odbs to
- * the linked list, so existing odbs will continue to exist for
- * the lifetime of the process.
- */
- r->objects->loaded_alternates = 0;
- prepare_alt_odb(r);
-
- for (odb = r->objects->odb; odb; odb = odb->next)
- odb_clear_loose_cache(odb);
-
- r->objects->approximate_object_count_valid = 0;
- r->objects->packed_git_initialized = 0;
- prepare_packed_git(r);
- obj_read_unlock();
-}
+ sort_packs(&store->packs.head, sort_pack);
+ for (struct packfile_list_entry *e = store->packs.head; e; e = e->next)
+ if (!e->next)
+ store->packs.tail = e;
-struct packed_git *get_packed_git(struct repository *r)
-{
- prepare_packed_git(r);
- return r->objects->packed_git;
+ store->initialized = true;
}
-struct multi_pack_index *get_multi_pack_index(struct repository *r)
+void packfile_store_reprepare(struct packfile_store *store)
{
- prepare_packed_git(r);
- return r->objects->multi_pack_index;
+ store->initialized = false;
+ packfile_store_prepare(store);
}
-struct multi_pack_index *get_local_multi_pack_index(struct repository *r)
+struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *store)
{
- struct multi_pack_index *m = get_multi_pack_index(r);
+ packfile_store_prepare(store);
- /* no need to iterate; we always put the local one first (if any) */
- if (m && m->local)
- return m;
+ for (struct odb_source *source = store->odb->sources; source; source = source->next) {
+ struct multi_pack_index *m = source->midx;
+ if (!m)
+ continue;
+ for (uint32_t i = 0; i < m->num_packs + m->num_packs_in_base; i++)
+ prepare_midx_pack(m, i);
+ }
- return NULL;
+ return store->packs.head;
}
-struct packed_git *get_all_packs(struct repository *r)
+/*
+ * Give a fast, rough count of the number of objects in the repository. This
+ * ignores loose objects completely. If you have a lot of them, then either
+ * you should repack because your performance will be awful, or they are
+ * all unreachable objects about to be pruned, in which case they're not really
+ * interesting as a measure of repo size in the first place.
+ */
+unsigned long repo_approximate_object_count(struct repository *r)
{
- struct multi_pack_index *m;
+ if (!r->objects->approximate_object_count_valid) {
+ struct odb_source *source;
+ unsigned long count = 0;
+ struct packed_git *p;
- prepare_packed_git(r);
- for (m = r->objects->multi_pack_index; m; m = m->next) {
- uint32_t i;
- for (i = 0; i < m->num_packs + m->num_packs_in_base; i++)
- prepare_midx_pack(r, m, i);
- }
+ odb_prepare_alternates(r->objects);
- return r->objects->packed_git;
-}
+ for (source = r->objects->sources; source; source = source->next) {
+ struct multi_pack_index *m = get_multi_pack_index(source);
+ if (m)
+ count += m->num_objects + m->num_objects_in_base;
+ }
-struct list_head *get_packed_git_mru(struct repository *r)
-{
- prepare_packed_git(r);
- return &r->objects->packed_git_mru;
+ repo_for_each_pack(r, p) {
+ if (p->multi_pack_index || open_pack_index(p))
+ continue;
+ count += p->num_objects;
+ }
+ r->objects->approximate_object_count = count;
+ r->objects->approximate_object_count_valid = 1;
+ }
+ return r->objects->approximate_object_count;
}
unsigned long unpack_object_header_buffer(const unsigned char *buf,
@@ -1165,7 +1218,7 @@ unsigned long get_size_from_delta(struct packed_git *p,
*
* Other worrying sections could be the call to close_pack_fd(),
* which can close packs even with in-use windows, and to
- * reprepare_packed_git(). Regarding the former, mmap doc says:
+ * odb_reprepare(). Regarding the former, mmap doc says:
* "closing the file descriptor does not unmap the region". And
* for the latter, it won't re-open already available packs.
*/
@@ -1227,11 +1280,11 @@ void mark_bad_packed_object(struct packed_git *p, const struct object_id *oid)
const struct packed_git *has_packed_and_bad(struct repository *r,
const struct object_id *oid)
{
- struct packed_git *p;
+ struct packfile_list_entry *e;
- for (p = r->objects->packed_git; p; p = p->next)
- if (oidset_contains(&p->bad_objects, oid))
- return p;
+ for (e = r->objects->packfiles->packs.head; e; e = e->next)
+ if (oidset_contains(&e->pack->bad_objects, oid))
+ return e->pack;
return NULL;
}
@@ -1321,7 +1374,7 @@ static int retry_bad_packed_offset(struct repository *r,
return OBJ_BAD;
nth_packed_object_id(&oid, p, pack_pos_to_index(p, pos));
mark_bad_packed_object(p, &oid);
- type = oid_object_info(r, &oid, NULL);
+ type = odb_read_object_info(r->objects, &oid, NULL);
if (type <= OBJ_NONE)
return OBJ_BAD;
return type;
@@ -1849,7 +1902,8 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset,
oi.typep = &type;
oi.sizep = &base_size;
oi.contentp = &base;
- if (oid_object_info_extended(r, &base_oid, &oi, 0) < 0)
+ if (odb_read_object_info_extended(r->objects, &base_oid,
+ &oi, 0) < 0)
base = NULL;
external_base = base;
@@ -2038,19 +2092,6 @@ int is_pack_valid(struct packed_git *p)
return !open_packed_git(p);
}
-struct packed_git *find_oid_pack(const struct object_id *oid,
- struct packed_git *packs)
-{
- struct packed_git *p;
-
- for (p = packs; p; p = p->next) {
- if (find_pack_entry_one(oid, p))
- return p;
- }
- return NULL;
-
-}
-
static int fill_pack_entry(const struct object_id *oid,
struct pack_entry *e,
struct packed_git *p)
@@ -2081,22 +2122,23 @@ static int fill_pack_entry(const struct object_id *oid,
int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e)
{
- struct list_head *pos;
- struct multi_pack_index *m;
+ struct packfile_list_entry *l;
- prepare_packed_git(r);
- if (!r->objects->packed_git && !r->objects->multi_pack_index)
- return 0;
+ packfile_store_prepare(r->objects->packfiles);
- for (m = r->objects->multi_pack_index; m; m = m->next) {
- if (fill_midx_entry(r, oid, e, m))
+ for (struct odb_source *source = r->objects->sources; source; source = source->next)
+ if (source->midx && fill_midx_entry(source->midx, oid, e))
return 1;
- }
- list_for_each(pos, &r->objects->packed_git_mru) {
- struct packed_git *p = list_entry(pos, struct packed_git, mru);
+ if (!r->objects->packfiles->packs.head)
+ return 0;
+
+ for (l = r->objects->packfiles->packs.head; l; l = l->next) {
+ struct packed_git *p = l->pack;
+
if (!p->multi_pack_index && fill_pack_entry(oid, e, p)) {
- list_move(&p->mru, &r->objects->packed_git_mru);
+ if (!r->objects->packfiles->skip_mru_updates)
+ packfile_list_prepend(&r->objects->packfiles->packs, p);
return 1;
}
}
@@ -2106,19 +2148,19 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa
static void maybe_invalidate_kept_pack_cache(struct repository *r,
unsigned flags)
{
- if (!r->objects->kept_pack_cache.packs)
+ if (!r->objects->packfiles->kept_cache.packs)
return;
- if (r->objects->kept_pack_cache.flags == flags)
+ if (r->objects->packfiles->kept_cache.flags == flags)
return;
- FREE_AND_NULL(r->objects->kept_pack_cache.packs);
- r->objects->kept_pack_cache.flags = 0;
+ FREE_AND_NULL(r->objects->packfiles->kept_cache.packs);
+ r->objects->packfiles->kept_cache.flags = 0;
}
struct packed_git **kept_pack_cache(struct repository *r, unsigned flags)
{
maybe_invalidate_kept_pack_cache(r, flags);
- if (!r->objects->kept_pack_cache.packs) {
+ if (!r->objects->packfiles->kept_cache.packs) {
struct packed_git **packs = NULL;
size_t nr = 0, alloc = 0;
struct packed_git *p;
@@ -2131,7 +2173,7 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags)
* covers, one kept and one not kept, but the midx returns only
* the non-kept version.
*/
- for (p = get_all_packs(r); p; p = p->next) {
+ repo_for_each_pack(r, p) {
if ((p->pack_keep && (flags & ON_DISK_KEEP_PACKS)) ||
(p->pack_keep_in_core && (flags & IN_CORE_KEEP_PACKS))) {
ALLOC_GROW(packs, nr + 1, alloc);
@@ -2141,11 +2183,11 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags)
ALLOC_GROW(packs, nr + 1, alloc);
packs[nr] = NULL;
- r->objects->kept_pack_cache.packs = packs;
- r->objects->kept_pack_cache.flags = flags;
+ r->objects->packfiles->kept_cache.packs = packs;
+ r->objects->packfiles->kept_cache.flags = flags;
}
- return r->objects->kept_pack_cache.packs;
+ return r->objects->packfiles->kept_cache.packs;
}
int find_kept_pack_entry(struct repository *r,
@@ -2228,7 +2270,8 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb,
int r = 0;
int pack_errors = 0;
- for (p = get_all_packs(repo); p; p = p->next) {
+ repo->objects->packfiles->skip_mru_updates = true;
+ repo_for_each_pack(repo, p) {
if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
continue;
if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) &&
@@ -2248,6 +2291,8 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb,
if (r)
break;
}
+ repo->objects->packfiles->skip_mru_updates = false;
+
return r ? r : pack_errors;
}
@@ -2342,3 +2387,31 @@ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *l
*len = hdr - out;
return 0;
}
+
+struct packfile_store *packfile_store_new(struct object_database *odb)
+{
+ struct packfile_store *store;
+ CALLOC_ARRAY(store, 1);
+ store->odb = odb;
+ strmap_init(&store->packs_by_path);
+ return store;
+}
+
+void packfile_store_free(struct packfile_store *store)
+{
+ for (struct packfile_list_entry *e = store->packs.head; e; e = e->next)
+ free(e->pack);
+ packfile_list_clear(&store->packs);
+
+ strmap_clear(&store->packs_by_path, 0);
+ free(store);
+}
+
+void packfile_store_close(struct packfile_store *store)
+{
+ for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) {
+ if (e->pack->do_not_close)
+ BUG("want to close pack marked 'do-not-close'");
+ close_pack(e->pack);
+ }
+}