diff options
Diffstat (limited to 'reftable/stack.c')
| -rw-r--r-- | reftable/stack.c | 655 |
1 files changed, 432 insertions, 223 deletions
diff --git a/reftable/stack.c b/reftable/stack.c index 84cf37a2ad..6dac015b47 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -8,7 +8,6 @@ https://developers.google.com/open-source/licenses/bsd #include "stack.h" -#include "../write-or-die.h" #include "system.h" #include "constants.h" #include "merged.h" @@ -17,7 +16,6 @@ https://developers.google.com/open-source/licenses/bsd #include "reftable-record.h" #include "reftable-merged.h" #include "writer.h" -#include "tempfile.h" static int stack_try_add(struct reftable_stack *st, int (*write_table)(struct reftable_writer *wr, @@ -31,70 +29,121 @@ static void reftable_addition_close(struct reftable_addition *add); static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, int reuse_open); -static void stack_filename(struct strbuf *dest, struct reftable_stack *st, - const char *name) +static int stack_filename(struct reftable_buf *dest, struct reftable_stack *st, + const char *name) { - strbuf_reset(dest); - strbuf_addstr(dest, st->reftable_dir); - strbuf_addstr(dest, "/"); - strbuf_addstr(dest, name); + int err; + reftable_buf_reset(dest); + if ((err = reftable_buf_addstr(dest, st->reftable_dir)) < 0 || + (err = reftable_buf_addstr(dest, "/")) < 0 || + (err = reftable_buf_addstr(dest, name)) < 0) + return err; + return 0; +} + +static int stack_fsync(const struct reftable_write_options *opts, int fd) +{ + if (opts->fsync) + return opts->fsync(fd); + return fsync(fd); } -static ssize_t reftable_fd_write(void *arg, const void *data, size_t sz) +static ssize_t reftable_write_data(int fd, const void *data, size_t size) { - int *fdp = (int *)arg; - return write_in_full(*fdp, data, sz); + size_t total_written = 0; + const char *p = data; + + while (total_written < size) { + ssize_t bytes_written = write(fd, p, size - total_written); + if (bytes_written < 0 && (errno == EAGAIN || errno == EINTR)) + continue; + if (bytes_written < 0) + return REFTABLE_IO_ERROR; + + total_written += bytes_written; + p += bytes_written; + } + + return total_written; } -static int reftable_fd_flush(void *arg) +struct fd_writer { + const struct reftable_write_options *opts; + int fd; +}; + +static ssize_t fd_writer_write(void *arg, const void *data, size_t sz) { - int *fdp = (int *)arg; + struct fd_writer *writer = arg; + return reftable_write_data(writer->fd, data, sz); +} - return fsync_component(FSYNC_COMPONENT_REFERENCE, *fdp); +static int fd_writer_flush(void *arg) +{ + struct fd_writer *writer = arg; + return stack_fsync(writer->opts, writer->fd); } int reftable_new_stack(struct reftable_stack **dest, const char *dir, const struct reftable_write_options *_opts) { - struct reftable_stack *p = reftable_calloc(1, sizeof(*p)); - struct strbuf list_file_name = STRBUF_INIT; - struct reftable_write_options opts = {0}; - int err = 0; + struct reftable_buf list_file_name = REFTABLE_BUF_INIT; + struct reftable_write_options opts = { 0 }; + struct reftable_stack *p; + int err; + + p = reftable_calloc(1, sizeof(*p)); + if (!p) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } if (_opts) opts = *_opts; if (opts.hash_id == 0) - opts.hash_id = GIT_SHA1_FORMAT_ID; + opts.hash_id = REFTABLE_HASH_SHA1; *dest = NULL; - strbuf_reset(&list_file_name); - strbuf_addstr(&list_file_name, dir); - strbuf_addstr(&list_file_name, "/tables.list"); + reftable_buf_reset(&list_file_name); + if ((err = reftable_buf_addstr(&list_file_name, dir)) < 0 || + (err = reftable_buf_addstr(&list_file_name, "/tables.list")) < 0) + goto out; - p->list_file = strbuf_detach(&list_file_name, NULL); + p->list_file = reftable_buf_detach(&list_file_name); p->list_fd = -1; - p->reftable_dir = xstrdup(dir); p->opts = opts; + p->reftable_dir = reftable_strdup(dir); + if (!p->reftable_dir) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } err = reftable_stack_reload_maybe_reuse(p, 1); - if (err < 0) { + if (err < 0) + goto out; + + *dest = p; + err = 0; + +out: + if (err < 0) reftable_stack_destroy(p); - } else { - *dest = p; - } return err; } static int fd_read_lines(int fd, char ***namesp) { - off_t size = lseek(fd, 0, SEEK_END); char *buf = NULL; int err = 0; + off_t size; + + size = lseek(fd, 0, SEEK_END); if (size < 0) { err = REFTABLE_IO_ERROR; goto done; } + err = lseek(fd, 0, SEEK_SET); if (err < 0) { err = REFTABLE_IO_ERROR; @@ -102,13 +151,29 @@ static int fd_read_lines(int fd, char ***namesp) } REFTABLE_ALLOC_ARRAY(buf, size + 1); - if (read_in_full(fd, buf, size) != size) { - err = REFTABLE_IO_ERROR; + if (!buf) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; goto done; } + + for (off_t total_read = 0; total_read < size; ) { + ssize_t bytes_read = read(fd, buf + total_read, size - total_read); + if (bytes_read < 0 && (errno == EAGAIN || errno == EINTR)) + continue; + if (bytes_read < 0 || !bytes_read) { + err = REFTABLE_IO_ERROR; + goto done; + } + + total_read += bytes_read; + } buf[size] = 0; - parse_names(buf, size, namesp); + *namesp = parse_names(buf, size); + if (!*namesp) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } done: reftable_free(buf); @@ -122,6 +187,8 @@ int read_lines(const char *filename, char ***namesp) if (fd < 0) { if (errno == ENOENT) { REFTABLE_CALLOC_ARRAY(*namesp, 1); + if (!*namesp) + return REFTABLE_OUT_OF_MEMORY_ERROR; return 0; } @@ -132,18 +199,18 @@ int read_lines(const char *filename, char ***namesp) return err; } -void reftable_stack_init_ref_iterator(struct reftable_stack *st, +int reftable_stack_init_ref_iterator(struct reftable_stack *st, struct reftable_iterator *it) { - merged_table_init_iter(reftable_stack_merged_table(st), - it, BLOCK_TYPE_REF); + return merged_table_init_iter(reftable_stack_merged_table(st), + it, BLOCK_TYPE_REF); } -void reftable_stack_init_log_iterator(struct reftable_stack *st, - struct reftable_iterator *it) +int reftable_stack_init_log_iterator(struct reftable_stack *st, + struct reftable_iterator *it) { - merged_table_init_iter(reftable_stack_merged_table(st), - it, BLOCK_TYPE_LOG); + return merged_table_init_iter(reftable_stack_merged_table(st), + it, BLOCK_TYPE_LOG); } struct reftable_merged_table * @@ -167,6 +234,10 @@ void reftable_stack_destroy(struct reftable_stack *st) { char **names = NULL; int err = 0; + + if (!st) + return; + if (st->merged) { reftable_merged_table_free(st->merged); st->merged = NULL; @@ -174,28 +245,32 @@ void reftable_stack_destroy(struct reftable_stack *st) err = read_lines(st->list_file, &names); if (err < 0) { - FREE_AND_NULL(names); + REFTABLE_FREE_AND_NULL(names); } if (st->readers) { - int i = 0; - struct strbuf filename = STRBUF_INIT; - for (i = 0; i < st->readers_len; i++) { + struct reftable_buf filename = REFTABLE_BUF_INIT; + + for (size_t i = 0; i < st->readers_len; i++) { const char *name = reader_name(st->readers[i]); - strbuf_reset(&filename); + int try_unlinking = 1; + + reftable_buf_reset(&filename); if (names && !has_name(names, name)) { - stack_filename(&filename, st, name); + if (stack_filename(&filename, st, name) < 0) + try_unlinking = 0; } reftable_reader_decref(st->readers[i]); - if (filename.len) { + if (try_unlinking && filename.len) { /* On Windows, can only unlink after closing. */ unlink(filename.buf); } } - strbuf_release(&filename); + + reftable_buf_release(&filename); st->readers_len = 0; - FREE_AND_NULL(st->readers); + REFTABLE_FREE_AND_NULL(st->readers); } if (st->list_fd >= 0) { @@ -203,20 +278,20 @@ void reftable_stack_destroy(struct reftable_stack *st) st->list_fd = -1; } - FREE_AND_NULL(st->list_file); - FREE_AND_NULL(st->reftable_dir); + REFTABLE_FREE_AND_NULL(st->list_file); + REFTABLE_FREE_AND_NULL(st->reftable_dir); reftable_free(st); free_names(names); } static struct reftable_reader **stack_copy_readers(struct reftable_stack *st, - int cur_len) + size_t cur_len) { struct reftable_reader **cur = reftable_calloc(cur_len, sizeof(*cur)); - int i = 0; - for (i = 0; i < cur_len; i++) { + if (!cur) + return NULL; + for (size_t i = 0; i < cur_len; i++) cur[i] = st->readers[i]; - } return cur; } @@ -225,18 +300,34 @@ static int reftable_stack_reload_once(struct reftable_stack *st, int reuse_open) { size_t cur_len = !st->merged ? 0 : st->merged->readers_len; - struct reftable_reader **cur = stack_copy_readers(st, cur_len); + struct reftable_reader **cur = NULL; struct reftable_reader **reused = NULL; - size_t reused_len = 0, reused_alloc = 0; - size_t names_len = names_length(names); - struct reftable_reader **new_readers = - reftable_calloc(names_len, sizeof(*new_readers)); + struct reftable_reader **new_readers = NULL; + size_t reused_len = 0, reused_alloc = 0, names_len; size_t new_readers_len = 0; struct reftable_merged_table *new_merged = NULL; - struct strbuf table_path = STRBUF_INIT; + struct reftable_buf table_path = REFTABLE_BUF_INIT; int err = 0; size_t i; + if (cur_len) { + cur = stack_copy_readers(st, cur_len); + if (!cur) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + } + + names_len = names_length(names); + + if (names_len) { + new_readers = reftable_calloc(names_len, sizeof(*new_readers)); + if (!new_readers) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + } + while (*names) { struct reftable_reader *rd = NULL; const char *name = *names++; @@ -256,7 +347,13 @@ static int reftable_stack_reload_once(struct reftable_stack *st, * thus need to keep them alive here, which we * do by bumping their refcount. */ - REFTABLE_ALLOC_GROW(reused, reused_len + 1, reused_alloc); + REFTABLE_ALLOC_GROW_OR_NULL(reused, + reused_len + 1, + reused_alloc); + if (!reused) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } reused[reused_len++] = rd; reftable_reader_incref(rd); break; @@ -265,7 +362,10 @@ static int reftable_stack_reload_once(struct reftable_stack *st, if (!rd) { struct reftable_block_source src = { NULL }; - stack_filename(&table_path, st, name); + + err = stack_filename(&table_path, st, name); + if (err < 0) + goto done; err = reftable_block_source_from_file(&src, table_path.buf); @@ -296,7 +396,11 @@ static int reftable_stack_reload_once(struct reftable_stack *st, for (i = 0; i < cur_len; i++) { if (cur[i]) { const char *name = reader_name(cur[i]); - stack_filename(&table_path, st, name); + + err = stack_filename(&table_path, st, name); + if (err < 0) + goto done; + reftable_reader_decref(cur[i]); unlink(table_path.buf); } @@ -329,7 +433,7 @@ done: reftable_free(new_readers); reftable_free(reused); reftable_free(cur); - strbuf_release(&table_path); + reftable_buf_release(&table_path); return err; } @@ -382,6 +486,10 @@ static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, } REFTABLE_CALLOC_ARRAY(names, 1); + if (!names) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } } else { err = fd_read_lines(fd, &names); if (err < 0) @@ -415,8 +523,8 @@ static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, close(fd); fd = -1; - delay = delay + (delay * rand()) / RAND_MAX + 1; - sleep_millisec(delay); + delay = delay + (delay * reftable_rand()) / UINT32_MAX + 1; + poll(NULL, 0, delay); } out: @@ -476,6 +584,10 @@ out: close(fd); free_names(names); free_names(names_after); + + if (st->opts.on_reload) + st->opts.on_reload(st->opts.on_reload_payload); + return err; } @@ -486,7 +598,6 @@ static int stack_uptodate(struct reftable_stack *st) { char **names = NULL; int err; - int i = 0; /* * When we have cached stat information available then we use it to @@ -526,7 +637,7 @@ static int stack_uptodate(struct reftable_stack *st) if (err < 0) return err; - for (i = 0; i < st->readers_len; i++) { + for (size_t i = 0; i < st->readers_len; i++) { if (!names[i]) { err = 1; goto done; @@ -574,18 +685,18 @@ int reftable_stack_add(struct reftable_stack *st, return 0; } -static void format_name(struct strbuf *dest, uint64_t min, uint64_t max) +static int format_name(struct reftable_buf *dest, uint64_t min, uint64_t max) { char buf[100]; - uint32_t rnd = (uint32_t)git_rand(); + uint32_t rnd = reftable_rand(); snprintf(buf, sizeof(buf), "0x%012" PRIx64 "-0x%012" PRIx64 "-%08x", min, max, rnd); - strbuf_reset(dest); - strbuf_addstr(dest, buf); + reftable_buf_reset(dest); + return reftable_buf_addstr(dest, buf); } struct reftable_addition { - struct lock_file tables_list_lock; + struct reftable_flock tables_list_lock; struct reftable_stack *stack; char **new_tables; @@ -599,15 +710,13 @@ static int reftable_stack_init_addition(struct reftable_addition *add, struct reftable_stack *st, unsigned int flags) { - struct strbuf lock_file_name = STRBUF_INIT; + struct reftable_buf lock_file_name = REFTABLE_BUF_INIT; int err; add->stack = st; - err = hold_lock_file_for_update_timeout(&add->tables_list_lock, - st->list_file, - LOCK_NO_DEREF, - st->opts.lock_timeout_ms); + err = flock_acquire(&add->tables_list_lock, st->list_file, + st->opts.lock_timeout_ms); if (err < 0) { if (errno == EEXIST) { err = REFTABLE_LOCK_ERROR; @@ -617,7 +726,7 @@ static int reftable_stack_init_addition(struct reftable_addition *add, goto done; } if (st->opts.default_permissions) { - if (chmod(get_lock_file_path(&add->tables_list_lock), + if (chmod(add->tables_list_lock.path, st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -641,18 +750,18 @@ static int reftable_stack_init_addition(struct reftable_addition *add, done: if (err) reftable_addition_close(add); - strbuf_release(&lock_file_name); + reftable_buf_release(&lock_file_name); return err; } static void reftable_addition_close(struct reftable_addition *add) { - struct strbuf nm = STRBUF_INIT; + struct reftable_buf nm = REFTABLE_BUF_INIT; size_t i; for (i = 0; i < add->new_tables_len; i++) { - stack_filename(&nm, add->stack, add->new_tables[i]); - unlink(nm.buf); + if (!stack_filename(&nm, add->stack, add->new_tables[i])) + unlink(nm.buf); reftable_free(add->new_tables[i]); add->new_tables[i] = NULL; } @@ -661,8 +770,8 @@ static void reftable_addition_close(struct reftable_addition *add) add->new_tables_len = 0; add->new_tables_cap = 0; - rollback_lock_file(&add->tables_list_lock); - strbuf_release(&nm); + flock_release(&add->tables_list_lock); + reftable_buf_release(&nm); } void reftable_addition_destroy(struct reftable_addition *add) @@ -676,8 +785,7 @@ void reftable_addition_destroy(struct reftable_addition *add) int reftable_addition_commit(struct reftable_addition *add) { - struct strbuf table_list = STRBUF_INIT; - int lock_file_fd = get_lock_file_fd(&add->tables_list_lock); + struct reftable_buf table_list = REFTABLE_BUF_INIT; int err = 0; size_t i; @@ -685,28 +793,31 @@ int reftable_addition_commit(struct reftable_addition *add) goto done; for (i = 0; i < add->stack->merged->readers_len; i++) { - strbuf_addstr(&table_list, add->stack->readers[i]->name); - strbuf_addstr(&table_list, "\n"); + if ((err = reftable_buf_addstr(&table_list, add->stack->readers[i]->name)) < 0 || + (err = reftable_buf_addstr(&table_list, "\n")) < 0) + goto done; } for (i = 0; i < add->new_tables_len; i++) { - strbuf_addstr(&table_list, add->new_tables[i]); - strbuf_addstr(&table_list, "\n"); + if ((err = reftable_buf_addstr(&table_list, add->new_tables[i])) < 0 || + (err = reftable_buf_addstr(&table_list, "\n")) < 0) + goto done; } - err = write_in_full(lock_file_fd, table_list.buf, table_list.len); - strbuf_release(&table_list); + err = reftable_write_data(add->tables_list_lock.fd, + table_list.buf, table_list.len); + reftable_buf_release(&table_list); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; } - err = fsync_component(FSYNC_COMPONENT_REFERENCE, lock_file_fd); + err = stack_fsync(&add->stack->opts, add->tables_list_lock.fd); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; } - err = commit_lock_file(&add->tables_list_lock); + err = flock_commit(&add->tables_list_lock); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -749,7 +860,11 @@ int reftable_stack_new_addition(struct reftable_addition **dest, { int err = 0; struct reftable_addition empty = REFTABLE_ADDITION_INIT; + REFTABLE_CALLOC_ARRAY(*dest, 1); + if (!*dest) + return REFTABLE_OUT_OF_MEMORY_ERROR; + **dest = empty; err = reftable_stack_init_addition(*dest, st, flags); if (err) { @@ -784,36 +899,47 @@ int reftable_addition_add(struct reftable_addition *add, void *arg), void *arg) { - struct strbuf temp_tab_file_name = STRBUF_INIT; - struct strbuf tab_file_name = STRBUF_INIT; - struct strbuf next_name = STRBUF_INIT; + struct reftable_buf temp_tab_file_name = REFTABLE_BUF_INIT; + struct reftable_buf tab_file_name = REFTABLE_BUF_INIT; + struct reftable_buf next_name = REFTABLE_BUF_INIT; struct reftable_writer *wr = NULL; - struct tempfile *tab_file = NULL; + struct reftable_tmpfile tab_file = REFTABLE_TMPFILE_INIT; + struct fd_writer writer = { + .opts = &add->stack->opts, + }; int err = 0; - int tab_fd; - strbuf_reset(&next_name); - format_name(&next_name, add->next_update_index, add->next_update_index); + reftable_buf_reset(&next_name); - stack_filename(&temp_tab_file_name, add->stack, next_name.buf); - strbuf_addstr(&temp_tab_file_name, ".temp.XXXXXX"); + err = format_name(&next_name, add->next_update_index, add->next_update_index); + if (err < 0) + goto done; - tab_file = mks_tempfile(temp_tab_file_name.buf); - if (!tab_file) { - err = REFTABLE_IO_ERROR; + err = stack_filename(&temp_tab_file_name, add->stack, next_name.buf); + if (err < 0) + goto done; + + err = reftable_buf_addstr(&temp_tab_file_name, ".temp.XXXXXX"); + if (err < 0) + goto done; + + err = tmpfile_from_pattern(&tab_file, temp_tab_file_name.buf); + if (err < 0) goto done; - } if (add->stack->opts.default_permissions) { - if (chmod(get_tempfile_path(tab_file), + if (chmod(tab_file.path, add->stack->opts.default_permissions)) { err = REFTABLE_IO_ERROR; goto done; } } - tab_fd = get_tempfile_fd(tab_file); - wr = reftable_new_writer(reftable_fd_write, reftable_fd_flush, &tab_fd, - &add->stack->opts); + writer.fd = tab_file.fd; + err = reftable_writer_new(&wr, fd_writer_write, fd_writer_flush, + &writer, &add->stack->opts); + if (err < 0) + goto done; + err = write_table(wr, arg); if (err < 0) goto done; @@ -826,39 +952,48 @@ int reftable_addition_add(struct reftable_addition *add, if (err < 0) goto done; - err = close_tempfile_gently(tab_file); - if (err < 0) { - err = REFTABLE_IO_ERROR; + err = tmpfile_close(&tab_file); + if (err < 0) goto done; - } if (wr->min_update_index < add->next_update_index) { err = REFTABLE_API_ERROR; goto done; } - format_name(&next_name, wr->min_update_index, wr->max_update_index); - strbuf_addstr(&next_name, ".ref"); - stack_filename(&tab_file_name, add->stack, next_name.buf); + err = format_name(&next_name, wr->min_update_index, wr->max_update_index); + if (err < 0) + goto done; + + err = reftable_buf_addstr(&next_name, ".ref"); + if (err < 0) + goto done; + + err = stack_filename(&tab_file_name, add->stack, next_name.buf); + if (err < 0) + goto done; /* On windows, this relies on rand() picking a unique destination name. Maybe we should do retry loop as well? */ - err = rename_tempfile(&tab_file, tab_file_name.buf); - if (err < 0) { - err = REFTABLE_IO_ERROR; + err = tmpfile_rename(&tab_file, tab_file_name.buf); + if (err < 0) + goto done; + + REFTABLE_ALLOC_GROW_OR_NULL(add->new_tables, add->new_tables_len + 1, + add->new_tables_cap); + if (!add->new_tables) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; goto done; } + add->new_tables[add->new_tables_len++] = reftable_buf_detach(&next_name); - REFTABLE_ALLOC_GROW(add->new_tables, add->new_tables_len + 1, - add->new_tables_cap); - add->new_tables[add->new_tables_len++] = strbuf_detach(&next_name, NULL); done: - delete_tempfile(&tab_file); - strbuf_release(&temp_tab_file_name); - strbuf_release(&tab_file_name); - strbuf_release(&next_name); + tmpfile_delete(&tab_file); + reftable_buf_release(&temp_tab_file_name); + reftable_buf_release(&tab_file_name); + reftable_buf_release(&next_name); reftable_writer_free(wr); return err; } @@ -875,35 +1010,46 @@ uint64_t reftable_stack_next_update_index(struct reftable_stack *st) static int stack_compact_locked(struct reftable_stack *st, size_t first, size_t last, struct reftable_log_expiry_config *config, - struct tempfile **tab_file_out) + struct reftable_tmpfile *tab_file_out) { - struct strbuf next_name = STRBUF_INIT; - struct strbuf tab_file_path = STRBUF_INIT; + struct reftable_buf next_name = REFTABLE_BUF_INIT; + struct reftable_buf tab_file_path = REFTABLE_BUF_INIT; struct reftable_writer *wr = NULL; - struct tempfile *tab_file; - int tab_fd, err = 0; + struct fd_writer writer= { + .opts = &st->opts, + }; + struct reftable_tmpfile tab_file = REFTABLE_TMPFILE_INIT; + int err = 0; - format_name(&next_name, - reftable_reader_min_update_index(st->readers[first]), - reftable_reader_max_update_index(st->readers[last])); - stack_filename(&tab_file_path, st, next_name.buf); - strbuf_addstr(&tab_file_path, ".temp.XXXXXX"); + err = format_name(&next_name, reftable_reader_min_update_index(st->readers[first]), + reftable_reader_max_update_index(st->readers[last])); + if (err < 0) + goto done; - tab_file = mks_tempfile(tab_file_path.buf); - if (!tab_file) { - err = REFTABLE_IO_ERROR; + err = stack_filename(&tab_file_path, st, next_name.buf); + if (err < 0) + goto done; + + err = reftable_buf_addstr(&tab_file_path, ".temp.XXXXXX"); + if (err < 0) + goto done; + + err = tmpfile_from_pattern(&tab_file, tab_file_path.buf); + if (err < 0) goto done; - } - tab_fd = get_tempfile_fd(tab_file); if (st->opts.default_permissions && - chmod(get_tempfile_path(tab_file), st->opts.default_permissions) < 0) { + chmod(tab_file.path, st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; } - wr = reftable_new_writer(reftable_fd_write, reftable_fd_flush, - &tab_fd, &st->opts); + writer.fd = tab_file.fd; + err = reftable_writer_new(&wr, fd_writer_write, fd_writer_flush, + &writer, &st->opts); + if (err < 0) + goto done; + err = stack_write_compact(st, wr, first, last, config); if (err < 0) goto done; @@ -912,18 +1058,18 @@ static int stack_compact_locked(struct reftable_stack *st, if (err < 0) goto done; - err = close_tempfile_gently(tab_file); + err = tmpfile_close(&tab_file); if (err < 0) goto done; *tab_file_out = tab_file; - tab_file = NULL; + tab_file = REFTABLE_TMPFILE_INIT; done: - delete_tempfile(&tab_file); + tmpfile_delete(&tab_file); reftable_writer_free(wr); - strbuf_release(&next_name); - strbuf_release(&tab_file_path); + reftable_buf_release(&next_name); + reftable_buf_release(&tab_file_path); return err; } @@ -942,15 +1088,20 @@ static int stack_write_compact(struct reftable_stack *st, for (size_t i = first; i <= last; i++) st->stats.bytes += st->readers[i]->size; - reftable_writer_set_limits(wr, st->readers[first]->min_update_index, - st->readers[last]->max_update_index); + err = reftable_writer_set_limits(wr, st->readers[first]->min_update_index, + st->readers[last]->max_update_index); + if (err < 0) + goto done; err = reftable_merged_table_new(&mt, st->readers + first, subtabs_len, st->opts.hash_id); if (err < 0) goto done; - merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); + err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); + if (err < 0) + goto done; + err = reftable_iterator_seek_ref(&it, ""); if (err < 0) goto done; @@ -975,7 +1126,10 @@ static int stack_write_compact(struct reftable_stack *st, } reftable_iterator_destroy(&it); - merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); + err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); + if (err < 0) + goto done; + err = reftable_iterator_seek_log(&it, ""); if (err < 0) goto done; @@ -1041,13 +1195,13 @@ static int stack_compact_range(struct reftable_stack *st, struct reftable_log_expiry_config *expiry, unsigned int flags) { - struct strbuf tables_list_buf = STRBUF_INIT; - struct strbuf new_table_name = STRBUF_INIT; - struct strbuf new_table_path = STRBUF_INIT; - struct strbuf table_name = STRBUF_INIT; - struct lock_file tables_list_lock = LOCK_INIT; - struct lock_file *table_locks = NULL; - struct tempfile *new_table = NULL; + struct reftable_buf tables_list_buf = REFTABLE_BUF_INIT; + struct reftable_buf new_table_name = REFTABLE_BUF_INIT; + struct reftable_buf new_table_path = REFTABLE_BUF_INIT; + struct reftable_buf table_name = REFTABLE_BUF_INIT; + struct reftable_flock tables_list_lock = REFTABLE_FLOCK_INIT; + struct reftable_flock *table_locks = NULL; + struct reftable_tmpfile new_table = REFTABLE_TMPFILE_INIT; int is_empty_table = 0, err = 0; size_t first_to_replace, last_to_replace; size_t i, nlocks = 0; @@ -1064,10 +1218,7 @@ static int stack_compact_range(struct reftable_stack *st, * Hold the lock so that we can read "tables.list" and lock all tables * which are part of the user-specified range. */ - err = hold_lock_file_for_update_timeout(&tables_list_lock, - st->list_file, - LOCK_NO_DEREF, - st->opts.lock_timeout_ms); + err = flock_acquire(&tables_list_lock, st->list_file, st->opts.lock_timeout_ms); if (err < 0) { if (errno == EEXIST) err = REFTABLE_LOCK_ERROR; @@ -1090,12 +1241,20 @@ static int stack_compact_range(struct reftable_stack *st, * older process is still busy compacting tables which are preexisting * from the point of view of the newer process. */ - REFTABLE_CALLOC_ARRAY(table_locks, last - first + 1); + REFTABLE_ALLOC_ARRAY(table_locks, last - first + 1); + if (!table_locks) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + for (i = 0; i < last - first + 1; i++) + table_locks[i] = REFTABLE_FLOCK_INIT; + for (i = last + 1; i > first; i--) { - stack_filename(&table_name, st, reader_name(st->readers[i - 1])); + err = stack_filename(&table_name, st, reader_name(st->readers[i - 1])); + if (err < 0) + goto done; - err = hold_lock_file_for_update(&table_locks[nlocks], - table_name.buf, LOCK_NO_DEREF); + err = flock_acquire(&table_locks[nlocks], table_name.buf, 0); if (err < 0) { /* * When the table is locked already we may do a @@ -1131,7 +1290,7 @@ static int stack_compact_range(struct reftable_stack *st, * run into file descriptor exhaustion when we compress a lot * of tables. */ - err = close_lock_file_gently(&table_locks[nlocks++]); + err = flock_close(&table_locks[nlocks++]); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -1143,7 +1302,7 @@ static int stack_compact_range(struct reftable_stack *st, * "tables.list" lock while compacting the locked tables. This allows * concurrent updates to the stack to proceed. */ - err = rollback_lock_file(&tables_list_lock); + err = flock_release(&tables_list_lock); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -1166,10 +1325,7 @@ static int stack_compact_range(struct reftable_stack *st, * "tables.list". We'll then replace the compacted range of tables with * the new table. */ - err = hold_lock_file_for_update_timeout(&tables_list_lock, - st->list_file, - LOCK_NO_DEREF, - st->opts.lock_timeout_ms); + err = flock_acquire(&tables_list_lock, st->list_file, st->opts.lock_timeout_ms); if (err < 0) { if (errno == EEXIST) err = REFTABLE_LOCK_ERROR; @@ -1179,7 +1335,7 @@ static int stack_compact_range(struct reftable_stack *st, } if (st->opts.default_permissions) { - if (chmod(get_lock_file_path(&tables_list_lock), + if (chmod(tables_list_lock.path, st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -1274,8 +1430,18 @@ static int stack_compact_range(struct reftable_stack *st, * thus have to allocate `readers_len + 1` many entries. */ REFTABLE_CALLOC_ARRAY(names, st->merged->readers_len + 1); - for (size_t i = 0; i < st->merged->readers_len; i++) - names[i] = xstrdup(st->readers[i]->name); + if (!names) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + + for (size_t i = 0; i < st->merged->readers_len; i++) { + names[i] = reftable_strdup(st->readers[i]->name); + if (!names[i]) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + } first_to_replace = first; last_to_replace = last; } @@ -1285,16 +1451,22 @@ static int stack_compact_range(struct reftable_stack *st, * it into place now. */ if (!is_empty_table) { - format_name(&new_table_name, st->readers[first]->min_update_index, - st->readers[last]->max_update_index); - strbuf_addstr(&new_table_name, ".ref"); - stack_filename(&new_table_path, st, new_table_name.buf); + err = format_name(&new_table_name, st->readers[first]->min_update_index, + st->readers[last]->max_update_index); + if (err < 0) + goto done; - err = rename_tempfile(&new_table, new_table_path.buf); - if (err < 0) { - err = REFTABLE_IO_ERROR; + err = reftable_buf_addstr(&new_table_name, ".ref"); + if (err < 0) + goto done; + + err = stack_filename(&new_table_path, st, new_table_name.buf); + if (err < 0) + goto done; + + err = tmpfile_rename(&new_table, new_table_path.buf); + if (err < 0) goto done; - } } /* @@ -1302,29 +1474,38 @@ static int stack_compact_range(struct reftable_stack *st, * have just written. In case the compacted table became empty we * simply skip writing it. */ - for (i = 0; i < first_to_replace; i++) - strbuf_addf(&tables_list_buf, "%s\n", names[i]); - if (!is_empty_table) - strbuf_addf(&tables_list_buf, "%s\n", new_table_name.buf); - for (i = last_to_replace + 1; names[i]; i++) - strbuf_addf(&tables_list_buf, "%s\n", names[i]); - - err = write_in_full(get_lock_file_fd(&tables_list_lock), - tables_list_buf.buf, tables_list_buf.len); + for (i = 0; i < first_to_replace; i++) { + if ((err = reftable_buf_addstr(&tables_list_buf, names[i])) < 0 || + (err = reftable_buf_addstr(&tables_list_buf, "\n")) < 0) + goto done; + } + if (!is_empty_table) { + if ((err = reftable_buf_addstr(&tables_list_buf, new_table_name.buf)) < 0 || + (err = reftable_buf_addstr(&tables_list_buf, "\n")) < 0) + goto done; + } + for (i = last_to_replace + 1; names[i]; i++) { + if ((err = reftable_buf_addstr(&tables_list_buf, names[i])) < 0 || + (err = reftable_buf_addstr(&tables_list_buf, "\n")) < 0) + goto done; + } + + err = reftable_write_data(tables_list_lock.fd, + tables_list_buf.buf, tables_list_buf.len); if (err < 0) { err = REFTABLE_IO_ERROR; unlink(new_table_path.buf); goto done; } - err = fsync_component(FSYNC_COMPONENT_REFERENCE, get_lock_file_fd(&tables_list_lock)); + err = stack_fsync(&st->opts, tables_list_lock.fd); if (err < 0) { err = REFTABLE_IO_ERROR; unlink(new_table_path.buf); goto done; } - err = commit_lock_file(&tables_list_lock); + err = flock_commit(&tables_list_lock); if (err < 0) { err = REFTABLE_IO_ERROR; unlink(new_table_path.buf); @@ -1345,23 +1526,28 @@ static int stack_compact_range(struct reftable_stack *st, * readers, so it is expected that unlinking tables may fail. */ for (i = 0; i < nlocks; i++) { - struct lock_file *table_lock = &table_locks[i]; - char *table_path = get_locked_file_path(table_lock); - unlink(table_path); - free(table_path); + struct reftable_flock *table_lock = &table_locks[i]; + + reftable_buf_reset(&table_name); + err = reftable_buf_add(&table_name, table_lock->path, + strlen(table_lock->path) - strlen(".lock")); + if (err) + continue; + + unlink(table_name.buf); } done: - rollback_lock_file(&tables_list_lock); + flock_release(&tables_list_lock); for (i = 0; table_locks && i < nlocks; i++) - rollback_lock_file(&table_locks[i]); + flock_release(&table_locks[i]); reftable_free(table_locks); - delete_tempfile(&new_table); - strbuf_release(&new_table_name); - strbuf_release(&new_table_path); - strbuf_release(&tables_list_buf); - strbuf_release(&table_name); + tmpfile_delete(&new_table); + reftable_buf_release(&new_table_name); + reftable_buf_release(&new_table_path); + reftable_buf_release(&tables_list_buf); + reftable_buf_release(&table_name); free_names(names); if (err == REFTABLE_LOCK_ERROR) @@ -1460,11 +1646,13 @@ struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) { - int version = (st->opts.hash_id == GIT_SHA1_FORMAT_ID) ? 1 : 2; + int version = (st->opts.hash_id == REFTABLE_HASH_SHA1) ? 1 : 2; int overhead = header_size(version) - 1; uint64_t *sizes; REFTABLE_CALLOC_ARRAY(sizes, st->merged->readers_len); + if (!sizes) + return NULL; for (size_t i = 0; i < st->merged->readers_len; i++) sizes[i] = st->readers[i]->size - overhead; @@ -1474,11 +1662,20 @@ static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) int reftable_stack_auto_compact(struct reftable_stack *st) { - uint64_t *sizes = stack_table_sizes_for_compaction(st); - struct segment seg = - suggest_compaction_segment(sizes, st->merged->readers_len, - st->opts.auto_compaction_factor); + struct segment seg; + uint64_t *sizes; + + if (st->merged->readers_len < 2) + return 0; + + sizes = stack_table_sizes_for_compaction(st); + if (!sizes) + return REFTABLE_OUT_OF_MEMORY_ERROR; + + seg = suggest_compaction_segment(sizes, st->merged->readers_len, + st->opts.auto_compaction_factor); reftable_free(sizes); + if (segment_size(&seg) > 0) return stack_compact_range(st, seg.start, seg.end - 1, NULL, STACK_COMPACT_RANGE_BEST_EFFORT); @@ -1498,7 +1695,10 @@ int reftable_stack_read_ref(struct reftable_stack *st, const char *refname, struct reftable_iterator it = { 0 }; int ret; - reftable_merged_table_init_ref_iterator(st->merged, &it); + ret = reftable_merged_table_init_ref_iterator(st->merged, &it); + if (ret) + goto out; + ret = reftable_iterator_seek_ref(&it, refname); if (ret) goto out; @@ -1525,7 +1725,10 @@ int reftable_stack_read_log(struct reftable_stack *st, const char *refname, struct reftable_iterator it = {0}; int err; - reftable_stack_init_log_iterator(st, &it); + err = reftable_stack_init_log_iterator(st, &it); + if (err) + goto done; + err = reftable_iterator_seek_log(&it, refname); if (err) goto done; @@ -1561,8 +1764,11 @@ static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max, uint64_t update_idx = 0; struct reftable_block_source src = { NULL }; struct reftable_reader *rd = NULL; - struct strbuf table_path = STRBUF_INIT; - stack_filename(&table_path, st, name); + struct reftable_buf table_path = REFTABLE_BUF_INIT; + + err = stack_filename(&table_path, st, name); + if (err < 0) + goto done; err = reftable_block_source_from_file(&src, table_path.buf); if (err < 0) @@ -1579,7 +1785,7 @@ static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max, unlink(table_path.buf); } done: - strbuf_release(&table_path); + reftable_buf_release(&table_path); } static int reftable_stack_clean_locked(struct reftable_stack *st) @@ -1593,14 +1799,12 @@ static int reftable_stack_clean_locked(struct reftable_stack *st) } while ((d = readdir(dir))) { - int i = 0; int found = 0; if (!is_table_name(d->d_name)) continue; - for (i = 0; !found && i < st->readers_len; i++) { + for (size_t i = 0; !found && i < st->readers_len; i++) found = !strcmp(reader_name(st->readers[i]), d->d_name); - } if (found) continue; @@ -1630,3 +1834,8 @@ done: reftable_addition_destroy(add); return err; } + +enum reftable_hash reftable_stack_hash_id(struct reftable_stack *st) +{ + return reftable_merged_table_hash_id(st->merged); +} |