diff options
Diffstat (limited to 't/unit-tests/t-reftable-stack.c')
| -rw-r--r-- | t/unit-tests/t-reftable-stack.c | 1394 |
1 files changed, 1394 insertions, 0 deletions
diff --git a/t/unit-tests/t-reftable-stack.c b/t/unit-tests/t-reftable-stack.c new file mode 100644 index 0000000000..72f6747064 --- /dev/null +++ b/t/unit-tests/t-reftable-stack.c @@ -0,0 +1,1394 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "test-lib.h" +#include "lib-reftable.h" +#include "reftable/merged.h" +#include "reftable/reader.h" +#include "reftable/reftable-error.h" +#include "reftable/stack.h" +#include <dirent.h> + +static void clear_dir(const char *dirname) +{ + struct strbuf path = REFTABLE_BUF_INIT; + strbuf_addstr(&path, dirname); + remove_dir_recursively(&path, 0); + strbuf_release(&path); +} + +static int count_dir_entries(const char *dirname) +{ + DIR *dir = opendir(dirname); + int len = 0; + struct dirent *d; + if (!dir) + return 0; + + while ((d = readdir(dir))) { + /* + * Besides skipping over "." and "..", we also need to + * skip over other files that have a leading ".". This + * is due to behaviour of NFS, which will rename files + * to ".nfs*" to emulate delete-on-last-close. + * + * In any case this should be fine as the reftable + * library will never write files with leading dots + * anyway. + */ + if (starts_with(d->d_name, ".")) + continue; + len++; + } + closedir(dir); + return len; +} + +/* + * Work linenumber into the tempdir, so we can see which tests forget to + * cleanup. + */ +static char *get_tmp_template(int linenumber) +{ + const char *tmp = getenv("TMPDIR"); + static char template[1024]; + snprintf(template, sizeof(template) - 1, "%s/stack_test-%d.XXXXXX", + tmp ? tmp : "/tmp", linenumber); + return template; +} + +static char *get_tmp_dir(int linenumber) +{ + char *dir = get_tmp_template(linenumber); + check(mkdtemp(dir) != NULL); + return dir; +} + +static void t_read_file(void) +{ + char *fn = get_tmp_template(__LINE__); + struct tempfile *tmp = mks_tempfile(fn); + int fd = get_tempfile_fd(tmp); + char out[1024] = "line1\n\nline2\nline3"; + int n, err; + char **names = NULL; + const char *want[] = { "line1", "line2", "line3" }; + + check_int(fd, >, 0); + n = write_in_full(fd, out, strlen(out)); + check_int(n, ==, strlen(out)); + err = close(fd); + check_int(err, >=, 0); + + err = read_lines(fn, &names); + check(!err); + + for (size_t i = 0; names[i]; i++) + check_str(want[i], names[i]); + free_names(names); + (void) remove(fn); + delete_tempfile(&tmp); +} + +static int write_test_ref(struct reftable_writer *wr, void *arg) +{ + struct reftable_ref_record *ref = arg; + reftable_writer_set_limits(wr, ref->update_index, ref->update_index); + return reftable_writer_add_ref(wr, ref); +} + +static void write_n_ref_tables(struct reftable_stack *st, + size_t n) +{ + int disable_auto_compact; + int err; + + disable_auto_compact = st->opts.disable_auto_compact; + st->opts.disable_auto_compact = 1; + + for (size_t i = 0; i < n; i++) { + struct reftable_ref_record ref = { + .update_index = reftable_stack_next_update_index(st), + .value_type = REFTABLE_REF_VAL1, + }; + char buf[128]; + + snprintf(buf, sizeof(buf), "refs/heads/branch-%04"PRIuMAX, (uintmax_t)i); + ref.refname = buf; + t_reftable_set_hash(ref.value.val1, i, GIT_SHA1_FORMAT_ID); + + err = reftable_stack_add(st, &write_test_ref, &ref); + check(!err); + } + + st->opts.disable_auto_compact = disable_auto_compact; +} + +struct write_log_arg { + struct reftable_log_record *log; + uint64_t update_index; +}; + +static int write_test_log(struct reftable_writer *wr, void *arg) +{ + struct write_log_arg *wla = arg; + + reftable_writer_set_limits(wr, wla->update_index, wla->update_index); + return reftable_writer_add_log(wr, wla->log); +} + +static void t_reftable_stack_add_one(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_buf scratch = REFTABLE_BUF_INIT; + int mask = umask(002); + struct reftable_write_options opts = { + .default_permissions = 0660, + }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record ref = { + .refname = (char *) "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + struct reftable_ref_record dest = { 0 }; + struct stat stat_result = { 0 }; + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + err = reftable_stack_add(st, write_test_ref, &ref); + check(!err); + + err = reftable_stack_read_ref(st, ref.refname, &dest); + check(!err); + check(reftable_ref_record_equal(&ref, &dest, GIT_SHA1_RAWSZ)); + check_int(st->readers_len, >, 0); + +#ifndef GIT_WINDOWS_NATIVE + check(!reftable_buf_addstr(&scratch, dir)); + check(!reftable_buf_addstr(&scratch, "/tables.list")); + err = stat(scratch.buf, &stat_result); + check(!err); + check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); + + reftable_buf_reset(&scratch); + check(!reftable_buf_addstr(&scratch, dir)); + check(!reftable_buf_addstr(&scratch, "/")); + /* do not try at home; not an external API for reftable. */ + check(!reftable_buf_addstr(&scratch, st->readers[0]->name)); + err = stat(scratch.buf, &stat_result); + check(!err); + check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); +#else + (void) stat_result; +#endif + + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + reftable_buf_release(&scratch); + clear_dir(dir); + umask(mask); +} + +static void t_reftable_stack_uptodate(void) +{ + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st1 = NULL; + struct reftable_stack *st2 = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err; + struct reftable_ref_record ref1 = { + .refname = (char *) "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + struct reftable_ref_record ref2 = { + .refname = (char *) "branch2", + .update_index = 2, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + + + /* simulate multi-process access to the same stack + by creating two stacks for the same directory. + */ + err = reftable_new_stack(&st1, dir, &opts); + check(!err); + + err = reftable_new_stack(&st2, dir, &opts); + check(!err); + + err = reftable_stack_add(st1, write_test_ref, &ref1); + check(!err); + + err = reftable_stack_add(st2, write_test_ref, &ref2); + check_int(err, ==, REFTABLE_OUTDATED_ERROR); + + err = reftable_stack_reload(st2); + check(!err); + + err = reftable_stack_add(st2, write_test_ref, &ref2); + check(!err); + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + clear_dir(dir); +} + +static void t_reftable_stack_transaction_api(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_addition *add = NULL; + + struct reftable_ref_record ref = { + .refname = (char *) "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + struct reftable_ref_record dest = { 0 }; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + reftable_addition_destroy(add); + + err = reftable_stack_new_addition(&add, st, 0); + check(!err); + + err = reftable_addition_add(add, write_test_ref, &ref); + check(!err); + + err = reftable_addition_commit(add); + check(!err); + + reftable_addition_destroy(add); + + err = reftable_stack_read_ref(st, ref.refname, &dest); + check(!err); + check_int(REFTABLE_REF_SYMREF, ==, dest.value_type); + check(reftable_ref_record_equal(&ref, &dest, GIT_SHA1_RAWSZ)); + + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void t_reftable_stack_transaction_with_reload(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_stack *st1 = NULL, *st2 = NULL; + int err; + struct reftable_addition *add = NULL; + struct reftable_ref_record refs[2] = { + { + .refname = (char *) "refs/heads/a", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = { '1' }, + }, + { + .refname = (char *) "refs/heads/b", + .update_index = 2, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = { '1' }, + }, + }; + struct reftable_ref_record ref = { 0 }; + + err = reftable_new_stack(&st1, dir, NULL); + check(!err); + err = reftable_new_stack(&st2, dir, NULL); + check(!err); + + err = reftable_stack_new_addition(&add, st1, 0); + check(!err); + err = reftable_addition_add(add, write_test_ref, &refs[0]); + check(!err); + err = reftable_addition_commit(add); + check(!err); + reftable_addition_destroy(add); + + /* + * The second stack is now outdated, which we should notice. We do not + * create the addition and lock the stack by default, but allow the + * reload to happen when REFTABLE_STACK_NEW_ADDITION_RELOAD is set. + */ + err = reftable_stack_new_addition(&add, st2, 0); + check_int(err, ==, REFTABLE_OUTDATED_ERROR); + err = reftable_stack_new_addition(&add, st2, REFTABLE_STACK_NEW_ADDITION_RELOAD); + check(!err); + err = reftable_addition_add(add, write_test_ref, &refs[1]); + check(!err); + err = reftable_addition_commit(add); + check(!err); + reftable_addition_destroy(add); + + for (size_t i = 0; i < ARRAY_SIZE(refs); i++) { + err = reftable_stack_read_ref(st2, refs[i].refname, &ref); + check(!err); + check(reftable_ref_record_equal(&refs[i], &ref, GIT_SHA1_RAWSZ)); + } + + reftable_ref_record_release(&ref); + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + clear_dir(dir); +} + +static void t_reftable_stack_transaction_api_performs_auto_compaction(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_write_options opts = {0}; + struct reftable_addition *add = NULL; + struct reftable_stack *st = NULL; + size_t n = 20; + int err; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + for (size_t i = 0; i <= n; i++) { + struct reftable_ref_record ref = { + .update_index = reftable_stack_next_update_index(st), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + char name[100]; + + snprintf(name, sizeof(name), "branch%04"PRIuMAX, (uintmax_t)i); + ref.refname = name; + + /* + * Disable auto-compaction for all but the last runs. Like this + * we can ensure that we indeed honor this setting and have + * better control over when exactly auto compaction runs. + */ + st->opts.disable_auto_compact = i != n; + + err = reftable_stack_new_addition(&add, st, 0); + check(!err); + + err = reftable_addition_add(add, write_test_ref, &ref); + check(!err); + + err = reftable_addition_commit(add); + check(!err); + + reftable_addition_destroy(add); + + /* + * The stack length should grow continuously for all runs where + * auto compaction is disabled. When enabled, we should merge + * all tables in the stack. + */ + if (i != n) + check_int(st->merged->readers_len, ==, i + 1); + else + check_int(st->merged->readers_len, ==, 1); + } + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void t_reftable_stack_auto_compaction_fails_gracefully(void) +{ + struct reftable_ref_record ref = { + .refname = (char *) "refs/heads/master", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = {0x01}, + }; + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st; + struct reftable_buf table_path = REFTABLE_BUF_INIT; + char *dir = get_tmp_dir(__LINE__); + int err; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + err = reftable_stack_add(st, write_test_ref, &ref); + check(!err); + check_int(st->merged->readers_len, ==, 1); + check_int(st->stats.attempts, ==, 0); + check_int(st->stats.failures, ==, 0); + + /* + * Lock the newly written table such that it cannot be compacted. + * Adding a new table to the stack should not be impacted by this, even + * though auto-compaction will now fail. + */ + check(!reftable_buf_addstr(&table_path, dir)); + check(!reftable_buf_addstr(&table_path, "/")); + check(!reftable_buf_addstr(&table_path, st->readers[0]->name)); + check(!reftable_buf_addstr(&table_path, ".lock")); + write_file_buf(table_path.buf, "", 0); + + ref.update_index = 2; + err = reftable_stack_add(st, write_test_ref, &ref); + check(!err); + check_int(st->merged->readers_len, ==, 2); + check_int(st->stats.attempts, ==, 1); + check_int(st->stats.failures, ==, 1); + + reftable_stack_destroy(st); + reftable_buf_release(&table_path); + clear_dir(dir); +} + +static int write_error(struct reftable_writer *wr UNUSED, void *arg) +{ + return *((int *)arg); +} + +static void t_reftable_stack_update_index_check(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record ref1 = { + .refname = (char *) "name1", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + struct reftable_ref_record ref2 = { + .refname = (char *) "name2", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + err = reftable_stack_add(st, write_test_ref, &ref1); + check(!err); + + err = reftable_stack_add(st, write_test_ref, &ref2); + check_int(err, ==, REFTABLE_API_ERROR); + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void t_reftable_stack_lock_failure(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + int err, i; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + for (i = -1; i != REFTABLE_EMPTY_TABLE_ERROR; i--) { + err = reftable_stack_add(st, write_error, &i); + check_int(err, ==, i); + } + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void t_reftable_stack_add(void) +{ + int err = 0; + struct reftable_write_options opts = { + .exact_log_message = 1, + .default_permissions = 0660, + .disable_auto_compact = 1, + }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + struct reftable_ref_record refs[2] = { 0 }; + struct reftable_log_record logs[2] = { 0 }; + struct reftable_buf path = REFTABLE_BUF_INIT; + struct stat stat_result; + size_t i, N = ARRAY_SIZE(refs); + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + for (i = 0; i < N; i++) { + char buf[256]; + snprintf(buf, sizeof(buf), "branch%02"PRIuMAX, (uintmax_t)i); + refs[i].refname = xstrdup(buf); + refs[i].update_index = i + 1; + refs[i].value_type = REFTABLE_REF_VAL1; + t_reftable_set_hash(refs[i].value.val1, i, GIT_SHA1_FORMAT_ID); + + logs[i].refname = xstrdup(buf); + logs[i].update_index = N + i + 1; + logs[i].value_type = REFTABLE_LOG_UPDATE; + logs[i].value.update.email = xstrdup("identity@invalid"); + t_reftable_set_hash(logs[i].value.update.new_hash, i, GIT_SHA1_FORMAT_ID); + } + + for (i = 0; i < N; i++) { + int err = reftable_stack_add(st, write_test_ref, &refs[i]); + check(!err); + } + + for (i = 0; i < N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, write_test_log, &arg); + check(!err); + } + + err = reftable_stack_compact_all(st, NULL); + check(!err); + + for (i = 0; i < N; i++) { + struct reftable_ref_record dest = { 0 }; + + int err = reftable_stack_read_ref(st, refs[i].refname, &dest); + check(!err); + check(reftable_ref_record_equal(&dest, refs + i, + GIT_SHA1_RAWSZ)); + reftable_ref_record_release(&dest); + } + + for (i = 0; i < N; i++) { + struct reftable_log_record dest = { 0 }; + int err = reftable_stack_read_log(st, refs[i].refname, &dest); + check(!err); + check(reftable_log_record_equal(&dest, logs + i, + GIT_SHA1_RAWSZ)); + reftable_log_record_release(&dest); + } + +#ifndef GIT_WINDOWS_NATIVE + check(!reftable_buf_addstr(&path, dir)); + check(!reftable_buf_addstr(&path, "/tables.list")); + err = stat(path.buf, &stat_result); + check(!err); + check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); + + reftable_buf_reset(&path); + check(!reftable_buf_addstr(&path, dir)); + check(!reftable_buf_addstr(&path, "/")); + /* do not try at home; not an external API for reftable. */ + check(!reftable_buf_addstr(&path, st->readers[0]->name)); + err = stat(path.buf, &stat_result); + check(!err); + check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); +#else + (void) stat_result; +#endif + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i < N; i++) { + reftable_ref_record_release(&refs[i]); + reftable_log_record_release(&logs[i]); + } + reftable_buf_release(&path); + clear_dir(dir); +} + +static void t_reftable_stack_iterator(void) +{ + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + struct reftable_ref_record refs[10] = { 0 }; + struct reftable_log_record logs[10] = { 0 }; + struct reftable_iterator it = { 0 }; + size_t N = ARRAY_SIZE(refs), i; + int err; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + for (i = 0; i < N; i++) { + refs[i].refname = xstrfmt("branch%02"PRIuMAX, (uintmax_t)i); + refs[i].update_index = i + 1; + refs[i].value_type = REFTABLE_REF_VAL1; + t_reftable_set_hash(refs[i].value.val1, i, GIT_SHA1_FORMAT_ID); + + logs[i].refname = xstrfmt("branch%02"PRIuMAX, (uintmax_t)i); + logs[i].update_index = i + 1; + logs[i].value_type = REFTABLE_LOG_UPDATE; + logs[i].value.update.email = xstrdup("johndoe@invalid"); + logs[i].value.update.message = xstrdup("commit\n"); + t_reftable_set_hash(logs[i].value.update.new_hash, i, GIT_SHA1_FORMAT_ID); + } + + for (i = 0; i < N; i++) { + err = reftable_stack_add(st, write_test_ref, &refs[i]); + check(!err); + } + + for (i = 0; i < N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + + err = reftable_stack_add(st, write_test_log, &arg); + check(!err); + } + + reftable_stack_init_ref_iterator(st, &it); + reftable_iterator_seek_ref(&it, refs[0].refname); + for (i = 0; ; i++) { + struct reftable_ref_record ref = { 0 }; + + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) + break; + check(!err); + check(reftable_ref_record_equal(&ref, &refs[i], GIT_SHA1_RAWSZ)); + reftable_ref_record_release(&ref); + } + check_int(i, ==, N); + + reftable_iterator_destroy(&it); + + err = reftable_stack_init_log_iterator(st, &it); + check(!err); + + reftable_iterator_seek_log(&it, logs[0].refname); + for (i = 0; ; i++) { + struct reftable_log_record log = { 0 }; + + err = reftable_iterator_next_log(&it, &log); + if (err > 0) + break; + check(!err); + check(reftable_log_record_equal(&log, &logs[i], GIT_SHA1_RAWSZ)); + reftable_log_record_release(&log); + } + check_int(i, ==, N); + + reftable_stack_destroy(st); + reftable_iterator_destroy(&it); + for (i = 0; i < N; i++) { + reftable_ref_record_release(&refs[i]); + reftable_log_record_release(&logs[i]); + } + clear_dir(dir); +} + +static void t_reftable_stack_log_normalize(void) +{ + int err = 0; + struct reftable_write_options opts = { + 0, + }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + struct reftable_log_record input = { + .refname = (char *) "branch", + .update_index = 1, + .value_type = REFTABLE_LOG_UPDATE, + .value = { + .update = { + .new_hash = { 1 }, + .old_hash = { 2 }, + }, + }, + }; + struct reftable_log_record dest = { + .update_index = 0, + }; + struct write_log_arg arg = { + .log = &input, + .update_index = 1, + }; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + input.value.update.message = (char *) "one\ntwo"; + err = reftable_stack_add(st, write_test_log, &arg); + check_int(err, ==, REFTABLE_API_ERROR); + + input.value.update.message = (char *) "one"; + err = reftable_stack_add(st, write_test_log, &arg); + check(!err); + + err = reftable_stack_read_log(st, input.refname, &dest); + check(!err); + check_str(dest.value.update.message, "one\n"); + + input.value.update.message = (char *) "two\n"; + arg.update_index = 2; + err = reftable_stack_add(st, write_test_log, &arg); + check(!err); + err = reftable_stack_read_log(st, input.refname, &dest); + check(!err); + check_str(dest.value.update.message, "two\n"); + + /* cleanup */ + reftable_stack_destroy(st); + reftable_log_record_release(&dest); + clear_dir(dir); +} + +static void t_reftable_stack_tombstone(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record refs[2] = { 0 }; + struct reftable_log_record logs[2] = { 0 }; + size_t i, N = ARRAY_SIZE(refs); + struct reftable_ref_record dest = { 0 }; + struct reftable_log_record log_dest = { 0 }; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + /* even entries add the refs, odd entries delete them. */ + for (i = 0; i < N; i++) { + const char *buf = "branch"; + refs[i].refname = xstrdup(buf); + refs[i].update_index = i + 1; + if (i % 2 == 0) { + refs[i].value_type = REFTABLE_REF_VAL1; + t_reftable_set_hash(refs[i].value.val1, i, + GIT_SHA1_FORMAT_ID); + } + + logs[i].refname = xstrdup(buf); + /* update_index is part of the key. */ + logs[i].update_index = 42; + if (i % 2 == 0) { + logs[i].value_type = REFTABLE_LOG_UPDATE; + t_reftable_set_hash(logs[i].value.update.new_hash, i, + GIT_SHA1_FORMAT_ID); + logs[i].value.update.email = + xstrdup("identity@invalid"); + } + } + for (i = 0; i < N; i++) { + int err = reftable_stack_add(st, write_test_ref, &refs[i]); + check(!err); + } + + for (i = 0; i < N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, write_test_log, &arg); + check(!err); + } + + err = reftable_stack_read_ref(st, "branch", &dest); + check_int(err, ==, 1); + reftable_ref_record_release(&dest); + + err = reftable_stack_read_log(st, "branch", &log_dest); + check_int(err, ==, 1); + reftable_log_record_release(&log_dest); + + err = reftable_stack_compact_all(st, NULL); + check(!err); + + err = reftable_stack_read_ref(st, "branch", &dest); + check_int(err, ==, 1); + + err = reftable_stack_read_log(st, "branch", &log_dest); + check_int(err, ==, 1); + reftable_ref_record_release(&dest); + reftable_log_record_release(&log_dest); + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i < N; i++) { + reftable_ref_record_release(&refs[i]); + reftable_log_record_release(&logs[i]); + } + clear_dir(dir); +} + +static void t_reftable_stack_hash_id(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + int err; + + struct reftable_ref_record ref = { + .refname = (char *) "master", + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "target", + .update_index = 1, + }; + struct reftable_write_options opts32 = { .hash_id = GIT_SHA256_FORMAT_ID }; + struct reftable_stack *st32 = NULL; + struct reftable_write_options opts_default = { 0 }; + struct reftable_stack *st_default = NULL; + struct reftable_ref_record dest = { 0 }; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + err = reftable_stack_add(st, write_test_ref, &ref); + check(!err); + + /* can't read it with the wrong hash ID. */ + err = reftable_new_stack(&st32, dir, &opts32); + check_int(err, ==, REFTABLE_FORMAT_ERROR); + + /* check that we can read it back with default opts too. */ + err = reftable_new_stack(&st_default, dir, &opts_default); + check(!err); + + err = reftable_stack_read_ref(st_default, "master", &dest); + check(!err); + + check(reftable_ref_record_equal(&ref, &dest, GIT_SHA1_RAWSZ)); + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + reftable_stack_destroy(st_default); + clear_dir(dir); +} + +static void t_suggest_compaction_segment(void) +{ + uint64_t sizes[] = { 512, 64, 17, 16, 9, 9, 9, 16, 2, 16 }; + struct segment min = + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes), 2); + check_int(min.start, ==, 1); + check_int(min.end, ==, 10); +} + +static void t_suggest_compaction_segment_nothing(void) +{ + uint64_t sizes[] = { 64, 32, 16, 8, 4, 2 }; + struct segment result = + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes), 2); + check_int(result.start, ==, result.end); +} + +static void t_reflog_expire(void) +{ + char *dir = get_tmp_dir(__LINE__); + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + struct reftable_log_record logs[20] = { 0 }; + size_t i, N = ARRAY_SIZE(logs) - 1; + int err; + struct reftable_log_expiry_config expiry = { + .time = 10, + }; + struct reftable_log_record log = { 0 }; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + for (i = 1; i <= N; i++) { + char buf[256]; + snprintf(buf, sizeof(buf), "branch%02"PRIuMAX, (uintmax_t)i); + + logs[i].refname = xstrdup(buf); + logs[i].update_index = i; + logs[i].value_type = REFTABLE_LOG_UPDATE; + logs[i].value.update.time = i; + logs[i].value.update.email = xstrdup("identity@invalid"); + t_reftable_set_hash(logs[i].value.update.new_hash, i, + GIT_SHA1_FORMAT_ID); + } + + for (i = 1; i <= N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, write_test_log, &arg); + check(!err); + } + + err = reftable_stack_compact_all(st, NULL); + check(!err); + + err = reftable_stack_compact_all(st, &expiry); + check(!err); + + err = reftable_stack_read_log(st, logs[9].refname, &log); + check_int(err, ==, 1); + + err = reftable_stack_read_log(st, logs[11].refname, &log); + check(!err); + + expiry.min_update_index = 15; + err = reftable_stack_compact_all(st, &expiry); + check(!err); + + err = reftable_stack_read_log(st, logs[14].refname, &log); + check_int(err, ==, 1); + + err = reftable_stack_read_log(st, logs[16].refname, &log); + check(!err); + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i <= N; i++) + reftable_log_record_release(&logs[i]); + clear_dir(dir); + reftable_log_record_release(&log); +} + +static int write_nothing(struct reftable_writer *wr, void *arg UNUSED) +{ + reftable_writer_set_limits(wr, 1, 1); + return 0; +} + +static void t_empty_add(void) +{ + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + int err; + char *dir = get_tmp_dir(__LINE__); + struct reftable_stack *st2 = NULL; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + err = reftable_stack_add(st, write_nothing, NULL); + check(!err); + + err = reftable_new_stack(&st2, dir, &opts); + check(!err); + clear_dir(dir); + reftable_stack_destroy(st); + reftable_stack_destroy(st2); +} + +static int fastlogN(uint64_t sz, uint64_t N) +{ + int l = 0; + if (sz == 0) + return 0; + for (; sz; sz /= N) + l++; + return l - 1; +} + +static void t_reftable_stack_auto_compaction(void) +{ + struct reftable_write_options opts = { + .disable_auto_compact = 1, + }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + int err; + size_t i, N = 100; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + for (i = 0; i < N; i++) { + char name[100]; + struct reftable_ref_record ref = { + .refname = name, + .update_index = reftable_stack_next_update_index(st), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + snprintf(name, sizeof(name), "branch%04"PRIuMAX, (uintmax_t)i); + + err = reftable_stack_add(st, write_test_ref, &ref); + check(!err); + + err = reftable_stack_auto_compact(st); + check(!err); + check(i < 2 || st->merged->readers_len < 2 * fastlogN(i, 2)); + } + + check_int(reftable_stack_compaction_stats(st)->entries_written, <, + (uint64_t)(N * fastlogN(N, 2))); + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void t_reftable_stack_auto_compaction_factor(void) +{ + struct reftable_write_options opts = { + .auto_compaction_factor = 5, + }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + int err; + size_t N = 100; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + for (size_t i = 0; i < N; i++) { + char name[20]; + struct reftable_ref_record ref = { + .refname = name, + .update_index = reftable_stack_next_update_index(st), + .value_type = REFTABLE_REF_VAL1, + }; + xsnprintf(name, sizeof(name), "branch%04"PRIuMAX, (uintmax_t)i); + + err = reftable_stack_add(st, &write_test_ref, &ref); + check(!err); + + check(i < 5 || st->merged->readers_len < 5 * fastlogN(i, 5)); + } + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void t_reftable_stack_auto_compaction_with_locked_tables(void) +{ + struct reftable_write_options opts = { + .disable_auto_compact = 1, + }; + struct reftable_stack *st = NULL; + struct reftable_buf buf = REFTABLE_BUF_INIT; + char *dir = get_tmp_dir(__LINE__); + int err; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + write_n_ref_tables(st, 5); + check_int(st->merged->readers_len, ==, 5); + + /* + * Given that all tables we have written should be roughly the same + * size, we expect that auto-compaction will want to compact all of the + * tables. Locking any of the tables will keep it from doing so. + */ + check(!reftable_buf_addstr(&buf, dir)); + check(!reftable_buf_addstr(&buf, "/")); + check(!reftable_buf_addstr(&buf, st->readers[2]->name)); + check(!reftable_buf_addstr(&buf, ".lock")); + write_file_buf(buf.buf, "", 0); + + /* + * When parts of the stack are locked, then auto-compaction does a best + * effort compaction of those tables which aren't locked. So while this + * would in theory compact all tables, due to the preexisting lock we + * only compact the newest two tables. + */ + err = reftable_stack_auto_compact(st); + check(!err); + check_int(st->stats.failures, ==, 0); + check_int(st->merged->readers_len, ==, 4); + + reftable_stack_destroy(st); + reftable_buf_release(&buf); + clear_dir(dir); +} + +static void t_reftable_stack_add_performs_auto_compaction(void) +{ + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + int err; + size_t i, n = 20; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + for (i = 0; i <= n; i++) { + struct reftable_ref_record ref = { + .update_index = reftable_stack_next_update_index(st), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = (char *) "master", + }; + char buf[128]; + + /* + * Disable auto-compaction for all but the last runs. Like this + * we can ensure that we indeed honor this setting and have + * better control over when exactly auto compaction runs. + */ + st->opts.disable_auto_compact = i != n; + + snprintf(buf, sizeof(buf), "branch-%04"PRIuMAX, (uintmax_t)i); + ref.refname = buf; + + err = reftable_stack_add(st, write_test_ref, &ref); + check(!err); + + /* + * The stack length should grow continuously for all runs where + * auto compaction is disabled. When enabled, we should merge + * all tables in the stack. + */ + if (i != n) + check_int(st->merged->readers_len, ==, i + 1); + else + check_int(st->merged->readers_len, ==, 1); + } + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void t_reftable_stack_compaction_with_locked_tables(void) +{ + struct reftable_write_options opts = { + .disable_auto_compact = 1, + }; + struct reftable_stack *st = NULL; + struct reftable_buf buf = REFTABLE_BUF_INIT; + char *dir = get_tmp_dir(__LINE__); + int err; + + err = reftable_new_stack(&st, dir, &opts); + check(!err); + + write_n_ref_tables(st, 3); + check_int(st->merged->readers_len, ==, 3); + + /* Lock one of the tables that we're about to compact. */ + check(!reftable_buf_addstr(&buf, dir)); + check(!reftable_buf_addstr(&buf, "/")); + check(!reftable_buf_addstr(&buf, st->readers[1]->name)); + check(!reftable_buf_addstr(&buf, ".lock")); + write_file_buf(buf.buf, "", 0); + + /* + * Compaction is expected to fail given that we were not able to + * compact all tables. + */ + err = reftable_stack_compact_all(st, NULL); + check_int(err, ==, REFTABLE_LOCK_ERROR); + check_int(st->stats.failures, ==, 1); + check_int(st->merged->readers_len, ==, 3); + + reftable_stack_destroy(st); + reftable_buf_release(&buf); + clear_dir(dir); +} + +static void t_reftable_stack_compaction_concurrent(void) +{ + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st1 = NULL, *st2 = NULL; + char *dir = get_tmp_dir(__LINE__); + int err; + + err = reftable_new_stack(&st1, dir, &opts); + check(!err); + write_n_ref_tables(st1, 3); + + err = reftable_new_stack(&st2, dir, &opts); + check(!err); + + err = reftable_stack_compact_all(st1, NULL); + check(!err); + + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + + check_int(count_dir_entries(dir), ==, 2); + clear_dir(dir); +} + +static void unclean_stack_close(struct reftable_stack *st) +{ + /* break abstraction boundary to simulate unclean shutdown. */ + for (size_t i = 0; i < st->readers_len; i++) + reftable_reader_decref(st->readers[i]); + st->readers_len = 0; + REFTABLE_FREE_AND_NULL(st->readers); +} + +static void t_reftable_stack_compaction_concurrent_clean(void) +{ + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st1 = NULL, *st2 = NULL, *st3 = NULL; + char *dir = get_tmp_dir(__LINE__); + int err; + + err = reftable_new_stack(&st1, dir, &opts); + check(!err); + write_n_ref_tables(st1, 3); + + err = reftable_new_stack(&st2, dir, &opts); + check(!err); + + err = reftable_stack_compact_all(st1, NULL); + check(!err); + + unclean_stack_close(st1); + unclean_stack_close(st2); + + err = reftable_new_stack(&st3, dir, &opts); + check(!err); + + err = reftable_stack_clean(st3); + check(!err); + check_int(count_dir_entries(dir), ==, 2); + + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + reftable_stack_destroy(st3); + + clear_dir(dir); +} + +static void t_reftable_stack_read_across_reload(void) +{ + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st1 = NULL, *st2 = NULL; + struct reftable_ref_record rec = { 0 }; + struct reftable_iterator it = { 0 }; + char *dir = get_tmp_dir(__LINE__); + int err; + + /* Create a first stack and set up an iterator for it. */ + err = reftable_new_stack(&st1, dir, &opts); + check(!err); + write_n_ref_tables(st1, 2); + check_int(st1->merged->readers_len, ==, 2); + reftable_stack_init_ref_iterator(st1, &it); + err = reftable_iterator_seek_ref(&it, ""); + check(!err); + + /* Set up a second stack for the same directory and compact it. */ + err = reftable_new_stack(&st2, dir, &opts); + check(!err); + check_int(st2->merged->readers_len, ==, 2); + err = reftable_stack_compact_all(st2, NULL); + check(!err); + check_int(st2->merged->readers_len, ==, 1); + + /* + * Verify that we can continue to use the old iterator even after we + * have reloaded its stack. + */ + err = reftable_stack_reload(st1); + check(!err); + check_int(st1->merged->readers_len, ==, 1); + err = reftable_iterator_next_ref(&it, &rec); + check(!err); + check_str(rec.refname, "refs/heads/branch-0000"); + err = reftable_iterator_next_ref(&it, &rec); + check(!err); + check_str(rec.refname, "refs/heads/branch-0001"); + err = reftable_iterator_next_ref(&it, &rec); + check_int(err, >, 0); + + reftable_ref_record_release(&rec); + reftable_iterator_destroy(&it); + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + clear_dir(dir); +} + +static void t_reftable_stack_reload_with_missing_table(void) +{ + struct reftable_write_options opts = { 0 }; + struct reftable_stack *st = NULL; + struct reftable_ref_record rec = { 0 }; + struct reftable_iterator it = { 0 }; + struct reftable_buf table_path = REFTABLE_BUF_INIT, content = REFTABLE_BUF_INIT; + char *dir = get_tmp_dir(__LINE__); + int err; + + /* Create a first stack and set up an iterator for it. */ + err = reftable_new_stack(&st, dir, &opts); + check(!err); + write_n_ref_tables(st, 2); + check_int(st->merged->readers_len, ==, 2); + reftable_stack_init_ref_iterator(st, &it); + err = reftable_iterator_seek_ref(&it, ""); + check(!err); + + /* + * Update the tables.list file with some garbage data, while reusing + * our old readers. This should trigger a partial reload of the stack, + * where we try to reuse our old readers. + */ + check(!reftable_buf_addstr(&content, st->readers[0]->name)); + check(!reftable_buf_addstr(&content, "\n")); + check(!reftable_buf_addstr(&content, st->readers[1]->name)); + check(!reftable_buf_addstr(&content, "\n")); + check(!reftable_buf_addstr(&content, "garbage\n")); + check(!reftable_buf_addstr(&table_path, st->list_file)); + check(!reftable_buf_addstr(&table_path, ".lock")); + write_file_buf(table_path.buf, content.buf, content.len); + err = rename(table_path.buf, st->list_file); + check(!err); + + err = reftable_stack_reload(st); + check_int(err, ==, -4); + check_int(st->merged->readers_len, ==, 2); + + /* + * Even though the reload has failed, we should be able to continue + * using the iterator. + */ + err = reftable_iterator_next_ref(&it, &rec); + check(!err); + check_str(rec.refname, "refs/heads/branch-0000"); + err = reftable_iterator_next_ref(&it, &rec); + check(!err); + check_str(rec.refname, "refs/heads/branch-0001"); + err = reftable_iterator_next_ref(&it, &rec); + check_int(err, >, 0); + + reftable_ref_record_release(&rec); + reftable_iterator_destroy(&it); + reftable_stack_destroy(st); + reftable_buf_release(&table_path); + reftable_buf_release(&content); + clear_dir(dir); +} + +int cmd_main(int argc UNUSED, const char *argv[] UNUSED) +{ + TEST(t_empty_add(), "empty addition to stack"); + TEST(t_read_file(), "read_lines works"); + TEST(t_reflog_expire(), "expire reflog entries"); + TEST(t_reftable_stack_add(), "add multiple refs and logs to stack"); + TEST(t_reftable_stack_add_one(), "add a single ref record to stack"); + TEST(t_reftable_stack_add_performs_auto_compaction(), "addition to stack triggers auto-compaction"); + TEST(t_reftable_stack_auto_compaction(), "stack must form geometric sequence after compaction"); + TEST(t_reftable_stack_auto_compaction_factor(), "auto-compaction with non-default geometric factor"); + TEST(t_reftable_stack_auto_compaction_fails_gracefully(), "failure on auto-compaction"); + TEST(t_reftable_stack_auto_compaction_with_locked_tables(), "auto compaction with locked tables"); + TEST(t_reftable_stack_compaction_concurrent(), "compaction with concurrent stack"); + TEST(t_reftable_stack_compaction_concurrent_clean(), "compaction with unclean stack shutdown"); + TEST(t_reftable_stack_compaction_with_locked_tables(), "compaction with locked tables"); + TEST(t_reftable_stack_hash_id(), "read stack with wrong hash ID"); + TEST(t_reftable_stack_iterator(), "log and ref iterator for reftable stack"); + TEST(t_reftable_stack_lock_failure(), "stack addition with lockfile failure"); + TEST(t_reftable_stack_log_normalize(), "log messages should be normalized"); + TEST(t_reftable_stack_read_across_reload(), "stack iterators work across reloads"); + TEST(t_reftable_stack_reload_with_missing_table(), "stack iteration with garbage tables"); + TEST(t_reftable_stack_tombstone(), "'tombstone' refs in stack"); + TEST(t_reftable_stack_transaction_api(), "update transaction to stack"); + TEST(t_reftable_stack_transaction_with_reload(), "transaction with reload"); + TEST(t_reftable_stack_transaction_api_performs_auto_compaction(), "update transaction triggers auto-compaction"); + TEST(t_reftable_stack_update_index_check(), "update transactions with equal update indices"); + TEST(t_reftable_stack_uptodate(), "stack must be reloaded before ref update"); + TEST(t_suggest_compaction_segment(), "suggest_compaction_segment with basic input"); + TEST(t_suggest_compaction_segment_nothing(), "suggest_compaction_segment with pre-compacted input"); + + return test_done(); +} |