//! Signed erasures at the end of a pull, sync or serve session: recording the
//! rows that verify against the pinned key, marking them applied, reclaiming
//! the erased bytes by a store rewrite, and the rows a push carries.
const std = @import("std");
const cli = @import("zit").cli;
const diagnostics = @import("../cli/diagnostics.zig");
const digestToWireName = zit.digest_name.digestToWireName;
const lockRewrite = @import("pack.zig").lockRewrite;
const logstore = @import("zit").logstore;
const plural = @import("sync_dialogue.zig").plural;
const reasonOf = diagnostics.reasonOf;
const rewriteAndSwap = @import("pack.zig").rewriteAndSwap;
const storage = @import("zit").storage;
const zit = @import("zit");
/// Whether `src` still holds reclaimable BYTES under `digest` — an interned,
/// non-phantom frame. This is exactly `packLog`'s own `dropped` predicate (it
/// counts non-phantom rids the filter drops), so "would be removed" here and "was
/// removed" there cannot disagree. No decode, so the refusal path stays cheap.
pub fn holdsBytes(src: *const logstore.LogStore, digest: []const u8) bool {
const rid = src.lookup(digest) orelse return false;
return !src.isPhantom(rid);
}
/// Decode a COMPLETE lowercase-hex artifact name (40 or 64 hex, the two forms
/// artifact-format §3.2 defines) into `out`, returning the used prefix, or null
/// when `hex` is not one of those two widths. The width, not `hash_kind`, decides
/// — the same rule `logstore.isCompleteDigest` enforces on the write end, so
/// nothing here interprets a digest.
pub fn shunHexToDigest(hex: []const u8, out: *[logstore.max_hash_len]u8) ?[]const u8 {
if (hex.len != 40 and hex.len != 64) return null;
const n = hex.len / 2;
std.debug.assert(n <= out.len);
_ = std.fmt.hexToBytes(out[0..n], hex) catch return null;
return out[0..n];
}
/// The project code of `log`'s live config, copied into `buf`; empty when the
/// store has none.
pub fn logProjectCode(gpa: std.mem.Allocator, log: *logstore.LogStore, buf: *[logstore.max_config_str]u8) error{ OutOfMemory, Storage }![]const u8 {
var owned = (log.readConfig(gpa) catch |e| return switch (e) {
error.OutOfMemory => error.OutOfMemory,
else => error.Storage,
}) orelse return buf[0..0];
defer owned.deinit(gpa);
const code = owned.config.project_code;
@memcpy(buf[0..code.len], code);
return buf[0..code.len];
}
/// Open the store beside `amb`'s root for an append, under the rewrite lock
/// then the writer lock, as every rewrite takes them. The caller closes all
/// three through `ErasureAppend.close`.
pub const ErasureAppend = struct {
rewrite_lock: zit.storelock.Lock,
store: cli.StoreAppend,
fs: storage.FileStorage,
log: logstore.LogStore,
pub fn close(self: *ErasureAppend, gpa: std.mem.Allocator, io: std.Io) void {
self.log.deinit(gpa);
self.store.close(io);
self.rewrite_lock.release(io);
}
};
pub fn openErasureAppend(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, out: *ErasureAppend) error{ RewriteBusy, StoreBusy, Storage, OutOfMemory, Corrupt }!void {
out.rewrite_lock = try lockRewrite(io, dir);
errdefer out.rewrite_lock.release(io);
out.store = cli.openStoreAppend(io, dir) catch |e| return switch (e) {
error.StoreBusy => error.StoreBusy,
else => error.Storage,
};
errdefer out.store.close(io);
const st = out.store.file.stat(io) catch return error.Storage;
out.fs = storage.FileStorage.init(io, out.store.file, st.size);
out.log = try logstore.LogStore.open(gpa, out.fs.storageFile(), 0);
}
/// The signed rows this store pushes on a push or sync: every row whose
/// statement verifies against the pinned key, in wire form. Arena-owned.
/// Empty when no key is pinned, which is every repository that never turned
/// the feature on.
pub fn erasurePushRows(arena: std.mem.Allocator, log: *const logstore.LogStore) error{OutOfMemory}![]const zit.sync.ShunRow {
if (log.erasureKey() == null) return &.{};
var out: std.ArrayList(zit.sync.ShunRow) = .empty;
for (log.shunList(), 0..) |row, i| {
switch (log.shunRowState(i).kind) {
.erase, .cancel => {},
.plain, .applied, .unverified => continue,
}
var hex: [2 * logstore.max_hash_len]u8 = undefined;
try out.append(arena, .{ .hash = try arena.dupe(u8, digestToWireName(row.hash, &hex)), .epoch = row.epoch, .reason = row.reason });
}
return out.items;
}
/// Whether `log` holds the bytes of a name a signed erasure has made active:
/// the condition for a reclaim at the end of a session, which covers both a
/// newly latched active row and a pending one whose not-before has passed.
pub fn erasedBytesHeld(log: *const logstore.LogStore) bool {
for (log.shunList()) |row| {
if (log.shunStatus(row.hash).state != .erased) continue;
if (holdsBytes(log, row.hash)) return true;
}
return false;
}
/// The end of a pull or sync session, and the place zit goes beyond Fossil's
/// clone-only adoption: record each latched `/shun` row that VERIFIES against
/// this repository's pinned key (an unsigned row, a row another key signed,
/// and a row with no key pinned are all ignored, so no peer can suppress
/// content here on its own say-so), then, when an active erasure's bytes are
/// still held, rewrite the store without them. Reports on stderr and never
/// changes the session's exit: the exchange itself succeeded. Returns how
/// many of the latched rows it could not fully apply (not recorded, or
/// recorded with their bytes still held), for `-F json`.
pub fn applySessionErasures(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
streams: ?cli.Streams,
verb: []const u8,
/// The session's own open store, when it has one: a session with no key
/// pinned, nothing new latched and nothing to mark or reclaim — every
/// session of a repository that never turned erasures on — is decided
/// off it and opens nothing more.
current_opt: ?*logstore.LogStore,
latched: []const zit.sync.OwnedShunRow,
/// Rewrite the store when an active erasure's bytes are held. `zit
/// serve` passes false from a request and reclaims between connections.
reclaim: bool,
) usize {
var write = true;
// The rows this session carried that verify, as the session's store
// judges them; every latched row when there is no session store.
var signed: usize = latched.len;
if (current_opt) |current| {
const key = current.erasureKey() orelse return 0;
var code_buf: [logstore.max_config_str]u8 = undefined;
const code = logProjectCode(gpa, current, &code_buf) catch return 0;
var fresh = false;
signed = 0;
for (latched) |row| {
if (zit.erasure.judge(key, .{ .project_code = code, .hash_hex = row.hash, .epoch = row.epoch }, row.reason) != .verified) continue;
signed += 1;
if (!rowHeld(current, row)) fresh = true;
}
write = fresh or erasureNeedsApplied(current);
if (!write and !(reclaim and erasedBytesHeld(current))) return 0;
}
var unreclaimed: usize = 0;
if (write) {
var ap: ErasureAppend = undefined;
openErasureAppend(gpa, io, dir, &ap) catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
defer ap.close(gpa, io);
// Re-judged against the store as it is under the lock: the key or
// the project code may have moved since the session opened it.
var recorded: usize = 0;
if (ap.log.erasureKey()) |lkey| {
var lcode_buf: [logstore.max_config_str]u8 = undefined;
const lcode = logProjectCode(gpa, &ap.log, &lcode_buf) catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
for (latched) |row| {
if (zit.erasure.judge(lkey, .{ .project_code = lcode, .hash_hex = row.hash, .epoch = row.epoch }, row.reason) != .verified) continue;
var raw: [logstore.max_hash_len]u8 = undefined;
const digest = shunHexToDigest(row.hash, &raw) orelse continue;
const before = ap.log.shunList().len;
ap.log.putShun(digest, row.reason, row.epoch) catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
recorded += ap.log.shunList().len - before;
}
}
// Each erasure in effect here is recorded as applied, so a later
// change of key cannot make it stop masking (the signed row then
// verifies nothing, but the applied row is plain).
recorded += markErasuresApplied(gpa, &ap.log) catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
if (recorded != 0) ap.log.commit() catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
if (!reclaim or !erasedBytesHeld(&ap.log)) return 0;
unreclaimed = carriedErasedHeld(&ap.log, latched);
} else unreclaimed = carriedErasedHeld(current_opt.?, latched);
// A reclaim, as `zit pack` would make: the same generation swap, dropping
// only what a signed erasure made active (a plain `--defer` shun waits for
// the pack its operator runs).
var src_file = cli.openStoreRead(io, dir) catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
defer src_file.close(io);
const st = src_file.stat(io) catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
var src_fs = storage.FileStorage.init(io, src_file, st.size);
var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
defer src.deinit(gpa);
const src_key = src.logKey() catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
const stats = rewriteAndSwap(gpa, io, dir, &src, src_key, zit.pack.erasedFilter(&src), &.{}) catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
if (streams) |s| if (stats.dropped != 0) s.err.print("zit {s}: erased by the repository owner: {d} artifact{s} removed\n", .{ verb, stats.dropped, plural(stats.dropped) }) catch {};
return 0;
}
/// How many of `latched` name an erased artifact whose bytes `log` holds.
fn carriedErasedHeld(log: *const logstore.LogStore, latched: []const zit.sync.OwnedShunRow) usize {
var n: usize = 0;
for (latched) |row| {
var raw: [logstore.max_hash_len]u8 = undefined;
const digest = shunHexToDigest(row.hash, &raw) orelse continue;
if (log.shunStatus(digest).state == .erased and holdsBytes(log, digest)) n += 1;
}
return n;
}
/// Record `erasure.applied_reason` for each name a signed erasure alone keeps
/// erased. Returns how many rows went in; the caller commits.
fn markErasuresApplied(gpa: std.mem.Allocator, log: *logstore.LogStore) !usize {
var names: std.ArrayList([]const u8) = .empty;
defer {
for (names.items) |n| gpa.free(n);
names.deinit(gpa);
}
for (log.shunList()) |row| {
if (!log.shunNeedsApplied(row.hash)) continue;
const dup = for (names.items) |n| {
if (std.mem.eql(u8, n, row.hash)) break true;
} else false;
if (!dup) try names.append(gpa, try gpa.dupe(u8, row.hash));
}
for (names.items) |name| try log.putShun(name, zit.erasure.applied_reason, log.erasure_now);
return names.items.len;
}
pub fn erasureNeedsApplied(log: *const logstore.LogStore) bool {
for (log.shunList()) |row| if (log.shunNeedsApplied(row.hash)) return true;
return false;
}
/// The session's exchange succeeded and its exit stays as the exchange set
/// it: every pull and sync asks for the peer's rows again, so the next
/// session records what this one could not. What did not happen is said.
fn erasureNotApplied(streams: ?cli.Streams, verb: []const u8, latched: usize, deferred: usize, e: anyerror) usize {
const s = streams orelse return deferred;
const why: []const u8 = switch (e) {
error.RewriteBusy => "another rewrite of this store is in progress",
error.StoreBusy => "another process holds the store's writer lock",
else => reasonOf(e),
};
s.err.print("zit {s}: the signed erasures this session carried ({d} row{s}) were not recorded: {s}; the next pull or sync records them\n", .{ verb, latched, plural(latched), why }) catch {};
return deferred;
}
fn erasureNotReclaimed(streams: ?cli.Streams, verb: []const u8, deferred: usize, e: anyerror) usize {
const s = streams orelse return deferred;
const why: []const u8 = switch (e) {
error.RewriteBusy => "another rewrite of this store is in progress",
error.TailChanged => "the store changed while it was being rewritten",
else => reasonOf(e),
};
s.err.print("zit {s}: an erasure is active but its bytes were not reclaimed: {s}; the next pull, sync or 'zit pack' reclaims them\n", .{ verb, why }) catch {};
return deferred;
}
/// Whether `log` already holds the wire row `row`, byte for byte.
fn rowHeld(log: *const logstore.LogStore, row: zit.sync.OwnedShunRow) bool {
for (log.shunList()) |held| {
if (held.epoch != row.epoch or !std.mem.eql(u8, held.reason, row.reason)) continue;
var hex: [2 * logstore.max_hash_len]u8 = undefined;
if (std.mem.eql(u8, digestToWireName(held.hash, &hex), row.hash)) return true;
}
return false;
}