Skip to content
zit Artifact [249b49d9ee]
Login

Artifact [249b49d9ee]

Artifact 249b49d9ee174c0d96fd1287026a713806846ef24da722db789ad867c05f6099:


//! Signed erasures at the end of a pull, sync or serve session: recording the
//! rows that verify against the pinned key, marking them applied, reclaiming
//! the erased bytes by a store rewrite, and the rows a push carries.

const std = @import("std");
const cli = @import("zit").cli;
const diagnostics = @import("../diagnostics.zig");
const digestToWireName = zit.digest_name.digestToWireName;
const lockRewrite = @import("sync.zig").lockRewrite;
const logstore = @import("zit").logstore;
const plural = @import("sync_dialogue.zig").plural;
const reasonOf = diagnostics.reasonOf;
const rewriteAndSwap = @import("sync.zig").rewriteAndSwap;
const storage = @import("zit").storage;
const zit = @import("zit");

/// Whether `src` still holds reclaimable BYTES under `digest` — an interned,
/// non-phantom frame. This is exactly `packLog`'s own `dropped` predicate (it
/// counts non-phantom rids the filter drops), so "would be removed" here and "was
/// removed" there cannot disagree. No decode, so the refusal path stays cheap.
pub fn holdsBytes(src: *const logstore.LogStore, digest: []const u8) bool {
    const rid = src.lookup(digest) orelse return false;
    return !src.isPhantom(rid);
}

/// Decode a COMPLETE lowercase-hex artifact name (40 or 64 hex, the two forms
/// artifact-format §3.2 defines) into `out`, returning the used prefix, or null
/// when `hex` is not one of those two widths. The width, not `hash_kind`, decides
/// — the same rule `logstore.isCompleteDigest` enforces on the write end, so
/// nothing here interprets a digest.
pub fn shunHexToDigest(hex: []const u8, out: *[logstore.max_hash_len]u8) ?[]const u8 {
    if (hex.len != 40 and hex.len != 64) return null;
    const n = hex.len / 2;
    std.debug.assert(n <= out.len);
    _ = std.fmt.hexToBytes(out[0..n], hex) catch return null;
    return out[0..n];
}

/// The project code of `log`'s live config, copied into `buf`; empty when the
/// store has none.
pub fn logProjectCode(gpa: std.mem.Allocator, log: *logstore.LogStore, buf: *[logstore.max_config_str]u8) error{ OutOfMemory, Storage }![]const u8 {
    var owned = (log.readConfig(gpa) catch |e| return switch (e) {
        error.OutOfMemory => error.OutOfMemory,
        else => error.Storage,
    }) orelse return buf[0..0];
    defer owned.deinit(gpa);
    const code = owned.config.project_code;
    @memcpy(buf[0..code.len], code);
    return buf[0..code.len];
}

/// Open the store beside `amb`'s root for an append, under the rewrite lock
/// then the writer lock, as every rewrite takes them. The caller closes all
/// three through `ErasureAppend.close`.
pub const ErasureAppend = struct {
    rewrite_lock: zit.storelock.Lock,
    store: cli.StoreAppend,
    fs: storage.FileStorage,
    log: logstore.LogStore,

    pub fn close(self: *ErasureAppend, gpa: std.mem.Allocator, io: std.Io) void {
        self.log.deinit(gpa);
        self.store.close(io);
        self.rewrite_lock.release(io);
    }
};

pub fn openErasureAppend(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, out: *ErasureAppend) error{ RewriteBusy, StoreBusy, Storage, OutOfMemory, Corrupt }!void {
    out.rewrite_lock = try lockRewrite(io, dir);
    errdefer out.rewrite_lock.release(io);
    out.store = cli.openStoreAppend(io, dir) catch |e| return switch (e) {
        error.StoreBusy => error.StoreBusy,
        else => error.Storage,
    };
    errdefer out.store.close(io);
    const st = out.store.file.stat(io) catch return error.Storage;
    out.fs = storage.FileStorage.init(io, out.store.file, st.size);
    out.log = try logstore.LogStore.open(gpa, out.fs.storageFile(), 0);
}

/// The signed rows this store pushes on a push or sync: every row whose
/// statement verifies against the pinned key, in wire form. Arena-owned.
/// Empty when no key is pinned, which is every repository that never turned
/// the feature on.
pub fn erasurePushRows(arena: std.mem.Allocator, log: *const logstore.LogStore) error{OutOfMemory}![]const zit.sync.ShunRow {
    if (log.erasureKey() == null) return &.{};
    var out: std.ArrayList(zit.sync.ShunRow) = .empty;
    for (log.shunList(), 0..) |row, i| {
        switch (log.shunRowState(i).kind) {
            .erase, .cancel => {},
            .plain, .applied, .unverified => continue,
        }
        var hex: [2 * logstore.max_hash_len]u8 = undefined;
        try out.append(arena, .{ .hash = try arena.dupe(u8, digestToWireName(row.hash, &hex)), .epoch = row.epoch, .reason = row.reason });
    }
    return out.items;
}

/// Whether `log` holds the bytes of a name a signed erasure has made active:
/// the condition for a reclaim at the end of a session, which covers both a
/// newly latched active row and a pending one whose not-before has passed.
pub fn erasedBytesHeld(log: *const logstore.LogStore) bool {
    for (log.shunList()) |row| {
        if (log.shunStatus(row.hash).state != .erased) continue;
        if (holdsBytes(log, row.hash)) return true;
    }
    return false;
}

/// The end of a pull or sync session, and the place zit goes beyond Fossil's
/// clone-only adoption: record each latched `/shun` row that VERIFIES against
/// this repository's pinned key (an unsigned row, a row another key signed,
/// and a row with no key pinned are all ignored, so no peer can suppress
/// content here on its own say-so), then, when an active erasure's bytes are
/// still held, rewrite the store without them. Reports on stderr and never
/// changes the session's exit: the exchange itself succeeded. Returns how
/// many of the latched rows it could not fully apply (not recorded, or
/// recorded with their bytes still held), for `-F json`.
pub fn applySessionErasures(
    gpa: std.mem.Allocator,
    io: std.Io,
    dir: std.Io.Dir,
    streams: ?cli.Streams,
    verb: []const u8,
    /// The session's own open store, when it has one: a session with no key
    /// pinned, nothing new latched and nothing to mark or reclaim — every
    /// session of a repository that never turned erasures on — is decided
    /// off it and opens nothing more.
    current_opt: ?*logstore.LogStore,
    latched: []const zit.sync.OwnedShunRow,
    /// Rewrite the store when an active erasure's bytes are held. `zit
    /// serve` passes false from a request and reclaims between connections.
    reclaim: bool,
) usize {
    var write = true;
    // The rows this session carried that verify, as the session's store
    // judges them; every latched row when there is no session store.
    var signed: usize = latched.len;
    if (current_opt) |current| {
        const key = current.erasureKey() orelse return 0;
        var code_buf: [logstore.max_config_str]u8 = undefined;
        const code = logProjectCode(gpa, current, &code_buf) catch return 0;
        var fresh = false;
        signed = 0;
        for (latched) |row| {
            if (zit.erasure.judge(key, .{ .project_code = code, .hash_hex = row.hash, .epoch = row.epoch }, row.reason) != .verified) continue;
            signed += 1;
            if (!rowHeld(current, row)) fresh = true;
        }
        write = fresh or erasureNeedsApplied(current);
        if (!write and !(reclaim and erasedBytesHeld(current))) return 0;
    }
    var unreclaimed: usize = 0;
    if (write) {
        var ap: ErasureAppend = undefined;
        openErasureAppend(gpa, io, dir, &ap) catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
        defer ap.close(gpa, io);
        // Re-judged against the store as it is under the lock: the key or
        // the project code may have moved since the session opened it.
        var recorded: usize = 0;
        if (ap.log.erasureKey()) |lkey| {
            var lcode_buf: [logstore.max_config_str]u8 = undefined;
            const lcode = logProjectCode(gpa, &ap.log, &lcode_buf) catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
            for (latched) |row| {
                if (zit.erasure.judge(lkey, .{ .project_code = lcode, .hash_hex = row.hash, .epoch = row.epoch }, row.reason) != .verified) continue;
                var raw: [logstore.max_hash_len]u8 = undefined;
                const digest = shunHexToDigest(row.hash, &raw) orelse continue;
                const before = ap.log.shunList().len;
                ap.log.putShun(digest, row.reason, row.epoch) catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
                recorded += ap.log.shunList().len - before;
            }
        }
        // Each erasure in effect here is recorded as applied, so a later
        // change of key cannot make it stop masking (the signed row then
        // verifies nothing, but the applied row is plain).
        recorded += markErasuresApplied(gpa, &ap.log) catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
        if (recorded != 0) ap.log.commit() catch |e| return erasureNotApplied(streams, verb, latched.len, signed, e);
        if (!reclaim or !erasedBytesHeld(&ap.log)) return 0;
        unreclaimed = carriedErasedHeld(&ap.log, latched);
    } else unreclaimed = carriedErasedHeld(current_opt.?, latched);
    // A reclaim, as `zit pack` would make: the same generation swap, dropping
    // only what a signed erasure made active (a plain `--defer` shun waits for
    // the pack its operator runs).
    var src_file = cli.openStoreRead(io, dir) catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
    defer src_file.close(io);
    const st = src_file.stat(io) catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
    var src_fs = storage.FileStorage.init(io, src_file, st.size);
    var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
    defer src.deinit(gpa);
    const src_key = src.logKey() catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
    const stats = rewriteAndSwap(gpa, io, dir, &src, src_key, zit.pack.erasedFilter(&src), &.{}) catch |e| return erasureNotReclaimed(streams, verb, unreclaimed, e);
    if (streams) |s| if (stats.dropped != 0) s.err.print("zit {s}: erased by the repository owner: {d} artifact{s} removed\n", .{ verb, stats.dropped, plural(stats.dropped) }) catch {};
    return 0;
}

/// How many of `latched` name an erased artifact whose bytes `log` holds.
fn carriedErasedHeld(log: *const logstore.LogStore, latched: []const zit.sync.OwnedShunRow) usize {
    var n: usize = 0;
    for (latched) |row| {
        var raw: [logstore.max_hash_len]u8 = undefined;
        const digest = shunHexToDigest(row.hash, &raw) orelse continue;
        if (log.shunStatus(digest).state == .erased and holdsBytes(log, digest)) n += 1;
    }
    return n;
}

/// Record `erasure.applied_reason` for each name a signed erasure alone keeps
/// erased. Returns how many rows went in; the caller commits.
fn markErasuresApplied(gpa: std.mem.Allocator, log: *logstore.LogStore) !usize {
    var names: std.ArrayList([]const u8) = .empty;
    defer {
        for (names.items) |n| gpa.free(n);
        names.deinit(gpa);
    }
    for (log.shunList()) |row| {
        if (!log.shunNeedsApplied(row.hash)) continue;
        const dup = for (names.items) |n| {
            if (std.mem.eql(u8, n, row.hash)) break true;
        } else false;
        if (!dup) try names.append(gpa, try gpa.dupe(u8, row.hash));
    }
    for (names.items) |name| try log.putShun(name, zit.erasure.applied_reason, log.erasure_now);
    return names.items.len;
}

pub fn erasureNeedsApplied(log: *const logstore.LogStore) bool {
    for (log.shunList()) |row| if (log.shunNeedsApplied(row.hash)) return true;
    return false;
}

/// The session's exchange succeeded and its exit stays as the exchange set
/// it: every pull and sync asks for the peer's rows again, so the next
/// session records what this one could not. What did not happen is said.
fn erasureNotApplied(streams: ?cli.Streams, verb: []const u8, latched: usize, deferred: usize, e: anyerror) usize {
    const s = streams orelse return deferred;
    const why: []const u8 = switch (e) {
        error.RewriteBusy => "another rewrite of this store is in progress",
        error.StoreBusy => "another process holds the store's writer lock",
        else => reasonOf(e),
    };
    s.err.print("zit {s}: the signed erasures this session carried ({d} row{s}) were not recorded: {s}; the next pull or sync records them\n", .{ verb, latched, plural(latched), why }) catch {};
    return deferred;
}

fn erasureNotReclaimed(streams: ?cli.Streams, verb: []const u8, deferred: usize, e: anyerror) usize {
    const s = streams orelse return deferred;
    const why: []const u8 = switch (e) {
        error.RewriteBusy => "another rewrite of this store is in progress",
        error.TailChanged => "the store changed while it was being rewritten",
        else => reasonOf(e),
    };
    s.err.print("zit {s}: an erasure is active but its bytes were not reclaimed: {s}; the next pull, sync or 'zit pack' reclaims them\n", .{ verb, why }) catch {};
    return deferred;
}

/// Whether `log` already holds the wire row `row`, byte for byte.
fn rowHeld(log: *const logstore.LogStore, row: zit.sync.OwnedShunRow) bool {
    for (log.shunList()) |held| {
        if (held.epoch != row.epoch or !std.mem.eql(u8, held.reason, row.reason)) continue;
        var hex: [2 * logstore.max_hash_len]u8 = undefined;
        if (std.mem.eql(u8, digestToWireName(held.hash, &hex), row.hash)) return true;
    }
    return false;
}