Artifact [315804eda6]

Artifact 315804eda65b53e1a2b7e2db91068e0534f4590bcbffd19afbb221c5e77eef5e:

• File src/sim/checkpoint_sweep.zig — part of check-in [e4026acd7b] at 2026-08-10 23:12:57 on branch sub-manifest — Add zit checkpoint: durable prefix resume for hybrid history

  .zit-cache/checkpoint pairs a validated log-prefix key with the prefix's crosslink record body; Repository.open resumes past the flat segment on a cache miss instead of re-parsing it. Witness shape under a closed emission gate creates zero artifacts; tree shape pins the anchor's file set as the canonical S-tree (batch-oracle byte-identical, one interior root node). verify re-derives every claim from real history; --no-cache bypasses both sidecars. New verb: zit checkpoint [show|build|verify|drop].

  pkgsrc (163k commits, 60k files): cold open 8.4s vs 352s full derive; verify passes; 366/367 tree nodes deduplicated against the imported tree. (user: dmestas, size: 10377) [annotate] [blame] [check-ins using]

//! Crash-recovery sweep for `zit checkpoint build` (hybrid-checkpoint Task 11,
//! the `checkout_sweep.zig` lineage applied to the checkpoint's write
//! boundaries). Every mutating op of a build gets a cut with a torn,
//! corrupted final write. Invariants per cut:
//!
//!   (i)   a witness build never mutates the store — `.zit` is byte-identical
//!         under every fault schedule;
//!   (ii)  no cut leaves a torn sidecar readable as valid (`tryLoad` yields a
//!         valid claim or nothing — the whole-file crc + atomic rename);
//!   (iii) the derived view after any cut equals a full bypass derive;
//!   (iv)  the healthy retry converges and `checkpoint verify` passes.
//!
//! Deterministic: the cut index seeds the fault stream, so any failure
//! replays verbatim.

const std = @import("std");
const zit = @import("zit");

const artifact = zit.artifact;
const checkpoint = zit.checkpoint;
const logstore = zit.logstore;
const storage = zit.storage;
const Repository = zit.Repository;

const faulty_dir = @import("faulty_dir.zig");

const testing = std.testing;

/// A tmp-backed seeded repository (the slim `Repo` shape every fixture here
/// carries): a real `.zit` built through `artifact.create`.
const Repo = struct {
    tmp: std.testing.TmpDir,
    fs: storage.FileStorage,
    log: logstore.LogStore,
    raw: std.Io.File,
    io: std.Io,

    fn init(self: *Repo, io: std.Io) !void {
        self.io = io;
        self.tmp = testing.tmpDir(.{});
        self.raw = try self.tmp.dir.createFile(io, ".zit", .{ .read = true });
        self.fs = storage.FileStorage.init(io, self.raw, 0);
        self.log = try logstore.LogStore.open(testing.allocator, self.fs.storageFile(), 0);
    }

    fn seal(self: *Repo) !void {
        try self.log.commit();
        self.log.deinit(testing.allocator);
        self.raw.close(self.io);
    }

    fn cleanup(self: *Repo) void {
        self.tmp.cleanup();
    }

    fn putRaw(self: *Repo, bytes: []const u8) ![64]u8 {
        var digest: [32]u8 = undefined;
        std.crypto.hash.sha3.Sha3_256.hash(bytes, &digest, .{});
        _ = try self.log.put(&digest, bytes);
        var hex: [64]u8 = undefined;
        for (digest, 0..) |b, i| {
            hex[i * 2] = "0123456789abcdef"[b >> 4];
            hex[i * 2 + 1] = "0123456789abcdef"[b & 0x0f];
        }
        return hex;
    }
};

fn baseCheckin(ts: []const u8, p: []const artifact.PCard, t: []const artifact.TCard, files: []const artifact.FCard) artifact.Manifest {
    return .{
        .arena_state = std.heap.ArenaAllocator.init(testing.allocator),
        .kind = .manifest,
        .a = null,
        .b = null,
        .c = .{ .comment = "c" },
        .d = .{ .timestamp = ts },
        .e = null,
        .g = null,
        .h = null,
        .i = null,
        .k = null,
        .l = null,
        .n = null,
        .r = null,
        .u = .{ .login = "u" },
        .z = null,
        .w = null,
        .f = files,
        .j = &.{},
        .m = &.{},
        .p = p,
        .q = &.{},
        .t = t,
    };
}

/// Seed: config (gating the sweep's shape), eight divergent-first-byte files
/// (so k=1 partitions), a root check-in and one child rewriting a file.
fn seed(repo: *Repo, enable_tree: bool) !void {
    const gpa = testing.allocator;
    try repo.log.putConfig(.{
        .hash_policy = 0,
        .default_branch = "trunk",
        .project_id = "p",
        .enable_submanifests = enable_tree,
        .submanifest_k = if (enable_tree) 1 else 0,
    });

    var names: [8][9]u8 = undefined;
    var blobs: [8][64]u8 = undefined;
    var cards: [8]artifact.FCard = undefined;
    for (0..8) |i| {
        var content_buf: [32]u8 = undefined;
        const content = try std.fmt.bufPrint(&content_buf, "content {d}\n", .{i});
        blobs[i] = try repo.putRaw(content);
        _ = try std.fmt.bufPrint(&names[i], "{c}{d:0>3}.txt", .{ 'a' + @as(u8, @intCast(i * 3)), i });
        cards[i] = .{ .filename = &names[i], .hash = &blobs[i], .perm = null, .old_name = null, .mode = .regular };
    }

    const tags = [_]artifact.TCard{.{ .sign = .propagate, .name = "branch", .target = null, .value = "trunk" }};
    var root_m = baseCheckin("2020-01-01T00:00:00", &.{}, &tags, &cards);
    defer root_m.deinit();
    const root_bytes = try artifact.create(&root_m, gpa);
    defer gpa.free(root_bytes);
    const root_hex = try repo.putRaw(root_bytes);

    blobs[0] = try repo.putRaw("rewritten\n");
    cards[0].hash = &blobs[0];
    var parent_buf = [_]artifact.PCard{.{ .parents = &root_hex, .count = 1 }};
    var child_m = baseCheckin("2020-01-02T00:00:00", &parent_buf, &.{}, &cards);
    defer child_m.deinit();
    const child_bytes = try artifact.create(&child_m, gpa);
    defer gpa.free(child_bytes);
    _ = try repo.putRaw(child_bytes);
}

fn readZit(gpa: std.mem.Allocator, repo: *Repo, io: std.Io) ![]u8 {
    return repo.tmp.dir.readFileAlloc(io, ".zit", gpa, .limited(1 << 24));
}

fn rewriteZit(io: std.Io, repo: *Repo, bytes: []const u8) !void {
    var f = try repo.tmp.dir.createFile(io, ".zit", .{ .truncate = true });
    defer f.close(io);
    try f.writePositionalAll(io, bytes, 0);
}

/// Invariants (ii) + (iii): the sidecar is valid-or-nothing, and the served
/// view equals a full bypass derive over the same log. A cut that TORE the
/// log's tail first gets the standard heal — the pre-existing contract is
/// that read-only opens refuse a torn tail until the next writer truncates
/// it, which is exactly what a real retry's RW open does.
fn checkServedEqualsFull(repo: *Repo, io: std.Io) !void {
    const gpa = testing.allocator;
    repo.tmp.dir.deleteFile(io, ".zit-cache/repo") catch {};
    var served: Repository = undefined;
    Repository.open(&served, gpa, io, repo.tmp.dir, .{}) catch |e| switch (e) {
        error.Corrupt => {
            healTornTail(repo, io) catch return error.TestUnexpectedResult;
            try Repository.open(&served, gpa, io, repo.tmp.dir, .{});
        },
        else => return e,
    };
    defer served.deinit(gpa);
    var full: Repository = undefined;
    try Repository.open(&full, gpa, io, repo.tmp.dir, .{ .bypass_cache = true });
    defer full.deinit(gpa);
    try testing.expectEqualStrings(full.view.main_branch, served.view.main_branch);
    var it_full = full.view.state.leaves();
    var it_served = served.view.state.leaves();
    while (true) {
        const a = it_full.next();
        const b = it_served.next();
        try testing.expectEqual(a == null, b == null);
        if (a == null) break;
        try testing.expectEqualStrings(full.view.hashOf(a.?), served.view.hashOf(b.?));
    }
}

/// The standard torn-tail heal a real retry performs: a read-write scan
/// truncates the incomplete final frame back to the last durable boundary.
fn healTornTail(repo: *Repo, io: std.Io) !void {
    var raw = try repo.tmp.dir.openFile(io, ".zit", .{ .mode = .read_write });
    defer raw.close(io);
    const st = try raw.stat(io);
    var fs = storage.FileStorage.init(io, raw, st.size);
    var log = try logstore.LogStore.open(testing.allocator, fs.storageFile(), 0);
    log.deinit(testing.allocator);
}

fn sweep(enable_tree: bool) !void {
    const io = testing.io;
    const gpa = testing.allocator;

    var repo: Repo = undefined;
    try repo.init(io);
    defer repo.cleanup();
    try seed(&repo, enable_tree);
    try repo.seal();
    const zit_before = try readZit(gpa, &repo, io);
    defer gpa.free(zit_before);

    // The healthy build's mutating-op count = the cut range.
    const total: u64 = blk: {
        var prng = std.Random.DefaultPrng.init(0xCB);
        var faulty = faulty_dir.FaultyDir.init(io, prng.random(), .{});
        faulty.install();
        defer faulty_dir.FaultyDir.uninstall();
        var r: Repository = undefined;
        try Repository.open(&r, gpa, faulty.io(), repo.tmp.dir, .{ .bypass_cache = true });
        defer r.deinit(gpa);
        _ = try checkpoint.build(gpa, faulty.io(), &r, repo.tmp.dir, null);
        break :blk faulty.op_index;
    };
    checkpoint.drop(io, repo.tmp.dir);
    try rewriteZit(io, &repo, zit_before);

    var k: u64 = 1;
    while (k <= total) : (k += 1) {
        var prng = std.Random.DefaultPrng.init(0xCB *% 31 +% k);
        // Clean-torn cuts, the gate discipline `checkout_sweep.run` uses: a
        // BIT-FLIPPED partial log frame is logstore's contracted Corrupt
        // refusal (§C — it cannot be told from real corruption), which is
        // that layer's own well-tested story, not this sweep's. A torn
        // SIDECAR is still fully exercised — the atomic temp+rename makes a
        // partial one unobservable, which invariant (ii) proves per cut.
        var faulty = faulty_dir.FaultyDir.init(io, prng.random(), .{
            .crash_at_op = k,
            .torn_write = true,
        });
        faulty.install();
        // The faulted build is EXPECTED to fail or silently swallow its
        // advisory tail; the invariants below must hold either way.
        blk: {
            var r: Repository = undefined;
            Repository.open(&r, gpa, faulty.io(), repo.tmp.dir, .{ .bypass_cache = true }) catch break :blk;
            defer r.deinit(gpa);
            _ = checkpoint.build(gpa, faulty.io(), &r, repo.tmp.dir, null) catch {};
        }
        faulty_dir.FaultyDir.uninstall();

        if (!enable_tree) { // (i): witness never touches the store
            const now = try readZit(gpa, &repo, io);
            defer gpa.free(now);
            try testing.expectEqualSlices(u8, zit_before, now);
        }
        try checkServedEqualsFull(&repo, io); // (ii) + (iii)

        checkpoint.drop(io, repo.tmp.dir);
        try rewriteZit(io, &repo, zit_before);
    }

    // (iv): the healthy retry converges and independently verifies.
    var r: Repository = undefined;
    try Repository.open(&r, gpa, io, repo.tmp.dir, .{ .bypass_cache = true });
    _ = try checkpoint.build(gpa, io, &r, repo.tmp.dir, null);
    r.deinit(gpa);
    const verdict = try checkpoint.verify(gpa, io, repo.tmp.dir);
    try testing.expectEqual(@as(usize, 8), verdict.ok.files);
}

test "checkpoint_sweep: a witness build survives every torn cut with the store untouched" {
    try sweep(false);
}

test "checkpoint_sweep: a tree build survives every torn cut and the retry verifies" {
    try sweep(true);
}