Artifact ea63da04a2f55640f515347b7b687e3ac7acc8e3bc66503827a86c94aa52f107:
- File
src/checkpoint_test.zig
— part of check-in
[2a8fdcff60]
at
2026-08-12 09:04:13
on branch sub-manifest
— checkpoint: retire the tree shape — sidecar only, structurally
The tree shape minted permanent SUBMANIFEST artifacts for benefits already had elsewhere: the rebuild win lives in the prefix record, verify's §6.2 oracle runs buildTree in memory, and the subtree-sharing bytes re-materialize at the first real S-emitting commit at identical cost. Minting early only created unreferenced permanent artifacts and the allow-tree/demotion/gate apparatus needed to keep automation from doing it by accident — all deleted with it. build no longer opens the log for writing or reads the emission settings at all; sidecar layout v3 (old sidecars miss and rebuild in seconds). Proven at scale: SHA-256 of pkgsrc's 593MB store is byte-identical across a full build; cold 4:12, rebuild 3.4s. (user: dmestas, size: 48633) [annotate] [blame] [check-ins using]
//! Checkpoint suite: container hit/miss/corruption matrix (Task 4), the //! file-set fingerprint (Task 5), the builder gate/refusal matrix (Task 6), //! the batch-oracle differential (Task 7), drift/invalidation and the //! paranoid bypass (Task 8), and the verifier (Task 9). Modelled on //! `repocache.zig`'s two load-bearing invariants: a validated HIT //! reconstructs exactly what a recompute produces, and a stale/corrupt/ //! wrong-anything file is a MISS, never partially trusted. const std = @import("std"); const artifact = @import("artifact.zig"); const checkpoint = @import("checkpoint.zig"); const logstore = @import("logstore.zig"); const storage = @import("storage.zig"); const crc32 = @import("crc32.zig"); const repocache = @import("repocache.zig"); const testing = std.testing; // --------------------------------------------------------------------------- // Container matrix (Task 4). // --------------------------------------------------------------------------- /// A tmp checkout root holding a real log file the prefix key validates /// against. The log carries opaque test blobs — the container never parses /// its body, so the record bytes here are arbitrary. const Fx = struct { tmp: std.testing.TmpDir, raw: std.Io.File, fs: storage.FileStorage, fn init(self: *Fx, io: std.Io) !void { self.tmp = std.testing.tmpDir(.{}); self.raw = try self.tmp.dir.createFile(io, "repo.zit", .{ .read = true }); self.fs = storage.FileStorage.init(io, self.raw, 0); } fn deinit(self: *Fx, io: std.Io) void { self.raw.close(io); self.tmp.cleanup(); } }; fn putBlob(store: *logstore.LogStore, seed: u8, bytes: []const u8) !logstore.Rid { const hash = [_]u8{seed} ** 32; return store.put(&hash, bytes); } fn testClaim(key: logstore.LogStore.LogKey) checkpoint.Claim { return .{ .checkin_hash = "a" ** 64, .fileset_fingerprint = [_]u8{0x5A} ** 32, .prefix_key = key, .shun_fp = 0, }; } fn rewriteRaw(io: std.Io, dir: std.Io.Dir, bytes: []const u8) !void { var f = try dir.createFile(io, checkpoint.checkpoint_filename, .{ .truncate = true }); defer f.close(io); try f.writePositionalAll(io, bytes, 0); } fn readRaw(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) ![]u8 { return dir.readFileAlloc(io, checkpoint.checkpoint_filename, gpa, .limited(1 << 20)); } test "checkpoint container: a validated hit round-trips the claim and body, and survives log growth" { const io = testing.io; var fx: Fx = undefined; try fx.init(io); defer fx.deinit(io); var store = try logstore.LogStore.open(testing.allocator, fx.fs.storageFile(), 0); defer store.deinit(testing.allocator); const rid1 = try putBlob(&store, 0x11, "first"); try store.commit(); const key = try store.logKeyAt(rid1); const claim = testClaim(key); checkpoint.store(testing.allocator, io, fx.tmp.dir, claim, "trunk", "RECORD-BODY"); // Hit at exactly the prefix. { var loaded = checkpoint.tryLoad(testing.allocator, io, fx.tmp.dir, fx.fs.storageFile()).?; defer loaded.deinit(testing.allocator); try testing.expectEqualStrings("a" ** 64, loaded.claim.checkin_hash); try testing.expectEqualSlices(u8, &([_]u8{0x5A} ** 32), &loaded.claim.fileset_fingerprint); try testing.expectEqual(key.end, loaded.claim.prefix_key.end); try testing.expectEqualStrings("trunk", loaded.main_branch); try testing.expectEqualStrings("RECORD-BODY", loaded.body); } // The log GROWS past the prefix: still a hit — the whole point. _ = try putBlob(&store, 0x22, "second"); try store.commit(); { var loaded = checkpoint.tryLoad(testing.allocator, io, fx.tmp.dir, fx.fs.storageFile()).?; defer loaded.deinit(testing.allocator); try testing.expectEqualStrings("RECORD-BODY", loaded.body); } // Truncated below the prefix: miss. try fx.fs.storageFile().truncate(key.end - 8); try testing.expect(checkpoint.tryLoad(testing.allocator, io, fx.tmp.dir, fx.fs.storageFile()) == null); } test "checkpoint container: every framing violation is a miss, never partial trust" { const io = testing.io; const gpa = testing.allocator; var fx: Fx = undefined; try fx.init(io); defer fx.deinit(io); var store = try logstore.LogStore.open(gpa, fx.fs.storageFile(), 0); defer store.deinit(gpa); const rid1 = try putBlob(&store, 0x11, "first"); try store.commit(); const key = try store.logKeyAt(rid1); checkpoint.store(gpa, io, fx.tmp.dir, testClaim(key), "trunk", "BODY"); const good = try readRaw(gpa, io, fx.tmp.dir); defer gpa.free(good); const Mutate = struct { name: []const u8, f: *const fn (b: []u8) void, refresh_crc: bool }; const cases = [_]Mutate{ .{ .name = "wrong magic", .f = struct { fn f(b: []u8) void { b[0] = 'X'; } }.f, .refresh_crc = false }, .{ .name = "wrong version", .f = struct { fn f(b: []u8) void { b[8] = 0xEE; } }.f, .refresh_crc = false }, .{ .name = "nonzero reserved", .f = struct { fn f(b: []u8) void { b[10] = 7; } }.f, .refresh_crc = true }, .{ .name = "rewritten prefix boundary crc", .f = struct { fn f(b: []u8) void { b[36] ^= 0xFF; // prefix_last_header_crc } }.f, .refresh_crc = true, }, .{ .name = "flipped payload byte", .f = struct { fn f(b: []u8) void { b[b.len - 6] ^= 0xFF; } }.f, .refresh_crc = false }, .{ .name = "flipped trailing crc", .f = struct { fn f(b: []u8) void { b[b.len - 1] ^= 0xFF; } }.f, .refresh_crc = false }, .{ .name = "body length past the file", .f = struct { fn f(b: []u8) void { b[79] +%= 5; // body_len low byte — with a refreshed crc, the SPAN check must reject } }.f, .refresh_crc = true, }, .{ .name = "impossible checkin-hash length", .f = struct { fn f(b: []u8) void { b[44] = 17; } }.f, .refresh_crc = true }, }; for (cases) |case| { const bad = try gpa.dupe(u8, good); defer gpa.free(bad); case.f(bad); if (case.refresh_crc) { var crc = crc32.Crc32.init(); crc.update(bad[0 .. bad.len - 4]); std.mem.writeInt(u32, bad[bad.len - 4 ..][0..4], crc.final(), .little); } try rewriteRaw(io, fx.tmp.dir, bad); if (checkpoint.tryLoad(gpa, io, fx.tmp.dir, fx.fs.storageFile()) != null) { std.debug.print("container accepted a mutated file: {s}\n", .{case.name}); return error.TestUnexpectedResult; } } // Shorter than the fixed header: miss before any allocation-driven parse. try rewriteRaw(io, fx.tmp.dir, good[0..20]); try testing.expect(checkpoint.tryLoad(gpa, io, fx.tmp.dir, fx.fs.storageFile()) == null); // Truncated mid-body and extended with garbage: exact accounting rejects. try rewriteRaw(io, fx.tmp.dir, good[0 .. good.len - 1]); try testing.expect(checkpoint.tryLoad(gpa, io, fx.tmp.dir, fx.fs.storageFile()) == null); const extended = try std.mem.concat(gpa, u8, &.{ good, "x" }); defer gpa.free(extended); try rewriteRaw(io, fx.tmp.dir, extended); try testing.expect(checkpoint.tryLoad(gpa, io, fx.tmp.dir, fx.fs.storageFile()) == null); } test "checkpoint container: an interrupted atomic write leaves the previous checkpoint intact, and drop is idempotent" { const io = testing.io; const gpa = testing.allocator; var fx: Fx = undefined; try fx.init(io); defer fx.deinit(io); var store = try logstore.LogStore.open(gpa, fx.fs.storageFile(), 0); defer store.deinit(gpa); const rid1 = try putBlob(&store, 0x11, "first"); try store.commit(); const key = try store.logKeyAt(rid1); checkpoint.store(gpa, io, fx.tmp.dir, testClaim(key), "trunk", "ORIGINAL"); // Begin a fresh atomic write, put bytes, then abandon WITHOUT replace: // deinit deletes the temp, and the reader only ever sees the real name. { var af = try fx.tmp.dir.createFileAtomic(io, checkpoint.checkpoint_filename, .{ .replace = true }); defer af.deinit(io); try af.file.writePositionalAll(io, "half-written garbage", 0); } var loaded = checkpoint.tryLoad(gpa, io, fx.tmp.dir, fx.fs.storageFile()).?; defer loaded.deinit(gpa); try testing.expectEqualStrings("ORIGINAL", loaded.body); checkpoint.drop(io, fx.tmp.dir); try testing.expect(checkpoint.tryLoad(gpa, io, fx.tmp.dir, fx.fs.storageFile()) == null); checkpoint.drop(io, fx.tmp.dir); // absent is success } // --------------------------------------------------------------------------- // File-set fingerprint (Task 5). // --------------------------------------------------------------------------- fn fcard(name: []const u8, hash: ?[]const u8, perm: ?[]const u8, old: ?[]const u8) artifact.FCard { return .{ .filename = name, .hash = hash, .perm = perm, .old_name = old, .mode = .regular }; } test "fileSetFingerprint: a pure structural function — stable, field-sensitive, escape-faithful" { const gpa = testing.allocator; const h1: []const u8 = "1" ** 64; const h2: []const u8 = "2" ** 64; // The empty file set has a defined, stable value: SHA3-256 of no bytes. const empty = try artifact.fileSetFingerprint(gpa, &.{}); const sha3_empty = "\xa7\xff\xc6\xf8\xbf\x1e\xd7\x66\x51\xc1\x47\x56\xa0\x61\xd6\x62" ++ "\xf5\x80\xff\x4d\xe4\x3b\x49\xfa\x82\xd8\x0a\x4b\x80\xf8\x43\x4a"; try testing.expectEqualSlices(u8, sha3_empty, &empty); // Pure: the same structural inputs give the same value; no content read. const base = [_]artifact.FCard{ fcard("a.txt", h1, null, null), fcard("b.txt", h2, null, null) }; const again = [_]artifact.FCard{ fcard("a.txt", h1, null, null), fcard("b.txt", h2, null, null) }; const fp = try artifact.fileSetFingerprint(gpa, &base); try testing.expectEqualSlices(u8, &fp, &(try artifact.fileSetFingerprint(gpa, &again))); // Every structural field is load-bearing: hash, perm token, old-name. const hash_diff = [_]artifact.FCard{ fcard("a.txt", h2, null, null), fcard("b.txt", h2, null, null) }; try testing.expect(!std.mem.eql(u8, &fp, &(try artifact.fileSetFingerprint(gpa, &hash_diff)))); const perm_diff = [_]artifact.FCard{ fcard("a.txt", h1, "x", null), fcard("b.txt", h2, null, null) }; try testing.expect(!std.mem.eql(u8, &fp, &(try artifact.fileSetFingerprint(gpa, &perm_diff)))); const old_diff = [_]artifact.FCard{ fcard("a.txt", h1, "w", "old.txt"), fcard("b.txt", h2, null, null) }; try testing.expect(!std.mem.eql(u8, &fp, &(try artifact.fileSetFingerprint(gpa, &perm_diff)))); try testing.expect(!std.mem.eql(u8, &(try artifact.fileSetFingerprint(gpa, &old_diff)), &(try artifact.fileSetFingerprint(gpa, &perm_diff)))); // §6.2 canonical escaping is part of the identity: a filename with a raw // SP ("a b") and one whose bytes literally spell the escape ("a\sb") are // different names and MUST fingerprint differently — only faithful // escaping keeps them apart. const spaced = [_]artifact.FCard{fcard("a b", h1, null, null)}; const literal = [_]artifact.FCard{fcard("a\\sb", h1, null, null)}; try testing.expect(!std.mem.eql( u8, &(try artifact.fileSetFingerprint(gpa, &spaced)), &(try artifact.fileSetFingerprint(gpa, &literal)), )); } // --------------------------------------------------------------------------- // Builder fixture: a real on-disk `.zit` seeded through `artifact.create`, // opened through `Repository.open` exactly as the CLI does (a slim copy of // repository_test.zig's `Repo`, narrowed to what the builder needs). // --------------------------------------------------------------------------- const repository = @import("repository.zig"); const repostate = @import("repostate.zig"); const repoview = @import("repoview.zig"); const Repository = repository.Repository; const hex_digits = "0123456789abcdef"; const Repo = struct { tmp: std.testing.TmpDir, fs: storage.FileStorage, log: logstore.LogStore, raw: std.Io.File, io: std.Io, fn init(self: *Repo, io: std.Io) !void { self.io = io; self.tmp = testing.tmpDir(.{}); self.raw = try self.tmp.dir.createFile(io, ".zit", .{ .read = true }); self.fs = storage.FileStorage.init(io, self.raw, 0); self.log = try logstore.LogStore.open(testing.allocator, self.fs.storageFile(), 0); } fn seal(self: *Repo) !void { try self.log.commit(); self.log.deinit(testing.allocator); self.raw.close(self.io); } fn cleanup(self: *Repo) void { self.tmp.cleanup(); } fn config(self: *Repo, enable: bool, k: u8) !void { try self.log.putConfig(.{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p", .enable_submanifests = enable, .submanifest_k = k, }); } fn putRaw(self: *Repo, bytes: []const u8) ![]u8 { var digest: [32]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(bytes, &digest, .{}); _ = try self.log.put(&digest, bytes); return hexName(&digest); } fn put(self: *Repo, m: *const artifact.Manifest) ![]u8 { const bytes = try artifact.create(m, testing.allocator); defer testing.allocator.free(bytes); return self.putRaw(bytes); } }; fn hexName(raw: []const u8) ![]u8 { const out = try testing.allocator.alloc(u8, raw.len * 2); for (raw, 0..) |b, i| { out[i * 2] = hex_digits[b >> 4]; out[i * 2 + 1] = hex_digits[b & 0x0f]; } return out; } fn baseManifest(kind: artifact.ArtifactType, ts: []const u8, p: []const artifact.PCard, t: []const artifact.TCard) artifact.Manifest { return .{ .arena_state = std.heap.ArenaAllocator.init(testing.allocator), .kind = kind, .a = null, .b = null, .c = if (kind == .manifest) .{ .comment = "c" } else null, .d = .{ .timestamp = ts }, .e = null, .g = null, .h = null, .i = null, .k = null, .l = null, .n = null, .r = null, .u = .{ .login = "u" }, .z = null, .w = null, .f = &.{}, .j = &.{}, .m = &.{}, .p = p, .q = &.{}, .t = t, }; } const FSpec = struct { ts: []const u8, parent: ?[]const u8 = null, baseline: ?[]const u8 = null, files: []const artifact.FCard = &.{}, }; fn putFiles(repo: *Repo, spec: FSpec) ![]u8 { var tag_buf: [1]artifact.TCard = undefined; var n: usize = 0; if (spec.parent == null) { tag_buf[n] = .{ .sign = .propagate, .name = "branch", .target = null, .value = "trunk" }; n += 1; } var parent_buf: [1]artifact.PCard = undefined; var p_slice: []const artifact.PCard = &.{}; if (spec.parent) |parent| { parent_buf[0] = .{ .parents = parent, .count = 1 }; p_slice = parent_buf[0..1]; } var m = baseManifest(.manifest, spec.ts, p_slice, tag_buf[0..n]); defer m.deinit(); m.f = spec.files; if (spec.baseline) |bh| m.b = .{ .hash = bh }; return repo.put(&m); } fn openRepo(out: *Repository, repo: *Repo, io: std.Io) !void { try Repository.open(out, testing.allocator, io, repo.tmp.dir, .{}); } /// Seed a linear file-bearing history: blobs `b0..n-1`, one root check-in /// tracking `width` files, then `extra` children each rewriting one file. /// Returns the tip's 64-hex name (caller frees). fn seedFileHistory(repo: *Repo, width: usize, extra: usize) ![]u8 { const gpa = testing.allocator; var names: std.ArrayList([]u8) = .empty; defer { for (names.items) |s| gpa.free(s); names.deinit(gpa); } var cards: std.ArrayList(artifact.FCard) = .empty; defer cards.deinit(gpa); var fname_bufs: std.ArrayList([]u8) = .empty; defer { for (fname_bufs.items) |s| gpa.free(s); fname_bufs.deinit(gpa); } for (0..width) |i| { const content = try std.fmt.allocPrint(gpa, "content of file {d}\n", .{i}); defer gpa.free(content); const blob = try repo.putRaw(content); try names.append(gpa, blob); const fname = try std.fmt.allocPrint(gpa, "dir{d:0>2}/file{d:0>4}.txt", .{ i % 7, i }); try fname_bufs.append(gpa, fname); try cards.append(gpa, .{ .filename = fname, .hash = blob, .perm = if (i % 9 == 0) "x" else null, .old_name = null, .mode = .regular }); } std.mem.sort(artifact.FCard, cards.items, {}, struct { fn lt(_: void, a: artifact.FCard, b: artifact.FCard) bool { return artifact.compareBytes(a.filename, b.filename) == .lt; } }.lt); var tip = try putFiles(repo, .{ .ts = "2020-01-01T00:00:00", .files = cards.items }); var day: usize = 2; for (0..extra) |i| { const content = try std.fmt.allocPrint(gpa, "rewritten {d}\n", .{i}); defer gpa.free(content); const blob = try repo.putRaw(content); try names.append(gpa, blob); cards.items[i % cards.items.len].hash = blob; var ts_buf: [32]u8 = undefined; const ts = try std.fmt.bufPrint(&ts_buf, "2020-01-{d:0>2}T00:00:00", .{day}); day += 1; const child = try putFiles(repo, .{ .ts = ts, .parent = tip, .files = cards.items }); gpa.free(tip); tip = child; } return tip; } fn readZit(gpa: std.mem.Allocator, repo: *Repo, io: std.Io) ![]u8 { return repo.tmp.dir.readFileAlloc(io, ".zit", gpa, .limited(1 << 26)); } // --------------------------------------------------------------------------- // Gate matrix and refusals (Task 6) + the batch oracle (Task 7). // --------------------------------------------------------------------------- test "checkpoint gate: whatever the emission setting, build writes the sidecar and never the store" { const io = testing.io; const gpa = testing.allocator; const Case = struct { enable: bool, k: u8 }; const cases = [_]Case{ .{ .enable = false, .k = 4 }, .{ .enable = true, .k = 0 }, .{ .enable = true, .k = 13 }, .{ .enable = true, .k = 2 }, }; for (cases) |case| { var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(case.enable, case.k); const tip = try seedFileHistory(&repo, 24, 3); defer gpa.free(tip); try repo.seal(); var r: Repository = undefined; try openRepo(&r, &repo, io); defer r.deinit(gpa); const census_before = try readZit(gpa, &repo, io); defer gpa.free(census_before); const report = try checkpoint.build(gpa, io, &r, repo.tmp.dir, null); const census_after = try readZit(gpa, &repo, io); defer gpa.free(census_after); try testing.expectEqual(@as(usize, 24), report.files); // Sidecar only: the store is byte-for-byte untouched, gate open or not. try testing.expectEqualSlices(u8, census_before, census_after); try testing.expectEqualStrings(tip, report.checkinHash()); // And the sidecar it wrote validates. var raw2 = repo.tmp.dir.openFile(io, ".zit", .{}) catch unreachable; defer raw2.close(io); const st2 = try raw2.stat(io); var fs2 = storage.FileStorage.init(io, raw2, st2.size); var loaded = checkpoint.tryLoad(gpa, io, repo.tmp.dir, fs2.storageFile()).?; defer loaded.deinit(gpa); try testing.expectEqualStrings(tip, loaded.claim.checkin_hash); } } test "checkpoint build: refusals — absent checkin, non-checkin target, off-mainline anchor, delta in prefix" { const io = testing.io; const gpa = testing.allocator; // Absent + off-mainline: a two-leaf history (root with two children). { var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(false, 0); const b1 = try repo.putRaw("blob one\n"); defer gpa.free(b1); const root = try putFiles(&repo, .{ .ts = "2020-01-01T00:00:00", .files = &.{.{ .filename = "a.txt", .hash = b1, .perm = null, .old_name = null, .mode = .regular }} }); defer gpa.free(root); const left = try putFiles(&repo, .{ .ts = "2020-01-02T00:00:00", .parent = root, .files = &.{.{ .filename = "a.txt", .hash = b1, .perm = null, .old_name = null, .mode = .regular }} }); defer gpa.free(left); const right = try putFiles(&repo, .{ .ts = "2020-01-03T00:00:00", .parent = root, .files = &.{.{ .filename = "b.txt", .hash = b1, .perm = null, .old_name = null, .mode = .regular }} }); defer gpa.free(right); try repo.seal(); var r: Repository = undefined; try openRepo(&r, &repo, io); defer r.deinit(gpa); try testing.expectError(error.CheckinNotFound, checkpoint.build(gpa, io, &r, repo.tmp.dir, "f" ** 64)); // A leaf is NOT an ancestor of its sibling leaf. try testing.expectError(error.CheckinNotAncestorOfAllLeaves, checkpoint.build(gpa, io, &r, repo.tmp.dir, left)); // The merge-base fold (null checkin) lands on the root and succeeds. const report = try checkpoint.build(gpa, io, &r, repo.tmp.dir, null); try testing.expectEqualStrings(root, report.checkinHash()); } // Delta manifest in the prefix. { var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(false, 0); const b1 = try repo.putRaw("base blob\n"); defer gpa.free(b1); const b2 = try repo.putRaw("delta blob\n"); defer gpa.free(b2); const root = try putFiles(&repo, .{ .ts = "2020-01-01T00:00:00", .files = &.{.{ .filename = "a.txt", .hash = b1, .perm = null, .old_name = null, .mode = .regular }} }); defer gpa.free(root); const child = try putFiles(&repo, .{ .ts = "2020-01-02T00:00:00", .parent = root, .baseline = root, .files = &.{.{ .filename = "b.txt", .hash = b2, .perm = null, .old_name = null, .mode = .regular }} }); defer gpa.free(child); try repo.seal(); var r: Repository = undefined; try openRepo(&r, &repo, io); defer r.deinit(gpa); try testing.expectError(error.DeltaManifestInPrefix, checkpoint.build(gpa, io, &r, repo.tmp.dir, null)); } } // --------------------------------------------------------------------------- // Drift/invalidation matrix and the paranoid bypass (Task 8). // --------------------------------------------------------------------------- /// Open through the facade and assert the derived view answers exactly as a /// full bypass derive over the same log — the "identical to a full derive" /// oracle every drift row requires. Also returns whether the checkpoint (vs /// a fallback path) could have served: the caller distinguishes HIT and MISS /// by deleting `.zit-cache/repo` first so only checkpoint or full derive can /// answer, then comparing `repoview.parsed_artifacts`. fn expectOpenEqualsFullDerive(repo: *Repo, io: std.Io) !u64 { const gpa = testing.allocator; repo.tmp.dir.deleteFile(io, repocache.cache_filename) catch {}; repoview.parsed_artifacts = 0; var served: Repository = undefined; try Repository.open(&served, gpa, io, repo.tmp.dir, .{}); defer served.deinit(gpa); const parses = repoview.parsed_artifacts; var full: Repository = undefined; try Repository.open(&full, gpa, io, repo.tmp.dir, .{ .bypass_cache = true }); defer full.deinit(gpa); // Observable equality over the derived surface the verbs read. try testing.expectEqualStrings(full.view.main_branch, served.view.main_branch); var it_full = full.view.state.leaves(); var it_served = served.view.state.leaves(); while (true) { const a = it_full.next(); const b = it_served.next(); try testing.expectEqual(a == null, b == null); if (a == null) break; try testing.expectEqualStrings(full.view.hashOf(a.?), served.view.hashOf(b.?)); } return parses; } test "checkpoint drift: every mutation is a MISS answered identically by the full derive; growth stays a HIT" { const io = testing.io; const gpa = testing.allocator; var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(true, 2); const tip = try seedFileHistory(&repo, 24, 3); defer gpa.free(tip); try repo.seal(); var r: Repository = undefined; try openRepo(&r, &repo, io); _ = try checkpoint.build(gpa, io, &r, repo.tmp.dir, null); // Sidecar-only: anchored at the tip, NOTHING lies past the prefix, so a // checkpointed open parses zero artifacts. const tail_artifacts: u64 = 0; r.deinit(gpa); // Baseline: the checkpoint serves, parsing ONLY the artifacts past the // prefix (the created tree nodes and nothing else). { const parses = try expectOpenEqualsFullDerive(&repo, io); try testing.expectEqual(tail_artifacts, parses); } // Growth: append a commit; still a hit (parses = tail only). { var raw2 = try repo.tmp.dir.openFile(io, ".zit", .{ .mode = .read_write }); const st2 = try raw2.stat(io); var fs2 = storage.FileStorage.init(io, raw2, st2.size); var log = try logstore.LogStore.open(gpa, fs2.storageFile(), 0); const blob_hash = [_]u8{0x77} ** 32; _ = try log.put(&blob_hash, "appended tail blob"); try log.commit(); log.deinit(gpa); raw2.close(io); const parses = try expectOpenEqualsFullDerive(&repo, io); try testing.expectEqual(tail_artifacts + 1, parses); } // Sidecar mutations: a fingerprint byte (caught by the whole-file crc — // open never re-enumerates the file set; a crc-consistent fingerprint lie // is `zit checkpoint verify`'s to catch), then the crc itself — each a // MISS (full derive parses everything) while the answers stay identical. const good = try readRaw(gpa, io, repo.tmp.dir); defer gpa.free(good); const all_parses: u64 = blk: { checkpoint.drop(io, repo.tmp.dir); break :blk try expectOpenEqualsFullDerive(&repo, io); }; for ([_]usize{ 48, good.len - 1 }) |flip| { // a fingerprint byte; the trailing crc const bad = try gpa.dupe(u8, good); defer gpa.free(bad); bad[flip] ^= 0xFF; try rewriteRaw(io, repo.tmp.dir, bad); const parses = try expectOpenEqualsFullDerive(&repo, io); try testing.expectEqual(all_parses, parses); } } test "checkpoint bypass: --no-cache never consults the sidecar and answers identically across valid, corrupt, and absent" { const io = testing.io; const gpa = testing.allocator; var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(false, 0); const tip = try seedFileHistory(&repo, 12, 2); defer gpa.free(tip); try repo.seal(); var r: Repository = undefined; try openRepo(&r, &repo, io); _ = try checkpoint.build(gpa, io, &r, repo.tmp.dir, null); r.deinit(gpa); const good = try readRaw(gpa, io, repo.tmp.dir); defer gpa.free(good); const corrupt = try gpa.dupe(u8, good); defer gpa.free(corrupt); corrupt[good.len / 2] ^= 0xFF; var reference: ?[]const u8 = null; defer if (reference) |s| gpa.free(s); for (0..3) |variant| { switch (variant) { 0 => try rewriteRaw(io, repo.tmp.dir, good), 1 => try rewriteRaw(io, repo.tmp.dir, corrupt), 2 => checkpoint.drop(io, repo.tmp.dir), else => unreachable, } checkpoint.tryload_consults = 0; var opened: Repository = undefined; try Repository.open(&opened, gpa, io, repo.tmp.dir, .{ .bypass_cache = true }); defer opened.deinit(gpa); // The paranoid promise: the sidecar was never consulted, let alone read. try testing.expectEqual(@as(u64, 0), checkpoint.tryload_consults); const tip_hash = blk: { var it = opened.view.state.leaves(); const leaf = it.next().?; try testing.expectEqual(@as(?repostate.Rid, null), it.next()); break :blk try gpa.dupe(u8, opened.view.hashOf(leaf)); }; if (reference) |want| { try testing.expectEqualStrings(want, tip_hash); gpa.free(tip_hash); } else reference = tip_hash; } } test "checkpoint drift: a claim whose anchors no longer resolve is a MISS under the §5.2 hard-failure discipline" { const io = testing.io; const gpa = testing.allocator; var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(true, 2); const tip = try seedFileHistory(&repo, 24, 2); defer gpa.free(tip); try repo.seal(); var r: Repository = undefined; try openRepo(&r, &repo, io); _ = try checkpoint.build(gpa, io, &r, repo.tmp.dir, null); const tail_artifacts: u64 = 0; // sidecar-only: nothing lies past a tip-anchored prefix r.deinit(gpa); const good = try readRaw(gpa, io, repo.tmp.dir); defer gpa.free(good); const all_parses = blk: { checkpoint.drop(io, repo.tmp.dir); break :blk try expectOpenEqualsFullDerive(&repo, io); }; try testing.expect(all_parses > tail_artifacts); // The anchor failure, spelled as a crc-honest claim rewrite: C absent. const ck_off = 83; // the v3 fixed prefix ends where checkin_hash begins const Case = struct { name: []const u8, off: usize, hex: []const u8 }; const absent = "e" ** 64; const cases = [_]Case{ .{ .name = "C absent", .off = ck_off, .hex = absent }, }; for (cases) |case| { const bad = try gpa.dupe(u8, good); defer gpa.free(bad); @memcpy(bad[case.off..][0..64], case.hex); var crc = crc32.Crc32.init(); crc.update(bad[0 .. bad.len - 4]); std.mem.writeInt(u32, bad[bad.len - 4 ..][0..4], crc.final(), .little); try rewriteRaw(io, repo.tmp.dir, bad); const parses = try expectOpenEqualsFullDerive(&repo, io); if (parses != all_parses) { std.debug.print("anchor case served from a dangling claim: {s}\n", .{case.name}); return error.TestUnexpectedResult; } } // Control: the untouched claim still resumes. try rewriteRaw(io, repo.tmp.dir, good); try testing.expectEqual(tail_artifacts, try expectOpenEqualsFullDerive(&repo, io)); } // --------------------------------------------------------------------------- // The verifier (Task 9). // --------------------------------------------------------------------------- test "checkpoint verify: a good checkpoint passes; every tampering earns its distinct verdict" { const io = testing.io; const gpa = testing.allocator; var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(true, 2); const tip = try seedFileHistory(&repo, 24, 2); defer gpa.free(tip); try repo.seal(); // No sidecar at all. try testing.expectEqual(checkpoint.Verdict.no_checkpoint, try checkpoint.verify(gpa, io, repo.tmp.dir)); var r: Repository = undefined; try openRepo(&r, &repo, io); const report = try checkpoint.build(gpa, io, &r, repo.tmp.dir, null); r.deinit(gpa); // The good checkpoint verifies ok, with the claim echoed back. { const verdict = try checkpoint.verify(gpa, io, repo.tmp.dir); try testing.expectEqualStrings(report.checkinHash(), verdict.ok.checkinHash()); try testing.expectEqual(@as(usize, 24), verdict.ok.files); } const good = try readRaw(gpa, io, repo.tmp.dir); defer gpa.free(good); const refresh = struct { fn crc(b: []u8) void { var c = crc32.Crc32.init(); c.update(b[0 .. b.len - 4]); std.mem.writeInt(u32, b[b.len - 4 ..][0..4], c.final(), .little); } }; // A crc-consistent fingerprint lie — invisible to the open path, and // exactly what verify exists to catch. { const bad = try gpa.dupe(u8, good); defer gpa.free(bad); bad[48] ^= 0xFF; refresh.crc(bad); try rewriteRaw(io, repo.tmp.dir, bad); try testing.expectEqual(checkpoint.Verdict.fileset_mismatch, try checkpoint.verify(gpa, io, repo.tmp.dir)); } // A crc-consistent record-body tamper: derivation_mismatch (§6.1). { const bad = try gpa.dupe(u8, good); defer gpa.free(bad); bad[bad.len - 8] ^= 0xFF; // inside the record body's tail refresh.crc(bad); try rewriteRaw(io, repo.tmp.dir, bad); try testing.expectEqual(checkpoint.Verdict.derivation_mismatch, try checkpoint.verify(gpa, io, repo.tmp.dir)); } // A bare corruption (crc broken): stale — the open path's silent miss, // named for the operator. { const bad = try gpa.dupe(u8, good); defer gpa.free(bad); bad[good.len / 2] ^= 0xFF; try rewriteRaw(io, repo.tmp.dir, bad); try testing.expectEqual(checkpoint.Verdict.stale, try checkpoint.verify(gpa, io, repo.tmp.dir)); } } // --------------------------------------------------------------------------- // Fault injection (Task 7 Step 3): every allocation-failure path surfaces // OutOfMemory without leaking, no torn sidecar survives, and a retry is // byte-identical. Already-stored immutable artifacts may remain — content // addressing makes them harmless and the retry deduplicates them. // --------------------------------------------------------------------------- test "checkpoint build: survives injected allocation failures and retries byte-identically" { const io = testing.io; const gpa = testing.allocator; var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(true, 1); const tip = try seedFileHistory(&repo, 8, 1); defer gpa.free(tip); try repo.seal(); // A manual failing-allocator sweep rather than checkAllAllocationFailures: // the advisory tails (`checkpoint.store`, the cache rewrite) deliberately // SWALLOW failures, which that harness reads as a bug. The invariants that // matter are asserted directly instead: every injected failure either // surfaces OutOfMemory or completes; the sidecar is never torn (tryLoad // yields a valid claim or nothing); and nothing leaks (the testing // allocator's leak check covers each iteration). var fail_index: usize = 0; while (fail_index < 400) : (fail_index += 1) { var failing = testing.FailingAllocator.init(gpa, .{ .fail_index = fail_index }); const alloc = failing.allocator(); var r0: Repository = undefined; const outcome: ?checkpoint.BuildError = blk: { Repository.open(&r0, alloc, io, repo.tmp.dir, .{ .bypass_cache = true }) catch |e| switch (e) { error.OutOfMemory => break :blk error.OutOfMemory, else => return e, }; defer r0.deinit(alloc); _ = checkpoint.build(alloc, io, &r0, repo.tmp.dir, null) catch |e| break :blk e; break :blk null; }; if (outcome) |e| try testing.expectEqual(checkpoint.BuildError.OutOfMemory, e); // Never a torn sidecar, whatever the cut point. var raw2 = repo.tmp.dir.openFile(io, ".zit", .{}) catch unreachable; const st2 = try raw2.stat(io); var fs2 = storage.FileStorage.init(io, raw2, st2.size); if (checkpoint.tryLoad(gpa, io, repo.tmp.dir, fs2.storageFile())) |loaded_const| { var loaded = loaded_const; loaded.deinit(gpa); } raw2.close(io); if (outcome == null and failing.has_induced_failure) break; // completed past the last injected point if (!failing.has_induced_failure) break; // fewer allocations than the index: done } // Whatever the injected failures left behind, tryLoad never trusts a torn // sidecar, and a clean rebuild lands the same bytes as a fresh build. var r: Repository = undefined; try Repository.open(&r, gpa, io, repo.tmp.dir, .{ .bypass_cache = true }); _ = try checkpoint.build(gpa, io, &r, repo.tmp.dir, null); r.deinit(gpa); const first = try readRaw(gpa, io, repo.tmp.dir); defer gpa.free(first); var r2: Repository = undefined; try Repository.open(&r2, gpa, io, repo.tmp.dir, .{ .bypass_cache = true }); _ = try checkpoint.build(gpa, io, &r2, repo.tmp.dir, null); r2.deinit(gpa); const second = try readRaw(gpa, io, repo.tmp.dir); defer gpa.free(second); try testing.expectEqualSlices(u8, first, second); } // --------------------------------------------------------------------------- // Hybrid-corpus qualification (Task 11): flat-then-tree history with the // S-card switch at 10%, 50%, and 90%, the checkpoint anchored both at the tip // and at the switch itself. The gate is the COUNTER, not the clock: a // checkpointed derive parses exactly the artifacts past the prefix and ZERO // below it, at every switch point; `verify` passes over the whole corpus. // (Wall-time scaling on repository-sized corpora is bench's measurement; a // timing "win" with a nonzero prefix-parse count would be a measurement bug, // which this gate makes impossible to miss.) // --------------------------------------------------------------------------- fn putTreeCheckin(repo: *Repo, ts: []const u8, parent: []const u8, cards: []const artifact.FCard) ![]u8 { const gpa = testing.allocator; var built = try artifact.buildTree(gpa, cards, 1, .sha3_256); defer built.deinit(); try testing.expect(!built.isFlat()); // the corpus names are chosen to partition for (built.subs) |sub| { var digest: [32]u8 = undefined; _ = try std.fmt.hexToBytes(&digest, sub.name); _ = try repo.log.putArtifact(&digest, sub.bytes, null); } var parent_buf: [1]artifact.PCard = undefined; parent_buf[0] = .{ .parents = parent, .count = 1 }; var m = baseManifest(.manifest, ts, parent_buf[0..1], &.{}); defer m.deinit(); m.s = built.s; return repo.put(&m); } test "checkpoint hybrid corpus: zero prefix parses at every switch point, anchored at tip and at the switch" { const io = testing.io; const gpa = testing.allocator; const total_commits: usize = 30; const width: usize = 16; for ([_]usize{ 3, 15, 27 }) |switch_at| { // ~10%, 50%, 90% of history var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(true, 1); // the final config state: emission on, k pinned // Divergent-first-byte names so k=1 partitions (see the probe note). var fname_bufs: std.ArrayList([]u8) = .empty; defer { for (fname_bufs.items) |s| gpa.free(s); fname_bufs.deinit(gpa); } var blob_names: std.ArrayList([]u8) = .empty; defer { for (blob_names.items) |s| gpa.free(s); blob_names.deinit(gpa); } var cards: std.ArrayList(artifact.FCard) = .empty; defer cards.deinit(gpa); for (0..width) |i| { const content = try std.fmt.allocPrint(gpa, "seed content {d}\n", .{i}); defer gpa.free(content); const blob = try repo.putRaw(content); try blob_names.append(gpa, blob); const fname = try std.fmt.allocPrint(gpa, "{c}{d:0>3}.txt", .{ 'a' + @as(u8, @intCast(i)), i }); try fname_bufs.append(gpa, fname); try cards.append(gpa, .{ .filename = fname, .hash = blob, .perm = null, .old_name = null, .mode = .regular }); } std.mem.sort(artifact.FCard, cards.items, {}, struct { fn lt(_: void, a: artifact.FCard, b: artifact.FCard) bool { return artifact.compareBytes(a.filename, b.filename) == .lt; } }.lt); var switch_hash: ?[]u8 = null; defer if (switch_hash) |s| gpa.free(s); var tip = try putFiles(&repo, .{ .ts = "2020-01-01T00:00:00", .files = cards.items }); for (1..total_commits) |i| { const content = try std.fmt.allocPrint(gpa, "rewrite {d}\n", .{i}); defer gpa.free(content); const blob = try repo.putRaw(content); try blob_names.append(gpa, blob); cards.items[i % width].hash = blob; var ts_buf: [32]u8 = undefined; const ts = try std.fmt.bufPrint(&ts_buf, "2020-{d:0>2}-{d:0>2}T00:00:00", .{ 1 + i / 27, 1 + i % 27 }); const child = if (i >= switch_at) try putTreeCheckin(&repo, ts, tip, cards.items) else try putFiles(&repo, .{ .ts = ts, .parent = tip, .files = cards.items }); gpa.free(tip); tip = child; if (i == switch_at) switch_hash = try gpa.dupe(u8, child); } try repo.seal(); // A full derive's parse census, and per-anchor prefix censuses. checkpoint.drop(io, repo.tmp.dir); repo.tmp.dir.deleteFile(io, repocache.cache_filename) catch {}; const all_parses = try expectOpenEqualsFullDerive(&repo, io); for ([_][]const u8{ tip, switch_hash.? }) |anchor| { var r: Repository = undefined; try Repository.open(&r, gpa, io, repo.tmp.dir, .{ .bypass_cache = true }); const report = try checkpoint.build(gpa, io, &r, repo.tmp.dir, anchor); r.deinit(gpa); _ = report; // The exact tail census: artifacts at or past the anchor's frame // end (including the checkpoint's own nodes) and NOTHING below. const expected_tail: u64 = blk: { var raw2 = try repo.tmp.dir.openFile(io, ".zit", .{}); defer raw2.close(io); const st2 = try raw2.stat(io); var fs2 = storage.FileStorage.init(io, raw2, st2.size); var log = try logstore.LogStore.openReadOnly(gpa, fs2.storageFile()); defer log.deinit(gpa); var digest: [32]u8 = undefined; _ = try std.fmt.hexToBytes(&digest, anchor); const key = try log.logKeyAt(log.lookup(&digest).?); const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var tail: u64 = 0; for (rids) |rid| { if (log.offsetOf(rid) >= key.end) tail += 1; } break :blk tail; }; const parses = try expectOpenEqualsFullDerive(&repo, io); try testing.expectEqual(expected_tail, parses); try testing.expect(parses < all_parses); // the prefix really was skipped const verdict = try checkpoint.verify(gpa, io, repo.tmp.dir); try testing.expectEqual(@as(usize, width), verdict.ok.files); } gpa.free(tip); } } // --------------------------------------------------------------------------- // Opportunistic advance: the open path's healing branches keep an existing // checkpoint fresh — witness shape only, never minting, behind the growth // and anchor-progress gates. // --------------------------------------------------------------------------- test "checkpoint advance: a stale checkpoint moves forward as witness on open — no artifacts, answers equal, gates honored" { const io = testing.io; const gpa = testing.allocator; var repo: Repo = undefined; try repo.init(io); defer repo.cleanup(); try repo.config(true, 1); // gate OPEN — proving the automatic path still mints nothing const tip0 = try seedFileHistory(&repo, 12, 2); defer gpa.free(tip0); try repo.seal(); // The operator's explicit build: tree shape, artifacts minted. var r0: Repository = undefined; try Repository.open(&r0, gpa, io, repo.tmp.dir, .{}); const built = try checkpoint.build(gpa, io, &r0, repo.tmp.dir, null); r0.deinit(gpa); const sidecar_before = try readRaw(gpa, io, repo.tmp.dir); defer gpa.free(sidecar_before); // Below the growth threshold: an open leaves the sidecar byte-untouched. var r1: Repository = undefined; try Repository.open(&r1, gpa, io, repo.tmp.dir, .{}); r1.deinit(gpa); { const now = try readRaw(gpa, io, repo.tmp.dir); defer gpa.free(now); try testing.expectEqualSlices(u8, sidecar_before, now); } // Grow the log past the threshold WITH anchor progress: fat commits, // appended through a local writable store (the fixture's own is sealed). var tip = try gpa.dupe(u8, tip0); defer gpa.free(tip); { var raw = try repo.tmp.dir.openFile(io, ".zit", .{ .mode = .read_write }); defer raw.close(io); const st = try raw.stat(io); var fs2 = storage.FileStorage.init(io, raw, st.size); var log = try logstore.LogStore.open(gpa, fs2.storageFile(), 0); defer log.deinit(gpa); const putLocal = struct { fn f(l: *logstore.LogStore, bytes: []const u8) ![64]u8 { var digest: [32]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(bytes, &digest, .{}); _ = try l.put(&digest, bytes); var hex: [64]u8 = undefined; for (digest, 0..) |b, i| { hex[i * 2] = hex_digits[b >> 4]; hex[i * 2 + 1] = hex_digits[b & 0x0f]; } return hex; } }.f; var day: usize = 3; const fat = try gpa.alloc(u8, 512 * 1024); defer gpa.free(fat); for (0..12) |i| { @memset(fat, 'a' + @as(u8, @intCast(i))); const blob_hex = try putLocal(&log, fat); var ts_buf: [32]u8 = undefined; const ts = try std.fmt.bufPrint(&ts_buf, "2020-02-{d:0>2}T00:00:00", .{day}); day += 1; var name_buf: [16]u8 = undefined; const fname = try std.fmt.bufPrint(&name_buf, "{c}fat.bin", .{'b' + @as(u8, @intCast(i))}); var parent_buf = [_]artifact.PCard{.{ .parents = tip, .count = 1 }}; var m = baseManifest(.manifest, ts, parent_buf[0..1], &.{}); defer m.deinit(); m.f = &.{.{ .filename = fname, .hash = &blob_hex, .perm = null, .old_name = null, .mode = .regular }}; const mbytes = try artifact.create(&m, gpa); defer gpa.free(mbytes); const child_hex = try putLocal(&log, mbytes); gpa.free(tip); tip = try gpa.dupe(u8, &child_hex); } try log.commit(); } // The healing open advances the sidecar: new anchor, witness shape, and // the log itself is byte-untouched (nothing minted, gate open or not). const zit_before = try readZit(gpa, &repo, io); defer gpa.free(zit_before); var r2: Repository = undefined; try Repository.open(&r2, gpa, io, repo.tmp.dir, .{}); r2.deinit(gpa); const zit_after = try readZit(gpa, &repo, io); defer gpa.free(zit_after); try testing.expectEqualSlices(u8, zit_before, zit_after); var raw2 = try repo.tmp.dir.openFile(io, ".zit", .{}); defer raw2.close(io); const st2 = try raw2.stat(io); var fs3 = storage.FileStorage.init(io, raw2, st2.size); var advanced = checkpoint.tryLoad(gpa, io, repo.tmp.dir, fs3.storageFile()).?; defer advanced.deinit(gpa); try testing.expectEqualStrings(tip, advanced.claim.checkin_hash); _ = built; // The advanced claim independently verifies, and a cold-cache open // resumes from it with answers equal to a full derive. const verdict = try checkpoint.verify(gpa, io, repo.tmp.dir); try testing.expectEqualStrings(tip, verdict.ok.checkinHash()); _ = try expectOpenEqualsFullDerive(&repo, io); }