Artifact 302e70e22ce02ed53782dd84e68d465dbd2e44608fc4009912b0a6e1c128d069:
- File
src/logstore_test.zig
— part of check-in
[dd71dd3006]
at
2026-07-29 13:18:34
on branch logstore-index
— logstore: checksum through the vendored crc32
The four frame checksums plus the file header. The open-time scan CRCs every payload byte, so this is ~93% of a cold open: 416 MB goes 1.22s to 292ms. No format change — the values are identical by construction. (user: dmestas, size: 102090) [annotate] [blame] [check-ins using] [more...]
//! Unit tests for the framed, content-addressed log (storage-track-plan.md §C/§E). //! //! Separated from the implementation per the kind-separation convention //! (mirrors the sibling `*_fuzz.zig` split). Every test here exercises only //! `logstore`'s public surface; tests reaching file-private decls stay inline //! in `logstore.zig`. const std = @import("std"); const logstore = @import("logstore.zig"); const crc32 = @import("crc32.zig"); const storage = @import("storage.zig"); // All four are files of the one `zit` module (this test aggregates into // root.zig's test block), so they are reached by relative import — the same // copies the module itself compiles, never a colliding second one. const delta = @import("delta.zig"); fn openTmp(tmp: anytype, io: std.Io, name: []const u8) !storage.FileStorage { const raw = try tmp.dir.createFile(io, name, .{ .read = true }); return storage.FileStorage.init(io, raw, 0); } /// Fault-injection seam for the rollback path (mirrors sim/FaultyFile, which /// cannot fail a shrinking truncate): a healthy in-memory file whose /// `truncate` ALWAYS fails — the device fault the offset-bookkeeping invariant /// guards against — and whose `append` optionally tears once armed. A healthy /// `append` lets a frame actually land before the failing rollback runs. const TruncateFailingFile = struct { gpa: std.mem.Allocator, bytes: std.ArrayList(u8) = .empty, /// When set, the next append persists a 1-byte prefix (a genuine torn /// write the failing truncate cannot remove) and returns error.Io. fail_append: bool = false, fn init(gpa: std.mem.Allocator) TruncateFailingFile { return .{ .gpa = gpa }; } fn deinit(self: *TruncateFailingFile) void { self.bytes.deinit(self.gpa); self.* = undefined; } fn storageFile(self: *TruncateFailingFile) storage.StorageFile { return .{ .context = self, .vtable = &vtable }; } const vtable: storage.StorageFile.VTable = .{ .append = appendImpl, .pread = preadImpl, .sync = syncImpl, .size = sizeImpl, .truncate = truncateImpl, }; fn appendImpl(ctx: *anyopaque, data: []const u8) storage.StorageError!void { const self: *TruncateFailingFile = @ptrCast(@alignCast(ctx)); if (self.fail_append and data.len > 0) { self.bytes.appendSlice(self.gpa, data[0..1]) catch return error.Io; return error.Io; // torn: a prefix lingers past the reported end } self.bytes.appendSlice(self.gpa, data) catch return error.Io; } fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize { const self: *TruncateFailingFile = @ptrCast(@alignCast(ctx)); if (offset >= self.bytes.items.len) return 0; const start: usize = @intCast(offset); const n = @min(self.bytes.items.len - start, buf.len); @memcpy(buf[0..n], self.bytes.items[start..][0..n]); return n; } fn syncImpl(ctx: *anyopaque) storage.StorageError!void { _ = ctx; } fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 { const self: *TruncateFailingFile = @ptrCast(@alignCast(ctx)); return self.bytes.items.len; } fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void { _ = ctx; _ = len; return error.Io; // the device fault under test } }; test "logstore §C: a failed rollback truncate after a torn append poisons the store (offset bookkeeping equals the file end)" { var tf = TruncateFailingFile.init(std.testing.allocator); defer tf.deinit(); const file = tf.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // Arm the append fault: the frame append tears (a prefix lingers) and the // rollback truncate then fails, so the file's physical end runs ahead of // `end`. The store must poison rather than append the next frame at a // diverged offset. tf.fail_append = true; try std.testing.expectError(error.Storage, store.put("h" ** 32, "torn payload")); try std.testing.expect(store.poisoned); // Even with the disk healed, the poisoned store refuses further writes: it // can no longer prove `end` equals the file end, so every put fails loudly // with error.Storage instead of corrupting the log at a wrong offset. tf.fail_append = false; try std.testing.expectError(error.Storage, store.put("g" ** 32, "second payload")); try std.testing.expect(store.poisoned); } test "logstore §C: a failed rollback truncate after an index-bookkeeping OOM poisons the store (offset bookkeeping equals the file end)" { // The new-entry insert path appends the frame, then records it in the // index. If that bookkeeping OOMs the frame is rolled back off disk; when // the rollback truncate ALSO fails, `end` can no longer be trusted to equal // the file's physical end, so the store poisons rather than let a later put // land at a diverged offset. Discover the fail_index that strikes the index // insert (frame already on disk) by scanning — no internal count is hard- // coded — then assert the poison and the loud subsequent failure. var found = false; var fail_index: usize = 0; while (fail_index < 64) : (fail_index += 1) { var fa = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = fail_index }); const gpa = fa.allocator(); var tf = TruncateFailingFile.init(std.testing.allocator); defer tf.deinit(); const file = tf.storageFile(); // open on an empty file allocates nothing, so fail_index indexes put's // own allocations directly. var store = logstore.LogStore.open(gpa, file, 0) catch continue; defer store.deinit(gpa); const before = try file.size(); if (store.put("h" ** 32, "rollback payload")) |_| { continue; // fail_index past this put's allocations: no failure induced } else |e| { const after = try file.size(); if (after == before) continue; // OOM struck before the frame landed // Frame landed, bookkeeping OOM'd, rollback truncate failed. try std.testing.expect(e == error.OutOfMemory); try std.testing.expect(store.poisoned); // Heal the allocator; a later write must still fail loudly. fa.fail_index = std.math.maxInt(usize); try std.testing.expectError(error.Storage, store.put("g" ** 32, "second payload")); found = true; break; } } try std.testing.expect(found); } fn writeRawHeader(file: storage.StorageFile, magic_bytes: []const u8, ver: u16, hk: u8, bad_crc: bool) !void { std.debug.assert(magic_bytes.len == 4); var buf: [logstore.header_len]u8 = [_]u8{0} ** logstore.header_len; @memcpy(buf[0..4], magic_bytes); std.mem.writeInt(u16, buf[4..6], ver, .little); buf[6] = hk; var crc = crc32.Crc32.init(); crc.update(buf[0..60]); var crc_val = crc.final(); if (bad_crc) crc_val ^= 1; std.mem.writeInt(u32, buf[60..64], crc_val, .little); try file.append(&buf); } test "logstore §C: open rejects a bad magic" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); try writeRawHeader(file, "XXXX", logstore.version, 0, false); try std.testing.expectError(error.Corrupt, logstore.LogStore.open(std.testing.allocator, file, 0)); } test "logstore §C: open re-initializes a torn file header instead of rejecting it" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); try file.append(&[_]u8{ 'z', 'i', 't' }); // 3 of 64 header bytes: nowhere near complete var store = try logstore.LogStore.open(std.testing.allocator, file, 3); defer store.deinit(std.testing.allocator); try std.testing.expectEqual(@as(u64, logstore.header_len), store.size()); try std.testing.expectEqual(@as(u8, 3), store.hash_kind); } test "logstore §C: open rejects a stored version other than this reader's" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); // Valid magic + crc but a version of 0. The writer only ever emits the // current `version`, so a header carrying any other value — older // included — is not a zit-written file and must be rejected, never // silently accepted. ("zitL" is the frozen §C magic.) try writeRawHeader(file, "zitL", 0, 5, false); try std.testing.expectError(error.Corrupt, logstore.LogStore.open(std.testing.allocator, file, 0)); } test "storage §B: truncate to a larger length extends the file and tracks size" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "trunc.bin"); defer fs.file.close(io); const file = fs.storageFile(); try file.append("0123"); try std.testing.expectEqual(@as(u64, 4), try file.size()); // setLength both shrinks and extends: the append watermark must track the // file's real length, never stay stale at the pre-truncate value. try file.truncate(9); try std.testing.expectEqual(@as(u64, 9), try file.size()); // A later append then lands at the true end, not over the extended tail. try file.append("Z"); var buf: [10]u8 = undefined; const n = try file.pread(0, &buf); try std.testing.expectEqual(@as(usize, 10), n); try std.testing.expectEqualSlices(u8, &[_]u8{ '0', '1', '2', '3', 0, 0, 0, 0, 0, 'Z' }, buf[0..n]); } test "logstore §C: put then content round-trips a raw frame" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const hash = "h" ** 32; const rid = try store.put(hash, "hello world"); try std.testing.expectEqualStrings(hash, store.hashOf(rid)); const got = (try store.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings("hello world", got); } test "logstore §C: a written frame occupies an 8-byte-aligned span" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); _ = try store.put("abcde", "xyz"); // unpadded = 26+5+3 = 34, align8 -> 40 try std.testing.expectEqual(@as(u64, 0), store.size() % 8); try std.testing.expectEqual(logstore.header_len + 40, store.size()); } test "logstore §C: content hash is the uncompressed address; hashOf returns it" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const h1 = "1" ** 16; const h2 = "2" ** 20; const r1 = try store.put(h1, "aaa"); const r2 = try store.put(h2, "bbbbbb"); try std.testing.expectEqualStrings(h1, store.hashOf(r1)); try std.testing.expectEqualStrings(h2, store.hashOf(r2)); const c1 = (try store.content(std.testing.allocator, r1)).?; defer std.testing.allocator.free(c1); try std.testing.expectEqualStrings("aaa", c1); } test "logstore §C: put is idempotent for an equal hash and appends nothing" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const hash = "dup" ** 8; const r1 = try store.put(hash, "payload-a"); const size_after_first = store.size(); const r2 = try store.put(hash, "payload-a"); try std.testing.expectEqual(r1, r2); try std.testing.expectEqual(size_after_first, store.size()); } test "logstore §C: intern yields a phantom; content is null and no frame is written" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const hash = "phantom-hash-example-value"; const size_before = store.size(); const rid = try store.intern(hash); try std.testing.expectEqual(size_before, store.size()); const got = try store.content(std.testing.allocator, rid); try std.testing.expectEqual(@as(?[]u8, null), got); } test "logstore §C: put upgrades an interned phantom in place, same rid, content now present" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); // intern fixes the rid with no frame; content is null while phantom. const h = "phantom-upgrade-hash-value-x"; const rid = try store.intern(h); try std.testing.expectEqual(@as(?[]u8, null), try store.content(std.testing.allocator, rid)); const size0 = store.size(); // put on the same hash upgrades the phantom in place: same rid, and it // appends the first frame (a phantom had none), so the log grows. const rid2 = try store.put(h, "content-arrived"); try std.testing.expectEqual(rid, rid2); try std.testing.expect(store.size() > size0); { const got = (try store.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings("content-arrived", got); } try std.testing.expectEqualStrings(h, store.hashOf(rid)); // The upgraded frame is real on disk, not just an in-memory flag: commit, // reopen, and the hash resolves to content-bearing bytes (§B inv1 for a // put that began life as a phantom). try store.commit(); store.deinit(std.testing.allocator); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); const rrid = reopened.lookup(h) orelse return error.TestUnexpectedResult; const got2 = (try reopened.content(std.testing.allocator, rrid)).?; defer std.testing.allocator.free(got2); try std.testing.expectEqualStrings("content-arrived", got2); try std.testing.expectEqualStrings(h, reopened.hashOf(rrid)); } test "logstore §C: persisted RID order follows frame order after phantom upgrade" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "order.bin"); defer fs.file.close(io); var store = try logstore.LogStore.open(std.testing.allocator, fs.storageFile(), 0); defer store.deinit(std.testing.allocator); _ = try store.intern("A"); _ = try store.put("B", "second rid, first persisted frame"); _ = try store.put("A", "first rid, second persisted frame"); const rids = try store.ridsInPersistedOrder(std.testing.allocator); defer std.testing.allocator.free(rids); try std.testing.expectEqual(@as(usize, 2), rids.len); try std.testing.expectEqualStrings("B", store.hashOf(rids[0])); try std.testing.expectEqualStrings("A", store.hashOf(rids[1])); } test "logstore §C(inv1): every committed put is readable and hash-exact after reopen" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); const Item = struct { hash: []const u8, bytes: []const u8 }; const items = [_]Item{ .{ .hash = "h-one" ** 4, .bytes = "one" }, .{ .hash = "h-two" ** 4, .bytes = "two-payload" }, .{ .hash = "h-three" ** 3, .bytes = "three!" }, }; var store = try logstore.LogStore.open(std.testing.allocator, file, 0); for (items) |it| _ = try store.put(it.hash, it.bytes); try store.commit(); store.deinit(std.testing.allocator); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); for (items) |it| { const rid = reopened.lookup(it.hash) orelse return error.TestUnexpectedResult; try std.testing.expectEqualStrings(it.hash, reopened.hashOf(rid)); const got = (try reopened.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(it.bytes, got); } } test "logstore §C(inv3): the index rebuilt from the log equals the written model" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); const hashes = [_][]const u8{ "r1" ** 8, "r2" ** 8, "r3" ** 8, "r4" ** 8 }; const bodies = [_][]const u8{ "b1", "bb2", "bbb3", "bbbb4" }; var store = try logstore.LogStore.open(std.testing.allocator, file, 0); for (hashes, bodies) |h, b| _ = try store.put(h, b); try store.commit(); store.deinit(std.testing.allocator); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); try std.testing.expectEqual(hashes.len, reopened.entries.items.len); for (hashes, bodies) |h, b| { const rid = reopened.lookup(h).?; const got = (try reopened.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(b, got); } } test "logstore §C: phantom hash is absent after reopen" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); const hash = "phantom-only-hash-value"; _ = try store.intern(hash); try store.commit(); store.deinit(std.testing.allocator); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); try std.testing.expect(reopened.lookup(hash) == null); } fn openNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); store.deinit(gpa); } fn openReadOnlyNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.openReadOnly(gpa, file); store.deinit(gpa); } test "logstore §C: open propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); _ = try seed.put("seed-hash-one-value", "seed payload one"); _ = try seed.put("seed-hash-two-value", "seed payload two, a bit longer"); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, openNoLeak, .{file}); } test "logstore §C: openReadOnly propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); { var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); defer seed.deinit(std.testing.allocator); _ = try seed.put("seed-hash-one-value", "seed payload one"); _ = try seed.put("seed-hash-two-value", "seed payload two, a bit longer"); try seed.commit(); } try std.testing.checkAllAllocationFailures(std.testing.allocator, openReadOnlyNoLeak, .{file}); } fn putNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const size = file.size() catch 0; var hash_buf: [8]u8 = undefined; std.mem.writeInt(u64, &hash_buf, size, .little); _ = try store.put(&hash_buf, "payload bytes exercised under simulated allocation failure"); } test "logstore §C: put propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); seed.deinit(std.testing.allocator); // header only: put() always takes the new-entry path try std.testing.checkAllAllocationFailures(std.testing.allocator, putNoLeak, .{file}); } fn putPhantomUpgradeNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); // Hash off the current file size so every run targets a fresh hash: intern // makes a live phantom this run, then put drives the phantom-upgrade branch // (a fixed hash would already be content-bearing on disk after the counting // run's append, sending put down the idempotent path instead). const size = file.size() catch 0; var hash_buf: [8]u8 = undefined; std.mem.writeInt(u64, &hash_buf, size, .little); _ = try store.intern(&hash_buf); _ = try store.put(&hash_buf, "payload upgrading a phantom under simulated allocation failure"); } test "logstore §C: put upgrading a phantom propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); seed.deinit(std.testing.allocator); // header only // On OOM in the upgrade's frame append the entry stays phantom (its fields // flip only after a successful append), so there is no leak and no // half-upgraded state at any fail point. try std.testing.checkAllAllocationFailures(std.testing.allocator, putPhantomUpgradeNoLeak, .{file}); } fn contentNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const bytes = try store.content(gpa, 1); if (bytes) |b| gpa.free(b); } test "logstore §C: content propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); _ = try seed.put("content-oom-test-hash", "payload bytes for the content allocation-failure test"); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, contentNoLeak, .{file}); } test "logstore §C/sync §4.1: putZlib then content round-trips the uncompressed bytes" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const hash = "zlib-round-trip-hash-value-0001"; const raw = "compress me: aaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbb aaaaaaaa"; const rid = try store.putZlib(hash, raw); const got = (try store.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(raw, got); } test "logstore §C/sync §4.1: a zlib frame payload is 4-byte-BE-uncompressed-length + RFC1950 stream" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const hash = "z" ** 20; const raw = "the payload whose uncompressed length the container declares"; _ = try store.putZlib(hash, raw); // First frame sits at header_len; its payload starts after the 22+hash_len // fixed header window (encodeFrame layout). const poff: u64 = logstore.header_len + 22 + hash.len; var head: [5]u8 = undefined; const n = try file.pread(poff, &head); try std.testing.expectEqual(@as(usize, 5), n); // §4.1 L1194: 4-byte big-endian uncompressed length. try std.testing.expectEqual(@as(u32, raw.len), std.mem.readInt(u32, head[0..4], .big)); // RFC1950 §2.2: the CMF byte's low nibble (CM) is 8 for a DEFLATE stream. try std.testing.expectEqual(@as(u8, 8), head[4] & 0x0f); } test "logstore §C: a zlib frame is addressed by its uncompressed hash" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const hash = "uncompressed-address-hash-00001"; const rid = try store.putZlib(hash, "content that gets compressed on the way to disk"); // The hash names the uncompressed bytes; storage compression must not // change the address (§4.1 L1206-1208). try std.testing.expectEqualStrings(hash, store.hashOf(rid)); try std.testing.expectEqual(logstore.FrameKind.zlib, store.kindOf(rid)); } test "logstore §C(inv1): a zlib frame reopens and still inflates" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); const hash = "zlib-reopen-hash-value-000000001"; const raw = "reopen and inflate me, please, exactly as written before the crash"; var store = try logstore.LogStore.open(std.testing.allocator, file, 0); _ = try store.putZlib(hash, raw); try store.commit(); store.deinit(std.testing.allocator); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); const rid = reopened.lookup(hash) orelse return error.TestUnexpectedResult; try std.testing.expectEqual(logstore.FrameKind.zlib, reopened.kindOf(rid)); const got = (try reopened.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(raw, got); } test "logstore §C: putZlib is idempotent for an equal hash" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const hash = "zlib-idempotent-hash-value-00001"; const r1 = try store.putZlib(hash, "payload-z"); const size_after = store.size(); const r2 = try store.putZlib(hash, "payload-z"); try std.testing.expectEqual(r1, r2); try std.testing.expectEqual(size_after, store.size()); } test "logstore §C: putZlib upgrades a phantom in place" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const hash = "zlib-phantom-upgrade-hash-000001"; const rid = try store.intern(hash); try std.testing.expectEqual(@as(?[]u8, null), try store.content(std.testing.allocator, rid)); const rid2 = try store.putZlib(hash, "content arriving for a zlib phantom"); try std.testing.expectEqual(rid, rid2); const got = (try store.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings("content arriving for a zlib phantom", got); } test "logstore §C/delta §4: putDelta then content applies the delta and round-trips the target" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const base = "the quick brown fox jumps over the lazy dog, and then again once more"; const target = "the quick brown fox jumps over the lazy dog, and then twice more, at last"; const d = try delta.create(std.testing.allocator, base, target); defer std.testing.allocator.free(d); const base_rid = try store.put("delta-base-hash-value-0000000001", base); const delta_rid = try store.putDelta("delta-target-hash-value-00000001", base_rid, d); try std.testing.expectEqual(logstore.FrameKind.delta, store.kindOf(delta_rid)); const got = (try store.content(std.testing.allocator, delta_rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(target, got); } test "logstore §C: a delta frame stores delta_of = the base frame byte offset" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const base = "base bytes long enough to admit a copy segment in the delta encoder"; const target = "base bytes long enough to admit a copy segment in the delta encoder too"; const d = try delta.create(std.testing.allocator, base, target); defer std.testing.allocator.free(d); const base_rid = try store.put("delta-of-base-hash-value-0000001", base); // first frame at header_len const delta_off = store.size(); const delta_rid = try store.putDelta("delta-of-target-hash-value-00001", base_rid, d); // Read delta_of straight from the frame header: it must be the base frame's // byte offset, i.e. header_len for the first frame. const hl = store.hashOf(delta_rid).len; var field: [8]u8 = undefined; _ = try file.pread(delta_off + 10 + hl, &field); try std.testing.expectEqual(@as(u64, logstore.header_len), std.mem.readInt(u64, &field, .little)); } test "logstore §C: a two-link delta chain (delta-of-delta) resolves in content()" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const a = "artifact A: a reasonably long base so the delta encoder emits copies"; const b = "artifact B: a reasonably long base so the delta encoder emits copies, v2"; const c = "artifact C: a reasonably long base so the delta encoder emits copies, v3!!"; const rid_a = try store.put("chain-hash-a-value-000000000001", a); const d_ab = try delta.create(std.testing.allocator, a, b); defer std.testing.allocator.free(d_ab); const rid_b = try store.putDelta("chain-hash-b-value-000000000001", rid_a, d_ab); const d_bc = try delta.create(std.testing.allocator, b, c); defer std.testing.allocator.free(d_bc); const rid_c = try store.putDelta("chain-hash-c-value-000000000001", rid_b, d_bc); const got = (try store.content(std.testing.allocator, rid_c)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(c, got); } test "logstore §C(inv1): a delta frame reopens and content still resolves" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); const base = "reopen base: bytes shared with the target so a copy is worthwhile here"; const target = "reopen base: bytes shared with the target so a copy is worthwhile now!!"; var store = try logstore.LogStore.open(std.testing.allocator, file, 0); const base_rid = try store.put("delta-reopen-base-hash-000000001", base); const d = try delta.create(std.testing.allocator, base, target); defer std.testing.allocator.free(d); _ = try store.putDelta("delta-reopen-target-hash-0000001", base_rid, d); try store.commit(); store.deinit(std.testing.allocator); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); const rid = reopened.lookup("delta-reopen-target-hash-0000001") orelse return error.TestUnexpectedResult; const got = (try reopened.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(target, got); } test "logstore §C/sync §7.2: wire() delta arm serves the stored container verbatim and names the basis" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const base = "verbatim base bytes shared with the target so a copy segment appears here"; const target = "verbatim base bytes shared with the target so a copy segment appears now!"; const base_hash = "verbatim-base-hash-value-0000001"; const base_rid = try store.put(base_hash, base); const d = try delta.create(std.testing.allocator, base, target); defer std.testing.allocator.free(d); const delta_off = store.size(); const delta_rid = try store.putDelta("verbatim-target-hash-value-00001", base_rid, d); var w = (try store.wire(std.testing.allocator, delta_rid)).?; defer w.deinit(std.testing.allocator); try std.testing.expect(w == .delta); // §7.2: the sender writes the stored bytes; it does not re-compress. The // served container must equal the on-disk payload byte-for-byte, and name // the delta basis (base) artifact. const plen = store.payloadLenOf(delta_rid); const on_disk = try std.testing.allocator.alloc(u8, plen); defer std.testing.allocator.free(on_disk); _ = try file.pread(delta_off + 22 + base_hash.len, on_disk); try std.testing.expectEqualSlices(u8, on_disk, w.delta.container); try std.testing.expectEqualStrings(base_hash, w.delta.baseHash()); // §7.2 L1142-1144 — USIZE is the reconstructed target size (delta §3.2 // header), NEVER the container's 4-byte prefix (the raw delta length). try std.testing.expectEqual(@as(u64, target.len), w.delta.target_size); const prefix = std.mem.readInt(u32, w.delta.container[0..4], .big); try std.testing.expect(w.delta.target_size != prefix); } test "logstore §7.2/§4.1: putContainer stores a full container verbatim and round-trips" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const content = "a full artifact whose §4.1 container is stored verbatim, not re-compressed"; // The §4.1 container a clone would receive on the wire: produce it via the // store's own encoder (putZlib) and extract the exact bytes through wire(). const src_rid = try store.putZlib("full-container-src-hash-0000001", content); var src_w = (try store.wire(std.testing.allocator, src_rid)).?; defer src_w.deinit(std.testing.allocator); const container = src_w.container(); // §7.7 precondition (test controls the name↔content pairing): store the // container verbatim under a fresh name. const rid = try store.putContainer("full-container-dst-hash-0000001", container); try std.testing.expectEqual(logstore.FrameKind.zlib, store.kindOf(rid)); // content() inflates the stored container back to the original plaintext. const got = (try store.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(content, got); // §7.2 — wire() re-serves the byte-identical container (retransmitted verbatim). var w = (try store.wire(std.testing.allocator, rid)).?; defer w.deinit(std.testing.allocator); try std.testing.expect(w == .full); try std.testing.expectEqualSlices(u8, container, w.container()); } test "logstore §7.2/§4.1: putDeltaContainer stores a received delta container verbatim; content() reconstructs the original" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const base = "delta-container base bytes long enough that the encoder emits a copy run"; const target = "delta-container base bytes long enough that the encoder emits a copy run!!"; const base_rid = try store.put("dc-base-hash-value-000000000001", base); const d = try delta.create(std.testing.allocator, base, target); defer std.testing.allocator.free(d); // The §4.1 delta container as delivered on the wire: putDelta encodes it and // wire() exposes the exact container bytes a cfile carries. const enc_rid = try store.putDelta("dc-encoded-hash-value-000000001", base_rid, d); var enc_w = (try store.wire(std.testing.allocator, enc_rid)).?; defer enc_w.deinit(std.testing.allocator); const container = enc_w.delta.container; // Store that container VERBATIM (not re-encoded) against the same base. const rid = try store.putDeltaContainer("dc-verbatim-hash-value-00000001", base_rid, container); try std.testing.expectEqual(logstore.FrameKind.delta, store.kindOf(rid)); // content() applies the stored delta to the base → the original target. const got = (try store.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(target, got); // §7.2 — wire() re-serves the byte-identical container, USIZE from the header. var w = (try store.wire(std.testing.allocator, rid)).?; defer w.deinit(std.testing.allocator); try std.testing.expect(w == .delta); try std.testing.expectEqualSlices(u8, container, w.delta.container); try std.testing.expectEqual(@as(u64, target.len), w.delta.target_size); } test "logstore §C: depthOf reports delta-chain depth and isPhantom distinguishes held from interned" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const a = "depth chain artifact A, long enough for the encoder to emit a copy run"; const b = "depth chain artifact A, long enough for the encoder to emit a copy run!"; const c = "depth chain artifact A, long enough for the encoder to emit a copy run!!"; const rid_a = try store.put("depth-a-hash-value-0000000000001", a); try std.testing.expectEqual(@as(u8, 0), store.depthOf(rid_a)); try std.testing.expect(!store.isPhantom(rid_a)); const d_ab = try delta.create(std.testing.allocator, a, b); defer std.testing.allocator.free(d_ab); const rid_b = try store.putDelta("depth-b-hash-value-0000000000001", rid_a, d_ab); try std.testing.expectEqual(@as(u8, 1), store.depthOf(rid_b)); const d_bc = try delta.create(std.testing.allocator, b, c); defer std.testing.allocator.free(d_bc); const rid_c = try store.putDelta("depth-c-hash-value-0000000000001", rid_b, d_bc); try std.testing.expectEqual(@as(u8, 2), store.depthOf(rid_c)); const p = try store.intern("depth-phantom-hash-value-0000001"); try std.testing.expect(store.isPhantom(p)); try std.testing.expectEqual(@as(u8, 0), store.depthOf(p)); } fn putContainerNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); // A single atomic append under a size-unique name (never idempotent, so // every run does identical work). putContainer treats its argument as // opaque §4.1 bytes (no decode), so a dummy container drives the alloc path. const size = file.size() catch 0; var hash_buf: [8]u8 = undefined; std.mem.writeInt(u64, &hash_buf, size, .little); _ = try store.putContainer(&hash_buf, "dummy container bytes for the putContainer allocation-failure probe"); } test "logstore §C: putContainer propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); // establish the header deterministically try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, putContainerNoLeak, .{file}); } fn putDeltaContainerNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const base_rid = store.lookup("oom-pdc-base-hash-value-00000001") orelse return error.TestUnexpectedResult; // A single atomic append under a size-unique name against the seeded base; // the container is opaque to the store. const size = file.size() catch 0; var hash_buf: [8]u8 = undefined; std.mem.writeInt(u64, &hash_buf, size, .little); _ = try store.putDeltaContainer(&hash_buf, base_rid, "dummy delta container bytes for the putDeltaContainer probe"); } test "logstore §C: putDeltaContainer propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); _ = try seed.put("oom-pdc-base-hash-value-00000001", "a base frame the retained delta references"); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, putDeltaContainerNoLeak, .{file}); } test "logstore §C: kindOf reports raw/zlib/delta" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); const base = "kindOf base bytes with enough shared run for a copy in the delta encoder"; const raw_rid = try store.put("kindof-raw-hash-value-0000000001", base); const zlib_rid = try store.putZlib("kindof-zlib-hash-value-000000001", "some zlib payload"); const d = try delta.create(std.testing.allocator, base, "kindOf base bytes with enough shared run for a copy in the delta encoder!"); defer std.testing.allocator.free(d); const delta_rid = try store.putDelta("kindof-delta-hash-value-00000001", raw_rid, d); try std.testing.expectEqual(logstore.FrameKind.raw, store.kindOf(raw_rid)); try std.testing.expectEqual(logstore.FrameKind.zlib, store.kindOf(zlib_rid)); try std.testing.expectEqual(logstore.FrameKind.delta, store.kindOf(delta_rid)); } test "logstore sync §7.2: putArtifact stores a delta iff materially smaller and chain-safe" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // A base with a long shared run, then a 1-byte mutation → a profitable // delta (raw delta far under 90% of the full artifact) → stored `.delta`. const base = "putArtifact base bytes, long enough that a small mutation deltas well below ninety percent of the full artifact size here"; const near = "putArtifact base bytes, long enough that a small mutation deltas well below ninety percent of the full artifact size here!"; const base_rid = try store.putArtifact("putartifact-base-hash-value-0001", base, null); try std.testing.expectEqual(logstore.FrameKind.zlib, store.kindOf(base_rid)); // no base → full const near_rid = try store.putArtifact("putartifact-near-hash-value-0001", near, base_rid); try std.testing.expectEqual(logstore.FrameKind.delta, store.kindOf(near_rid)); // Unrelated/incompressible content against the same base → the delta is not // materially smaller → stored full (`.zlib`). var rng = std.Random.DefaultPrng.init(0xABCDEF); var noise: [128]u8 = undefined; rng.random().bytes(&noise); const noise_rid = try store.putArtifact("putartifact-noise-hash-value-001", &noise, base_rid); try std.testing.expectEqual(logstore.FrameKind.zlib, store.kindOf(noise_rid)); // A phantom base is ineligible → silently skipped (never an error). With no // profitable base for this incompressible content anywhere in the window, // it falls through to the full form. const ph = try store.intern("putartifact-phantom-base-hash-01"); var noise2: [128]u8 = undefined; rng.random().bytes(&noise2); const ph_child = try store.putArtifact("putartifact-phantom-child-hash01", &noise2, ph); try std.testing.expectEqual(logstore.FrameKind.zlib, store.kindOf(ph_child)); // Idempotent: a re-put of a held hash keeps its stored form, appends nothing. const end_before = store.size(); const again = try store.putArtifact("putartifact-near-hash-value-0001", near, base_rid); try std.testing.expectEqual(near_rid, again); try std.testing.expectEqual(end_before, store.size()); } test "logstore sync §7.2: putArtifact never creates a delta past the write-depth cap" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // Build a long chain via putArtifact: each 1-byte mutation deltas against // an eligible recent base. Best-of-N (§5.2) may keep the chain going by // choosing a shallower recent frame rather than the deepest hint, but the // write policy guarantees no locally created frame ever exceeds // `delta_chain_write_max` depth (so the read walk stays shallow), deltas are // still produced under the cap, and every frame reconstructs exactly. var current = try std.testing.allocator.dupe(u8, "write-cap chain base bytes, long enough that each single-character mutation deltas well under the profitability ratio here 000"); defer std.testing.allocator.free(current); var prev = try store.putArtifact("writecap-base-hash-value-0000001", current, null); var saw_delta = false; var i: usize = 0; // Drive well past the cap so a naive single-base policy would blow through it. while (i <= logstore.delta_chain_write_max + 4) : (i += 1) { var next_buf: [160]u8 = undefined; const next = try std.fmt.bufPrint(&next_buf, "write-cap chain base bytes, long enough that each single-character mutation deltas well under the profitability ratio here {d:0>3}", .{i + 1}); var hbuf: [32]u8 = undefined; std.mem.writeInt(u64, hbuf[0..8], i, .little); @memcpy(hbuf[8..], "writecap-chain-hash-paddingval--"[0..24]); const rid = try store.putArtifact(&hbuf, next, prev); // The write-cap invariant: a locally created frame never exceeds the cap. try std.testing.expect(store.depthOf(rid) <= logstore.delta_chain_write_max); if (store.kindOf(rid) == .delta) saw_delta = true; // content() must still reconstruct at every depth. const got = (try store.content(std.testing.allocator, rid)).?; defer std.testing.allocator.free(got); try std.testing.expectEqualStrings(next, got); std.testing.allocator.free(current); current = try std.testing.allocator.dupe(u8, next); prev = rid; } try std.testing.expect(saw_delta); // best-of-N keeps deltifying under the cap } fn putArtifactNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const base_rid = store.lookup("oom-putartifact-base-hash-00001") orelse return error.TestUnexpectedResult; // A fresh, size-derived hash every run — so a persisted frame from the // reference run never turns a later run into an idempotent short-circuit // (which would change the allocation count → NondeterministicMemoryUsage). const size = file.size() catch 0; var hash_buf: [8]u8 = undefined; std.mem.writeInt(u64, &hash_buf, size, .little); _ = try store.putArtifact(&hash_buf, "putArtifact OOM target bytes with a long shared run so a profitable delta is attempted here!", base_rid); } test "logstore §C: putArtifact propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); _ = try seed.putArtifact("oom-putartifact-base-hash-00001", "putArtifact OOM target bytes with a long shared run so a profitable delta is attempted here", null); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, putArtifactNoLeak, .{file}); } test "logstore §C: a delta chain deeper than max_delta_chain is error.Corrupt, not unbounded recursion" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // Build a chain of exactly max_delta_chain deltas over a raw base: it must // resolve. One more link must be rejected by the depth bound (no unbounded // recursion), proving the bound is enforced from the read side. var current: []u8 = try std.testing.allocator.dupe(u8, "chain base bytes, long enough to permit copies across every link here"); defer std.testing.allocator.free(current); var prev_rid = try store.put("depth-base-hash-value-0000000000", current); var last_ok_rid: logstore.Rid = prev_rid; var i: usize = 0; while (i < logstore.max_delta_chain + 1) : (i += 1) { var next_buf: [96]u8 = undefined; const next = try std.fmt.bufPrint(&next_buf, "chain base bytes, long enough to permit copies across every link {d:0>3}", .{i}); const d = try delta.create(std.testing.allocator, current, next); defer std.testing.allocator.free(d); var hbuf: [32]u8 = undefined; std.mem.writeInt(u64, hbuf[0..8], i, .little); @memcpy(hbuf[8..], "depth-chain-hash-padding-value--"[0..24]); const rid = try store.putDelta(&hbuf, prev_rid, d); std.testing.allocator.free(current); current = try std.testing.allocator.dupe(u8, next); prev_rid = rid; if (i + 1 == logstore.max_delta_chain) last_ok_rid = rid; // top of a length-max_delta_chain chain } // A chain of exactly max_delta_chain deltas resolves. const ok = (try store.content(std.testing.allocator, last_ok_rid)).?; defer std.testing.allocator.free(ok); // One link deeper (max_delta_chain + 1) trips the bound. try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, prev_rid)); } // OOM coverage for every public allocating fn added this slice. fn putZlibNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const size = file.size() catch 0; var hash_buf: [8]u8 = undefined; std.mem.writeInt(u64, &hash_buf, size, .little); _ = try store.putZlib(&hash_buf, "payload compressed under simulated allocation failure"); } test "logstore §C: putZlib propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, putZlibNoLeak, .{file}); } fn putDeltaNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const base_rid = store.lookup("oom-delta-base-hash-value-000001") orelse return error.TestUnexpectedResult; const size = file.size() catch 0; var hash_buf: [8]u8 = undefined; std.mem.writeInt(u64, &hash_buf, size, .little); _ = try store.putDelta(&hash_buf, base_rid, "raw delta bytes under simulated allocation failure"); } test "logstore §C: putDelta propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); _ = try seed.put("oom-delta-base-hash-value-000001", "a base frame the delta references"); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, putDeltaNoLeak, .{file}); } fn contentZlibNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const rid = store.lookup("oom-zlib-content-hash-value-0001") orelse return error.TestUnexpectedResult; const bytes = try store.content(gpa, rid); if (bytes) |b| gpa.free(b); } test "logstore §C: content on a zlib frame propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); _ = try seed.putZlib("oom-zlib-content-hash-value-0001", "payload for the zlib content allocation-failure probe"); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, contentZlibNoLeak, .{file}); } fn contentDeltaNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const rid = store.lookup("oom-delta-content-target-hash-01") orelse return error.TestUnexpectedResult; const bytes = try store.content(gpa, rid); if (bytes) |b| gpa.free(b); } test "logstore §C: content on a delta chain propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); const base = "oom delta base bytes, long enough that the encoder emits a copy segment"; const target = "oom delta base bytes, long enough that the encoder emits a copy segment!!"; var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); const base_rid = try seed.put("oom-delta-content-base-hash-0001", base); const d = try delta.create(std.testing.allocator, base, target); defer std.testing.allocator.free(d); _ = try seed.putDelta("oom-delta-content-target-hash-01", base_rid, d); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, contentDeltaNoLeak, .{file}); } fn wireDeltaNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const rid = store.lookup("oom-stored-delta-target-hash-001") orelse return error.TestUnexpectedResult; var w = (try store.wire(gpa, rid)) orelse return error.TestUnexpectedResult; w.deinit(gpa); } test "logstore §C: wire() on a delta frame propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); const base = "stored-delta base bytes, long enough that the encoder emits a copy here"; const target = "stored-delta base bytes, long enough that the encoder emits a copy here!!"; var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); const base_rid = try seed.put("oom-stored-delta-base-hash-00001", base); const d = try delta.create(std.testing.allocator, base, target); defer std.testing.allocator.free(d); _ = try seed.putDelta("oom-stored-delta-target-hash-001", base_rid, d); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, wireDeltaNoLeak, .{file}); } // --------------------------------------------------------------------------- // Config frames (storage-track-plan.md §E: repo-local config in the file). // --------------------------------------------------------------------------- test "logstore §E: putConfig then readConfig round-trips branch, project id, and opaque hash policy" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // hash_policy 7 is opaque: it round-trips byte-for-byte, never interpreted. // The sync project code (§9.1) is a fourth round-tripped field. try store.putConfig(.{ .hash_policy = 7, .default_branch = "trunk", .project_id = "proj-abc", .project_code = "aa87ea41b0bb05ba528f7a3bdba700d7e23d6318", }); var oc = (try store.readConfig(std.testing.allocator)).?; defer oc.deinit(std.testing.allocator); try std.testing.expectEqual(@as(u8, 7), oc.config.hash_policy); try std.testing.expectEqualStrings("trunk", oc.config.default_branch); try std.testing.expectEqualStrings("proj-abc", oc.config.project_id); try std.testing.expectEqualStrings("aa87ea41b0bb05ba528f7a3bdba700d7e23d6318", oc.config.project_code); } test "logstore §E/sync §6.3: the derived login credential round-trips; the plaintext never enters the stored bytes" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // DECISION-3: the 40-hex derived secret is stored, never the plaintext // password it was derived from. This secret is an arbitrary 40-hex value // (the storage layer never derives it — sync §6.3 does). const secret_hex = "da39a3ee5e6b4b0d3255bfef95601890afd80709"; const plaintext = "hunter2-the-password"; try store.putConfig(.{ .hash_policy = 1, .default_branch = "trunk", .project_id = "proj", .project_code = "aa87ea41b0bb05ba528f7a3bdba700d7e23d6318", .login_user = "alice", .login_secret_hex = secret_hex, }); try store.commit(); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); var oc = (try reopened.readConfig(std.testing.allocator)).?; defer oc.deinit(std.testing.allocator); try std.testing.expectEqualStrings("alice", oc.config.login_user); try std.testing.expectEqualStrings(secret_hex, oc.config.login_secret_hex); try std.testing.expectEqual(@as(usize, 40), oc.config.login_secret_hex.len); // The plaintext password appears nowhere in the persisted repository bytes. const raw = try tmp.dir.readFileAlloc(io, "log.bin", std.testing.allocator, .unlimited); defer std.testing.allocator.free(raw); try std.testing.expect(std.mem.indexOf(u8, raw, plaintext) == null); } test "config logstore §E: a saved remote_url survives putConfig commit+reopen with every field intact" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // sync §4/§6 — the saved remote is a local config value written alongside // the credential; it must survive a commit+reopen without disturbing the // other fields (the whole §E record is a lossless round-trip through the log). const secret_hex = "da39a3ee5e6b4b0d3255bfef95601890afd80709"; try store.putConfig(.{ .hash_policy = 3, .default_branch = "trunk", .project_id = "proj", .project_code = "aa87ea41b0bb05ba528f7a3bdba700d7e23d6318", .login_user = "alice", .login_secret_hex = secret_hex, .remote_url = "http://h.example/xfer", }); try store.commit(); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); var oc = (try reopened.readConfig(std.testing.allocator)).?; defer oc.deinit(std.testing.allocator); try std.testing.expectEqualStrings("http://h.example/xfer", oc.config.remote_url); try std.testing.expectEqualStrings("trunk", oc.config.default_branch); try std.testing.expectEqualStrings("proj", oc.config.project_id); try std.testing.expectEqualStrings("aa87ea41b0bb05ba528f7a3bdba700d7e23d6318", oc.config.project_code); try std.testing.expectEqualStrings("alice", oc.config.login_user); try std.testing.expectEqualStrings(secret_hex, oc.config.login_secret_hex); } test "logstore §E/sync §6.3: a config with no credential round-trips as empty (forward-compatible)" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // No login fields supplied — they default to empty and must decode back // as empty, exactly as an older three-field frame would. try store.putConfig(.{ .hash_policy = 1, .default_branch = "trunk", .project_id = "p" }); var oc = (try store.readConfig(std.testing.allocator)).?; defer oc.deinit(std.testing.allocator); try std.testing.expectEqual(@as(usize, 0), oc.config.login_user.len); try std.testing.expectEqual(@as(usize, 0), oc.config.login_secret_hex.len); } test "logstore §E: the last config frame wins, and survives commit+reopen" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); try store.putConfig(.{ .hash_policy = 1, .default_branch = "trunk", .project_id = "p1" }); try store.putConfig(.{ .hash_policy = 2, .default_branch = "main", .project_id = "p2" }); { var oc = (try store.readConfig(std.testing.allocator)).?; defer oc.deinit(std.testing.allocator); try std.testing.expectEqual(@as(u8, 2), oc.config.hash_policy); try std.testing.expectEqualStrings("main", oc.config.default_branch); } try store.commit(); store.deinit(std.testing.allocator); var reopened = try logstore.LogStore.open(std.testing.allocator, file, 0); defer reopened.deinit(std.testing.allocator); var oc = (try reopened.readConfig(std.testing.allocator)).?; defer oc.deinit(std.testing.allocator); try std.testing.expectEqual(@as(u8, 2), oc.config.hash_policy); try std.testing.expectEqualStrings("main", oc.config.default_branch); try std.testing.expectEqualStrings("p2", oc.config.project_id); } test "logstore §E: a fresh log has no config; readConfig returns null" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); try std.testing.expect((try store.readConfig(std.testing.allocator)) == null); } fn configNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); try store.putConfig(.{ .hash_policy = 3, .default_branch = "branch-name", .project_id = "project-id" }); // config_offset flips only after a successful append, so an OOM mid-put // leaves no config to read and no half-state; a successful put must read // back cleanly with no leak. var oc = (try store.readConfig(gpa)) orelse return error.TestUnexpectedResult; oc.deinit(gpa); } test "logstore §E: putConfig and readConfig propagate allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); seed.deinit(std.testing.allocator); // header only try std.testing.checkAllAllocationFailures(std.testing.allocator, configNoLeak, .{file}); } test "logstore §E: cloneInto copies + derives — the clone is an equal repository (same artifact set, same config, byte-identical log)" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; const alloc = std.testing.allocator; var src_fs = try openTmp(&tmp, io, "ship.zit"); defer src_fs.file.close(io); const src_file = src_fs.storageFile(); var dst_fs = try openTmp(&tmp, io, "clone.zit"); defer dst_fs.file.close(io); const dst_file = dst_fs.storageFile(); const a1 = "artifact one: a base blob long enough to feed the delta encoder a copy segment"; const a2 = "artifact two: separate content routed through the fossil zlib container path here"; const a3 = "artifact one: a base blob long enough to feed the delta encoder a copy segment, extended tail"; const cfg: logstore.Config = .{ .hash_policy = 6, .default_branch = "trunk", .project_id = "ship-proj" }; var r1: logstore.Rid = undefined; var r2: logstore.Rid = undefined; var r3: logstore.Rid = undefined; { var store = try logstore.LogStore.open(alloc, src_file, 9); defer store.deinit(alloc); r1 = try store.put("clone-A1", a1); r2 = try store.putZlib("clone-A2", a2); const d = try delta.create(alloc, a1, a3); defer alloc.free(d); r3 = try store.putDelta("clone-A3", r1, d); try store.putConfig(cfg); try store.commit(); } var clone = try logstore.LogStore.cloneInto(alloc, src_file, dst_file); defer clone.deinit(alloc); // Byte-identical log: only the authoritative artifact set ships (no // derived tables), so the copy is exact and the header round-trips. // Compare every byte, not just the length — an equal size proves // nothing about content. const total = try src_file.size(); try std.testing.expectEqual(total, clone.size()); const src_bytes = try alloc.alloc(u8, @intCast(total)); defer alloc.free(src_bytes); const dst_bytes = try alloc.alloc(u8, @intCast(total)); defer alloc.free(dst_bytes); try std.testing.expectEqual(@as(usize, @intCast(total)), try src_file.pread(0, src_bytes)); try std.testing.expectEqual(@as(usize, @intCast(total)), try dst_file.pread(0, dst_bytes)); try std.testing.expectEqualSlices(u8, src_bytes, dst_bytes); try std.testing.expectEqual(@as(u8, 9), clone.hash_kind); // Same artifact set: every content hash resolves to the same rid and the // same reconstructed bytes — the delta resolves against its copied base. const cases = [_]struct { hash: []const u8, rid: logstore.Rid, want: []const u8 }{ .{ .hash = "clone-A1", .rid = r1, .want = a1 }, .{ .hash = "clone-A2", .rid = r2, .want = a2 }, .{ .hash = "clone-A3", .rid = r3, .want = a3 }, }; for (cases) |c| { try std.testing.expectEqual(c.rid, clone.lookup(c.hash).?); const got = (try clone.content(alloc, clone.lookup(c.hash).?)).?; defer alloc.free(got); try std.testing.expectEqualStrings(c.want, got); } try std.testing.expectEqual(@as(usize, 3), clone.entries.items.len); // Config recovered through the fresh open — the file is self-contained. var oc = (try clone.readConfig(alloc)).?; defer oc.deinit(alloc); try std.testing.expectEqual(cfg.hash_policy, oc.config.hash_policy); try std.testing.expectEqualStrings(cfg.default_branch, oc.config.default_branch); try std.testing.expectEqualStrings(cfg.project_id, oc.config.project_id); } test "logstore §E: cloning a header-only log yields an equal empty repository" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; const alloc = std.testing.allocator; var src_fs = try openTmp(&tmp, io, "empty-ship.zit"); defer src_fs.file.close(io); const src_file = src_fs.storageFile(); var dst_fs = try openTmp(&tmp, io, "empty-clone.zit"); defer dst_fs.file.close(io); const dst_file = dst_fs.storageFile(); // Source carries a header and no content frames. var seed = try logstore.LogStore.open(alloc, src_file, 0); seed.deinit(alloc); var clone = try logstore.LogStore.cloneInto(alloc, src_file, dst_file); defer clone.deinit(alloc); try std.testing.expectEqual(@as(u64, logstore.header_len), clone.size()); try std.testing.expectEqual(@as(usize, 0), clone.entries.items.len); try std.testing.expect((try clone.readConfig(alloc)) == null); } test "logstore §E: a damaged clone degrades gracefully via torn-tail truncation" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; const alloc = std.testing.allocator; var src_fs = try openTmp(&tmp, io, "torn-ship.zit"); defer src_fs.file.close(io); const src_file = src_fs.storageFile(); var dst_fs = try openTmp(&tmp, io, "torn-clone.zit"); defer dst_fs.file.close(io); const dst_file = dst_fs.storageFile(); var cut_base: u64 = undefined; { var store = try logstore.LogStore.open(alloc, src_file, 0); defer store.deinit(alloc); _ = try store.put("clone-B4-a", "alpha payload for the damaged-clone recovery test"); try store.putConfig(.{ .hash_policy = 3, .default_branch = "trunk", .project_id = "keep" }); try store.commit(); cut_base = store.size(); // boundary after A + config, before B _ = try store.put("clone-B4-b", "beta payload that the torn tail truncates away on derive"); try store.commit(); } // A damaged copy: only a byte-prefix ending 5 bytes into B's frame. The // derive step (open) is the same one cloneInto uses, so it drops exactly // the torn tail and recovers A + config (§E L119). var buf: [4096]u8 = undefined; const prefix: usize = @intCast(cut_base + 5); std.debug.assert(prefix <= buf.len); const n = try src_file.pread(0, buf[0..prefix]); try std.testing.expectEqual(prefix, n); try dst_file.append(buf[0..n]); var clone = try logstore.LogStore.open(alloc, dst_file, 0); defer clone.deinit(alloc); try std.testing.expectEqual(cut_base, clone.size()); try std.testing.expect(clone.lookup("clone-B4-a") != null); try std.testing.expect(clone.lookup("clone-B4-b") == null); var oc = (try clone.readConfig(alloc)).?; defer oc.deinit(alloc); try std.testing.expectEqualStrings("trunk", oc.config.default_branch); try std.testing.expectEqualStrings("keep", oc.config.project_id); } fn cloneNoLeak(gpa: std.mem.Allocator, source: storage.StorageFile, tmp: *std.testing.TmpDir, io: std.Io) !void { // A fresh (truncating) dest each pass, so a retried invocation starts from // an empty capability — cloneInto's documented precondition. var dst_fs = try openTmp(tmp, io, "clone-noleak.zit"); defer dst_fs.file.close(io); var store = try logstore.LogStore.cloneInto(gpa, source, dst_fs.storageFile()); defer store.deinit(gpa); const c = (try store.content(gpa, store.lookup("clone-A1").?)).?; gpa.free(c); } test "logstore §E: cloneInto propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var src_fs = try openTmp(&tmp, io, "noleak-ship.zit"); defer src_fs.file.close(io); const src_file = src_fs.storageFile(); { var store = try logstore.LogStore.open(std.testing.allocator, src_file, 0); defer store.deinit(std.testing.allocator); _ = try store.put("clone-A1", "the base blob for the clone allocation-failure sweep"); try store.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid" }); try store.commit(); } try std.testing.checkAllAllocationFailures(std.testing.allocator, cloneNoLeak, .{ src_file, &tmp, io }); } // --------------------------------------------------------------------------- // Verbatim cfile serving (sync §4.1/§7.2): wire() serves a `.zlib` frame's // stored §4.1 container back for full-form retransmission. // --------------------------------------------------------------------------- test "logstore sync §7.2: wire() is null for raw/phantom, full for zlib, delta for delta" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var store = try logstore.LogStore.open(std.testing.allocator, file, 0); defer store.deinit(std.testing.allocator); // raw → no stored wire form (serve falls back to compressing content()). const raw_rid = try store.put("wire-raw-frame-hash-00000000001", "raw stored bytes"); try std.testing.expect((try store.wire(std.testing.allocator, raw_rid)) == null); // phantom → no frame, no wire. const ph_rid = try store.intern("wire-phantom-hash-000000000001"); try std.testing.expect((try store.wire(std.testing.allocator, ph_rid)) == null); // zlib → full form: the stored §4.1 container (with its 4-byte length prefix). const zlib_rid = try store.putZlib("wire-zlib-frame-hash-0000000001", "zlib stored bytes, long enough to encode"); { var w = (try store.wire(std.testing.allocator, zlib_rid)).?; defer w.deinit(std.testing.allocator); try std.testing.expect(w == .full); try std.testing.expect(w.full.len >= 4); // §4.1 — the container carries its length prefix } // delta → delta form: the stored delta container plus the basis hash and the // reconstructed-target size, retransmitted verbatim (sync §7.2). const base = "delta wire base bytes, long enough that the encoder emits a copy"; const target = "delta wire base bytes, long enough that the encoder emits a copy!!"; const base_rid = try store.put("wire-delta-base-hash-0000000001", base); const d = try delta.create(std.testing.allocator, base, target); defer std.testing.allocator.free(d); const delta_rid = try store.putDelta("wire-delta-target-hash-00000001", base_rid, d); { var w = (try store.wire(std.testing.allocator, delta_rid)).?; defer w.deinit(std.testing.allocator); try std.testing.expect(w == .delta); try std.testing.expectEqual(@as(u64, target.len), w.delta.target_size); try std.testing.expectEqualStrings("wire-delta-base-hash-0000000001", w.delta.baseHash()); } } // OOM coverage for the verbatim-serve public allocating fns. fn wireZlibNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void { var store = try logstore.LogStore.open(gpa, file, 0); defer store.deinit(gpa); const rid = store.lookup("wire-oom-zlib-hash-0000000000001") orelse return error.TestUnexpectedResult; var w = (try store.wire(gpa, rid)) orelse return error.TestUnexpectedResult; w.deinit(gpa); } test "logstore sync §7.2: wire on a zlib frame propagates allocation failure" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); var seed = try logstore.LogStore.open(std.testing.allocator, file, 0); _ = try seed.putZlib("wire-oom-zlib-hash-0000000000001", "payload for the wire() allocation-failure probe"); try seed.commit(); seed.deinit(std.testing.allocator); try std.testing.checkAllAllocationFailures(std.testing.allocator, wireZlibNoLeak, .{file}); } // Fix-slice tests: the derivation sweep's read path — a delta-base plaintext // memo (PlaintextCache) that leaves reconstructed bytes byte-identical, and the // openReadOnly whole-frame-CRC-verify-once guarantee that contentCached relies // on. Native storage-engine framing, so the oracle is this file's own // round-trip invariant: reconstruct(store) == the bytes that were stored. /// Build a depth-`links` delta chain plus one extra delta forking off the base, /// returning the rids alongside their known plaintexts. `base`..`base+links` /// are the linear chain (raw base, then one delta per level); the last rid is a /// second delta over the base (a shared base, so a sweep hits the memo). fn buildDeltaChain(store: *logstore.LogStore, links: usize, out_rids: []logstore.Rid, out_plain: [][]u8) !void { const gpa = std.testing.allocator; var current = try gpa.dupe(u8, "chain base payload, repeated enough that small edits delta-encode compactly across every link of the chain"); out_plain[0] = current; var hbuf: [32]u8 = undefined; @memcpy(&hbuf, "delta-chain-base-hash-value-0000"); out_rids[0] = try store.put(&hbuf, current); var i: usize = 1; while (i <= links) : (i += 1) { var next = try gpa.dupe(u8, current); next[i % next.len] = @intCast('A' + (i % 26)); // a small, valid edit const d = try delta.create(gpa, current, next); defer gpa.free(d); std.mem.writeInt(u64, hbuf[0..8], i, .little); @memcpy(hbuf[8..], "elta-chain-link-hash-pad"[0..24]); out_rids[i] = try store.putDelta(&hbuf, out_rids[i - 1], d); out_plain[i] = next; current = next; } // One more delta whose base is the raw base rid (a fork sharing the base). var fork = try gpa.dupe(u8, out_plain[0]); fork[0] = 'Z'; const df = try delta.create(gpa, out_plain[0], fork); defer gpa.free(df); @memcpy(&hbuf, "delta-chain-fork-hash-value-0000"); out_rids[links + 1] = try store.putDelta(&hbuf, out_rids[0], df); out_plain[links + 1] = fork; } test "logstore §7.x: contentCached over a delta chain reconstructs byte-identical plaintext (openReadOnly CRC-verify-once)" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const file = fs.storageFile(); const gpa = std.testing.allocator; const links = 8; var rids: [links + 2]logstore.Rid = undefined; var plain: [links + 2][]u8 = undefined; { var w = try logstore.LogStore.open(gpa, file, 0); defer w.deinit(gpa); try buildDeltaChain(&w, links, &rids, &plain); try w.commit(); } defer for (plain) |p| gpa.free(p); // Reopen read-only: scan whole-frame-CRC-verifies every frame once, so // contentCached's reads skip the redundant CRC (verified_on_open). var ro = try logstore.LogStore.openReadOnly(gpa, file); defer ro.deinit(gpa); var cache = logstore.PlaintextCache.init(); defer cache.deinit(gpa); // Resolve in persisted order (the sweep's order): each rid's cached bytes // must equal both the from-scratch content() and the known plaintext. for (rids, 0..) |rid, i| { const cached = (try ro.contentCached(gpa, rid, &cache)).?; defer gpa.free(cached); const fresh = (try ro.content(gpa, rid)).?; defer gpa.free(fresh); try std.testing.expectEqualSlices(u8, plain[i], cached); try std.testing.expectEqualSlices(u8, plain[i], fresh); } } test "logstore §7.x: PlaintextCache is byte-exact and bounded under LRU eviction" { const gpa = std.testing.allocator; var cache = logstore.PlaintextCache.init(); defer cache.deinit(gpa); // A tiny byte budget forces eviction on nearly every insert. cache.max_bytes = 32; cache.max_entries = 4; var payload: [8]u8 = undefined; var off: u64 = 100; while (off < 200) : (off += 1) { @memset(&payload, @intCast(off & 0xff)); try cache.put(gpa, off, &payload); // A just-inserted key is the MRU and must be retrievable and exact. const got = cache.get(off).?; try std.testing.expectEqualSlices(u8, &payload, got); // Bounds hold after every insert: never over budget, never over count. try std.testing.expect(cache.bytes_used <= cache.max_bytes); try std.testing.expect(@as(usize, cache.map.count()) <= cache.max_entries); } // An item larger than the whole budget is silently not cached. var big: [64]u8 = undefined; @memset(&big, 7); try cache.put(gpa, 9999, &big); try std.testing.expect(cache.get(9999) == null); // A duplicate key is a no-op (base offsets are unique keys upstream). @memset(&payload, 42); try cache.put(gpa, 199, &payload); // 199 already resident from the loop const dup = cache.get(199).?; try std.testing.expect(dup[0] == @as(u8, 199 & 0xff)); // original bytes kept } test "logstore §5: pickBase runs best-of-N and keeps the smallest delta, not the first" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const gpa = std.testing.allocator; var store = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer store.deinit(gpa); // `near` is a one-byte mutation of the target → a tiny delta; `far` shares // almost nothing → a large delta. Both are eligible content frames. const target = "The quick brown fox jumps over the lazy dog, then keeps running across the whole wide meadow twice over."; const near = "The quick brown fox jumps over the lazy dog, then keeps running across the whole wide meadow twice over!"; const far = "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt labore dolore aa."; const far_rid = try store.putArtifact("pb-far-base-hash-value-000000001", far, null); const near_rid = try store.putArtifact("pb-near-base-hash-value-00000001", near, null); // Offer `far` at higher priority than `near`: best-of-N must still pick // `near` because its delta is strictly smaller. var cands = [_]logstore.Rid{ far_rid, near_rid }; const pick = (try store.pickBase(gpa, "pb-target-hash-value-00000000001", target, &cands)).?; defer gpa.free(pick.delta); try std.testing.expectEqual(near_rid, pick.base); // The returned delta reconstructs the target from the chosen base, and it // clears the 75% bar. const near_bytes = (try store.content(gpa, near_rid)).?; defer gpa.free(near_bytes); const rebuilt = try delta.apply(gpa, near_bytes, pick.delta); defer gpa.free(rebuilt); try std.testing.expectEqualStrings(target, rebuilt); try std.testing.expect(pick.delta.len * 4 < target.len * 3); } test "logstore §5: pickBase returns null when no candidate delta clears 75%" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const gpa = std.testing.allocator; var store = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer store.deinit(gpa); // A base that shares little with the (incompressible) target: the best // possible delta is no smaller than the artifact, so nothing is accepted. const base = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa base bytes with a long run"; const base_rid = try store.putArtifact("pb-noacc-base-hash-value-0000001", base, null); var noise: [160]u8 = undefined; var rng = std.Random.DefaultPrng.init(0xBADF00D); rng.random().bytes(&noise); var cands = [_]logstore.Rid{base_rid}; try std.testing.expect((try store.pickBase(gpa, "pb-noacc-target-hash-value-00001", &noise, &cands)) == null); } test "logstore §5: pickBase excludes a candidate already at the write-depth cap" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const gpa = std.testing.allocator; var store = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer store.deinit(gpa); // Build a chain of exactly `delta_chain_write_max` deltas so the deepest // frame sits AT the write cap. Each link is a one-character mutation, so a // profitable delta is always available — the only reason to reject the // deepest frame as a base is the depth cap itself. var cur = try gpa.dupe(u8, "depth-cap chain base bytes, long enough that each single-character mutation deltas well under the ratio 000"); defer gpa.free(cur); var prev = try store.put("pb-cap-base-hash-value-000000001", cur); var deepest = prev; var i: usize = 0; while (i < logstore.delta_chain_write_max) : (i += 1) { var next_buf: [160]u8 = undefined; const next = try std.fmt.bufPrint(&next_buf, "depth-cap chain base bytes, long enough that each single-character mutation deltas well under the ratio {d:0>3}", .{i + 1}); const d = try delta.create(gpa, cur, next); defer gpa.free(d); var hbuf: [32]u8 = undefined; std.mem.writeInt(u64, hbuf[0..8], i, .little); @memcpy(hbuf[8..], "pb-cap-chain-hash-paddin"[0..24]); deepest = try store.putDelta(&hbuf, prev, d); prev = deepest; gpa.free(cur); cur = try gpa.dupe(u8, next); } try std.testing.expectEqual(logstore.delta_chain_write_max, store.depthOf(deepest)); // A one-char mutation of the deepest content WOULD delta tiny — but the // deepest frame is at the cap, so `pickBase` excludes it and returns null. var tgt_buf: [160]u8 = undefined; const tgt = try std.fmt.bufPrint(&tgt_buf, "{s}X", .{cur}); var cands = [_]logstore.Rid{deepest}; try std.testing.expect((try store.pickBase(gpa, "pb-cap-target-hash-value-0000001", tgt, &cands)) == null); } test "logstore §5: pickBase silently skips phantom and self candidates" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const gpa = std.testing.allocator; var store = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer store.deinit(gpa); const target = "pickBase self/phantom exclusion target bytes, long enough to be well over the fifty-byte source floor here."; // A phantom candidate carries no content → skipped → no accepted base. const ph = try store.intern("pb-phantom-cand-hash-value-00001"); var c_ph = [_]logstore.Rid{ph}; try std.testing.expect((try store.pickBase(gpa, "pb-phantom-target-hash-value-001", target, &c_ph)) == null); // The artifact's own frame offered as its own base → excluded (a self-delta // is degenerate), even though `target` would delta perfectly against it. const self_hash = "pb-self-hash-value-000000000001"; const self_rid = try store.putArtifact(self_hash, target, null); var c_self = [_]logstore.Rid{self_rid}; try std.testing.expect((try store.pickBase(gpa, self_hash, target, &c_self)) == null); // NOTE: a `.config` candidate is structurally impossible — `putConfig` // never interns a frame into `entries`, so no RID ever resolves to config; // `pickBase`'s `kind == .config` guard is belt-and-suspenders. } test "logstore §5.2: putArtifact picks the best base from the recent window over a worse hint" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const gpa = std.testing.allocator; var store = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer store.deinit(gpa); // A poor hint (`far`) and a great recent frame (`near`). putArtifact's // window is hint + recent frames, and best-of-N must store the delta // against `near`, proving the write path is wired to the size search. const target = "Sphinx of black quartz judge my vow, and then repeat that whole pangram twice more for a good long run here."; const near = "Sphinx of black quartz judge my vow, and then repeat that whole pangram twice more for a good long run here!"; const far = "Pack my box with five dozen liquor jugs while the band plays on through the long and rather tedious night oo."; const far_rid = try store.putArtifact("pa-far-base-hash-value-000000001", far, null); const near_rid = try store.putArtifact("pa-near-base-hash-value-00000001", near, null); // Pass `far` as the lineage hint; `near` is only in the recent window. const tgt_rid = try store.putArtifact("pa-target-hash-value-00000000001", target, far_rid); try std.testing.expectEqual(logstore.FrameKind.delta, store.kindOf(tgt_rid)); // The stored delta names `near` as its base, not the `far` hint. var w = (try store.wire(gpa, tgt_rid)).?; defer w.deinit(gpa); try std.testing.expectEqualStrings(store.hashOf(near_rid), w.delta.baseHash()); // And content still round-trips exactly through the chosen chain. const got = (try store.content(gpa, tgt_rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(target, got); } test "logstore §5.2: putArtifact's backward window scan is bounded by max_base_scan, not phantom density" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const gpa = std.testing.allocator; var store = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer store.deinit(gpa); // One real content frame, then a run of phantom frames longer than the scan // window. `target` is a one-byte mutation of `base` — it would delta-encode // well under 75% IF the backward walk reached `base`. But the walk must stop // after `max_base_scan` frames no matter how many phantoms it steps over, so // the sole content base sits past the window and is never offered. Without // the scan cap the walk would `continue` over every phantom to reach `base` // (the hidden O(n^2) this bound closes), storing a delta instead. const base = "Sphinx of black quartz judge my vow, and then repeat that whole pangram twice more for a good long run here."; const target = "Sphinx of black quartz judge my vow, and then repeat that whole pangram twice more for a good long run here!"; _ = try store.putArtifact("scan-base-hash-0001", base, null); // Intern comfortably more phantoms than the scan cap so `base` is strictly // outside the window (each phantom `continue`s without filling a slot). var i: usize = 0; var hbuf: [32]u8 = undefined; while (i < logstore.max_base_scan + 4) : (i += 1) { const h = std.fmt.bufPrint(&hbuf, "scan-phantom-hash-{d:0>13}", .{i}) catch unreachable; _ = try store.intern(h); } // No lineage hint: the only base candidate is the content frame, now beyond // the scan window, so pickBase sees no eligible candidate → full-form store. const tgt_rid = try store.putArtifact("scan-tgt-hash-0001", target, null); try std.testing.expectEqual(logstore.FrameKind.zlib, store.kindOf(tgt_rid)); // The invariant holds regardless of the base choice: content round-trips // byte-exact. const got = (try store.content(gpa, tgt_rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(target, got); } test "logstore §5: an idempotent re-put does zero encode work (no allocation)" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const gpa = std.testing.allocator; var store = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer store.deinit(gpa); const h = "idem-artifact-hash-value-0000001"; const bytes = "idempotent artifact bytes long enough to exceed fifty and be worth a delta attempt against a base here."; const first = try store.putArtifact(h, bytes, null); const size_before = store.size(); // Re-put the identical hash under an allocator that fails on its FIRST // allocation: an idempotent hit must short-circuit before any encode work, // so zero allocations occur and the call still succeeds. A single stray // allocation would surface as error.OutOfMemory here. var failing = std.testing.FailingAllocator.init(gpa, .{ .fail_index = 0 }); store.gpa = failing.allocator(); const again = try store.putArtifact(h, bytes, first); store.gpa = gpa; // restore before deinit frees with the real allocator try std.testing.expectEqual(first, again); try std.testing.expectEqual(size_before, store.size()); // nothing appended try std.testing.expectEqual(@as(usize, 0), failing.allocated_bytes); } test "logstore §5 property: pickBase yields the smallest eligible delta and it reconstructs" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const io = std.testing.io; var fs = try openTmp(&tmp, io, "log.bin"); defer fs.file.close(io); const gpa = std.testing.allocator; var store = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer store.deinit(gpa); var prng = std.Random.DefaultPrng.init(0x5EEDB45E); const rnd = prng.random(); // A fixed pool of 4 eligible content frames (all >= 64 bytes, sharing a // long common prefix so mutations delta well), stored full via putArtifact. const pool_n = 4; var pool: [pool_n]logstore.Rid = undefined; var pool_bufs: [pool_n][192]u8 = undefined; var pool_lens: [pool_n]usize = undefined; for (0..pool_n) |p| { const len = 96 + rnd.uintAtMost(usize, 96); rnd.bytes(pool_bufs[p][0..len]); @memset(pool_bufs[p][0..48], 0xC7); // shared prefix → deltable structure pool_lens[p] = len; var hbuf: [32]u8 = undefined; std.mem.writeInt(u64, hbuf[0..8], p, .little); @memcpy(hbuf[8..], "prop-pool-frame-paddingX"[0..24]); pool[p] = try store.putArtifact(&hbuf, pool_bufs[p][0..len], null); } var target_buf: [192]u8 = undefined; var iter: usize = 0; while (iter < 300) : (iter += 1) { // Half the time derive the target from a pool frame (a profitable // delta exists); half the time use fresh noise (usually unprofitable). var tlen: usize = undefined; if (rnd.boolean()) { const src = rnd.uintLessThan(usize, pool_n); tlen = pool_lens[src]; @memcpy(target_buf[0..tlen], pool_bufs[src][0..tlen]); const flips = 1 + rnd.uintAtMost(usize, 3); for (0..flips) |_| { const pos = rnd.uintLessThan(usize, tlen); target_buf[pos] +%= rnd.intRangeAtMost(u8, 1, 255); // nonzero → the byte changes } } else { tlen = 64 + rnd.uintAtMost(usize, 96); rnd.bytes(target_buf[0..tlen]); } const target = target_buf[0..tlen]; // Candidates: the whole pool, in pool order (a distinct target hash, so // every pool frame is eligible: content-bearing, depth 0, not self). var cands: [pool_n]logstore.Rid = pool; const pick = try store.pickBase(gpa, "prop-target-hash-value-000000001", target, &cands); // Independently compute the smallest delta over the same eligible set. var min_len: usize = std.math.maxInt(usize); var min_base: logstore.Rid = 0; for (cands) |c| { const cb = (try store.content(gpa, c)).?; defer gpa.free(cb); const d = try delta.create(gpa, cb, target); defer gpa.free(d); if (d.len < min_len) { min_len = d.len; min_base = c; } } const should_accept = min_len * 4 < target.len * 3; // the §5 75% bar if (should_accept) { const got = pick.?; defer gpa.free(got.delta); // pickBase returns exactly the global minimum, at the first // (highest-priority) base achieving it. try std.testing.expectEqual(min_len, got.delta.len); try std.testing.expectEqual(min_base, got.base); // …and that delta reconstructs the target byte-for-byte. const bb = (try store.content(gpa, got.base)).?; defer gpa.free(bb); const rebuilt = try delta.apply(gpa, bb, got.delta); defer gpa.free(rebuilt); try std.testing.expectEqualSlices(u8, target, rebuilt); } else { try std.testing.expect(pick == null); } } }