//! Storage §C: the framed, append-only, content-addressed log over the
//! `StorageFile` capability (storage-track-plan.md §C). One file: a 64-byte
//! header followed by 8-byte-aligned frames, each self-verifying via a
//! header checksum (over the fixed fields) plus a whole-frame checksum
//! (over header+payload). The header checksum is what makes torn-vs-
//! corrupt discrimination exact and size-independent — see
//! storage-slice.md's v2 rework note for why a v1 without it was unsound.
//!
//! No RFC section behind this file: native storage-engine design. The
//! oracle is storage-track-plan.md §C plus the four crash-recovery
//! invariants exercised by sim/storage_sweep.zig's DST sweep.
//!
//! A second, narrower checksum guards the frame's fixed 6-byte prefix
//! (`len|flags|hash_len`) on its own, in a 10-byte window whose size
//! never depends on `hash_len`. Without it, scanning a frame whose
//! `hash_len` byte is itself the bitflipped one has no way to know how
//! many bytes the rest of the header needs before it can check the
//! header checksum -- a corrupted `hash_len` that happens to demand more
//! bytes than the file has left reads exactly like a torn tail. The
//! prefix checksum removes that ambiguity: `hash_len` is proven correct
//! (or rejected outright) from a fixed-size read, before it is ever used
//! to size anything else.
const std = @import("std");
const storage = @import("storage.zig");
const delta = @import("zit").delta;
const flate = std.compress.flate;
/// Local artifact id: dense, 1-based, assigned in intern/put order. 0 is
/// never issued (mirrors repostate.zig's MemStore).
pub const Rid = u64;
/// FILE header size: magic(4) | version u16(2) | hash_kind u8(1) |
/// reserved(53) | crc32(4).
pub const header_len: u64 = 64;
/// Explicit payload bound (storage-track-plan.md §C).
pub const max_payload: u64 = 1 << 30;
pub const max_hash_len: u8 = 32;
pub const min_hash_len: u8 = 1;
/// Every frame's on-disk span is a multiple of this.
pub const frame_align: u64 = 8;
/// Readers accept any stored version `<= version` (freeze seam, §E).
pub const version: u16 = 1;
/// Smallest possible frame span: hash_len=1, payload=0 ->
/// align8(26+1+0) = 32.
const min_frame: u64 = 32;
/// Explicit upper bound on delta-chain resolution: a `content()` on a
/// delta frame walks at most this many `delta_of` hops before rejecting
/// the chain as corrupt. Caps read recursion and read-amplification;
/// generous versus the short chains a real store produces.
pub const max_delta_chain: u8 = 32;
const magic = "zitL";
/// Payload encoding. `raw` = uncompressed bytes; `zlib` = a fossil zlib
/// container (4-byte BE uncompressed length + RFC1950 stream, sync §4.1);
/// `delta` = a zlib-contained delta whose `delta_of` is the byte offset of
/// the base frame (storage-track-plan.md §C). Any other flag byte is an
/// unknown kind and rejected on open. The content hash always names the
/// UNCOMPRESSED, fully-reconstructed bytes, so encoding never affects
/// naming (sync §4.1 L1206-1208).
pub const FrameKind = enum(u8) { raw = 0, zlib = 1, delta = 2, _ };
pub const OpenError = error{ OutOfMemory, Storage, Corrupt };
/// No read happens on the write path, so `Corrupt` can't occur here —
/// advertising it would be dead mechanism.
pub const PutError = error{ OutOfMemory, Storage };
/// A phantom is purely in-memory bookkeeping: no I/O, so no `Storage`.
pub const InternError = error{OutOfMemory};
pub const ContentError = error{ OutOfMemory, Storage, Corrupt };
pub const CommitError = error{Storage};
/// The framed, content-addressed log. `open` rebuilds `entries`/`index`
/// from the file each time — there is no persistent side index yet
/// (storage-track-plan.md §C: "O(n)-on-open correct at this scale").
pub const LogStore = struct {
gpa: std.mem.Allocator,
file: storage.StorageFile,
hash_kind: u8,
/// Structural end of the log: header + every complete frame written so
/// far. Always a multiple of `frame_align`.
end: u64,
entries: std.ArrayList(Entry) = .empty,
index: std.StringHashMapUnmanaged(Rid) = .empty,
const Entry = struct {
/// Owned duplicate; also the index's key (stable across `entries`
/// reallocation since it's a separate heap allocation).
hash: []u8,
/// Frame start offset. Meaningless while `phantom`.
offset: u64,
/// On-disk (container/wire) payload length: raw bytes for `raw`,
/// the zlib-container length for `zlib`/`delta`.
payload_len: u32,
/// Payload encoding of this frame. `.raw` while `phantom` (no frame).
kind: FrameKind,
/// A phantom was `intern`ed but never `put`: no frame on disk.
phantom: bool,
};
/// Opens (creating a fresh header on an empty file, or re-initializing
/// a torn one) and recovers the in-memory index by scanning frames.
/// `hash_kind` seeds a brand-new header; an existing header's stored
/// value wins on reopen (opaque, round-tripped, never compared).
pub fn open(gpa: std.mem.Allocator, file: storage.StorageFile, hash_kind: u8) OpenError!LogStore {
const file_size = file.size() catch return error.Storage;
var self: LogStore = .{ .gpa = gpa, .file = file, .hash_kind = hash_kind, .end = header_len };
errdefer self.deinit(gpa);
if (file_size == 0) {
try writeHeader(file, hash_kind);
std.debug.assert(self.end % frame_align == 0);
return self;
}
if (file_size < header_len) {
// Torn header: nothing before byte `header_len` can be trusted
// as a frame boundary either, so start clean.
file.truncate(0) catch return error.Storage;
try writeHeader(file, hash_kind);
std.debug.assert(self.end % frame_align == 0);
return self;
}
self.hash_kind = try validateHeader(file);
try scan(&self);
std.debug.assert(self.end % frame_align == 0);
return self;
}
pub fn deinit(self: *LogStore, gpa: std.mem.Allocator) void {
for (self.entries.items) |e| gpa.free(e.hash);
self.entries.deinit(gpa);
self.index.deinit(gpa);
self.* = undefined;
}
/// Resolves `hash` to its rid, creating a phantom (no I/O) when
/// unknown. Phantoms never reach disk.
pub fn intern(self: *LogStore, hash: []const u8) InternError!Rid {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
if (self.index.get(hash)) |rid| return rid;
try self.entries.ensureUnusedCapacity(self.gpa, 1);
const owned = try self.gpa.dupe(u8, hash);
errdefer self.gpa.free(owned);
const rid: Rid = self.entries.items.len + 1;
try self.index.put(self.gpa, owned, rid);
self.entries.appendAssumeCapacity(.{ .hash = owned, .offset = 0, .payload_len = 0, .kind = .raw, .phantom = true });
return rid;
}
/// Stores `bytes` verbatim (a `raw` frame) under `hash`. Idempotent for
/// an already-content-bearing hash (returns the existing rid, appends
/// nothing); upgrades a phantom in place.
pub fn put(self: *LogStore, hash: []const u8, bytes: []const u8) PutError!Rid {
std.debug.assert(bytes.len <= max_payload);
return self.insert(hash, bytes, .raw, 0);
}
/// Stores `bytes` compressed into a fossil zlib container (§4.1, default
/// level) under `hash`. The hash still names the UNCOMPRESSED bytes, so
/// storage compression never changes an artifact's address (§4.1
/// L1206-1208). Idempotent + phantom-upgrade like `put`.
pub fn putZlib(self: *LogStore, hash: []const u8, bytes: []const u8) PutError!Rid {
std.debug.assert(bytes.len <= max_payload);
const container = try containerEncode(self.gpa, bytes);
defer self.gpa.free(container);
return self.insert(hash, container, .zlib, 0);
}
/// Stores `raw_delta` (a delta reconstructing this artifact from the frame
/// at `base_rid`) as a zlib-contained `delta` frame; `delta_of` records
/// the base frame's byte offset. The stored container IS the wire payload
/// a `cfile` serves verbatim (§7.2). Idempotent + phantom-upgrade like
/// `put`.
pub fn putDelta(self: *LogStore, hash: []const u8, base_rid: Rid, raw_delta: []const u8) PutError!Rid {
std.debug.assert(base_rid >= 1 and base_rid <= self.entries.items.len);
const base = &self.entries.items[@intCast(base_rid - 1)];
std.debug.assert(!base.phantom);
std.debug.assert(base.offset >= header_len and base.offset < self.end);
std.debug.assert(raw_delta.len >= 1 and raw_delta.len <= max_payload);
const container = try containerEncode(self.gpa, raw_delta);
defer self.gpa.free(container);
return self.insert(hash, container, .delta, base.offset);
}
/// Shared insert core: append one frame carrying `payload` under `hash`
/// with encoding `kind` (and `delta_of` for `.delta`), then update the
/// index. Idempotent for an already-content-bearing hash; upgrades a
/// phantom in place. On a bookkeeping failure after the frame landed, the
/// file is truncated back so disk and memory stay in lockstep.
fn insert(self: *LogStore, hash: []const u8, payload: []const u8, kind: FrameKind, delta_of: u64) PutError!Rid {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
std.debug.assert(payload.len <= max_payload);
if (self.index.get(hash)) |rid| {
const entry = &self.entries.items[@intCast(rid - 1)];
if (!entry.phantom) return rid;
const res = try self.appendFrame(hash, payload, kind, delta_of);
entry.offset = res.offset;
entry.payload_len = res.len;
entry.kind = kind;
entry.phantom = false;
return rid;
}
try self.entries.ensureUnusedCapacity(self.gpa, 1);
const owned = try self.gpa.dupe(u8, hash);
errdefer self.gpa.free(owned);
const res = try self.appendFrame(hash, payload, kind, delta_of);
const rid: Rid = self.entries.items.len + 1;
self.index.put(self.gpa, owned, rid) catch |e| {
// Frame already landed on disk but bookkeeping failed: undo the
// write so the file matches in-memory state exactly.
self.file.truncate(res.offset) catch {};
self.end = res.offset;
return e;
};
self.entries.appendAssumeCapacity(.{ .hash = owned, .offset = res.offset, .payload_len = res.len, .kind = kind, .phantom = false });
return rid;
}
/// The payload encoding of `rid` (`.raw` for a phantom). Lets a serve
/// loop pick a full-content versus delta cfile without a frame read.
pub fn kindOf(self: *const LogStore, rid: Rid) FrameKind {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
return self.entries.items[@intCast(rid - 1)].kind;
}
/// The on-disk (container/wire) payload length of `rid`; 0 for a phantom.
/// Symmetric with `hashOf`; the DST sweep uses it to bound the crc-covered
/// span of a frame whose uncompressed content differs in length.
pub fn payloadLenOf(self: *const LogStore, rid: Rid) u32 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
return self.entries.items[@intCast(rid - 1)].payload_len;
}
/// Owned uncompressed bytes for a rid, or null while it's a phantom.
/// Re-reads and re-verifies the frame (both checksums, embedded hash) on
/// every call, then decodes by kind: `raw` copied out, `zlib` inflated,
/// `delta` resolved by walking `delta_of` and applying via `delta.apply`.
/// Never trusts a stale scan result.
pub fn content(self: *LogStore, gpa: std.mem.Allocator, rid: Rid) ContentError!?[]u8 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
const entry = &self.entries.items[@intCast(rid - 1)];
if (entry.phantom) return null;
return try self.contentAtOffset(gpa, entry.offset, max_delta_chain, entry.hash);
}
/// Reconstructs the uncompressed content of the frame at `offset`.
/// `depth_remaining` bounds delta recursion: each `delta` hop consumes
/// one, and a delta reached at zero is `error.Corrupt` (never unbounded
/// recursion). `expect_hash`, when non-null, requires the frame's embedded
/// hash to match (the top-level integrity check); recursion into a base
/// passes null since a base carries its own name.
fn contentAtOffset(self: *LogStore, gpa: std.mem.Allocator, offset: u64, depth_remaining: u8, expect_hash: ?[]const u8) ContentError![]u8 {
std.debug.assert(depth_remaining <= max_delta_chain);
const fv = try self.readFrameAt(gpa, offset);
if (expect_hash) |eh| {
if (!std.mem.eql(u8, fv.hash[0..fv.hash_len], eh)) {
gpa.free(fv.payload);
return error.Corrupt;
}
}
switch (fv.kind) {
.raw => return fv.payload, // ownership transfers to the caller
.zlib => {
defer gpa.free(fv.payload);
return containerDecode(gpa, fv.payload, max_payload);
},
.delta => {
defer gpa.free(fv.payload);
if (depth_remaining == 0) return error.Corrupt;
// A base must be an earlier, in-range frame: a forward or
// self reference is corruption, and a mid-frame/garbage
// offset fails `readFrameAt`'s checksums below.
if (fv.delta_of < header_len or fv.delta_of >= offset) return error.Corrupt;
const raw_delta = try containerDecode(gpa, fv.payload, max_payload);
defer gpa.free(raw_delta);
const base = try self.contentAtOffset(gpa, fv.delta_of, depth_remaining - 1, null);
defer gpa.free(base);
return delta.apply(gpa, base, raw_delta) catch |e| switch (e) {
error.OutOfMemory => error.OutOfMemory,
error.MalformedDelta, error.ChecksumMismatch => error.Corrupt,
};
},
_ => unreachable, // readFrameAt rejects any flag byte outside {raw,zlib,delta}
}
}
/// One fully validated frame read from `offset`: kind, `delta_of`, the
/// embedded hash, and the owned on-disk payload (raw bytes, or the zlib
/// container for `zlib`/`delta`). Caller owns `payload`.
const FrameView = struct {
kind: FrameKind,
delta_of: u64,
hash: [max_hash_len]u8,
hash_len: u8,
payload: []u8,
};
/// Reads and checksum-verifies the frame at `offset`, returning its
/// decoded fields and owned payload. Unlike scan, a short read here is
/// `error.Corrupt` (the frame was known to exist): the file lied or lost
/// bytes. Any flag byte outside {raw,zlib,delta} is rejected.
fn readFrameAt(self: *LogStore, gpa: std.mem.Allocator, offset: u64) ContentError!FrameView {
var prefix: [prefix_len]u8 = undefined;
const np = self.file.pread(offset, &prefix) catch return error.Storage;
if (np < prefix_len) return error.Corrupt;
const payload_len = std.mem.readInt(u32, prefix[0..4], .little);
const flags = prefix[4];
const hash_len = prefix[5];
if (hash_len < min_hash_len or hash_len > max_hash_len) return error.Corrupt;
var pcrc = std.hash.Crc32.init();
pcrc.update(prefix[0..6]);
if (pcrc.final() != std.mem.readInt(u32, prefix[6..][0..4], .little)) return error.Corrupt;
if (flags > @intFromEnum(FrameKind.delta)) return error.Corrupt;
const hl: usize = hash_len;
const fixed_and_crc_len: usize = 22 + hl;
var hbuf: [22 + max_hash_len]u8 = undefined;
const nh = self.file.pread(offset, hbuf[0..fixed_and_crc_len]) catch return error.Storage;
if (nh < fixed_and_crc_len) return error.Corrupt;
var hcrc = std.hash.Crc32.init();
hcrc.update(hbuf[0 .. 18 + hl]);
if (hcrc.final() != std.mem.readInt(u32, hbuf[18 + hl ..][0..4], .little)) return error.Corrupt;
const pl: usize = payload_len;
if (pl > max_payload) return error.Corrupt; // wire-derived bound before allocation
const payload = try gpa.alloc(u8, pl);
errdefer gpa.free(payload);
const payload_offset: u64 = offset + fixed_and_crc_len;
const npay = self.file.pread(payload_offset, payload) catch return error.Storage;
if (npay < pl) return error.Corrupt;
var trailer: [4]u8 = undefined;
const nt = self.file.pread(payload_offset + pl, &trailer) catch return error.Storage;
if (nt < 4) return error.Corrupt;
var fcrc = std.hash.Crc32.init();
fcrc.update(hbuf[0..fixed_and_crc_len]);
fcrc.update(payload);
if (fcrc.final() != std.mem.readInt(u32, &trailer, .little)) return error.Corrupt;
var fv: FrameView = .{
.kind = @enumFromInt(flags),
.delta_of = std.mem.readInt(u64, hbuf[10 + hl ..][0..8], .little),
.hash = undefined,
.hash_len = hash_len,
.payload = payload,
};
@memcpy(fv.hash[0..hl], hbuf[10..][0..hl]);
return fv;
}
/// A stored delta served byte-for-byte (§7.2 "the sender does not
/// re-compress the payload; it writes the stored bytes"): the on-disk
/// zlib-container payload plus the base artifact's hash. Caller frees
/// `wire`.
pub const StoredDelta = struct {
wire: []u8,
base_hash_buf: [max_hash_len]u8,
base_hash_len: u8,
pub fn baseHash(self: *const StoredDelta) []const u8 {
return self.base_hash_buf[0..self.base_hash_len];
}
};
/// Serves the `delta` frame at `rid` verbatim for a cfile (§7.2): its
/// stored container bytes, uncompressed and unmodified, plus the base
/// hash read from the base frame. Precondition: `kindOf(rid) == .delta`.
pub fn storedDelta(self: *LogStore, gpa: std.mem.Allocator, rid: Rid) ContentError!StoredDelta {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
const entry = &self.entries.items[@intCast(rid - 1)];
std.debug.assert(!entry.phantom and entry.kind == .delta);
const fv = try self.readFrameAt(gpa, entry.offset);
errdefer gpa.free(fv.payload);
std.debug.assert(fv.kind == .delta);
// The base frame's own header carries its hash; reading it also
// checksum-validates that `delta_of` still points at a real frame.
const base = try self.readFrameAt(gpa, fv.delta_of);
gpa.free(base.payload);
var out: StoredDelta = .{ .wire = fv.payload, .base_hash_buf = undefined, .base_hash_len = base.hash_len };
@memcpy(out.base_hash_buf[0..base.hash_len], base.hash[0..base.hash_len]);
return out;
}
/// Borrows the hash naming `rid`. Every rid, phantom or not, has one.
pub fn hashOf(self: *const LogStore, rid: Rid) []const u8 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
return self.entries.items[@intCast(rid - 1)].hash;
}
pub fn lookup(self: *const LogStore, hash: []const u8) ?Rid {
return self.index.get(hash);
}
pub fn commit(self: *LogStore) CommitError!void {
self.file.sync() catch return error.Storage;
}
/// Structural end of the log (header + every complete frame written so
/// far), for callers that track write progress without duplicating
/// frame-layout arithmetic (e.g. the DST sweep's crash-recovery model).
pub fn size(self: *const LogStore) u64 {
return self.end;
}
const AppendResult = struct { offset: u64, len: u32 };
/// Encodes one frame and appends it in a single `StorageFile.append`
/// call — one call is one atomic torn-write draw point, matching the
/// crash simulator's fault granularity. On failure, truncates back to
/// the prior structural end so a partial write never lingers.
fn appendFrame(self: *LogStore, hash: []const u8, bytes: []const u8, kind: FrameKind, delta_of: u64) PutError!AppendResult {
const hl: u64 = hash.len;
const pl: u64 = bytes.len;
const frame_len = alignUp8(u64, 26 + hl + pl);
std.debug.assert(frame_len <= max_payload + 64);
const frame = try self.gpa.alloc(u8, @intCast(frame_len));
defer self.gpa.free(frame);
_ = encodeFrame(frame, hash, bytes, @intFromEnum(kind), delta_of);
const offset = self.end;
if (self.file.append(frame)) |_| {
self.end += frame_len;
std.debug.assert(self.end % frame_align == 0);
return .{ .offset = offset, .len = @intCast(pl) };
} else |_| {
self.file.truncate(self.end) catch {};
return error.Storage;
}
}
};
fn writeHeader(file: storage.StorageFile, hash_kind: u8) OpenError!void {
var buf: [header_len]u8 = [_]u8{0} ** header_len;
@memcpy(buf[0..4], magic);
std.mem.writeInt(u16, buf[4..6], version, .little);
buf[6] = hash_kind;
var crc = std.hash.Crc32.init();
crc.update(buf[0..60]);
std.mem.writeInt(u32, buf[60..64], crc.final(), .little);
file.append(&buf) catch return error.Storage;
}
/// Returns the stored `hash_kind` on success. Rejects: bad magic, a stored
/// version newer than this reader, or a header crc mismatch.
fn validateHeader(file: storage.StorageFile) OpenError!u8 {
var buf: [header_len]u8 = undefined;
const n = file.pread(0, &buf) catch return error.Storage;
if (n < header_len) return error.Corrupt;
if (!std.mem.eql(u8, buf[0..4], magic)) return error.Corrupt;
const stored_version = std.mem.readInt(u16, buf[4..6], .little);
if (stored_version > version) return error.Corrupt;
const stored_crc = std.mem.readInt(u32, buf[60..64], .little);
var crc = std.hash.Crc32.init();
crc.update(buf[0..60]);
if (crc.final() != stored_crc) return error.Corrupt;
return buf[6];
}
/// Outcome of reading one frame at a scan offset: either a torn tail (the
/// file ends mid-frame — truncate and stop) or a validated frame.
const ScanOutcome = union(enum) {
torn,
frame: struct { hash: [max_hash_len]u8, hash_len: u8, frame_len: u64, payload_len: u32, kind: FrameKind },
};
/// Fixed prefix window: len(4)|flags(1)|hash_len(1)|prefix_crc32(4). Its
/// size does not depend on `hash_len` — that is what lets `hash_len` be
/// proven correct (or rejected) before it is used to size anything else.
const prefix_len: usize = 10;
/// Reads and validates exactly one frame at `offset`. Two-stage: first the
/// fixed-size prefix window (hash_len-independent, so a short read there
/// is unambiguously a torn write — nothing about its size can be thrown
/// off by corruption), which proves `hash_len` trustworthy via its own
/// crc; only then is `hash_len` used to size the rest of the header read,
/// at which point the same "short read = torn, checksum fails = corrupt"
/// reasoning from storage-slice.md v2 applies validly (append() only
/// ever writes one frame per call and a torn write is a clean prefix —
/// never damaged bytes; a full window whose checksum fails can only be a
/// bitflip on already-landed bytes).
fn scanOneFrame(file: storage.StorageFile, offset: u64, total: u64) OpenError!ScanOutcome {
var prefix: [prefix_len]u8 = undefined;
const n0 = file.pread(offset, &prefix) catch return error.Storage;
if (n0 < prefix_len) return .torn;
const payload_len = std.mem.readInt(u32, prefix[0..4], .little);
const flags = prefix[4];
const hash_len = prefix[5];
if (hash_len < min_hash_len or hash_len > max_hash_len) return error.Corrupt;
var pcrc = std.hash.Crc32.init();
pcrc.update(prefix[0..6]);
if (pcrc.final() != std.mem.readInt(u32, prefix[6..][0..4], .little)) return error.Corrupt;
// A prefix-verified flag byte is trustworthy (a torn write is a clean
// truncation, never damaged bytes), so any kind outside {raw,zlib,delta}
// is a genuine unknown-kind frame, rejected — not a torn tail.
if (flags > @intFromEnum(FrameKind.delta)) return error.Corrupt;
const fixed_and_crc_len: usize = 22 + @as(usize, hash_len);
var hbuf: [22 + max_hash_len]u8 = undefined;
const n1 = file.pread(offset, hbuf[0..fixed_and_crc_len]) catch return error.Storage;
if (n1 < fixed_and_crc_len) return .torn;
var hcrc = std.hash.Crc32.init();
hcrc.update(hbuf[0 .. 18 + @as(usize, hash_len)]);
const stored_header_crc = std.mem.readInt(u32, hbuf[18 + @as(usize, hash_len) ..][0..4], .little);
if (hcrc.final() != stored_header_crc) return error.Corrupt;
const frame_len = alignUp8(u64, 26 + @as(u64, hash_len) + payload_len);
if (offset + frame_len > total) return .torn;
var crc = std.hash.Crc32.init();
crc.update(hbuf[0..fixed_and_crc_len]);
var remaining: u64 = payload_len;
var pos = offset + @as(u64, fixed_and_crc_len);
var chunk: [4096]u8 = undefined;
while (remaining > 0) {
const take: usize = @intCast(@min(remaining, @as(u64, chunk.len)));
const got = file.pread(pos, chunk[0..take]) catch return error.Storage;
std.debug.assert(got == take); // bounds check above guarantees `total` covers this
crc.update(chunk[0..take]);
pos += take;
remaining -= take;
}
var trailer: [4]u8 = undefined;
const nt = file.pread(pos, &trailer) catch return error.Storage;
std.debug.assert(nt == 4);
if (crc.final() != std.mem.readInt(u32, &trailer, .little)) return error.Corrupt;
var hash: [max_hash_len]u8 = undefined;
@memcpy(hash[0..hash_len], hbuf[10..][0..hash_len]);
return .{ .frame = .{ .hash = hash, .hash_len = hash_len, .frame_len = frame_len, .payload_len = payload_len, .kind = @enumFromInt(flags) } };
}
/// Rebuilds `self.entries`/`self.index` by scanning frames from
/// `header_len`. A torn tail truncates the file to the last valid
/// boundary and stops; a corrupt frame propagates `error.Corrupt`
/// unconditionally (never truncated over).
fn scan(self: *LogStore) OpenError!void {
const total = self.file.size() catch return error.Storage;
const max_frames = (total - header_len) / min_frame + 1;
var offset: u64 = header_len;
var frames: u64 = 0;
while (offset < total) {
frames += 1;
std.debug.assert(frames <= max_frames);
switch (try scanOneFrame(self.file, offset, total)) {
.torn => {
self.file.truncate(offset) catch return error.Storage;
self.end = offset;
return;
},
.frame => |f| {
const owned = try self.gpa.dupe(u8, f.hash[0..f.hash_len]);
errdefer self.gpa.free(owned);
try self.entries.append(self.gpa, .{ .hash = owned, .offset = offset, .payload_len = f.payload_len, .kind = f.kind, .phantom = false });
errdefer _ = self.entries.pop();
const rid: Rid = self.entries.items.len;
try self.index.put(self.gpa, owned, rid);
const next = offset + f.frame_len;
std.debug.assert(next > offset);
offset = next;
},
}
}
self.end = offset;
}
/// Encodes one frame into `buf` (which must be at least the padded frame
/// span) and returns the written slice. Shared by the production write
/// path and by tests that hand-craft torn/corrupt fixtures.
///
/// Layout (all little-endian): len(4) | flags(1) | hash_len(1) |
/// prefix_crc32(4) = crc32(bytes[0..6)) | hash(hash_len) | delta_of(8) |
/// header_crc32(4) = crc32(bytes[0..18+hash_len)) | payload(len) |
/// frame_crc32(4) = crc32(bytes[0..22+hash_len+len)). `delta_of` is the
/// base frame's byte offset for a `delta` frame, and 0 otherwise.
fn encodeFrame(buf: []u8, hash: []const u8, payload: []const u8, flags: u8, delta_of: u64) []const u8 {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
std.debug.assert(payload.len <= max_payload);
const hl: usize = hash.len;
const pl: usize = payload.len;
const hash_len: u8 = @intCast(hl);
const payload_len: u32 = @intCast(pl);
const unpadded: usize = 26 + hl + pl;
const frame_len: usize = @intCast(alignUp8(u64, unpadded));
std.debug.assert(buf.len >= frame_len);
std.mem.writeInt(u32, buf[0..4], payload_len, .little);
buf[4] = flags;
buf[5] = hash_len;
var pcrc = std.hash.Crc32.init();
pcrc.update(buf[0..6]);
std.mem.writeInt(u32, buf[6..][0..4], pcrc.final(), .little);
@memcpy(buf[10..][0..hl], hash);
std.mem.writeInt(u64, buf[10 + hl ..][0..8], delta_of, .little);
var hcrc = std.hash.Crc32.init();
hcrc.update(buf[0 .. 18 + hl]);
std.mem.writeInt(u32, buf[18 + hl ..][0..4], hcrc.final(), .little);
@memcpy(buf[22 + hl ..][0..pl], payload);
var fcrc = std.hash.Crc32.init();
fcrc.update(buf[0 .. 22 + hl + pl]);
std.mem.writeInt(u32, buf[22 + hl + pl ..][0..4], fcrc.final(), .little);
if (frame_len > unpadded) @memset(buf[unpadded..frame_len], 0);
return buf[0..frame_len];
}
fn alignUp8(comptime T: type, n: T) T {
const a: T = frame_align;
return (n + (a - 1)) / a * a;
}
/// `Compress.init` asserts its output writer already holds more than 8 bytes
/// of buffer; `Writer.Allocating` starts empty, so pre-size before init.
/// This headroom covers the 4-byte length prefix and 2-byte zlib header
/// written first.
const compress_out_min: usize = 64;
/// sync §4.1 L1192-1197: the fossil zlib container — a 4-byte big-endian
/// uncompressed length, then an RFC1950 zlib stream (2-byte header, DEFLATE
/// body, 4-byte Adler-32 trailer) at the default level. The only compression
/// in the store. Kept private to the storage layer (no layer-inverting import
/// of the protocol module); sync carries its own cited copy.
fn containerEncode(gpa: std.mem.Allocator, bytes: []const u8) error{OutOfMemory}![]u8 {
std.debug.assert(bytes.len <= max_payload); // max_payload < 2^32, fits the length field
var aw = std.Io.Writer.Allocating.init(gpa);
defer aw.deinit();
aw.ensureUnusedCapacity(compress_out_min) catch return error.OutOfMemory;
aw.writer.writeInt(u32, @intCast(bytes.len), .big) catch return error.OutOfMemory;
const window = gpa.alloc(u8, flate.max_window_len) catch return error.OutOfMemory;
defer gpa.free(window);
var c = flate.Compress.init(&aw.writer, window, .zlib, .level_6) catch return error.OutOfMemory;
c.writer.writeAll(bytes) catch return error.OutOfMemory;
c.finish() catch return error.OutOfMemory;
const out = aw.toOwnedSlice() catch return error.OutOfMemory;
std.debug.assert(std.mem.readInt(u32, out[0..4], .big) == bytes.len);
return out;
}
/// Decodes a `containerEncode` container. `max_output` bounds the declared
/// length (a lying header is rejected before any inflate work) and the stream
/// is pumped no further than `declared + 1` bytes, so a length lie cannot
/// drive unbounded allocation and an overrun past `declared` is caught.
/// Malformed input (short header, length mismatch, bad stream) is
/// `error.Corrupt`; decode is compression-level-independent (§4.1 L1204-1206).
fn containerDecode(gpa: std.mem.Allocator, wire: []const u8, max_output: usize) ContentError![]u8 {
if (wire.len < 4) return error.Corrupt;
const declared = std.mem.readInt(u32, wire[0..4], .big);
if (declared > max_output) return error.Corrupt;
var fr = std.Io.Reader.fixed(wire[4..]);
const window = gpa.alloc(u8, flate.max_window_len) catch return error.OutOfMemory;
defer gpa.free(window);
var dc = flate.Decompress.init(&fr, .zlib, window);
var aw = std.Io.Writer.Allocating.init(gpa);
defer aw.deinit();
const cap: usize = @as(usize, declared) + 1;
var total: usize = 0;
// stream() may make progress without emitting bytes (parsing a block
// header), so 0 is not completion — only EndOfStream or reaching `cap`
// ends the pump. The iteration count is still explicitly bounded so a
// stalling stdlib regression surfaces as Corrupt rather than a hang.
const max_iters: usize = wire.len + cap + 64;
var iters: usize = 0;
while (total < cap and iters < max_iters) : (iters += 1) {
const n = dc.reader.stream(&aw.writer, .limited(cap - total)) catch |err| switch (err) {
error.EndOfStream => break,
error.WriteFailed => return error.OutOfMemory,
error.ReadFailed => return error.Corrupt,
};
total += n;
}
if (iters >= max_iters) return error.Corrupt;
if (total != declared) return error.Corrupt; // inflated length must equal the declared length
const out = aw.toOwnedSlice() catch return error.OutOfMemory;
std.debug.assert(out.len == declared);
return out;
}
// ---------------------------------------------------------------------------
// Tests (plan section C — storage-track-plan.md / storage-slice.md; no RFC
// section behind this file).
// ---------------------------------------------------------------------------
fn openTmp(tmp: anytype, io: std.Io, name: []const u8) !storage.FileStorage {
const raw = try tmp.dir.createFile(io, name, .{ .read = true });
return storage.FileStorage.init(io, raw, 0);
}
fn writeRawHeader(file: storage.StorageFile, magic_bytes: []const u8, ver: u16, hk: u8, bad_crc: bool) !void {
std.debug.assert(magic_bytes.len == 4);
var buf: [header_len]u8 = [_]u8{0} ** header_len;
@memcpy(buf[0..4], magic_bytes);
std.mem.writeInt(u16, buf[4..6], ver, .little);
buf[6] = hk;
var crc = std.hash.Crc32.init();
crc.update(buf[0..60]);
var crc_val = crc.final();
if (bad_crc) crc_val ^= 1;
std.mem.writeInt(u32, buf[60..64], crc_val, .little);
try file.append(&buf);
}
test "logstore §C: header round-trips magic/version/hash_kind through open" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store1 = try LogStore.open(std.testing.allocator, file, 7);
store1.deinit(std.testing.allocator);
var raw: [header_len]u8 = undefined;
const n = try file.pread(0, &raw);
try std.testing.expectEqual(@as(usize, header_len), n);
try std.testing.expectEqualStrings(magic, raw[0..4]);
try std.testing.expectEqual(@as(u16, version), std.mem.readInt(u16, raw[4..6], .little));
try std.testing.expectEqual(@as(u8, 7), raw[6]);
for (raw[7..60]) |b| try std.testing.expectEqual(@as(u8, 0), b);
var crc = std.hash.Crc32.init();
crc.update(raw[0..60]);
try std.testing.expectEqual(crc.final(), std.mem.readInt(u32, raw[60..64], .little));
var store2 = try LogStore.open(std.testing.allocator, file, 0); // hint ignored on reopen
defer store2.deinit(std.testing.allocator);
try std.testing.expectEqual(@as(u8, 7), store2.hash_kind);
}
test "logstore §C: open rejects a bad magic" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
try writeRawHeader(file, "XXXX", version, 0, false);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: open rejects a corrupt file-header crc" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
try writeRawHeader(file, magic, version, 0, true);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: open rejects a version newer than this reader" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
try writeRawHeader(file, magic, version + 1, 0, false);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: open re-initializes a torn file header instead of rejecting it" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
try file.append(&[_]u8{ 'z', 'i', 't' }); // 3 of 64 header bytes: nowhere near complete
var store = try LogStore.open(std.testing.allocator, file, 3);
defer store.deinit(std.testing.allocator);
try std.testing.expectEqual(@as(u64, header_len), store.size());
try std.testing.expectEqual(@as(u8, 3), store.hash_kind);
}
test "logstore §C: put then content round-trips a raw frame" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const hash = "h" ** 32;
const rid = try store.put(hash, "hello world");
try std.testing.expectEqualStrings(hash, store.hashOf(rid));
const got = (try store.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings("hello world", got);
}
test "logstore §C: a written frame occupies an 8-byte-aligned span" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
_ = try store.put("abcde", "xyz"); // unpadded = 26+5+3 = 34, align8 -> 40
try std.testing.expectEqual(@as(u64, 0), store.size() % 8);
try std.testing.expectEqual(header_len + 40, store.size());
}
test "logstore §C: the frame header crc covers len/flags/hash_len/hash/delta_of" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator); // header only
var buf: [64]u8 = undefined;
const frame = encodeFrame(&buf, "aaaa", "payload", @intFromEnum(FrameKind.raw), 0);
var corrupted: [64]u8 = undefined;
@memcpy(corrupted[0..frame.len], frame);
corrupted[10] ^= 0xff; // inside the hash bytes, part of the fixed header
try file.append(corrupted[0..frame.len]);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: content hash is the uncompressed address; hashOf returns it" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const h1 = "1" ** 16;
const h2 = "2" ** 20;
const r1 = try store.put(h1, "aaa");
const r2 = try store.put(h2, "bbbbbb");
try std.testing.expectEqualStrings(h1, store.hashOf(r1));
try std.testing.expectEqualStrings(h2, store.hashOf(r2));
const c1 = (try store.content(std.testing.allocator, r1)).?;
defer std.testing.allocator.free(c1);
try std.testing.expectEqualStrings("aaa", c1);
}
test "logstore §C: put is idempotent for an equal hash and appends nothing" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const hash = "dup" ** 8;
const r1 = try store.put(hash, "payload-a");
const size_after_first = store.size();
const r2 = try store.put(hash, "payload-a");
try std.testing.expectEqual(r1, r2);
try std.testing.expectEqual(size_after_first, store.size());
}
test "logstore §C: intern yields a phantom; content is null and no frame is written" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const hash = "phantom-hash-example-value";
const size_before = store.size();
const rid = try store.intern(hash);
try std.testing.expectEqual(size_before, store.size());
const got = try store.content(std.testing.allocator, rid);
try std.testing.expectEqual(@as(?[]u8, null), got);
}
test "logstore §C: put upgrades an interned phantom in place, same rid, content now present" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
// intern fixes the rid with no frame; content is null while phantom.
const h = "phantom-upgrade-hash-value-x";
const rid = try store.intern(h);
try std.testing.expectEqual(@as(?[]u8, null), try store.content(std.testing.allocator, rid));
const size0 = store.size();
// put on the same hash upgrades the phantom in place: same rid, and it
// appends the first frame (a phantom had none), so the log grows.
const rid2 = try store.put(h, "content-arrived");
try std.testing.expectEqual(rid, rid2);
try std.testing.expect(store.size() > size0);
{
const got = (try store.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings("content-arrived", got);
}
try std.testing.expectEqualStrings(h, store.hashOf(rid));
// The upgraded frame is real on disk, not just an in-memory flag: commit,
// reopen, and the hash resolves to content-bearing bytes (§B inv1 for a
// put that began life as a phantom).
try store.commit();
store.deinit(std.testing.allocator);
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
const rrid = reopened.lookup(h) orelse return error.TestUnexpectedResult;
const got2 = (try reopened.content(std.testing.allocator, rrid)).?;
defer std.testing.allocator.free(got2);
try std.testing.expectEqualStrings("content-arrived", got2);
try std.testing.expectEqualStrings(h, reopened.hashOf(rrid));
}
test "logstore §C(inv1): every committed put is readable and hash-exact after reopen" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
const Item = struct { hash: []const u8, bytes: []const u8 };
const items = [_]Item{
.{ .hash = "h-one" ** 4, .bytes = "one" },
.{ .hash = "h-two" ** 4, .bytes = "two-payload" },
.{ .hash = "h-three" ** 3, .bytes = "three!" },
};
var store = try LogStore.open(std.testing.allocator, file, 0);
for (items) |it| _ = try store.put(it.hash, it.bytes);
try store.commit();
store.deinit(std.testing.allocator);
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
for (items) |it| {
const rid = reopened.lookup(it.hash) orelse return error.TestUnexpectedResult;
try std.testing.expectEqualStrings(it.hash, reopened.hashOf(rid));
const got = (try reopened.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings(it.bytes, got);
}
}
test "logstore §C(inv2): a genuinely torn (prefix-truncated) final frame truncates to exactly the last valid boundary" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
_ = try store.put("frame-one-hash", "payload-one");
try store.commit();
const boundary = store.size();
store.deinit(std.testing.allocator);
var buf: [96]u8 = undefined;
const frame2 = encodeFrame(&buf, "frame-two-hash", "payload-two", @intFromEnum(FrameKind.raw), 0);
try file.append(frame2[0..10]); // short of even the fixed header+crc window
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
try std.testing.expectEqual(boundary, reopened.size());
try std.testing.expect(reopened.lookup("frame-two-hash") == null);
}
test "logstore §C(inv2): a torn payload/crc tail truncates exactly" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
_ = try store.put("frame-a-hash-here", "hello-payload");
try store.commit();
const boundary = store.size();
store.deinit(std.testing.allocator);
var buf: [128]u8 = undefined;
const frame2 = encodeFrame(&buf, "frame-b-hash-value", "a longer second payload for this test", @intFromEnum(FrameKind.raw), 0);
const header_window = 22 + "frame-b-hash-value".len;
try file.append(frame2[0 .. header_window + 3]); // full header+crc landed; payload/crc badly short
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
try std.testing.expectEqual(boundary, reopened.size());
}
test "logstore §C(inv3): the index rebuilt from the log equals the written model" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
const hashes = [_][]const u8{ "r1" ** 8, "r2" ** 8, "r3" ** 8, "r4" ** 8 };
const bodies = [_][]const u8{ "b1", "bb2", "bbb3", "bbbb4" };
var store = try LogStore.open(std.testing.allocator, file, 0);
for (hashes, bodies) |h, b| _ = try store.put(h, b);
try store.commit();
store.deinit(std.testing.allocator);
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
try std.testing.expectEqual(hashes.len, reopened.entries.items.len);
for (hashes, bodies) |h, b| {
const rid = reopened.lookup(h).?;
const got = (try reopened.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings(b, got);
}
}
test "logstore §C(inv4): a bitflipped header on a durable non-final frame surfaces error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
var buf1: [64]u8 = undefined;
const f1 = encodeFrame(&buf1, "frame-one-abc", "payload-one", @intFromEnum(FrameKind.raw), 0);
var corrupted: [64]u8 = undefined;
@memcpy(corrupted[0..f1.len], f1);
corrupted[10] ^= 0xff;
try file.append(corrupted[0..f1.len]);
var buf2: [64]u8 = undefined;
const f2 = encodeFrame(&buf2, "frame-two-def", "payload-two", @intFromEnum(FrameKind.raw), 0);
try file.append(f2);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C(inv4): a bitflipped complete final frame is error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
var buf: [64]u8 = undefined;
const f = encodeFrame(&buf, "final-frame-hash", "payload-for-final", @intFromEnum(FrameKind.raw), 0);
var corrupted: [64]u8 = undefined;
@memcpy(corrupted[0..f.len], f);
corrupted[6] ^= 0xff;
try file.append(corrupted[0..f.len]); // complete, fully landed, nothing follows
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C(inv4): a corrupted payload on a complete frame surfaces error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
var buf: [96]u8 = undefined;
const f = encodeFrame(&buf, "payload-hash-value", "some payload bytes here", @intFromEnum(FrameKind.raw), 0);
var corrupted: [96]u8 = undefined;
@memcpy(corrupted[0..f.len], f);
const header_window = 22 + "payload-hash-value".len;
corrupted[header_window] ^= 0xff; // inside the payload, not the fixed header
try file.append(corrupted[0..f.len]);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C(inv4): a bitflipped hash_len that would need more bytes than the file has surfaces error.Corrupt, not a torn tail" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// hash_len=1, payload="" -> frame_len = align8(26+1+0) = 32, fully
// landed, nothing follows. Corrupting hash_len to an in-range value
// (20) makes the naive "read 18+hash_len bytes, short = torn" reading
// demand 38 bytes from a 32-byte frame -- exactly the false-torn gap
// the prefix checksum closes: it fails on this corrupted byte before
// any window-size decision is made from it.
var buf: [64]u8 = undefined;
const f = encodeFrame(&buf, "h", "", @intFromEnum(FrameKind.raw), 0);
var corrupted: [64]u8 = undefined;
@memcpy(corrupted[0..f.len], f);
corrupted[5] = 20;
try file.append(corrupted[0..f.len]);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: a crc-valid frame with an unknown flag kind (not raw/zlib/delta) is rejected on open" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// flags=3 is one past `delta`: both crcs are computed correctly over it,
// so the frame is byte-intact -- it is simply an encoding this reader does
// not know, which must fail closed rather than be interpreted.
var buf: [64]u8 = undefined;
const f = encodeFrame(&buf, "unknown-flag-hash", "some payload bytes", 3, 0);
try file.append(f);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: phantom hash is absent after reopen" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
const hash = "phantom-only-hash-value";
_ = try store.intern(hash);
try store.commit();
store.deinit(std.testing.allocator);
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
try std.testing.expect(reopened.lookup(hash) == null);
}
fn openNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
store.deinit(gpa);
}
test "logstore §C: open propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
_ = try seed.put("seed-hash-one-value", "seed payload one");
_ = try seed.put("seed-hash-two-value", "seed payload two, a bit longer");
try seed.commit();
seed.deinit(std.testing.allocator);
try std.testing.checkAllAllocationFailures(std.testing.allocator, openNoLeak, .{file});
}
fn putNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
defer store.deinit(gpa);
const size = file.size() catch 0;
var hash_buf: [8]u8 = undefined;
std.mem.writeInt(u64, &hash_buf, size, .little);
_ = try store.put(&hash_buf, "payload bytes exercised under simulated allocation failure");
}
test "logstore §C: put propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator); // header only: put() always takes the new-entry path
try std.testing.checkAllAllocationFailures(std.testing.allocator, putNoLeak, .{file});
}
fn putPhantomUpgradeNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
defer store.deinit(gpa);
// Hash off the current file size so every run targets a fresh hash: intern
// makes a live phantom this run, then put drives the phantom-upgrade branch
// (a fixed hash would already be content-bearing on disk after the counting
// run's append, sending put down the idempotent path instead).
const size = file.size() catch 0;
var hash_buf: [8]u8 = undefined;
std.mem.writeInt(u64, &hash_buf, size, .little);
_ = try store.intern(&hash_buf);
_ = try store.put(&hash_buf, "payload upgrading a phantom under simulated allocation failure");
}
test "logstore §C: put upgrading a phantom propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator); // header only
// On OOM in the upgrade's frame append the entry stays phantom (its fields
// flip only after a successful append), so there is no leak and no
// half-upgraded state at any fail point.
try std.testing.checkAllAllocationFailures(std.testing.allocator, putPhantomUpgradeNoLeak, .{file});
}
fn contentNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
defer store.deinit(gpa);
const bytes = try store.content(gpa, 1);
if (bytes) |b| gpa.free(b);
}
test "logstore §C: content propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
_ = try seed.put("content-oom-test-hash", "payload bytes for the content allocation-failure test");
try seed.commit();
seed.deinit(std.testing.allocator);
try std.testing.checkAllAllocationFailures(std.testing.allocator, contentNoLeak, .{file});
}
test "logstore fuzz: open never crashes on garbage bytes appended after a valid header" {
var prng = std.Random.DefaultPrng.init(0xC0FFEE);
const random = prng.random();
const max_iters = 200;
var iter: usize = 0;
while (iter < max_iters) : (iter += 1) {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
var garbage: [256]u8 = undefined;
const n = random.uintLessThan(usize, garbage.len + 1);
random.bytes(garbage[0..n]);
try file.append(garbage[0..n]);
if (LogStore.open(std.testing.allocator, file, 0)) |store| {
var opened = store;
opened.deinit(std.testing.allocator);
} else |err| switch (err) {
error.OutOfMemory, error.Storage, error.Corrupt => {},
}
}
}
// ---------------------------------------------------------------------------
// zlib and delta frames (storage-track-plan.md §C; container sync §4.1;
// verbatim serve sync §7.2; delta apply delta §4).
// ---------------------------------------------------------------------------
/// Appends one hand-crafted frame (valid crcs) to `file`, for fixtures that
/// need a payload/flag/delta_of a normal put would never produce.
fn appendCrafted(gpa: std.mem.Allocator, file: storage.StorageFile, hash: []const u8, payload: []const u8, kind: FrameKind, delta_of: u64) !void {
const frame_len: usize = @intCast(alignUp8(u64, 26 + hash.len + payload.len));
const buf = try gpa.alloc(u8, frame_len);
defer gpa.free(buf);
const f = encodeFrame(buf, hash, payload, @intFromEnum(kind), delta_of);
try file.append(f);
}
test "logstore §C/sync §4.1: putZlib then content round-trips the uncompressed bytes" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const hash = "zlib-round-trip-hash-value-0001";
const raw = "compress me: aaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbb aaaaaaaa";
const rid = try store.putZlib(hash, raw);
const got = (try store.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings(raw, got);
}
test "logstore §C/sync §4.1: a zlib frame payload is 4-byte-BE-uncompressed-length + RFC1950 stream" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const hash = "z" ** 20;
const raw = "the payload whose uncompressed length the container declares";
_ = try store.putZlib(hash, raw);
// First frame sits at header_len; its payload starts after the 22+hash_len
// fixed header window (encodeFrame layout).
const poff: u64 = header_len + 22 + hash.len;
var head: [5]u8 = undefined;
const n = try file.pread(poff, &head);
try std.testing.expectEqual(@as(usize, 5), n);
// §4.1 L1194: 4-byte big-endian uncompressed length.
try std.testing.expectEqual(@as(u32, raw.len), std.mem.readInt(u32, head[0..4], .big));
// RFC1950 §2.2: the CMF byte's low nibble (CM) is 8 for a DEFLATE stream.
try std.testing.expectEqual(@as(u8, 8), head[4] & 0x0f);
}
test "logstore §C: a zlib frame is addressed by its uncompressed hash" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const hash = "uncompressed-address-hash-00001";
const rid = try store.putZlib(hash, "content that gets compressed on the way to disk");
// The hash names the uncompressed bytes; storage compression must not
// change the address (§4.1 L1206-1208).
try std.testing.expectEqualStrings(hash, store.hashOf(rid));
try std.testing.expectEqual(FrameKind.zlib, store.kindOf(rid));
}
test "logstore §C(inv1): a zlib frame reopens and still inflates" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
const hash = "zlib-reopen-hash-value-000000001";
const raw = "reopen and inflate me, please, exactly as written before the crash";
var store = try LogStore.open(std.testing.allocator, file, 0);
_ = try store.putZlib(hash, raw);
try store.commit();
store.deinit(std.testing.allocator);
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
const rid = reopened.lookup(hash) orelse return error.TestUnexpectedResult;
try std.testing.expectEqual(FrameKind.zlib, reopened.kindOf(rid));
const got = (try reopened.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings(raw, got);
}
test "logstore §C: putZlib is idempotent for an equal hash" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const hash = "zlib-idempotent-hash-value-00001";
const r1 = try store.putZlib(hash, "payload-z");
const size_after = store.size();
const r2 = try store.putZlib(hash, "payload-z");
try std.testing.expectEqual(r1, r2);
try std.testing.expectEqual(size_after, store.size());
}
test "logstore §C: putZlib upgrades a phantom in place" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const hash = "zlib-phantom-upgrade-hash-000001";
const rid = try store.intern(hash);
try std.testing.expectEqual(@as(?[]u8, null), try store.content(std.testing.allocator, rid));
const rid2 = try store.putZlib(hash, "content arriving for a zlib phantom");
try std.testing.expectEqual(rid, rid2);
const got = (try store.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings("content arriving for a zlib phantom", got);
}
test "logstore §C: content on a malformed zlib container is error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// A crc-valid zlib frame whose payload is not a valid container: two bytes
// is short of even the 4-byte length header, so containerDecode rejects it.
const hash = "malformed-zlib-container-hash-01";
try appendCrafted(std.testing.allocator, file, hash, "\x00\x00", .zlib, 0);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup(hash).?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
test "logstore §C/sync §4.1: a zlib container whose declared length exceeds max_payload is rejected without a huge allocation" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// Declared length 0xFFFFFFFF is far past max_payload; containerDecode must
// reject on the declared value before inflating anything.
const hash = "oversize-declared-length-hash-01";
try appendCrafted(std.testing.allocator, file, hash, "\xff\xff\xff\xff\x78\x9c", .zlib, 0);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup(hash).?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
test "logstore §C/delta §4: putDelta then content applies the delta and round-trips the target" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const base = "the quick brown fox jumps over the lazy dog, and then again once more";
const target = "the quick brown fox jumps over the lazy dog, and then twice more, at last";
const d = try delta.create(std.testing.allocator, base, target);
defer std.testing.allocator.free(d);
const base_rid = try store.put("delta-base-hash-value-0000000001", base);
const delta_rid = try store.putDelta("delta-target-hash-value-00000001", base_rid, d);
try std.testing.expectEqual(FrameKind.delta, store.kindOf(delta_rid));
const got = (try store.content(std.testing.allocator, delta_rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings(target, got);
}
test "logstore §C: a delta frame stores delta_of = the base frame byte offset" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const base = "base bytes long enough to admit a copy segment in the delta encoder";
const target = "base bytes long enough to admit a copy segment in the delta encoder too";
const d = try delta.create(std.testing.allocator, base, target);
defer std.testing.allocator.free(d);
const base_rid = try store.put("delta-of-base-hash-value-0000001", base); // first frame at header_len
const delta_off = store.size();
const delta_rid = try store.putDelta("delta-of-target-hash-value-00001", base_rid, d);
// Read delta_of straight from the frame header: it must be the base frame's
// byte offset, i.e. header_len for the first frame.
const hl = store.hashOf(delta_rid).len;
var field: [8]u8 = undefined;
_ = try file.pread(delta_off + 10 + hl, &field);
try std.testing.expectEqual(@as(u64, header_len), std.mem.readInt(u64, &field, .little));
}
test "logstore §C: a two-link delta chain (delta-of-delta) resolves in content()" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const a = "artifact A: a reasonably long base so the delta encoder emits copies";
const b = "artifact B: a reasonably long base so the delta encoder emits copies, v2";
const c = "artifact C: a reasonably long base so the delta encoder emits copies, v3!!";
const rid_a = try store.put("chain-hash-a-value-000000000001", a);
const d_ab = try delta.create(std.testing.allocator, a, b);
defer std.testing.allocator.free(d_ab);
const rid_b = try store.putDelta("chain-hash-b-value-000000000001", rid_a, d_ab);
const d_bc = try delta.create(std.testing.allocator, b, c);
defer std.testing.allocator.free(d_bc);
const rid_c = try store.putDelta("chain-hash-c-value-000000000001", rid_b, d_bc);
const got = (try store.content(std.testing.allocator, rid_c)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings(c, got);
}
test "logstore §C(inv1): a delta frame reopens and content still resolves" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
const base = "reopen base: bytes shared with the target so a copy is worthwhile here";
const target = "reopen base: bytes shared with the target so a copy is worthwhile now!!";
var store = try LogStore.open(std.testing.allocator, file, 0);
const base_rid = try store.put("delta-reopen-base-hash-000000001", base);
const d = try delta.create(std.testing.allocator, base, target);
defer std.testing.allocator.free(d);
_ = try store.putDelta("delta-reopen-target-hash-0000001", base_rid, d);
try store.commit();
store.deinit(std.testing.allocator);
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
const rid = reopened.lookup("delta-reopen-target-hash-0000001") orelse return error.TestUnexpectedResult;
const got = (try reopened.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualStrings(target, got);
}
test "logstore §C/sync §7.2: storedDelta serves the stored container bytes verbatim" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const base = "verbatim base bytes shared with the target so a copy segment appears here";
const target = "verbatim base bytes shared with the target so a copy segment appears now!";
const base_hash = "verbatim-base-hash-value-0000001";
const base_rid = try store.put(base_hash, base);
const d = try delta.create(std.testing.allocator, base, target);
defer std.testing.allocator.free(d);
const delta_off = store.size();
const delta_rid = try store.putDelta("verbatim-target-hash-value-00001", base_rid, d);
const sd = try store.storedDelta(std.testing.allocator, delta_rid);
defer std.testing.allocator.free(sd.wire);
// §7.2: the sender writes the stored bytes; it does not re-compress. The
// served wire must equal the on-disk payload byte-for-byte, and name the
// delta basis (base) artifact.
const plen = store.payloadLenOf(delta_rid);
const on_disk = try std.testing.allocator.alloc(u8, plen);
defer std.testing.allocator.free(on_disk);
_ = try file.pread(delta_off + 22 + base_hash.len, on_disk);
try std.testing.expectEqualSlices(u8, on_disk, sd.wire);
try std.testing.expectEqualStrings(base_hash, sd.baseHash());
}
test "logstore §C: kindOf reports raw/zlib/delta" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const base = "kindOf base bytes with enough shared run for a copy in the delta encoder";
const raw_rid = try store.put("kindof-raw-hash-value-0000000001", base);
const zlib_rid = try store.putZlib("kindof-zlib-hash-value-000000001", "some zlib payload");
const d = try delta.create(std.testing.allocator, base, "kindOf base bytes with enough shared run for a copy in the delta encoder!");
defer std.testing.allocator.free(d);
const delta_rid = try store.putDelta("kindof-delta-hash-value-00000001", raw_rid, d);
try std.testing.expectEqual(FrameKind.raw, store.kindOf(raw_rid));
try std.testing.expectEqual(FrameKind.zlib, store.kindOf(zlib_rid));
try std.testing.expectEqual(FrameKind.delta, store.kindOf(delta_rid));
}
test "logstore §C/delta §4: content on a delta whose inflated payload is not a valid delta is error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// A valid raw base, then a crc-valid delta frame whose container inflates
// to bytes that are not a conforming delta -> delta.apply rejects -> Corrupt.
try appendCrafted(std.testing.allocator, file, "garbage-delta-base-hash-00000001", "base content here", .raw, 0);
const container = try containerEncode(std.testing.allocator, "this is not a delta at all");
defer std.testing.allocator.free(container);
try appendCrafted(std.testing.allocator, file, "garbage-delta-hash-value-0000001", container, .delta, header_len);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup("garbage-delta-hash-value-0000001").?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
test "logstore §C: a delta chain deeper than max_delta_chain is error.Corrupt, not unbounded recursion" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
// Build a chain of exactly max_delta_chain deltas over a raw base: it must
// resolve. One more link must be rejected by the depth bound (no unbounded
// recursion), proving the bound is enforced from the read side.
var current: []u8 = try std.testing.allocator.dupe(u8, "chain base bytes, long enough to permit copies across every link here");
defer std.testing.allocator.free(current);
var prev_rid = try store.put("depth-base-hash-value-0000000000", current);
var last_ok_rid: Rid = prev_rid;
var i: usize = 0;
while (i < max_delta_chain + 1) : (i += 1) {
var next_buf: [96]u8 = undefined;
const next = try std.fmt.bufPrint(&next_buf, "chain base bytes, long enough to permit copies across every link {d:0>3}", .{i});
const d = try delta.create(std.testing.allocator, current, next);
defer std.testing.allocator.free(d);
var hbuf: [32]u8 = undefined;
std.mem.writeInt(u64, hbuf[0..8], i, .little);
@memcpy(hbuf[8..], "depth-chain-hash-padding-value--"[0..24]);
const rid = try store.putDelta(&hbuf, prev_rid, d);
std.testing.allocator.free(current);
current = try std.testing.allocator.dupe(u8, next);
prev_rid = rid;
if (i + 1 == max_delta_chain) last_ok_rid = rid; // top of a length-max_delta_chain chain
}
// A chain of exactly max_delta_chain deltas resolves.
const ok = (try store.content(std.testing.allocator, last_ok_rid)).?;
defer std.testing.allocator.free(ok);
// One link deeper (max_delta_chain + 1) trips the bound.
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, prev_rid));
}
test "logstore §C: content on a delta whose delta_of is out of range or mid-frame is error.Corrupt" {
const cases = [_]struct { name: []const u8, delta_of: u64 }{
.{ .name = "below-header", .delta_of = 0 },
.{ .name = "forward-ref", .delta_of = 1 << 20 },
.{ .name = "mid-base-frame", .delta_of = header_len + 3 },
};
for (cases) |c| {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
try appendCrafted(std.testing.allocator, file, "bad-delta-of-base-hash-000000001", "base content bytes", .raw, 0);
const container = try containerEncode(std.testing.allocator, "1\ndelta-ish payload; only delta_of matters here");
defer std.testing.allocator.free(container);
try appendCrafted(std.testing.allocator, file, "bad-delta-of-hash-value-00000001", container, .delta, c.delta_of);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup("bad-delta-of-hash-value-00000001").?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
}
// OOM coverage for every public allocating fn added this slice.
fn putZlibNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
defer store.deinit(gpa);
const size = file.size() catch 0;
var hash_buf: [8]u8 = undefined;
std.mem.writeInt(u64, &hash_buf, size, .little);
_ = try store.putZlib(&hash_buf, "payload compressed under simulated allocation failure");
}
test "logstore §C: putZlib propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
try std.testing.checkAllAllocationFailures(std.testing.allocator, putZlibNoLeak, .{file});
}
fn putDeltaNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
defer store.deinit(gpa);
const base_rid = store.lookup("oom-delta-base-hash-value-000001") orelse return error.TestUnexpectedResult;
const size = file.size() catch 0;
var hash_buf: [8]u8 = undefined;
std.mem.writeInt(u64, &hash_buf, size, .little);
_ = try store.putDelta(&hash_buf, base_rid, "raw delta bytes under simulated allocation failure");
}
test "logstore §C: putDelta propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
_ = try seed.put("oom-delta-base-hash-value-000001", "a base frame the delta references");
try seed.commit();
seed.deinit(std.testing.allocator);
try std.testing.checkAllAllocationFailures(std.testing.allocator, putDeltaNoLeak, .{file});
}
fn contentZlibNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
defer store.deinit(gpa);
const rid = store.lookup("oom-zlib-content-hash-value-0001") orelse return error.TestUnexpectedResult;
const bytes = try store.content(gpa, rid);
if (bytes) |b| gpa.free(b);
}
test "logstore §C: content on a zlib frame propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
_ = try seed.putZlib("oom-zlib-content-hash-value-0001", "payload for the zlib content allocation-failure probe");
try seed.commit();
seed.deinit(std.testing.allocator);
try std.testing.checkAllAllocationFailures(std.testing.allocator, contentZlibNoLeak, .{file});
}
fn contentDeltaNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
defer store.deinit(gpa);
const rid = store.lookup("oom-delta-content-target-hash-01") orelse return error.TestUnexpectedResult;
const bytes = try store.content(gpa, rid);
if (bytes) |b| gpa.free(b);
}
test "logstore §C: content on a delta chain propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
const base = "oom delta base bytes, long enough that the encoder emits a copy segment";
const target = "oom delta base bytes, long enough that the encoder emits a copy segment!!";
var seed = try LogStore.open(std.testing.allocator, file, 0);
const base_rid = try seed.put("oom-delta-content-base-hash-0001", base);
const d = try delta.create(std.testing.allocator, base, target);
defer std.testing.allocator.free(d);
_ = try seed.putDelta("oom-delta-content-target-hash-01", base_rid, d);
try seed.commit();
seed.deinit(std.testing.allocator);
try std.testing.checkAllAllocationFailures(std.testing.allocator, contentDeltaNoLeak, .{file});
}
fn storedDeltaNoLeak(gpa: std.mem.Allocator, file: storage.StorageFile) !void {
var store = try LogStore.open(gpa, file, 0);
defer store.deinit(gpa);
const rid = store.lookup("oom-stored-delta-target-hash-001") orelse return error.TestUnexpectedResult;
const sd = try store.storedDelta(gpa, rid);
gpa.free(sd.wire);
}
test "logstore §C: storedDelta propagates allocation failure" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
const base = "stored-delta base bytes, long enough that the encoder emits a copy here";
const target = "stored-delta base bytes, long enough that the encoder emits a copy here!!";
var seed = try LogStore.open(std.testing.allocator, file, 0);
const base_rid = try seed.put("oom-stored-delta-base-hash-00001", base);
const d = try delta.create(std.testing.allocator, base, target);
defer std.testing.allocator.free(d);
_ = try seed.putDelta("oom-stored-delta-target-hash-001", base_rid, d);
try seed.commit();
seed.deinit(std.testing.allocator);
try std.testing.checkAllAllocationFailures(std.testing.allocator, storedDeltaNoLeak, .{file});
}
// Seeded deterministic fuzz: decode/apply on adversarial wire bytes must
// yield a value or a declared error, never a crash or a hang.
test "logstore §C/sync §4.1 fuzz: zlib round-trips and a garbage zlib container never crashes" {
var prng = std.Random.DefaultPrng.init(0x2172B);
const random = prng.random();
var iter: usize = 0;
while (iter < 150) : (iter += 1) {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
// Invariant: content(putZlib(x)) == x for any bytes.
var raw: [128]u8 = undefined;
const rn = random.uintLessThan(usize, raw.len + 1);
random.bytes(raw[0..rn]);
var hbuf: [8]u8 = undefined;
std.mem.writeInt(u64, &hbuf, iter, .little);
const rid = try store.putZlib(&hbuf, raw[0..rn]);
const got = (try store.content(std.testing.allocator, rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualSlices(u8, raw[0..rn], got);
// A crc-valid zlib frame with a random payload must never crash content.
var garbage: [64]u8 = undefined;
const gn = random.uintLessThan(usize, garbage.len + 1);
random.bytes(garbage[0..gn]);
var ghash: [9]u8 = undefined;
ghash[0] = 'g';
std.mem.writeInt(u64, ghash[1..9], iter, .little);
try appendCrafted(std.testing.allocator, file, &ghash, garbage[0..gn], .zlib, 0);
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
const grid = reopened.lookup(&ghash).?;
if (reopened.content(std.testing.allocator, grid)) |maybe| {
if (maybe) |b| std.testing.allocator.free(b);
} else |err| switch (err) {
error.OutOfMemory, error.Storage, error.Corrupt => {},
}
}
}
test "logstore §C/delta §4 fuzz: delta round-trips and a garbage delta payload or delta_of never crashes" {
var prng = std.Random.DefaultPrng.init(0xD317A);
const random = prng.random();
var iter: usize = 0;
while (iter < 150) : (iter += 1) {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
// Invariant: content(putDelta(create(base,target))) == target.
var base: [96]u8 = undefined;
const bn = 16 + random.uintLessThan(usize, base.len - 16 + 1);
random.bytes(base[0..bn]);
var target_buf: [128]u8 = undefined;
@memcpy(target_buf[0..bn], base[0..bn]);
target_buf[random.uintLessThan(usize, bn)] +%= 1; // small mutation keeps a shared run
const target = target_buf[0..bn];
const d = try delta.create(std.testing.allocator, base[0..bn], target);
defer std.testing.allocator.free(d);
var bhash: [9]u8 = undefined;
bhash[0] = 'b';
std.mem.writeInt(u64, bhash[1..9], iter, .little);
const base_rid = try store.put(&bhash, base[0..bn]);
var thash: [9]u8 = undefined;
thash[0] = 't';
std.mem.writeInt(u64, thash[1..9], iter, .little);
const delta_rid = try store.putDelta(&thash, base_rid, d);
const got = (try store.content(std.testing.allocator, delta_rid)).?;
defer std.testing.allocator.free(got);
try std.testing.expectEqualSlices(u8, target, got);
// A crc-valid delta frame with a random container payload and a random
// delta_of must never crash content (Corrupt/OOM/value only).
var garbage: [64]u8 = undefined;
const gn = random.uintLessThan(usize, garbage.len + 1);
random.bytes(garbage[0..gn]);
var ghash: [9]u8 = undefined;
ghash[0] = 'x';
std.mem.writeInt(u64, ghash[1..9], iter, .little);
try appendCrafted(std.testing.allocator, file, &ghash, garbage[0..gn], .delta, random.int(u64));
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
const grid = reopened.lookup(&ghash).?;
if (reopened.content(std.testing.allocator, grid)) |maybe| {
if (maybe) |b| std.testing.allocator.free(b);
} else |err| switch (err) {
error.OutOfMemory, error.Storage, error.Corrupt => {},
}
}
}