//! The framed, append-only, content-addressed log over the `StorageFile`
//! capability. One file: a 64-byte header followed by 8-byte-aligned
//! frames, each self-verifying via a header checksum (over the fixed
//! fields) plus a whole-frame checksum (over header+payload). The header
//! checksum is what makes torn-vs-corrupt discrimination exact and
//! size-independent: without it, a torn write (a clean prefix, safe to
//! truncate) and a corrupted length field (unsafe to trust) both read as
//! "not enough of a frame," and nothing short of decoding payload bytes
//! that may not exist could tell them apart.
//!
//! No RFC section behind this file: native storage-engine design. The
//! oracle is this file's own framing invariants, checked by
//! sim/storage_sweep.zig's DST sweep: every acknowledged put stays
//! readable and hash-verifies; a torn tail truncates to exactly the last
//! complete frame; the index rebuilt on open matches the write-time
//! model; and corruption always surfaces as an error, never as silently
//! wrong bytes.
//!
//! A second, narrower checksum guards the frame's fixed 6-byte prefix
//! (`len|flags|hash_len`) on its own, in a 10-byte window whose size
//! never depends on `hash_len`. Without it, scanning a frame whose
//! `hash_len` byte is itself the bitflipped one has no way to know how
//! many bytes the rest of the header needs before it can check the
//! header checksum -- a corrupted `hash_len` that happens to demand more
//! bytes than the file has left reads exactly like a torn tail. The
//! prefix checksum removes that ambiguity: `hash_len` is proven correct
//! (or rejected outright) from a fixed-size read, before it is ever used
//! to size anything else.
const std = @import("std");
const storage = @import("storage.zig");
const delta = @import("delta.zig");
/// The §4.1 zlib container codec — the leaf module shared with `repostate` and
/// `sync`. The store compresses artifacts through it (the hash still names the
/// UNCOMPRESSED bytes, §4.1 L1206-1208) and inflates them back via
/// `decodeContainer` below.
const codec = @import("container.zig");
/// Local artifact id: dense, 1-based, assigned in intern/put order. 0 is
/// never issued (mirrors repostate.zig's MemStore).
pub const Rid = u64;
/// FILE header size: magic(4) | version u16(2) | hash_kind u8(1) |
/// reserved(53) | crc32(4).
pub const header_len: u64 = 64;
/// Explicit payload bound: keeps a single frame's read/decode work
/// bounded no matter how large a check-in gets.
pub const max_payload: u64 = 1 << 30;
pub const max_hash_len: u8 = 32;
pub const min_hash_len: u8 = 1;
/// Every frame's on-disk span is a multiple of this.
pub const frame_align: u64 = 8;
/// Frozen on-disk format version — the `.zit` ship-format identity. A reader
/// accepts only this exact stored version and rejects any other, older
/// included (`validateHeader`): the writer emits exactly this value, so a
/// header carrying anything else is not a zit-written file. An unknown frame
/// flag-kind is likewise a clean reject, never UB (`scanOneFrame`). Bumped
/// only on a layout change — a copied `.zit` file is a complete,
/// self-verifying repository, so this field is what absorbs any future layout
/// evolution; widening to a compatibility range belongs to that change, not
/// to a version this reader has never produced.
pub const version: u16 = 1;
/// Smallest possible frame span: hash_len=1, payload=0 ->
/// align8(26+1+0) = 32.
const min_frame: u64 = 32;
/// Explicit upper bound on delta-chain resolution: a `content()` on a
/// delta frame walks at most this many `delta_of` hops before rejecting
/// the chain as corrupt. Caps read recursion and read-amplification;
/// generous versus the short chains a real store produces.
pub const max_delta_chain: u8 = 32;
/// The deepest delta chain `putArtifact` will CREATE (write policy), kept
/// strictly below the READ bound `max_delta_chain` so a retained foreign
/// chain (a peer with no write cap) still has headroom to resolve. A base
/// already at this depth is stored full, so a locally-produced chain never
/// exceeds this and `content()`'s read walk stays shallow.
pub const delta_chain_write_max: u8 = 16;
/// fossil `N_NEIGHBOR` (content.c): the most delta bases `pickBase` will ever
/// try for one artifact. Bounds the write-path fan-out to at most this many
/// `delta.create` runs per put — the whole reason the candidate window is a
/// fixed slice, not the unbounded log.
pub const max_base_candidates: usize = 5;
/// The most frames `putArtifact`'s backward window walk will ever step over to
/// gather its chronological candidates, independent of phantom density. The
/// candidate cap (`max_base_candidates`) bounds deltas EXAMINED, not frames
/// SCANNED: a run of trailing phantom frames (a normal sync/clone artifact of
/// interned forward-references) is skipped by `continue` without filling a
/// candidate slot, so without a scan bound one put could step over the entire
/// store to gather < 5 real bases — hidden O(n^2) over N puts. This caps the
/// per-put scan to a fixed window: at most this many entries are inspected,
/// so the write-path cost stays O(1) in store size no matter how dense the
/// phantoms. Generous versus `max_base_candidates` so an ordinary phantom run
/// never starves the window of real bases; a base past the window only costs
/// compactness, never correctness (candidates are hints).
pub const max_base_scan: usize = 64;
/// fossil's per-source floor (content.c:908): a candidate whose reconstructed
/// content is shorter than this is skipped BEFORE `delta.create` — the cheap
/// length early-out. A delta's fixed framing overhead swamps any saving a
/// sub-50-byte source could offer, so running the encoder is wasted work.
const delta_min_source: u64 = 50;
/// The §5 accept ratio (fossil content.c:914): a delta is worth storing only
/// when it is STRICTLY under 75% of the uncompressed artifact — `delta.len * 4
/// < bytes.len * 3`. Integer math, no float (`delta_accept_den`/`num` = 4/3),
/// replacing the former looser 9/10 write-path threshold.
const delta_accept_num: u64 = 3;
const delta_accept_den: u64 = 4;
/// The §5 accept test in one pure predicate (fossil content.c:914): is a delta
/// of `delta_len` bytes worth storing for a `target_len`-byte artifact? True
/// iff strictly under 75% — `delta_len * 4 < target_len * 3`. Integer math on
/// u64, overflow-free by a wide margin: `target_len <= max_payload` (< 2^30, the
/// caller's asserted bound) so `target_len * 3 < 2^32`; `delta_len` is a delta
/// of a `<= max_payload` target, exceeding it only by bounded framing overhead,
/// so it stays far below 2^31 and `delta_len * 4` far below 2^33 — neither
/// product approaches u64's range. Exact at the integer boundary — 75/100
/// rejects, 74/100 accepts.
fn deltaProfitable(delta_len: usize, target_len: usize) bool {
return @as(u64, delta_len) * delta_accept_den < @as(u64, target_len) * delta_accept_num;
}
/// The winning base and the raw (uncompressed) delta reconstructing the
/// artifact from it — `pickBase`'s result. The caller owns `delta` and stores
/// it (e.g. via `putDelta`), or frees it if it chooses not to.
pub const Pick = struct { base: Rid, delta: []u8 };
const magic = "zitL";
/// Payload encoding. `raw` = uncompressed bytes; `zlib` = a fossil zlib
/// container (4-byte BE uncompressed length + RFC1950 stream, sync §4.1);
/// `delta` = a zlib-contained delta whose `delta_of` is the byte offset of
/// the base frame, so resolving a chain link costs one seek and never a
/// second index lookup; `config` = a distinguished singleton repo-config
/// record, NOT content-addressed and never interned. Any other flag byte
/// is an unknown kind and rejected on open. The
/// content hash always names the UNCOMPRESSED, fully-reconstructed bytes, so
/// encoding never affects naming (sync §4.1 L1206-1208).
pub const FrameKind = enum(u8) { raw = 0, zlib = 1, delta = 2, config = 3, _ };
/// Repo-local config: default-branch name, project id, the sync project
/// code (§9.1 `projectcode`, the repository's shared identity — distinct
/// from `project_id`), and an opaque hash-policy byte the storage layer
/// round-trips but never interprets (exactly like the header's
/// `hash_kind`). It has a home IN the file as one distinguished `config`
/// frame; "last config frame wins" is derived by scan order, reusing the
/// log's own append-prefix property.
pub const max_config_str: usize = 255;
/// Fixed config record bound: `[hash_policy u8][branch_len u16][branch]
/// [pid_len u16][pid][pcode_len u16][pcode][login_len u16][login]
/// [secret_len u16][secret][url_len u16][url][seen_submanifest u8]
/// [enable_submanifests u8][submanifest_k u8][submanifest_auto_retune u8]
/// [settled_cost_shape 32]` — six length-prefixed strings after the policy
/// byte, then the four submanifest settings bytes and, when the analysis is
/// settled, its fixed-width SHA-256 cost-shape digest. The digest is absent
/// from legacy and unsettled records, so readers distinguish "scan once" from
/// a real all-zero digest.
pub const max_config_payload: usize = 1 + 6 * (2 + max_config_str) + 4 + 32; // = 1579
/// A config frame is never keyed by hash — it is found only via
/// `config_offset`, never through `index`/`lookup` — so its 1-byte embedded
/// hash is inert bookkeeping (`appendFrame` requires some hash). It is never
/// an index key and never compared.
const config_sentinel: []const u8 = "\x00";
/// One repo config. Both slices borrow a backing buffer (`OwnedConfig`);
/// `hash_policy` is opaque to storage.
pub const Config = struct {
hash_policy: u8,
default_branch: []const u8,
project_id: []const u8,
/// sync §9.1 — the repository's shared project code (`projectcode`,
/// an artifact-hash-shaped identifier). Empty on a repo that predates
/// it; `init` generates one and a clone adopts the origin's. Defaulted
/// so existing construction sites need not restate it.
project_code: []const u8 = "",
/// sync §6.3 — the login name a `zit push` authenticates as, or empty
/// when no credential is stored. Paired with `login_secret_hex`.
login_user: []const u8 = "",
/// sync §6.3/DECISION-3 — the 40-hex derived shared secret
/// (`SHA1(project-code "/" login "/" password)`), NOT the plaintext
/// password: it is all a signer needs (§6.4) and is scoped to this
/// repo+login. Empty when no credential is stored; exactly 40 bytes
/// otherwise. Persisted only under 0600 file permissions by the caller.
login_secret_hex: []const u8 = "",
/// sync §4/§6 — the default peer URL a bare `zit sync`/`pull`/`push`
/// targets, or empty when no remote is saved. A LOCAL, non-synced value:
/// it never crosses the wire, it just spares the operator from retyping
/// the endpoint. Anonymous only — a credential never lives in the URL.
/// Defaulted so existing construction sites need not restate it.
remote_url: []const u8 = "",
/// draft-fossil-submanifests-00 §8.1 — "When the first S-bearing artifact
/// is stored, an implementation SHOULD record a local seen-submanifest
/// indicator so operational surfaces can warn before the setting is enabled
/// or a pre-S client is granted access."
///
/// It is also what makes §7.1's phantoms survive a restart at a cost of
/// zero for every repository predating this draft. A phantom is an interned
/// entry with no frame — `intern`'s "Phantoms never reach disk" — so an
/// S-reference gap is re-derived by rescanning the artifacts that name it,
/// and this flag is the durable answer to "is there anything to rescan?".
/// False on every existing repository, so the rescan is unreachable there.
/// One-way: set when the first S reference is stored, never cleared, since
/// artifact names are immutable ([FossilArtifact] §7.6) and a stored
/// S-bearing artifact cannot be retired.
seen_submanifest: bool = false,
/// draft-fossil-submanifests-00 §8.1 — the `enable-submanifests` setting,
/// "a project-scope boolean, default false". The one operator act that lets
/// this repository's writer emit S cards (§8.1) and the one that decides
/// whether its server sends the §7.2 `avoid-submanifests` pragma.
///
/// KNOWN GAP (recorded, not papered over): §6.3 makes both this and
/// `submanifest_k` PROJECT scope — "configuration propagating to clones ...
/// is REQUIRED: sharing by content address works only while every writer
/// partitions identically". zit's config frame is repo-LOCAL: nothing in
/// sync §9 carries it, and `clone` adopts only the origin's project code.
/// So a clone of an enabled repository comes up disabled and unpinned, and
/// an operator who enables it must re-pin the same k by hand or the two
/// repositories' trees stop sharing sub-manifests. §6.4 bounds the damage
/// to cost, not correctness — "Repositories pinned to different values
/// exchange artifacts correctly ... so the propagation of Section 6.3 need
/// be reliable only for cost" — which is why this ships as a stated gap
/// rather than a blocker. Closing it needs a sync §9 config-card group and
/// belongs with that work, not here.
enable_submanifests: bool = false,
/// draft-fossil-submanifests-00 §6.3 — the pinned partition parameter
/// (`submanifest-k`), or 0 when this repository has none pinned.
///
/// §6.3 fixes the legal range at 1..12 inclusive, so 0 is outside it and
/// cannot collide with a pinned value; it is the encoding of "no pinned
/// parameter", which §8.1 forbids enabling emission on. An out-of-range
/// value that reaches a writer is NOT repaired here — §6.3 requires the
/// writer to "fall back to emitting flat manifests rather than substitute a
/// parameter of its own", which storage cannot do on its behalf, so this
/// layer round-trips whatever byte it was given (like `hash_policy`) and
/// `repository.submanifest` makes the decision.
submanifest_k: u8 = 0,
/// draft-fossil-submanifests-00 §6.4, as amended by erratum
/// submanifests-00-s6.4-opt-in-auto-retune — "an implementation MAY
/// re-tune automatically... only when the repository's
/// submanifest-auto-retune setting -- a project-scope boolean, default
/// false -- is true; enabling that setting is itself the
/// operator-initiated act that authorizes every automatic re-tune it
/// subsequently triggers." Manual `retune --k N` is unaffected by this
/// setting either way — it is the automatic path's gate alone.
submanifest_auto_retune: bool = false,
/// Last locally settled cost-analysis input shape. Null forces one real
/// scan; storage does not decide when that scan is settled and never
/// substitutes an all-zero sentinel.
submanifest_settled_cost_shape: ?[32]u8 = null,
};
/// A `readConfig` result: the owned frame payload plus a `Config` whose
/// slices point into it. Mirrors `OwnedConfig`'s owned-backing shape.
pub const OwnedConfig = struct {
backing: []u8,
config: Config,
pub fn deinit(self: *OwnedConfig, gpa: std.mem.Allocator) void {
gpa.free(self.backing);
self.* = undefined;
}
};
pub const OpenError = error{ OutOfMemory, Storage, Corrupt };
/// No read happens on the write path, so `Corrupt` can't occur here —
/// advertising it would be dead mechanism.
pub const PutError = error{ OutOfMemory, Storage };
/// A phantom is purely in-memory bookkeeping: no I/O, so no `Storage`.
pub const InternError = error{OutOfMemory};
pub const ContentError = error{ OutOfMemory, Storage, Corrupt };
pub const CommitError = error{Storage};
/// The framed, content-addressed log. `open` rebuilds `entries`/`index`
/// from the file each time — there is no persistent side index yet.
/// O(n) on open is correct at this scale; a persistent, regenerable
/// sidecar is a noted future extension, built only if a benchmark shows
/// open time missing its target at large artifact counts.
pub const LogStore = struct {
gpa: std.mem.Allocator,
file: storage.StorageFile,
hash_kind: u8,
/// Structural end of the log: header + every complete frame written so
/// far. Always a multiple of `frame_align`.
end: u64,
/// Total complete frames (content + config) seen by the scan, and the byte
/// offset of the last one. Populated by `scan`; the derivation-cache key
/// (`logKey`) reads them so a cached verb can validate the log's tail in
/// O(1) instead of re-walking every frame. Zero/`header_len` on an empty log.
frame_count: u64 = 0,
last_frame_offset: u64 = header_len,
entries: std.ArrayList(Entry) = .empty,
index: std.StringHashMapUnmanaged(Rid) = .empty,
/// Byte offset of the live (latest in scan order) config frame, or null
/// when the log carries none. Config is found ONLY through this — never
/// via `index`/`entries` (§E).
config_offset: ?u64 = null,
/// Set when a rollback truncate fails and `end` can no longer be proven
/// equal to the file's physical end. Every append refuses to run while it
/// is set, so a later frame never lands at a diverged offset (the storage
/// invariant: offset bookkeeping equals the file end). One-way: a store
/// only ever goes healthy -> poisoned, never back.
poisoned: bool = false,
/// Set once `openReadOnly` has scanned and whole-frame-CRC-verified every
/// complete frame in the file (and proven `end == file_size`, so no writer
/// can extend it). While set, `readFrameAt` trusts that verification and
/// skips the redundant per-read whole-frame CRC — a read-only verb must not
/// CRC the entire log a second time after open already did. Every frame's
/// CRC is thus verified exactly once (at scan), never zero times: the header
/// prefix/header CRCs still run on every read, and a short payload read is
/// still corruption. The append path (`open`) leaves this false, so writes
/// and post-write reads keep full at-use verification.
verified_on_open: bool = false,
const Entry = struct {
/// Owned duplicate; also the index's key (stable across `entries`
/// reallocation since it's a separate heap allocation).
hash: []u8,
/// Frame start offset. Meaningless while `phantom`.
offset: u64,
/// On-disk (container/wire) payload length: raw bytes for `raw`,
/// the zlib-container length for `zlib`/`delta`.
payload_len: u32,
/// Payload encoding of this frame. `.raw` while `phantom` (no frame).
kind: FrameKind,
/// A phantom was `intern`ed but never `put`: no frame on disk.
phantom: bool,
/// Delta-chain depth: 0 for a phantom or a non-delta frame; one more
/// than the base's depth for a `.delta` frame. Set at `insert`/`scan`
/// and read by `putArtifact` to enforce `delta_chain_write_max`, so a
/// locally-created chain never grows past the write cap.
depth: u8 = 0,
};
/// Number of unique artifact holdings rebuilt from the verified log scan.
/// Config frames are deliberately excluded because they are not artifacts.
pub fn artifactCount(self: *const LogStore) u64 {
return @intCast(self.entries.items.len);
}
/// Largest retained delta-chain depth among real artifact holdings.
pub fn maxArtifactDeltaDepth(self: *const LogStore) u8 {
var max_depth: u8 = 0;
for (self.entries.items) |entry| {
if (!entry.phantom) max_depth = @max(max_depth, entry.depth);
}
return max_depth;
}
/// Opens for append, creating or repairing the file when necessary, and
/// rebuilds the in-memory index by scanning every complete frame.
pub fn open(gpa: std.mem.Allocator, file: storage.StorageFile, hash_kind: u8) OpenError!LogStore {
const file_size = file.size() catch return error.Storage;
var self: LogStore = .{ .gpa = gpa, .file = file, .hash_kind = hash_kind, .end = header_len };
errdefer self.deinit(gpa);
if (file_size == 0) {
try writeHeader(file, hash_kind);
std.debug.assert(self.end % frame_align == 0);
return self;
}
if (file_size < header_len) {
// Torn header: nothing before byte `header_len` can be trusted
// as a frame boundary either, so start clean.
file.truncate(0) catch return error.Storage;
try writeHeader(file, hash_kind);
std.debug.assert(self.end % frame_align == 0);
return self;
}
self.hash_kind = try validateHeader(file);
try scan(&self, true);
std.debug.assert(self.end % frame_align == 0);
return self;
}
/// Opens an existing log strictly for reads. Unlike `open`, this path never
/// creates, rewrites, or truncates storage: an empty file, torn header, or
/// torn frame tail is corruption. Complete-frame checksum and encoding
/// faults are rejected by the same scanner used by the append path.
pub fn openReadOnly(gpa: std.mem.Allocator, file: storage.StorageFile) OpenError!LogStore {
const file_size = file.size() catch return error.Storage;
if (file_size < header_len) return error.Corrupt;
var self: LogStore = .{ .gpa = gpa, .file = file, .hash_kind = 0, .end = header_len };
errdefer self.deinit(gpa);
self.hash_kind = try validateHeader(file);
try scan(&self, false);
std.debug.assert(self.end == file_size);
std.debug.assert(self.end % frame_align == 0);
// scan() whole-frame-CRC-verified every frame and proved end == file
// size; reads may now skip the redundant per-frame CRC (see the field).
self.verified_on_open = true;
return self;
}
pub fn deinit(self: *LogStore, gpa: std.mem.Allocator) void {
for (self.entries.items) |e| gpa.free(e.hash);
self.entries.deinit(gpa);
self.index.deinit(gpa);
self.* = undefined;
}
/// The append-only-log identity a derivation cache is keyed on: the
/// structural end offset, the total complete-frame count, the byte offset of
/// the last complete frame, and that frame's fixed-header CRC. The log only
/// ever grows by whole frames, so a cache built at key K stays valid exactly
/// while the live log still matches K: an equal `end` (file size) proves no
/// append and no truncation changed the length, and the last frame still
/// sitting at `last_frame_offset` with header CRC `last_header_crc` and
/// ending at `end` pins the truncate-then-reappend-to-the-same-length corner
/// that `end` alone would miss. `frame_count` is recorded for the identity
/// but, under the append-only invariant, is subsumed by `end` — see
/// `validateTail`, which stays O(1) by checking only the tail. Whole-log
/// payload integrity remains `openReadOnly`'s job on the miss/rebuild path,
/// so a cache hit will not detect interior bitrot that preserves the file
/// size and tail frame; deleting the `.zit-cache` (or any append, which
/// forces a rebuild) restores full-log detection.
pub const LogKey = struct {
end: u64,
frame_count: u64,
last_frame_offset: u64,
last_header_crc: u32,
};
pub const KeyError = error{ Storage, Corrupt };
/// The `LogKey` for this already-scanned store — the value a cache write
/// records. `end`, `frame_count`, and `last_frame_offset` come from the scan
/// (`open`/`openReadOnly` already walked every frame); only the last frame's
/// header CRC needs one extra small read, so this is O(1), never a re-walk.
/// An empty log (no frames) keys on `end == header_len` alone. Never mutates
/// storage.
pub fn logKey(self: *const LogStore) KeyError!LogKey {
if (self.frame_count == 0) {
return .{ .end = self.end, .frame_count = 0, .last_frame_offset = self.end, .last_header_crc = 0 };
}
const info = try tailHeaderInfo(self.file, self.last_frame_offset);
// The recorded last frame must end exactly at the structural end.
std.debug.assert(info.frame_end == self.end);
return .{
.end = self.end,
.frame_count = self.frame_count,
.last_frame_offset = self.last_frame_offset,
.last_header_crc = info.header_crc,
};
}
/// O(1) cache-freshness check: is `file` still the append-only log `key` was
/// computed over? True iff the file size equals `key.end` and — for a
/// non-empty log — the last frame still occupies `key.last_frame_offset`
/// with the same header CRC and end. Reads at most one frame header (plus a
/// stat); no full scan, so a cached verb stays sub-second even on a
/// multi-gigabyte log. Any I/O failure, a shrunk/grown file, or a changed
/// tail is a miss (false), never an error — a cache is advisory.
pub fn validateTail(file: storage.StorageFile, key: LogKey) bool {
const file_size = file.size() catch return false;
if (file_size != key.end) return false; // any append or truncation changes the length
if (key.frame_count == 0) return key.end == header_len; // empty log: length is the whole identity
// The recorded tail offset must be a real interior frame boundary.
if (key.last_frame_offset < header_len or key.last_frame_offset >= file_size) return false;
const info = tailHeaderInfo(file, key.last_frame_offset) catch return false;
return info.frame_end == key.end and info.header_crc == key.last_header_crc;
}
const TailInfo = struct { frame_end: u64, header_crc: u32 };
/// Read and CRC-verify the fixed header of the frame at `offset`, returning
/// its end offset and header CRC. A torn or corrupt header is `error.Corrupt`
/// (`readFrameHeader` already validates the prefix and header CRCs).
fn tailHeaderInfo(file: storage.StorageFile, offset: u64) KeyError!TailInfo {
const fh = switch (readFrameHeader(file, offset) catch |e| switch (e) {
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
}) {
.torn => return error.Corrupt, // a recorded frame offset is never torn
.header => |h| h,
};
var hcrc = std.hash.Crc32.init();
hcrc.update(fh.header_bytes[0 .. 18 + @as(usize, fh.hash_len)]);
const frame_len = frameSpan(fh.hash_len, fh.payload_len);
return .{ .frame_end = offset + frame_len, .header_crc = hcrc.final() };
}
/// Clone = copy + derive: byte-copies `source` into `dest`, then opens
/// `dest` — rebuilding the index and recovering
/// config with no network (no network capability is even in scope here).
/// The result is an independent, equal repository: same artifact set, same
/// config, byte-identical log. A damaged copy needs no clone-specific
/// error — `open`'s torn-tail truncation drops exactly the incomplete
/// tail, so the copy degrades gracefully through the shared derive path.
/// Precondition (documented, not asserted — assertions carry no I/O):
/// `dest` is empty. `hash_kind` is a don't-care: `dest` is non-empty after
/// the copy, so `open` reads the copied header's stored kind.
pub fn cloneInto(
gpa: std.mem.Allocator,
source: storage.StorageFile,
dest: storage.StorageFile,
) OpenError!LogStore {
const src_size = source.size() catch return error.Storage;
var chunk: [64 * 1024]u8 = undefined;
var copied: u64 = 0;
// Bounded: `copied` rises by `n > 0` every pass, so the loop runs at
// most ceil(src_size / chunk.len) times before `copied == src_size`.
while (copied < src_size) {
const n = source.pread(copied, &chunk) catch return error.Storage;
// A 0-count before EOF on a stable file is a device failure, not a
// boundary — return rather than spin (no unbounded loop).
if (n == 0) return error.Storage;
dest.append(chunk[0..n]) catch return error.Storage;
copied += n;
}
std.debug.assert(copied == src_size);
return open(gpa, dest, 0);
}
/// Resolves `hash` to its rid, creating a phantom (no I/O) when
/// unknown. Phantoms never reach disk.
pub fn intern(self: *LogStore, hash: []const u8) InternError!Rid {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
if (self.index.get(hash)) |rid| return rid;
try self.entries.ensureUnusedCapacity(self.gpa, 1);
const owned = try self.gpa.dupe(u8, hash);
errdefer self.gpa.free(owned);
const rid: Rid = self.entries.items.len + 1;
try self.index.put(self.gpa, owned, rid);
self.entries.appendAssumeCapacity(.{ .hash = owned, .offset = 0, .payload_len = 0, .kind = .raw, .phantom = true });
return rid;
}
/// Stores `bytes` verbatim (a `raw` frame) under `hash`. Idempotent for
/// an already-content-bearing hash (returns the existing rid, appends
/// nothing); upgrades a phantom in place.
pub fn put(self: *LogStore, hash: []const u8, bytes: []const u8) PutError!Rid {
std.debug.assert(bytes.len <= max_payload);
return self.insert(hash, bytes, .raw, 0, 0);
}
/// Store `bytes` under `hash`, delta-encoded against the size-minimizing
/// base `pickBase` (§5) chooses from a small candidate window, else as a
/// §4.1 zlib container. The window is the caller's lineage `base` hint
/// (priority) plus the most recent content frames in log order — a bounded
/// best-of-N search (fossil `extra_deltification`), accepting a delta only
/// when it is under 75% of the artifact. Every candidate is a HINT, never a
/// requirement: a null/phantom/config/unreadable/over-deep/unprofitable
/// base is silently skipped — there is no error path for an ineligible
/// base, only a compactness difference. Idempotent + phantom-upgrade like
/// `put`/`putZlib`: an already content-bearing `hash` keeps its stored
/// form and no encode work runs. This one method hides the whole storage
/// policy so callers pass only the base hint they naturally hold.
pub fn putArtifact(self: *LogStore, hash: []const u8, bytes: []const u8, base: ?Rid) PutError!Rid {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
std.debug.assert(bytes.len <= max_payload);
// Idempotent: return before any encode work when the artifact is held.
const held: ?Rid = self.index.get(hash);
if (held) |rid| {
if (!self.entries.items[@intCast(rid - 1)].phantom) return rid;
}
var pri_buf: [1]Rid = undefined;
const priority: []const Rid = if (base) |b| blk: {
pri_buf[0] = b;
break :blk pri_buf[0..1];
} else pri_buf[0..0];
var cand_buf: [max_base_candidates]Rid = undefined;
const n = self.gatherCandidates(priority, cand_buf[0..]);
if (try self.pickBase(self.gpa, hash, bytes, cand_buf[0..n])) |pick| {
defer self.gpa.free(pick.delta);
return try self.putDelta(hash, pick.base, pick.delta);
}
return self.putZlib(hash, bytes);
}
/// Fill `cand_buf` with the best-of-N base candidate window (§5.2) for a
/// write: the caller's `priority` hints first (in order — a lineage parent,
/// or the §6.2 same-path predecessors pack derives), then the most recent
/// content frames in log order. Insertion order IS log order, so a backward
/// walk over `entries` yields the chronological window for free — no manifest
/// parsing on the write path. Priority candidates are deduped against each
/// other and the chronological walk skips any rid already listed. Returns the
/// count written (≤ `max_base_candidates`); `pickBase` re-checks full
/// eligibility, so this only gathers plausible rids. Shared by `putArtifact`
/// and `putArtifactPacked` so the two write paths pick from an identical
/// window shape.
fn gatherCandidates(self: *LogStore, priority: []const Rid, cand_buf: []Rid) usize {
std.debug.assert(cand_buf.len >= max_base_candidates);
// The priority window is bounded to the candidate cap: both callers
// (putArtifact's 1-elem hint, pack's fixed-size hint buffer) structurally
// hold it, and the `for (priority)` loop below only caps entries ADDED,
// not iterations — state the upper bound locally so a future caller with a
// duplicate-heavy slice cannot silently scan past the window.
std.debug.assert(priority.len <= max_base_candidates);
var n: usize = 0;
// Priority hints fill first, in caller order, bounded to the window and
// deduped so a hint that repeats (or coincides across lanes) costs no slot.
for (priority) |b| {
if (n >= max_base_candidates) break;
std.debug.assert(b >= 1 and b <= self.entries.items.len);
if (ridListed(cand_buf[0..n], b)) continue;
cand_buf[n] = b;
n += 1;
}
// Newest-first; `idx` strictly decreases to 0. Two independent bounds
// stop the walk: `n < max_base_candidates` (window full) and
// `scanned < max_base_scan` (frames inspected). The scan cap is the one
// that bites under load — a run of trailing phantom frames `continue`s
// without filling a slot, so without it a single put could step over the
// whole store to gather < 5 candidates (hidden O(n^2) over N puts). The
// cap holds the per-put scan to a fixed window regardless of phantom
// density; a real base past the window only costs compactness. A held
// artifact that reaches this walk is necessarily a phantom (a
// content-bearing hit returns early in the caller), so the phantom skip
// already excludes "self"; the priority-dedup skips any hint already
// listed.
var idx: usize = self.entries.items.len;
var scanned: usize = 0;
while (idx > 0 and n < max_base_candidates and scanned < max_base_scan) {
idx -= 1;
scanned += 1;
const rid: Rid = @intCast(idx + 1);
if (self.entries.items[idx].phantom) continue;
if (ridListed(cand_buf[0..n], rid)) continue; // priority hint already listed
cand_buf[n] = rid;
n += 1;
}
return n;
}
/// Whether `rid` already appears in `listed` — the bounded (≤
/// `max_base_candidates`) dedup guard `gatherCandidates` runs so a priority
/// hint and the chronological walk never offer the same base twice.
fn ridListed(listed: []const Rid, rid: Rid) bool {
for (listed) |r| if (r == rid) return true;
return false;
}
/// The outcome of `putArtifactPacked`: either a fresh re-encoding won the
/// §6.1 monotonic-shrink comparison and was written (`rewrote`), or the
/// existing source frame is smaller-or-equal and NOTHING was written
/// (`kept`) — the caller then copies that existing frame verbatim.
pub const PackPut = union(enum) { rewrote: Rid, kept };
/// `putArtifact` with the §6.1 monotonic-shrink guard for `zit pack`: gather
/// the same best-of-N window, run `pickBase`, and encode the winning form —
/// but COMMIT the new frame only when its on-disk span is STRICTLY smaller
/// than `floor_span` (the artifact's existing stored-frame span in the source
/// store, from `frameSpanOf`). A tie or a larger re-encoding returns `.kept`
/// and writes nothing, so the caller re-stores the existing frame — often a
/// compact wire delta whose base fossil already chose well — verbatim.
/// `pickBase`'s <75%-of-FULL accept is a FIRST-encoding rule; in pack the
/// reference size is the EXISTING frame, not the full artifact, so this guard
/// keeps every REWRITTEN frame non-worsening: this frame is size-neutral or
/// smaller, never larger. (The whole-store guarantee has one exception outside
/// this function — pack's `copySrcFrame` §3.3 deep-chain fallback can enlarge a
/// KEPT frame; see pack.zig.) `priority` is the caller's ordered base hints
/// (a lineage parent, or the §6.2 same-path predecessors), offered ahead of
/// the chronological window. Idempotent + phantom-upgrade like `putArtifact`
/// (a re-put of a held artifact returns `.rewrote` with its existing rid).
pub fn putArtifactPacked(self: *LogStore, hash: []const u8, bytes: []const u8, priority: []const Rid, floor_span: u64) PutError!PackPut {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
std.debug.assert(bytes.len <= max_payload);
std.debug.assert(floor_span >= min_frame);
// Idempotent: return before any encode work when the artifact is held.
if (self.index.get(hash)) |rid| {
if (!self.entries.items[@intCast(rid - 1)].phantom) return .{ .rewrote = rid };
}
var cand_buf: [max_base_candidates]Rid = undefined;
const n = self.gatherCandidates(priority, cand_buf[0..]);
// The best new encoding, compressed to its on-disk container form so its
// span is comparable to the existing frame's. A profitable delta compares
// its zlib-contained delta; otherwise the fallback is a full zlib
// container (exactly what `putZlib` would store).
if (try self.pickBase(self.gpa, hash, bytes, cand_buf[0..n])) |pick| {
defer self.gpa.free(pick.delta);
const container = codec.encode(self.gpa, pick.delta, .default) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.InputTooLarge => unreachable, // pick.delta <= max_payload + bounded overhead
};
defer self.gpa.free(container);
// Keep the smaller; the existing frame wins ties (strict `<`).
if (frameSpan(hash.len, container.len) < floor_span) {
return .{ .rewrote = try self.putDeltaContainer(hash, pick.base, container) };
}
return .kept;
}
const container = codec.encode(self.gpa, bytes, .default) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.InputTooLarge => unreachable, // asserted bytes.len <= max_payload (< 2^32)
};
defer self.gpa.free(container);
if (frameSpan(hash.len, container.len) < floor_span) {
return .{ .rewrote = try self.putContainer(hash, container) };
}
return .kept;
}
/// The §5 delta-base policy as ONE deep module: choose the size-minimizing
/// base for storing `bytes` (named `hash`) from `candidates`, or null ⇒
/// store the full form. `candidates` are RIDs the caller believes are
/// plausible bases (lineage parent, chronological neighbors, same-path
/// predecessors), in priority order; at most `max_base_candidates` are
/// examined. This owns the WHOLE decision — the caller owns only WHICH
/// candidates to offer:
/// * best-of-N: run `delta.create` (delta.zig:294) against every ELIGIBLE
/// candidate and keep the SMALLEST delta (ties resolve to the earlier,
/// higher-priority candidate via the strict `<`);
/// * eligibility: content-bearing (non-phantom), non-config, chain-depth
/// headroom (`depth < delta_chain_write_max`), and never the artifact
/// itself (a self-delta is degenerate) — all free checks, run before
/// any I/O so an ineligible candidate costs nothing;
/// * cheap length early-out: a candidate whose content is under
/// `delta_min_source` bytes is skipped BEFORE `delta.create` (fossil
/// content.c:908);
/// * accept iff the smallest delta clears the 75% bar (`deltaProfitable`,
/// fossil content.c:914).
/// On accept, returns the winning base rid and the raw delta bytes (caller
/// owns and stores them, e.g. via `putDelta`); otherwise null. Ineligible /
/// unreadable candidates are silently skipped — a base is a hint, never a
/// requirement, so there is no error path for a bad candidate. Only
/// `OutOfMemory`/`Storage` escape.
pub fn pickBase(
self: *LogStore,
gpa: std.mem.Allocator,
hash: []const u8,
bytes: []const u8,
candidates: []const Rid,
) PutError!?Pick {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
std.debug.assert(bytes.len <= max_payload);
var best: ?[]u8 = null;
var best_base: Rid = 0;
// Free the retained best delta if an allocation/storage fault escapes
// mid-search; the success and reject paths hand it off / free it
// explicitly, so this only fires on an error return.
errdefer if (best) |b| gpa.free(b);
// Bounded fan-out: at most `max_base_candidates` `delta.create` runs,
// even if the caller over-offers (fossil's N_NEIGHBOR = 5).
const limit = @min(candidates.len, max_base_candidates);
var i: usize = 0;
while (i < limit) : (i += 1) {
const cand = candidates[i];
std.debug.assert(cand >= 1 and cand <= self.entries.items.len);
const entry = &self.entries.items[@intCast(cand - 1)];
// Eligibility — all free, no I/O: skip a phantom, the config
// singleton, a base already at the write-chain cap, and the
// artifact itself.
if (entry.phantom) continue;
if (entry.kind == .config) continue;
if (entry.depth >= delta_chain_write_max) continue;
if (std.mem.eql(u8, entry.hash, hash)) continue;
const base_bytes = self.content(gpa, cand) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => continue, // an unreadable base is not a basis
} orelse continue; // a phantom carries no content
defer gpa.free(base_bytes);
// Cheap length early-out (fossil content.c:908): a source too small
// to ever pay for a delta's framing is skipped before the encoder.
if (base_bytes.len < delta_min_source) continue;
const raw = delta.create(gpa, base_bytes, bytes) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.TargetTooLarge => continue, // unencodable target; skip
};
// Keep the strictly smaller delta; a tie keeps the earlier (higher
// priority) candidate. The loser is freed immediately so at most
// one delta buffer is ever live.
if (best == null or raw.len < best.?.len) {
if (best) |b| gpa.free(b);
best = raw;
best_base = cand;
} else {
gpa.free(raw);
}
}
const smallest = best orelse return null;
// Accept iff strictly under 75% of the uncompressed artifact. The
// globally smallest delta is <= every other, so if any candidate clears
// 75% the smallest does too — testing only the smallest is exact.
if (!deltaProfitable(smallest.len, bytes.len)) {
gpa.free(smallest);
return null;
}
return .{ .base = best_base, .delta = smallest };
}
/// Stores `bytes` compressed into a fossil zlib container (§4.1, default
/// level) under `hash`. The hash still names the UNCOMPRESSED bytes, so
/// storage compression never changes an artifact's address (§4.1
/// L1206-1208). Idempotent + phantom-upgrade like `put`.
pub fn putZlib(self: *LogStore, hash: []const u8, bytes: []const u8) PutError!Rid {
std.debug.assert(bytes.len <= max_payload);
const container = codec.encode(self.gpa, bytes, .default) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.InputTooLarge => unreachable, // asserted bytes.len <= max_payload (< 2^32)
};
defer self.gpa.free(container);
return self.insert(hash, container, .zlib, 0, 0);
}
/// Store an already-encoded full-form §4.1 zlib `container` verbatim under
/// `hash` as a `.zlib` frame, so a clone that received this exact container
/// re-serves it byte-for-byte (sync §7.2/§4.1 L858-860: "retransmitted
/// verbatim"). Unlike `putZlib` — which compresses raw plaintext — the
/// wire bytes are preserved, not re-created. PRECONDITION (§7.7 L1332):
/// the caller MUST have verified `hash` names the bytes `container`
/// inflates to; this store performs no hash recompute (it lacks the
/// plaintext to). Idempotent + phantom-upgrade like `put`.
pub fn putContainer(self: *LogStore, hash: []const u8, container: []const u8) PutError!Rid {
// §4.1 — a container carries at least its 4-byte length prefix.
std.debug.assert(container.len >= 4 and container.len <= max_payload);
return self.insert(hash, container, .zlib, 0, 0);
}
/// Stores `raw_delta` (a delta reconstructing this artifact from the frame
/// at `base_rid`) as a zlib-contained `delta` frame by encoding it into a
/// §4.1 container, then storing that container verbatim via
/// `putDeltaContainer`. The ingest path (`putArtifact`) uses this to store
/// a locally created delta. Idempotent + phantom-upgrade like `put`.
pub fn putDelta(self: *LogStore, hash: []const u8, base_rid: Rid, raw_delta: []const u8) PutError!Rid {
std.debug.assert(raw_delta.len >= 1 and raw_delta.len <= max_payload);
const container = codec.encode(self.gpa, raw_delta, .default) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.InputTooLarge => unreachable, // asserted raw_delta.len <= max_payload (< 2^32)
};
defer self.gpa.free(container);
return self.putDeltaContainer(hash, base_rid, container);
}
/// Store an already-§4.1-encoded delta `container` (a compressed
/// [FossilDelta] reconstructing this artifact from the frame at `base_rid`)
/// verbatim as a `.delta` frame; `delta_of` records the base frame's byte
/// offset. The stored container IS the wire payload a `cfile` serves
/// verbatim (§7.2), so a clone that received this exact delta container
/// re-serves it byte-for-byte — unlike `putDelta`, the container is NOT
/// re-encoded. PRECONDITION (§7.7 L1332): the caller MUST have verified
/// `hash` names the bytes this container reconstructs to (the delta applied
/// to `base_rid`'s content); this store performs no hash recompute.
/// Idempotent + phantom-upgrade like `put`.
pub fn putDeltaContainer(self: *LogStore, hash: []const u8, base_rid: Rid, container: []const u8) PutError!Rid {
std.debug.assert(base_rid >= 1 and base_rid <= self.entries.items.len);
const base = &self.entries.items[@intCast(base_rid - 1)];
std.debug.assert(!base.phantom);
std.debug.assert(base.offset >= header_len and base.offset < self.end);
// §4.1 — a delta container carries at least its 4-byte length prefix.
std.debug.assert(container.len >= 4 and container.len <= max_payload);
const base_off = base.offset;
// Bookkeeping only: the WRITE cap is `putArtifact`'s job (it refuses a
// base at `delta_chain_write_max`); the verbatim caller (`persistStore`)
// enforces the READ bound `max_delta_chain` for a retained foreign
// chain. `putDeltaContainer` itself is the low-level primitive and
// stays permissive. Saturating so depth never wraps on a pathologically
// deep foreign chain.
const depth: u8 = base.depth +| 1;
return self.insert(hash, container, .delta, base_off, depth);
}
/// Shared insert core: append one frame carrying `payload` under `hash`
/// with encoding `kind` (and `delta_of` for `.delta`), then update the
/// index. Idempotent for an already-content-bearing hash; upgrades a
/// phantom in place. On a bookkeeping failure after the frame landed, the
/// file is truncated back so disk and memory stay in lockstep; a rollback
/// truncate that itself fails poisons the store.
fn insert(self: *LogStore, hash: []const u8, payload: []const u8, kind: FrameKind, delta_of: u64, depth: u8) PutError!Rid {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
std.debug.assert(payload.len <= max_payload);
std.debug.assert((depth == 0) == (kind != .delta)); // depth is delta-only
if (self.index.get(hash)) |rid| {
const entry = &self.entries.items[@intCast(rid - 1)];
if (!entry.phantom) return rid;
const res = try self.appendFrame(hash, payload, kind, delta_of);
entry.offset = res.offset;
entry.payload_len = res.len;
entry.kind = kind;
entry.phantom = false;
entry.depth = depth;
return rid;
}
try self.entries.ensureUnusedCapacity(self.gpa, 1);
const owned = try self.gpa.dupe(u8, hash);
errdefer self.gpa.free(owned);
const prior_frame_count = self.frame_count;
const prior_last_frame_offset = self.last_frame_offset;
const res = try self.appendFrame(hash, payload, kind, delta_of);
const rid: Rid = self.entries.items.len + 1;
self.index.put(self.gpa, owned, rid) catch |e| {
// Frame already landed on disk but bookkeeping failed: undo the
// write so disk and every tail-key field return to the exact
// pre-append state. If truncate itself fails, poison rather than
// permit any later append against a physically diverged tail.
self.file.truncate(res.offset) catch {
self.poisoned = true;
};
self.end = res.offset;
self.frame_count = prior_frame_count;
self.last_frame_offset = prior_last_frame_offset;
return e;
};
self.entries.appendAssumeCapacity(.{ .hash = owned, .offset = res.offset, .payload_len = res.len, .kind = kind, .phantom = false, .depth = depth });
return rid;
}
/// The payload encoding of `rid` (`.raw` for a phantom). Lets a serve
/// loop pick a full-content versus delta cfile without a frame read.
pub fn kindOf(self: *const LogStore, rid: Rid) FrameKind {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
return self.entries.items[@intCast(rid - 1)].kind;
}
/// The on-disk (container/wire) payload length of `rid`; 0 for a phantom.
/// Symmetric with `hashOf`; the DST sweep uses it to bound the crc-covered
/// span of a frame whose uncompressed content differs in length.
pub fn payloadLenOf(self: *const LogStore, rid: Rid) u32 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
return self.entries.items[@intCast(rid - 1)].payload_len;
}
/// The total on-disk span (aligned prefix + hash + payload) of `rid`'s
/// frame — the byte cost `zit pack`'s §6.1 monotonic-shrink guard uses as
/// the floor when deciding whether a re-encoding is worth writing. Asserts a
/// real (non-phantom) frame; a phantom carries no frame to measure.
pub fn frameSpanOf(self: *const LogStore, rid: Rid) u64 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
const entry = &self.entries.items[@intCast(rid - 1)];
std.debug.assert(!entry.phantom);
return frameSpan(entry.hash.len, entry.payload_len);
}
/// The delta-chain depth of `rid`: 0 for a non-delta frame (or a phantom),
/// one more than its base for a `.delta` frame. Lets the verbatim persist
/// path (`persistStore`) refuse a retained delta whose base already sits at
/// the `max_delta_chain` READ bound, so a stored chain never exceeds what
/// `content()` can walk.
pub fn depthOf(self: *const LogStore, rid: Rid) u8 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
return self.entries.items[@intCast(rid - 1)].depth;
}
/// The byte offset of `rid`'s frame in the log file — the stable key a
/// `PlaintextCache` uses to memoise a frame's reconstructed plaintext across
/// a derivation sweep. Stable because the log is append-only: a written
/// frame never moves, so its offset names it for the sweep's lifetime.
pub fn offsetOf(self: *const LogStore, rid: Rid) u64 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
std.debug.assert(!self.entries.items[@intCast(rid - 1)].phantom); // a phantom has no frame
return self.entries.items[@intCast(rid - 1)].offset;
}
/// Whether `rid` is a phantom (interned but carrying no frame). Lets a
/// caller distinguish "held on disk" from "announced but absent" without
/// reconstructing content.
pub fn isPhantom(self: *const LogStore, rid: Rid) bool {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
return self.entries.items[@intCast(rid - 1)].phantom;
}
/// The delta-chain depth of the entry whose frame starts at `offset`, for
/// recovering a delta's depth during the open-time scan. Entries are
/// appended in ascending offset order there (real frames only, no
/// phantoms), so a binary search resolves the base with no auxiliary map.
/// Returns 0 when no entry starts at `offset` (a dangling `delta_of` that
/// `content()` rejects). Precondition: `entries` is offset-sorted — true
/// only mid-`scan`, which is this method's sole caller.
fn depthForBaseOffset(self: *const LogStore, offset: u64) u8 {
const items = self.entries.items;
var lo: usize = 0;
var hi: usize = items.len;
while (lo < hi) {
const mid = lo + (hi - lo) / 2;
const o = items[mid].offset;
if (o == offset) return items[mid].depth;
if (o < offset) lo = mid + 1 else hi = mid;
}
return 0;
}
/// Owned uncompressed bytes for a rid, or null while it's a phantom.
/// Re-reads and re-verifies the frame (both checksums, embedded hash) on
/// every call, then decodes by kind: `raw` copied out, `zlib` inflated,
/// `delta` resolved by walking `delta_of` and applying via `delta.apply`.
/// Never trusts a stale scan result.
pub fn content(self: *LogStore, gpa: std.mem.Allocator, rid: Rid) ContentError!?[]u8 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
const entry = &self.entries.items[@intCast(rid - 1)];
if (entry.phantom) return null;
return try self.contentAtOffset(gpa, entry.offset, max_delta_chain, entry.hash, null);
}
/// Like `content`, but resolves delta bases through `cache` — a bounded
/// plaintext memo the caller owns for the span of one derivation sweep. A
/// full sweep over delta chains would otherwise re-reconstruct each base
/// once per referencing delta (the k(k+1)/2 read-amplification the plan
/// projects at ~8.5x on depth-16 chains); memoizing a base's plaintext on
/// first use makes every later reference a lookup, so each frame is fully
/// reconstructed at most about twice regardless of chain depth. A `.delta`
/// frame's own plaintext is also memoized so a deeper delta built on it
/// hits; `.raw`/`.zlib` top-level results are never inserted, so an
/// all-raw repository pays zero cache overhead. Identical bytes to
/// `content(rid)`; the cache changes only the work, never the answer.
pub fn contentCached(self: *LogStore, gpa: std.mem.Allocator, rid: Rid, cache: *PlaintextCache) ContentError!?[]u8 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
const entry = &self.entries.items[@intCast(rid - 1)];
if (entry.phantom) return null;
const bytes = try self.contentAtOffset(gpa, entry.offset, max_delta_chain, entry.hash, cache);
// Keep a resolved delta's plaintext hot for any deeper delta that names
// this frame as its base; raw/zlib top-level frames are left uncached so
// a delta-free sweep never dups a byte. Best-effort: a failed insert
// just means no memo, never a wrong or missing answer.
if (entry.kind == .delta) cache.put(gpa, entry.offset, bytes) catch {};
return bytes;
}
/// Reconstructs the uncompressed content of the frame at `offset`.
/// `depth_remaining` bounds delta recursion: each `delta` hop consumes
/// one, and a delta reached at zero is `error.Corrupt` (never unbounded
/// recursion). `expect_hash`, when non-null, requires the frame's embedded
/// hash to match (the top-level integrity check); recursion into a base
/// passes null since a base carries its own name. `cache`, when non-null,
/// memoizes delta-base plaintext (see `contentCached`).
fn contentAtOffset(self: *LogStore, gpa: std.mem.Allocator, offset: u64, depth_remaining: u8, expect_hash: ?[]const u8, cache: ?*PlaintextCache) ContentError![]u8 {
std.debug.assert(depth_remaining <= max_delta_chain);
const fv = try self.readFrameAt(gpa, offset);
if (expect_hash) |eh| {
if (!std.mem.eql(u8, fv.hash[0..fv.hash_len], eh)) {
gpa.free(fv.payload);
return error.Corrupt;
}
}
switch (fv.kind) {
.raw => return fv.payload, // ownership transfers to the caller
.zlib => {
defer gpa.free(fv.payload);
return decodeContainer(gpa, fv.payload, max_payload);
},
.delta => {
defer gpa.free(fv.payload);
if (depth_remaining == 0) return error.Corrupt;
// A base must be an earlier, in-range frame: a forward or
// self reference is corruption, and a mid-frame/garbage
// offset fails `readFrameAt`'s checksums below.
if (fv.delta_of < header_len or fv.delta_of >= offset) return error.Corrupt;
const raw_delta = try decodeContainer(gpa, fv.payload, max_payload);
defer gpa.free(raw_delta);
// A memoized base: apply against the borrowed plaintext, never
// re-reading or re-reconstructing the base frame this sweep.
if (cache) |c| {
if (c.get(fv.delta_of)) |hit| {
return delta.apply(gpa, hit, raw_delta) catch |e| switch (e) {
error.OutOfMemory => error.OutOfMemory,
error.MalformedDelta, error.ChecksumMismatch => error.Corrupt,
};
}
}
const base = try self.contentAtOffset(gpa, fv.delta_of, depth_remaining - 1, null, cache);
defer gpa.free(base);
// A base is referenced by at least this delta, so memoize it.
if (cache) |c| c.put(gpa, fv.delta_of, base) catch {};
return delta.apply(gpa, base, raw_delta) catch |e| switch (e) {
error.OutOfMemory => error.OutOfMemory,
error.MalformedDelta, error.ChecksumMismatch => error.Corrupt,
};
},
.config => {
// A delta base (or content lookup) resolving to a config
// frame is corruption: config is not content-addressed and
// never a base (§E).
gpa.free(fv.payload);
return error.Corrupt;
},
_ => unreachable, // readFrameAt rejects any flag byte outside {raw,zlib,delta,config}
}
}
/// One fully validated frame read from `offset`: kind, `delta_of`, the
/// embedded hash, and the owned on-disk payload (raw bytes, or the zlib
/// container for `zlib`/`delta`). Caller owns `payload`.
const FrameView = struct {
kind: FrameKind,
delta_of: u64,
hash: [max_hash_len]u8,
hash_len: u8,
payload: []u8,
};
/// Reads and checksum-verifies the frame at `offset`, returning its
/// decoded fields and owned payload. Shares the fixed-header validation
/// (`readFrameHeader`) with the open-time scanner; unlike the scanner, a
/// short read here is `error.Corrupt` — the frame was known to exist, so
/// missing bytes mean the file lied or lost data, not a torn tail to
/// recover. Any flag byte outside {raw,zlib,delta,config} is rejected.
fn readFrameAt(self: *LogStore, gpa: std.mem.Allocator, offset: u64) ContentError!FrameView {
const fh = switch (try readFrameHeader(self.file, offset)) {
.torn => return error.Corrupt,
.header => |h| h,
};
const pl: usize = fh.payload_len;
if (pl > max_payload) return error.Corrupt; // wire-derived bound before allocation
const payload = try gpa.alloc(u8, pl);
errdefer gpa.free(payload);
const payload_offset: u64 = offset + fh.fixed_and_crc_len;
const npay = self.file.pread(payload_offset, payload) catch return error.Storage;
if (npay < pl) return error.Corrupt;
// Whole-frame CRC: skipped only when `openReadOnly` already verified
// every frame at scan time (the file is proven immutable-length there).
// The header prefix/header CRCs in `readFrameHeader` always run, so a
// frame is never read wholly unchecked; a short payload read above is
// still corruption regardless.
if (!self.verified_on_open) {
var trailer: [4]u8 = undefined;
const nt = self.file.pread(payload_offset + pl, &trailer) catch return error.Storage;
if (nt < 4) return error.Corrupt;
var fcrc = std.hash.Crc32.init();
fcrc.update(fh.header_bytes[0..fh.fixed_and_crc_len]);
fcrc.update(payload);
if (fcrc.final() != std.mem.readInt(u32, &trailer, .little)) return error.Corrupt;
}
return .{
.kind = fh.kind,
.delta_of = fh.delta_of,
.hash = fh.hash,
.hash_len = fh.hash_len,
.payload = payload,
};
}
/// A stored artifact's cfile wire payload (sync §7.2), served verbatim.
/// Two forms, matching the two stored container kinds:
/// * `full` — a §4.1 zlib container of the whole artifact; USIZE is its
/// own 4-byte big-endian prefix (`writeCfileVerbatim`).
/// * `delta` — a §4.1 zlib container of a [FossilDelta], plus the base
/// artifact's hash (DELTASRC) and the reconstructed-target size read
/// from the delta header (delta §3.2). USIZE is that `target_size` —
/// NEVER the container's 4-byte prefix, which is the raw delta's length
/// (sync §7.2 L1142-1144: "in delta form it is not delta size").
/// `deinit` frees the owned `container` in either arm.
pub const Wire = union(enum) {
full: []u8,
delta: Delta,
pub const Delta = struct {
container: []u8,
base_hash_buf: [max_hash_len]u8,
base_hash_len: u8,
/// delta §3.2 — the reconstructed target length; the cfile USIZE.
target_size: u64,
pub fn baseHash(self: *const Delta) []const u8 {
return self.base_hash_buf[0..self.base_hash_len];
}
};
pub fn container(self: *const Wire) []const u8 {
return switch (self.*) {
.full => |c| c,
.delta => |d| d.container,
};
}
pub fn deinit(self: *Wire, gpa: std.mem.Allocator) void {
switch (self.*) {
.full => |c| gpa.free(c),
.delta => |d| gpa.free(d.container),
}
self.* = undefined;
}
};
/// The stored cfile payload for `rid` served verbatim (sync §4.1/§7.2), or
/// null when the frame holds no stored container: a `.raw` frame or a
/// phantom (the serve loop then falls back to compressing `content()`). A
/// `.zlib` frame yields the `full` arm; a `.delta` frame yields the `delta`
/// arm, carrying the base hash (read from the base frame) and the target
/// size (read from the delta header — one small inflate). The stored
/// container IS the wire payload (normative §4.1/§7.2), so this exposes no
/// representation the RFC does not already fix. Caller owns the `Wire`.
pub fn wire(self: *LogStore, gpa: std.mem.Allocator, rid: Rid) ContentError!?Wire {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
const entry = &self.entries.items[@intCast(rid - 1)];
if (entry.phantom) return null;
switch (entry.kind) {
.raw => return null,
.zlib => {
const fv = try self.readFrameAt(gpa, entry.offset);
std.debug.assert(fv.kind == .zlib);
return .{ .full = fv.payload };
},
.delta => {
const fv = try self.readFrameAt(gpa, entry.offset);
errdefer gpa.free(fv.payload);
std.debug.assert(fv.kind == .delta);
// Structural phantom safety (D3): a stored delta's base ALWAYS
// precedes it in the log; a forward/self reference is corruption.
if (fv.delta_of < header_len or fv.delta_of >= entry.offset) return error.Corrupt;
// The reconstructed-target size (the §7.2 USIZE) comes from the
// delta's own header, not the container prefix (that is the raw
// delta length). One small inflate reads it without applying a
// single segment.
const raw_delta = try decodeContainer(gpa, fv.payload, max_payload);
defer gpa.free(raw_delta);
const target_size = delta.targetSize(raw_delta) catch return error.Corrupt;
// The base frame's own header carries its hash; reading it also
// checksum-validates that `delta_of` still points at a real frame.
const base = try self.readFrameAt(gpa, fv.delta_of);
gpa.free(base.payload);
var out: Wire = .{ .delta = .{
.container = fv.payload,
.base_hash_buf = undefined,
.base_hash_len = base.hash_len,
.target_size = target_size,
} };
@memcpy(out.delta.base_hash_buf[0..base.hash_len], base.hash[0..base.hash_len]);
return out;
},
// A config frame is never an artifact rid (never interned), so it
// never reaches here; any other kind is rejected on open.
.config, _ => unreachable,
}
}
/// Borrows the hash naming `rid`. Every rid, phantom or not, has one.
pub fn hashOf(self: *const LogStore, rid: Rid) []const u8 {
std.debug.assert(rid >= 1 and rid <= self.entries.items.len);
return self.entries.items[@intCast(rid - 1)].hash;
}
/// Returns caller-owned non-phantom artifact RIDs in the order their
/// frames were persisted. Dense RID order is deliberately not used:
/// `intern` may create a phantom before a later `put` writes its frame.
pub fn ridsInPersistedOrder(self: *const LogStore, gpa: std.mem.Allocator) error{OutOfMemory}![]Rid {
var count: usize = 0;
for (self.entries.items) |entry| {
if (!entry.phantom) count += 1;
}
const rids = try gpa.alloc(Rid, count);
count = 0;
for (self.entries.items, 0..) |entry, i| {
if (entry.phantom) continue;
rids[count] = @intCast(i + 1);
count += 1;
}
std.mem.sort(Rid, rids, self, ridOffsetLessThan);
return rids;
}
fn ridOffsetLessThan(self: *const LogStore, a: Rid, b: Rid) bool {
return self.entries.items[@intCast(a - 1)].offset < self.entries.items[@intCast(b - 1)].offset;
}
pub fn lookup(self: *const LogStore, hash: []const u8) ?Rid {
return self.index.get(hash);
}
/// Resolve a lowercase-hex artifact-name prefix to the single non-phantom
/// rid whose full name starts with it, or null when nothing matches
/// (artifact §3.6 prefix lookup). `error.Ambiguous` when more than one
/// stored artifact shares the prefix — a prefix matching several artifacts
/// is REFUSED, never silently narrowed to one (repo-state §8.3 item 8).
/// General-purpose: this queries EVERY held artifact, file blobs included,
/// since a blob is a non-phantom frame here just like a manifest. No
/// allocation — each entry's raw digest is hex-encoded into a fixed stack
/// buffer. Bound: `self.entries.items.len`. Preconditions (asserted): the
/// prefix is 4..2*max_hash_len bytes and every byte is a lowercase hex
/// digit — the caller canonicalizes; a non-hex or out-of-band prefix is the
/// caller's not-found family, never reaches here.
pub fn resolveHexPrefix(self: *const LogStore, prefix: []const u8) error{Ambiguous}!?Rid {
std.debug.assert(prefix.len >= 4 and prefix.len <= 2 * max_hash_len);
for (prefix) |c| std.debug.assert((c >= '0' and c <= '9') or (c >= 'a' and c <= 'f'));
const lower = "0123456789abcdef";
var found: ?Rid = null;
for (self.entries.items, 0..) |entry, i| {
if (entry.phantom) continue; // announced but no frame — carries no content
var hexbuf: [2 * max_hash_len]u8 = undefined;
const hex_len = entry.hash.len * 2;
std.debug.assert(hex_len <= hexbuf.len);
for (entry.hash, 0..) |b, j| {
hexbuf[j * 2] = lower[b >> 4];
hexbuf[j * 2 + 1] = lower[b & 0x0f];
}
const hex = hexbuf[0..hex_len];
if (hex.len < prefix.len) continue; // a longer prefix cannot name a shorter hash
if (std.mem.startsWith(u8, hex, prefix)) {
if (found != null) return error.Ambiguous; // §8.3 item 8: >1 match refuses
found = @intCast(i + 1); // rids are 1-based (entries index + 1)
}
}
if (found) |rid| std.debug.assert(!self.entries.items[@intCast(rid - 1)].phantom);
return found;
}
pub fn commit(self: *LogStore) CommitError!void {
self.file.sync() catch return error.Storage;
}
/// Appends a new config frame and makes it the live config (§E). Latest
/// wins by scan order, so an edit is one append — no rewrite, no mutable
/// pointer. On a torn append `appendFrame` rolls the file back and
/// `config_offset` is left untouched, so failure leaves the prior config
/// live.
pub fn putConfig(self: *LogStore, cfg: Config) PutError!void {
std.debug.assert(cfg.default_branch.len <= max_config_str);
std.debug.assert(cfg.project_id.len <= max_config_str);
std.debug.assert(cfg.project_code.len <= max_config_str);
std.debug.assert(cfg.login_user.len <= max_config_str);
std.debug.assert(cfg.login_secret_hex.len <= max_config_str);
// sync §6.3 — a stored secret is either absent or the full 40-hex
// derived value; no other width is ever persisted.
std.debug.assert(cfg.login_secret_hex.len == 0 or cfg.login_secret_hex.len == 40);
std.debug.assert(cfg.remote_url.len <= max_config_str);
var buf: [max_config_payload]u8 = undefined;
const payload = encodeConfig(&buf, cfg);
const res = try self.appendFrame(config_sentinel, payload, .config, 0);
std.debug.assert(res.offset + payload.len < self.end); // frame landed before the new end
self.config_offset = res.offset;
}
/// The live config, or null when the log carries none (§E: absent config
/// means the caller — CLI/repostate — owns the default; storage bakes
/// none). Re-reads and re-verifies the frame on every call. Caller owns
/// the returned `OwnedConfig`.
pub fn readConfig(self: *LogStore, gpa: std.mem.Allocator) ContentError!?OwnedConfig {
const off = self.config_offset orelse return null;
const fv = try self.readFrameAt(gpa, off);
errdefer gpa.free(fv.payload);
if (fv.kind != .config) return error.Corrupt;
const parsed = try parseConfig(fv.payload);
return .{ .backing = fv.payload, .config = parsed };
}
/// Structural end of the log (header + every complete frame written so
/// far), for callers that track write progress without duplicating
/// frame-layout arithmetic (e.g. the DST sweep's crash-recovery model).
pub fn size(self: *const LogStore) u64 {
return self.end;
}
/// The number of interned frames (phantom or real) held so far — lets
/// `pack.packLog` assert its destination is a just-opened empty log before
/// it re-bases anything into it (§6.1 base-availability precondition: a
/// hint translated via `dst.lookup` must only ever resolve a frame THIS
/// pack wrote, never a pre-existing unrelated one).
pub fn entryCount(self: *const LogStore) usize {
return self.entries.items.len;
}
const AppendResult = struct { offset: u64, len: u32 };
/// Encodes one frame and appends it in a single `StorageFile.append`
/// call — one call is one atomic torn-write draw point, matching the
/// crash simulator's fault granularity. On failure, truncates back to
/// the prior structural end so a partial write never lingers; if that
/// rollback truncate itself fails the store is poisoned and refuses every
/// later append.
fn appendFrame(self: *LogStore, hash: []const u8, bytes: []const u8, kind: FrameKind, delta_of: u64) PutError!AppendResult {
if (self.poisoned) return error.Storage;
// A read-only store proved every frame's whole-frame CRC at scan time and
// reads skip the redundant re-CRC; appending here would add a frame that
// never gets that check. The append path always opens via `open`.
std.debug.assert(!self.verified_on_open);
const hl: u64 = hash.len;
const pl: u64 = bytes.len;
const frame_len = frameSpan(hl, pl);
std.debug.assert(frame_len <= max_payload + 64);
const frame = try self.gpa.alloc(u8, @intCast(frame_len));
defer self.gpa.free(frame);
_ = encodeFrame(frame, hash, bytes, @intFromEnum(kind), delta_of);
const offset = self.end;
if (self.file.append(frame)) |_| {
self.end += frame_len;
self.frame_count += 1;
self.last_frame_offset = offset;
std.debug.assert(self.end % frame_align == 0);
return .{ .offset = offset, .len = @intCast(pl) };
} else |_| {
// A torn append may have left a prefix; if the rollback truncate
// can't remove it, `end` no longer matches the file end — poison.
self.file.truncate(self.end) catch {
self.poisoned = true;
};
return error.Storage;
}
}
};
/// A bounded plaintext memo for a single derivation sweep: base frame offset to
/// its reconstructed plaintext. Owned by the sweep caller (repoview.load), not
/// by the store — see `LogStore.contentCached`. Explicitly bounded on BOTH a
/// byte budget and an entry count; an insert that would exceed either evicts
/// least-recently-used entries first, and a single item larger than the whole
/// byte budget is simply never cached (the budget never grows). No unbounded
/// allocation and no unbounded memoization — deleting the cache can only change
/// speed, never any answer.
pub const PlaintextCache = struct {
/// Default resident-byte budget: comfortably holds a delta family's hot
/// bases without letting the memo grow without limit.
pub const default_max_bytes: usize = 64 << 20;
/// Default hard cap on resident entries, bounding per-entry metadata even
/// under many tiny items.
pub const default_max_entries: usize = 4096;
const Node = struct { bytes: []u8, last_used: u64 };
map: std.AutoHashMapUnmanaged(u64, Node) = .empty,
bytes_used: usize = 0,
/// Resident plaintext never exceeds this many bytes (fixed for a cache's
/// lifetime; the field form lets a test drive eviction with a small budget).
max_bytes: usize = default_max_bytes,
/// Resident entry count never exceeds this.
max_entries: usize = default_max_entries,
/// Monotonic recency clock; the entry with the smallest stamp is the LRU
/// victim. u64 never wraps within a single sweep's insert/lookup count.
tick: u64 = 0,
/// Observability only (never gates logic): the number of `get` calls that
/// returned a memoised plaintext. A persist/derivation sweep over delta
/// chains proves basis reuse by this rising above zero — a re-reconstruction
/// that the memo replaced with a lookup.
hits: u64 = 0,
pub fn init() PlaintextCache {
return .{};
}
pub fn deinit(self: *PlaintextCache, gpa: std.mem.Allocator) void {
var it = self.map.valueIterator();
while (it.next()) |n| gpa.free(n.bytes);
self.map.deinit(gpa);
self.* = undefined;
}
/// The memoized plaintext for `offset`, or null; bumps its LRU recency.
/// The returned slice is borrowed and valid until the next `put`/`deinit`.
pub fn get(self: *PlaintextCache, offset: u64) ?[]const u8 {
const n = self.map.getPtr(offset) orelse return null;
self.tick += 1;
self.hits += 1;
n.last_used = self.tick;
return n.bytes;
}
/// Memoize a private copy of `src` under `offset`, evicting LRU entries to
/// stay within both bounds. A no-op when `offset` is already resident (base
/// offsets are unique keys) or when `src` alone exceeds the byte budget.
/// Only `error.OutOfMemory` escapes.
pub fn put(self: *PlaintextCache, gpa: std.mem.Allocator, offset: u64, src: []const u8) error{OutOfMemory}!void {
if (self.map.contains(offset)) return;
if (src.len > self.max_bytes) return; // never grow the budget for one item
// Evict until the newcomer fits under both caps (bounded by the
// resident count — each pass removes exactly one entry).
while (self.bytes_used + src.len > self.max_bytes or @as(usize, self.map.count()) >= self.max_entries) {
if (!self.evictOne(gpa)) break; // nothing left to evict
}
if (self.bytes_used + src.len > self.max_bytes) return; // still over: skip
const owned = try gpa.dupe(u8, src);
errdefer gpa.free(owned);
self.tick += 1;
try self.map.put(gpa, offset, .{ .bytes = owned, .last_used = self.tick });
self.bytes_used += owned.len;
}
/// Evict the least-recently-used entry; false when the map is empty.
fn evictOne(self: *PlaintextCache, gpa: std.mem.Allocator) bool {
var victim_key: ?u64 = null;
var victim_tick: u64 = std.math.maxInt(u64);
var victim_len: usize = 0;
var it = self.map.iterator();
while (it.next()) |e| {
if (e.value_ptr.last_used < victim_tick) {
victim_tick = e.value_ptr.last_used;
victim_key = e.key_ptr.*;
victim_len = e.value_ptr.bytes.len;
}
}
const k = victim_key orelse return false;
const n = self.map.getPtr(k).?;
gpa.free(n.bytes);
_ = self.map.remove(k);
std.debug.assert(self.bytes_used >= victim_len);
self.bytes_used -= victim_len;
return true;
}
};
fn writeHeader(file: storage.StorageFile, hash_kind: u8) OpenError!void {
var buf: [header_len]u8 = [_]u8{0} ** header_len;
@memcpy(buf[0..4], magic);
std.mem.writeInt(u16, buf[4..6], version, .little);
buf[6] = hash_kind;
var crc = std.hash.Crc32.init();
crc.update(buf[0..60]);
std.mem.writeInt(u32, buf[60..64], crc.final(), .little);
file.append(&buf) catch return error.Storage;
}
/// Returns the stored `hash_kind` on success. Rejects: bad magic, a stored
/// version other than this reader's, or a header crc mismatch.
fn validateHeader(file: storage.StorageFile) OpenError!u8 {
var buf: [header_len]u8 = undefined;
const n = file.pread(0, &buf) catch return error.Storage;
if (n < header_len) return error.Corrupt;
if (!std.mem.eql(u8, buf[0..4], magic)) return error.Corrupt;
const stored_version = std.mem.readInt(u16, buf[4..6], .little);
if (stored_version != version) return error.Corrupt;
const stored_crc = std.mem.readInt(u32, buf[60..64], .little);
var crc = std.hash.Crc32.init();
crc.update(buf[0..60]);
if (crc.final() != stored_crc) return error.Corrupt;
return buf[6];
}
/// Outcome of reading one frame at a scan offset: either a torn tail (the
/// file ends mid-frame — truncate and stop) or a validated frame.
const ScanOutcome = union(enum) {
torn,
frame: struct { hash: [max_hash_len]u8, hash_len: u8, frame_len: u64, payload_len: u32, kind: FrameKind, delta_of: u64 },
};
/// Fixed prefix window: len(4)|flags(1)|hash_len(1)|prefix_crc32(4). Its
/// size does not depend on `hash_len` — that is what lets `hash_len` be
/// proven correct (or rejected) before it is used to size anything else.
const prefix_len: usize = 10;
/// A frame's fully validated fixed header — everything before the payload.
/// The payload and its whole-frame crc are the caller's concern: the
/// content reader allocates and returns the payload, the open-time scanner
/// streams it for the crc without allocating.
const FrameHeader = struct {
payload_len: u32,
kind: FrameKind,
delta_of: u64,
hash: [max_hash_len]u8,
hash_len: u8,
/// Span of prefix+hash+delta_of+header_crc, i.e. the payload's offset
/// relative to the frame start; the whole-frame crc is seeded over
/// exactly this many bytes of `header_bytes`.
fixed_and_crc_len: usize,
/// The verified header bytes, so the caller seeds the whole-frame crc
/// without re-reading them.
header_bytes: [22 + max_hash_len]u8,
};
/// Either a torn tail (a short read on the prefix or header window) or a
/// validated fixed header.
const HeaderScan = union(enum) {
torn,
header: FrameHeader,
};
/// Reads and checksum-verifies a frame's fixed header at `offset`, shared by
/// the content reader (`readFrameAt`) and the open-time scanner
/// (`scanOneFrame`). Two-stage: first the fixed-size prefix window
/// (hash_len-independent, so a short read there is unambiguously a torn write
/// — nothing about its size can be thrown off by corruption), which proves
/// `hash_len` trustworthy via its own crc; only then is `hash_len` used to
/// size the rest of the header read, at which point the same "short read =
/// torn, checksum fails = corrupt" reasoning applies validly (append() only
/// ever writes one frame per call and a torn write is a clean prefix — never
/// damaged bytes; a full window whose checksum fails can only be a bitflip on
/// already-landed bytes). A short read on either window is `.torn`; the
/// callers decide its disposition (the scanner recovers, the content reader
/// treats it as `error.Corrupt`). Any flag byte outside
/// {raw,zlib,delta,config} is rejected.
fn readFrameHeader(file: storage.StorageFile, offset: u64) error{ Corrupt, Storage }!HeaderScan {
var prefix: [prefix_len]u8 = undefined;
const n0 = file.pread(offset, &prefix) catch return error.Storage;
if (n0 < prefix_len) return .torn;
const payload_len = std.mem.readInt(u32, prefix[0..4], .little);
const flags = prefix[4];
const hash_len = prefix[5];
if (hash_len < min_hash_len or hash_len > max_hash_len) return error.Corrupt;
var pcrc = std.hash.Crc32.init();
pcrc.update(prefix[0..6]);
if (pcrc.final() != std.mem.readInt(u32, prefix[6..][0..4], .little)) return error.Corrupt;
// A prefix-verified flag byte is trustworthy (a torn write is a clean
// truncation, never damaged bytes), so any kind outside
// {raw,zlib,delta,config} is a genuine unknown-kind frame, rejected — not
// a torn tail.
if (flags > @intFromEnum(FrameKind.config)) return error.Corrupt;
const hl: usize = hash_len;
const fixed_and_crc_len: usize = 22 + hl;
var hbuf: [22 + max_hash_len]u8 = undefined;
const n1 = file.pread(offset, hbuf[0..fixed_and_crc_len]) catch return error.Storage;
if (n1 < fixed_and_crc_len) return .torn;
var hcrc = std.hash.Crc32.init();
hcrc.update(hbuf[0 .. 18 + hl]);
if (hcrc.final() != std.mem.readInt(u32, hbuf[18 + hl ..][0..4], .little)) return error.Corrupt;
var fh: FrameHeader = .{
.payload_len = payload_len,
.kind = @enumFromInt(flags),
.delta_of = std.mem.readInt(u64, hbuf[10 + hl ..][0..8], .little),
.hash = undefined,
.hash_len = hash_len,
.fixed_and_crc_len = fixed_and_crc_len,
.header_bytes = undefined,
};
@memcpy(fh.hash[0..hl], hbuf[10..][0..hl]);
@memcpy(fh.header_bytes[0..fixed_and_crc_len], hbuf[0..fixed_and_crc_len]);
return .{ .header = fh };
}
/// Reads and validates exactly one frame at `offset`, for the open-time scan.
/// Delegates the fixed-header validation to `readFrameHeader`; a torn header,
/// a frame that would run past `total`, or a torn payload/trailer all report
/// `.torn` (recoverable tail), while a failed whole-frame crc is
/// `error.Corrupt`. The payload is streamed through the crc in bounded chunks
/// so a large frame never forces a full-payload allocation.
fn scanOneFrame(file: storage.StorageFile, offset: u64, total: u64) OpenError!ScanOutcome {
const fh = switch (try readFrameHeader(file, offset)) {
.torn => return .torn,
.header => |h| h,
};
const frame_len = frameSpan(fh.hash_len, fh.payload_len);
if (offset + frame_len > total) return .torn;
var crc = std.hash.Crc32.init();
crc.update(fh.header_bytes[0..fh.fixed_and_crc_len]);
var remaining: u64 = fh.payload_len;
var pos = offset + @as(u64, fh.fixed_and_crc_len);
var chunk: [4096]u8 = undefined;
while (remaining > 0) {
const take: usize = @intCast(@min(remaining, @as(u64, chunk.len)));
const got = file.pread(pos, chunk[0..take]) catch return error.Storage;
if (got < take) return .torn;
crc.update(chunk[0..take]);
pos += take;
remaining -= take;
}
var trailer: [4]u8 = undefined;
const nt = file.pread(pos, &trailer) catch return error.Storage;
if (nt < trailer.len) return .torn;
if (crc.final() != std.mem.readInt(u32, &trailer, .little)) return error.Corrupt;
return .{ .frame = .{ .hash = fh.hash, .hash_len = fh.hash_len, .frame_len = frame_len, .payload_len = fh.payload_len, .kind = fh.kind, .delta_of = fh.delta_of } };
}
/// Rebuilds `self.entries`/`self.index` by scanning frames from
/// `header_len`. With `recover_torn`, a torn tail truncates to the last valid
/// boundary; otherwise it is reported as corruption without mutating storage.
/// A corrupt complete frame always propagates `error.Corrupt`.
fn scan(self: *LogStore, recover_torn: bool) OpenError!void {
const total = self.file.size() catch return error.Storage;
const max_frames = (total - header_len) / min_frame + 1;
var offset: u64 = header_len;
var frames: u64 = 0;
while (offset < total) {
frames += 1;
std.debug.assert(frames <= max_frames);
switch (try scanOneFrame(self.file, offset, total)) {
.torn => {
if (!recover_torn) return error.Corrupt;
self.file.truncate(offset) catch return error.Storage;
self.end = offset;
return;
},
.frame => |f| {
// Every complete frame — content or config — advances the
// cache-key tail bookkeeping.
self.frame_count += 1;
self.last_frame_offset = offset;
if (f.kind == .config) {
// Config is not interned: it never enters entries/index,
// and "latest wins" is just the last one scan reaches (§E).
if (self.config_offset) |prev| std.debug.assert(offset > prev);
self.config_offset = offset;
} else {
// Delta depth is recovered from the base frame, which — by
// the append-only, strictly-backward `delta_of` invariant —
// was scanned earlier and is already in `entries` (offset
// order). A base whose offset is not found is a dangling
// delta `content()` will reject; depth 0 is a safe placeholder.
const depth: u8 = if (f.kind == .delta)
self.depthForBaseOffset(f.delta_of) +| 1
else
0;
const owned = try self.gpa.dupe(u8, f.hash[0..f.hash_len]);
errdefer self.gpa.free(owned);
try self.entries.append(self.gpa, .{ .hash = owned, .offset = offset, .payload_len = f.payload_len, .kind = f.kind, .phantom = false, .depth = depth });
errdefer _ = self.entries.pop();
const rid: Rid = self.entries.items.len;
try self.index.put(self.gpa, owned, rid);
}
const next = offset + f.frame_len;
std.debug.assert(next > offset);
offset = next;
},
}
}
self.end = offset;
}
/// Encodes one frame into `buf` (which must be at least the padded frame
/// span) and returns the written slice. Shared by the production write
/// path and by tests that hand-craft torn/corrupt fixtures.
///
/// Layout (all little-endian): len(4) | flags(1) | hash_len(1) |
/// prefix_crc32(4) = crc32(bytes[0..6)) | hash(hash_len) | delta_of(8) |
/// header_crc32(4) = crc32(bytes[0..18+hash_len)) | payload(len) |
/// frame_crc32(4) = crc32(bytes[0..22+hash_len+len)). `delta_of` is the
/// base frame's byte offset for a `delta` frame, and 0 otherwise.
fn encodeFrame(buf: []u8, hash: []const u8, payload: []const u8, flags: u8, delta_of: u64) []const u8 {
std.debug.assert(hash.len >= min_hash_len and hash.len <= max_hash_len);
std.debug.assert(payload.len <= max_payload);
const hl: usize = hash.len;
const pl: usize = payload.len;
const hash_len: u8 = @intCast(hl);
const payload_len: u32 = @intCast(pl);
const unpadded: usize = frame_fixed_overhead + hl + pl;
const frame_len: usize = @intCast(frameSpan(hl, pl));
std.debug.assert(buf.len >= frame_len);
std.mem.writeInt(u32, buf[0..4], payload_len, .little);
buf[4] = flags;
buf[5] = hash_len;
var pcrc = std.hash.Crc32.init();
pcrc.update(buf[0..6]);
std.mem.writeInt(u32, buf[6..][0..4], pcrc.final(), .little);
@memcpy(buf[10..][0..hl], hash);
std.mem.writeInt(u64, buf[10 + hl ..][0..8], delta_of, .little);
var hcrc = std.hash.Crc32.init();
hcrc.update(buf[0 .. 18 + hl]);
std.mem.writeInt(u32, buf[18 + hl ..][0..4], hcrc.final(), .little);
@memcpy(buf[22 + hl ..][0..pl], payload);
var fcrc = std.hash.Crc32.init();
fcrc.update(buf[0 .. 22 + hl + pl]);
std.mem.writeInt(u32, buf[22 + hl + pl ..][0..4], fcrc.final(), .little);
if (frame_len > unpadded) @memset(buf[unpadded..frame_len], 0);
return buf[0..frame_len];
}
fn alignUp8(comptime T: type, n: T) T {
const a: T = frame_align;
return (n + (a - 1)) / a * a;
}
/// The fixed per-frame overhead (bytes) that brackets the variable hash+payload
/// on disk: len(4) + flags(1) + hash_len(1) + prefix_crc32(4) + delta_of(8) +
/// header_crc32(4) + frame_crc32(4) = 26 (see `encodeFrame`'s layout). Named
/// once so every on-disk span — `frameSpan`, `appendFrame`, and the decode/scan
/// paths — derives the same number and pack's §6.1 monotonic floor can never
/// silently drift from the span `appendFrame` commits.
const frame_fixed_overhead = 26;
/// The aligned on-disk span of a frame carrying `hash_len`-byte hash and
/// `payload_len`-byte payload — the same `alignUp8(frame_fixed_overhead + hl +
/// pl)` `appendFrame` commits. Used by `frameSpanOf` and pack's §6.1
/// monotonic-shrink guard to compare a re-encoding's byte cost against the frame
/// it would replace, and by the scan/decode paths to advance frame-to-frame.
fn frameSpan(hash_len: u64, payload_len: u64) u64 {
return alignUp8(u64, frame_fixed_overhead + hash_len + payload_len);
}
/// Encodes `cfg` into `buf` as `[hash_policy u8][branch_len u16 LE][branch]
/// [pid_len u16 LE][pid][pcode_len u16 LE][pcode][login_len u16 LE][login]
/// [secret_len u16 LE][secret][url_len u16 LE][url]` and returns the written
/// slice (§E fixed field order — deterministic, not a hash-map dump). Every
/// string field is always written (length zero when empty), so decode
/// consumption stays exact. Fixed bounds, no unbounded copy.
fn encodeConfig(buf: []u8, cfg: Config) []const u8 {
std.debug.assert(buf.len >= max_config_payload);
std.debug.assert(cfg.default_branch.len <= max_config_str);
std.debug.assert(cfg.project_id.len <= max_config_str);
std.debug.assert(cfg.project_code.len <= max_config_str);
std.debug.assert(cfg.login_user.len <= max_config_str);
std.debug.assert(cfg.login_secret_hex.len <= max_config_str);
std.debug.assert(cfg.remote_url.len <= max_config_str);
const bl: usize = cfg.default_branch.len;
const pl: usize = cfg.project_id.len;
const cl: usize = cfg.project_code.len;
const ul: usize = cfg.login_user.len;
const sl: usize = cfg.login_secret_hex.len;
const rl: usize = cfg.remote_url.len;
buf[0] = cfg.hash_policy;
std.mem.writeInt(u16, buf[1..3], @intCast(bl), .little);
@memcpy(buf[3..][0..bl], cfg.default_branch);
std.mem.writeInt(u16, buf[3 + bl ..][0..2], @intCast(pl), .little);
@memcpy(buf[5 + bl ..][0..pl], cfg.project_id);
std.mem.writeInt(u16, buf[5 + bl + pl ..][0..2], @intCast(cl), .little);
@memcpy(buf[7 + bl + pl ..][0..cl], cfg.project_code);
std.mem.writeInt(u16, buf[7 + bl + pl + cl ..][0..2], @intCast(ul), .little);
@memcpy(buf[9 + bl + pl + cl ..][0..ul], cfg.login_user);
std.mem.writeInt(u16, buf[9 + bl + pl + cl + ul ..][0..2], @intCast(sl), .little);
@memcpy(buf[11 + bl + pl + cl + ul ..][0..sl], cfg.login_secret_hex);
std.mem.writeInt(u16, buf[11 + bl + pl + cl + ul + sl ..][0..2], @intCast(rl), .little);
@memcpy(buf[13 + bl + pl + cl + ul + sl ..][0..rl], cfg.remote_url);
// The four existing submanifest settings bytes are always written as one
// block. A pre-settings frame ends earlier and parses with defaults. The
// optional digest is local persistence, present only after a settled scan.
const submanifest_pos = 13 + bl + pl + cl + ul + sl + rl;
buf[submanifest_pos] = @intFromBool(cfg.seen_submanifest);
buf[submanifest_pos + 1] = @intFromBool(cfg.enable_submanifests);
buf[submanifest_pos + 2] = cfg.submanifest_k;
buf[submanifest_pos + 3] = @intFromBool(cfg.submanifest_auto_retune);
if (cfg.submanifest_settled_cost_shape) |digest| {
std.debug.assert(digest.len == 32);
@memcpy(buf[submanifest_pos + 4 ..][0..digest.len], &digest);
return buf[0 .. submanifest_pos + 4 + digest.len];
}
return buf[0 .. submanifest_pos + 4];
}
/// Parses an `encodeConfig` payload. Every LE length read is bounds-checked
/// against the remaining slice before use, each length is capped at
/// `max_config_str`, and total consumption must be exact (trailing bytes are
/// corruption). Borrows `payload`. All wire-derived, validated before use.
fn parseConfig(payload: []const u8) error{Corrupt}!Config {
if (payload.len < 1) return error.Corrupt; // no hash_policy byte
const hash_policy = payload[0];
var pos: usize = 1;
if (payload.len - pos < 2) return error.Corrupt;
const blen: usize = std.mem.readInt(u16, payload[pos..][0..2], .little);
pos += 2;
if (blen > max_config_str) return error.Corrupt;
if (payload.len - pos < blen) return error.Corrupt;
const branch = payload[pos..][0..blen];
pos += blen;
if (payload.len - pos < 2) return error.Corrupt;
const plen: usize = std.mem.readInt(u16, payload[pos..][0..2], .little);
pos += 2;
if (plen > max_config_str) return error.Corrupt;
if (payload.len - pos < plen) return error.Corrupt;
const pid = payload[pos..][0..plen];
pos += plen;
if (payload.len - pos < 2) return error.Corrupt;
const clen: usize = std.mem.readInt(u16, payload[pos..][0..2], .little);
pos += 2;
if (clen > max_config_str) return error.Corrupt;
if (payload.len - pos < clen) return error.Corrupt;
const pcode = payload[pos..][0..clen];
pos += clen;
// sync §6.3 — the login credential fields are an optional trailing block:
// a frame written before they existed ends exactly here, and parses with
// an empty credential (forward-compatible), never as corruption.
if (pos == payload.len) {
return .{ .hash_policy = hash_policy, .default_branch = branch, .project_id = pid, .project_code = pcode };
}
if (payload.len - pos < 2) return error.Corrupt;
const ulen: usize = std.mem.readInt(u16, payload[pos..][0..2], .little);
pos += 2;
if (ulen > max_config_str) return error.Corrupt;
if (payload.len - pos < ulen) return error.Corrupt;
const login = payload[pos..][0..ulen];
pos += ulen;
if (payload.len - pos < 2) return error.Corrupt;
const slen: usize = std.mem.readInt(u16, payload[pos..][0..2], .little);
pos += 2;
if (slen > max_config_str) return error.Corrupt;
if (payload.len - pos < slen) return error.Corrupt;
const secret = payload[pos..][0..slen];
pos += slen;
// sync §4/§6 — the saved remote URL is another optional trailing block:
// a frame written before it existed ends exactly here and parses with an
// empty remote (forward-compatible), never as corruption.
if (pos == payload.len) {
return .{ .hash_policy = hash_policy, .default_branch = branch, .project_id = pid, .project_code = pcode, .login_user = login, .login_secret_hex = secret };
}
if (payload.len - pos < 2) return error.Corrupt;
const rlen: usize = std.mem.readInt(u16, payload[pos..][0..2], .little);
pos += 2;
if (rlen > max_config_str) return error.Corrupt;
if (payload.len - pos < rlen) return error.Corrupt;
const url = payload[pos..][0..rlen];
pos += rlen;
// Submanifest settings are the final optional trailing block. A frame
// written before them ends here and parses with defaults. The local
// settled-digest extension accepts exactly thirty-six bytes: four settings
// plus a fixed 32-byte digest. Any other width is corrupt, because guessing
// a future field would silently skip a required scan.
var seen_submanifest = false;
var enable_submanifests = false;
var submanifest_k: u8 = 0;
var submanifest_auto_retune = false;
var submanifest_settled_cost_shape: ?[32]u8 = null;
if (pos != payload.len) {
const trailing = payload.len - pos;
const digest_width = 4 + @as(usize, 32);
if (trailing != 1 and trailing != 3 and trailing != 4 and trailing != digest_width) {
return error.Corrupt;
}
seen_submanifest = payload[pos] != 0;
if (trailing >= 3) {
enable_submanifests = payload[pos + 1] != 0;
submanifest_k = payload[pos + 2];
}
if (trailing >= 4) {
submanifest_auto_retune = payload[pos + 3] != 0;
}
if (trailing == digest_width) {
var digest: [32]u8 = undefined;
@memcpy(&digest, payload[pos + 4 ..][0..digest.len]);
std.debug.assert(digest.len == 32);
submanifest_settled_cost_shape = digest;
}
pos += trailing;
}
if (pos != payload.len) return error.Corrupt; // no trailing bytes
return .{
.hash_policy = hash_policy,
.default_branch = branch,
.project_id = pid,
.project_code = pcode,
.login_user = login,
.login_secret_hex = secret,
.remote_url = url,
.seen_submanifest = seen_submanifest,
.enable_submanifests = enable_submanifests,
.submanifest_k = submanifest_k,
.submanifest_auto_retune = submanifest_auto_retune,
.submanifest_settled_cost_shape = submanifest_settled_cost_shape,
};
}
/// Map the canonical §4.1 decode error set (`codec.DecodeError`) into this
/// layer's `ContentError`: a malformed container — short header, a declared
/// capacity over `max_output`, or a corrupt RFC1950 stream — is `Corrupt`;
/// OOM propagates. The §4.1 codec itself is the leaf `codec` module
/// (`container.zig`), shared with `repostate` and `sync`.
fn decodeContainer(gpa: std.mem.Allocator, wire: []const u8, max_output: usize) ContentError![]u8 {
return codec.decode(gpa, wire, max_output) catch |e| switch (e) {
error.OutOfMemory => error.OutOfMemory,
error.TruncatedHeader, error.InvalidStream, error.OutputTooLarge => error.Corrupt,
};
}
// ---------------------------------------------------------------------------
// Framing tests (no RFC section behind this file; the oracle is the module
// doc's crash-recovery invariants above).
// ---------------------------------------------------------------------------
fn openTmp(tmp: anytype, io: std.Io, name: []const u8) !storage.FileStorage {
const raw = try tmp.dir.createFile(io, name, .{ .read = true });
return storage.FileStorage.init(io, raw, 0);
}
fn writeRawHeader(file: storage.StorageFile, magic_bytes: []const u8, ver: u16, hk: u8, bad_crc: bool) !void {
std.debug.assert(magic_bytes.len == 4);
var buf: [header_len]u8 = [_]u8{0} ** header_len;
@memcpy(buf[0..4], magic_bytes);
std.mem.writeInt(u16, buf[4..6], ver, .little);
buf[6] = hk;
var crc = std.hash.Crc32.init();
crc.update(buf[0..60]);
var crc_val = crc.final();
if (bad_crc) crc_val ^= 1;
std.mem.writeInt(u32, buf[60..64], crc_val, .little);
try file.append(&buf);
}
test "logstore §C: header round-trips magic/version/hash_kind through open" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store1 = try LogStore.open(std.testing.allocator, file, 7);
store1.deinit(std.testing.allocator);
var raw: [header_len]u8 = undefined;
const n = try file.pread(0, &raw);
try std.testing.expectEqual(@as(usize, header_len), n);
try std.testing.expectEqualStrings(magic, raw[0..4]);
try std.testing.expectEqual(@as(u16, version), std.mem.readInt(u16, raw[4..6], .little));
try std.testing.expectEqual(@as(u8, 7), raw[6]);
for (raw[7..60]) |b| try std.testing.expectEqual(@as(u8, 0), b);
var crc = std.hash.Crc32.init();
crc.update(raw[0..60]);
try std.testing.expectEqual(crc.final(), std.mem.readInt(u32, raw[60..64], .little));
var store2 = try LogStore.open(std.testing.allocator, file, 0); // hint ignored on reopen
defer store2.deinit(std.testing.allocator);
try std.testing.expectEqual(@as(u8, 7), store2.hash_kind);
}
test "logstore: logKey + validateTail track the append-only tail in O(1)" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
// Empty log: no complete frames, end at the header boundary; validateTail
// accepts the empty key exactly at that length. logKey is defined over an
// already-scanned store, so `open` (which scans, and for an empty file just
// writes the header) is the store the key is read from.
const empty = blk: {
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const k = try store.logKey();
break :blk k;
};
try std.testing.expectEqual(@as(u64, 0), empty.frame_count);
try std.testing.expectEqual(@as(u64, header_len), empty.end);
try std.testing.expect(LogStore.validateTail(file, empty));
// Append two frames through a writable store, then read the key from a
// freshly-scanned store — the exact shape production keys on (`repocache`
// computes the LogKey over an `openReadOnly` scan, never a mid-append
// writable store, whose tail bookkeeping is only established by a scan).
{
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
_ = try store.put(&([_]u8{1} ** 32), "payload one");
_ = try store.put(&([_]u8{2} ** 32), "payload two two");
try store.commit();
}
var scan2 = try LogStore.openReadOnly(std.testing.allocator, file);
const k2 = try scan2.logKey();
scan2.deinit(std.testing.allocator);
try std.testing.expectEqual(@as(u64, 2), k2.frame_count);
try std.testing.expect(LogStore.validateTail(file, k2)); // still fresh
try std.testing.expect(!LogStore.validateTail(file, empty)); // stale (shorter) key is a miss
// Appending one frame advances the key: the old key no longer validates.
{
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
_ = try store.put(&([_]u8{3} ** 32), "third");
try store.commit();
}
try std.testing.expect(!LogStore.validateTail(file, k2)); // grown log ⇒ miss
var scan3 = try LogStore.openReadOnly(std.testing.allocator, file);
const k3 = try scan3.logKey();
scan3.deinit(std.testing.allocator);
try std.testing.expectEqual(@as(u64, 3), k3.frame_count);
try std.testing.expect(k3.end > k2.end);
try std.testing.expect(LogStore.validateTail(file, k3));
}
test "logstore §3.6/§8.3-8: resolveHexPrefix resolves a unique prefix, refuses an ambiguous one, misses an unknown" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
// Two artifacts sharing the "abcd" nibble prefix and one distinct, so
// "abcd" is ambiguous while "abcd00"/"ef" each name exactly one.
const h_shared_a = [_]u8{ 0xab, 0xcd, 0x00 } ++ [_]u8{0} ** 29; // abcd00...
const h_shared_b = [_]u8{ 0xab, 0xcd, 0x11 } ++ [_]u8{0} ** 29; // abcd11...
const h_other = [_]u8{ 0xef, 0x01 } ++ [_]u8{0} ** 30; // ef01...
{
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
_ = try store.put(&h_shared_a, "alpha");
_ = try store.put(&h_shared_b, "bravo");
_ = try store.put(&h_other, "other");
try store.commit();
}
var store = try LogStore.openReadOnly(std.testing.allocator, file);
defer store.deinit(std.testing.allocator);
// Unique 6-char prefix → the exact rid; the full name round-trips.
const rid_a = (try store.resolveHexPrefix("abcd00")).?;
try std.testing.expectEqualSlices(u8, &h_shared_a, store.hashOf(rid_a));
// A distinct 4-char prefix names the third artifact uniquely.
const rid_e = (try store.resolveHexPrefix("ef01")).?;
try std.testing.expectEqualSlices(u8, &h_other, store.hashOf(rid_e));
// Ambiguous 4-char prefix → refused, never one silently picked.
try std.testing.expectError(error.Ambiguous, store.resolveHexPrefix("abcd"));
// A prefix no artifact carries → null (the not-found family, not an error).
try std.testing.expectEqual(@as(?Rid, null), try store.resolveHexPrefix("dead"));
}
test "logstore §C: open rejects a corrupt file-header crc" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
try writeRawHeader(file, magic, version, 0, true);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: open rejects a version newer than this reader" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
try writeRawHeader(file, magic, version + 1, 0, false);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: the frame header crc covers len/flags/hash_len/hash/delta_of" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator); // header only
var buf: [64]u8 = undefined;
const frame = encodeFrame(&buf, "aaaa", "payload", @intFromEnum(FrameKind.raw), 0);
var corrupted: [64]u8 = undefined;
@memcpy(corrupted[0..frame.len], frame);
corrupted[10] ^= 0xff; // inside the hash bytes, part of the fixed header
try file.append(corrupted[0..frame.len]);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C(inv2): a genuinely torn (prefix-truncated) final frame truncates to exactly the last valid boundary" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
_ = try store.put("frame-one-hash", "payload-one");
try store.commit();
const boundary = store.size();
store.deinit(std.testing.allocator);
var buf: [96]u8 = undefined;
const frame2 = encodeFrame(&buf, "frame-two-hash", "payload-two", @intFromEnum(FrameKind.raw), 0);
try file.append(frame2[0..10]); // short of even the fixed header+crc window
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
try std.testing.expectEqual(boundary, reopened.size());
try std.testing.expect(reopened.lookup("frame-two-hash") == null);
}
test "logstore §C(inv2): a torn payload/crc tail truncates exactly" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
_ = try store.put("frame-a-hash-here", "hello-payload");
try store.commit();
const boundary = store.size();
store.deinit(std.testing.allocator);
var buf: [128]u8 = undefined;
const frame2 = encodeFrame(&buf, "frame-b-hash-value", "a longer second payload for this test", @intFromEnum(FrameKind.raw), 0);
const header_window = 22 + "frame-b-hash-value".len;
try file.append(frame2[0 .. header_window + 3]); // full header+crc landed; payload/crc badly short
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
try std.testing.expectEqual(boundary, reopened.size());
}
test "logstore §C(inv4): a bitflipped header on a durable non-final frame surfaces error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
var buf1: [64]u8 = undefined;
const f1 = encodeFrame(&buf1, "frame-one-abc", "payload-one", @intFromEnum(FrameKind.raw), 0);
var corrupted: [64]u8 = undefined;
@memcpy(corrupted[0..f1.len], f1);
corrupted[10] ^= 0xff;
try file.append(corrupted[0..f1.len]);
var buf2: [64]u8 = undefined;
const f2 = encodeFrame(&buf2, "frame-two-def", "payload-two", @intFromEnum(FrameKind.raw), 0);
try file.append(f2);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C(inv4): a bitflipped complete final frame is error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
var buf: [64]u8 = undefined;
const f = encodeFrame(&buf, "final-frame-hash", "payload-for-final", @intFromEnum(FrameKind.raw), 0);
var corrupted: [64]u8 = undefined;
@memcpy(corrupted[0..f.len], f);
corrupted[6] ^= 0xff;
try file.append(corrupted[0..f.len]); // complete, fully landed, nothing follows
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C(inv4): a corrupted payload on a complete frame surfaces error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
var buf: [96]u8 = undefined;
const f = encodeFrame(&buf, "payload-hash-value", "some payload bytes here", @intFromEnum(FrameKind.raw), 0);
var corrupted: [96]u8 = undefined;
@memcpy(corrupted[0..f.len], f);
const header_window = 22 + "payload-hash-value".len;
corrupted[header_window] ^= 0xff; // inside the payload, not the fixed header
try file.append(corrupted[0..f.len]);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C(inv4): a bitflipped hash_len that would need more bytes than the file has surfaces error.Corrupt, not a torn tail" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// hash_len=1, payload="" -> frame_len = align8(26+1+0) = 32, fully
// landed, nothing follows. Corrupting hash_len to an in-range value
// (20) makes the naive "read 18+hash_len bytes, short = torn" reading
// demand 38 bytes from a 32-byte frame -- exactly the false-torn gap
// the prefix checksum closes: it fails on this corrupted byte before
// any window-size decision is made from it.
var buf: [64]u8 = undefined;
const f = encodeFrame(&buf, "h", "", @intFromEnum(FrameKind.raw), 0);
var corrupted: [64]u8 = undefined;
@memcpy(corrupted[0..f.len], f);
corrupted[5] = 20;
try file.append(corrupted[0..f.len]);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
test "logstore §C: a crc-valid frame with an unknown flag kind (not raw/zlib/delta) is rejected on open" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// flags=4 is one past `config`: both crcs are computed correctly over it,
// so the frame is byte-intact -- it is simply an encoding this reader does
// not know, which must fail closed rather than be interpreted.
var buf: [64]u8 = undefined;
const f = encodeFrame(&buf, "unknown-flag-hash", "some payload bytes", 4, 0);
try file.append(f);
try std.testing.expectError(error.Corrupt, LogStore.open(std.testing.allocator, file, 0));
}
// ---------------------------------------------------------------------------
// zlib and delta frames (container sync §4.1; verbatim serve sync §7.2;
// delta apply delta §4).
// ---------------------------------------------------------------------------
/// Appends one hand-crafted frame (valid crcs) to `file`, for fixtures that
/// need a payload/flag/delta_of a normal put would never produce.
fn appendCrafted(gpa: std.mem.Allocator, file: storage.StorageFile, hash: []const u8, payload: []const u8, kind: FrameKind, delta_of: u64) !void {
const frame_len: usize = @intCast(frameSpan(hash.len, payload.len));
const buf = try gpa.alloc(u8, frame_len);
defer gpa.free(buf);
const f = encodeFrame(buf, hash, payload, @intFromEnum(kind), delta_of);
try file.append(f);
}
test "logstore §C: content on a malformed zlib container is error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// A crc-valid zlib frame whose payload is not a valid container: two bytes
// is short of even the 4-byte length header, so decodeContainer rejects it.
const hash = "malformed-zlib-container-hash-01";
try appendCrafted(std.testing.allocator, file, hash, "\x00\x00", .zlib, 0);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup(hash).?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
test "logstore §C/sync §4.1: a zlib container whose declared length exceeds max_payload is rejected without a huge allocation" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// Declared length 0xFFFFFFFF is far past max_payload; decodeContainer must
// reject on the declared value before inflating anything.
const hash = "oversize-declared-length-hash-01";
try appendCrafted(std.testing.allocator, file, hash, "\xff\xff\xff\xff\x78\x9c", .zlib, 0);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup(hash).?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
test "logstore §C/delta §4: content on a delta whose inflated payload is not a valid delta is error.Corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
// A valid raw base, then a crc-valid delta frame whose container inflates
// to bytes that are not a conforming delta -> delta.apply rejects -> Corrupt.
try appendCrafted(std.testing.allocator, file, "garbage-delta-base-hash-00000001", "base content here", .raw, 0);
const container = try codec.encode(std.testing.allocator, "this is not a delta at all", .default);
defer std.testing.allocator.free(container);
try appendCrafted(std.testing.allocator, file, "garbage-delta-hash-value-0000001", container, .delta, header_len);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup("garbage-delta-hash-value-0000001").?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
test "logstore §C: content on a delta whose delta_of is out of range or mid-frame is error.Corrupt" {
const cases = [_]struct { name: []const u8, delta_of: u64 }{
.{ .name = "below-header", .delta_of = 0 },
.{ .name = "forward-ref", .delta_of = 1 << 20 },
.{ .name = "mid-base-frame", .delta_of = header_len + 3 },
};
for (cases) |c| {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator);
try appendCrafted(std.testing.allocator, file, "bad-delta-of-base-hash-000000001", "base content bytes", .raw, 0);
const container = try codec.encode(std.testing.allocator, "1\ndelta-ish payload; only delta_of matters here", .default);
defer std.testing.allocator.free(container);
try appendCrafted(std.testing.allocator, file, "bad-delta-of-hash-value-00000001", container, .delta, c.delta_of);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup("bad-delta-of-hash-value-00000001").?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
}
// Seeded deterministic fuzz: decode/apply on adversarial wire bytes must
// yield a value or a declared error, never a crash or a hang.
test "logstore §C/sync §4.1 fuzz: a garbage zlib container never crashes content" {
var prng = std.Random.DefaultPrng.init(0x2172B);
const random = prng.random();
var iter: usize = 0;
while (iter < 150) : (iter += 1) {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
store.deinit(std.testing.allocator); // header only; craft the frame directly
// A crc-valid zlib frame with a random payload must never crash content.
var garbage: [64]u8 = undefined;
const gn = random.uintLessThan(usize, garbage.len + 1);
random.bytes(garbage[0..gn]);
var ghash: [9]u8 = undefined;
ghash[0] = 'g';
std.mem.writeInt(u64, ghash[1..9], iter, .little);
try appendCrafted(std.testing.allocator, file, &ghash, garbage[0..gn], .zlib, 0);
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
const grid = reopened.lookup(&ghash).?;
if (reopened.content(std.testing.allocator, grid)) |maybe| {
if (maybe) |b| std.testing.allocator.free(b);
} else |err| switch (err) {
error.OutOfMemory, error.Storage, error.Corrupt => {},
}
}
}
test "logstore §C/delta §4 fuzz: a garbage delta payload or delta_of never crashes content" {
var prng = std.Random.DefaultPrng.init(0xD317A);
const random = prng.random();
var iter: usize = 0;
while (iter < 150) : (iter += 1) {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
store.deinit(std.testing.allocator); // header only; craft the frame directly
// A crc-valid delta frame with a random container payload and a random
// delta_of must never crash content (Corrupt/OOM/value only).
var garbage: [64]u8 = undefined;
const gn = random.uintLessThan(usize, garbage.len + 1);
random.bytes(garbage[0..gn]);
var ghash: [9]u8 = undefined;
ghash[0] = 'x';
std.mem.writeInt(u64, ghash[1..9], iter, .little);
try appendCrafted(std.testing.allocator, file, &ghash, garbage[0..gn], .delta, random.int(u64));
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
const grid = reopened.lookup(&ghash).?;
if (reopened.content(std.testing.allocator, grid)) |maybe| {
if (maybe) |b| std.testing.allocator.free(b);
} else |err| switch (err) {
error.OutOfMemory, error.Storage, error.Corrupt => {},
}
}
}
test "logstore §C: a config frame is not interned — rids stay content-only and contiguous" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const r1 = try store.put("config-interleave-hash-one-0001", "one");
try store.putConfig(.{ .hash_policy = 0, .default_branch = "trunk", .project_id = "pid" });
const r2 = try store.put("config-interleave-hash-two-0001", "two");
// The config frame consumed a byte range but no rid: content rids stay
// dense and contiguous, and the sentinel hash is never an index key.
try std.testing.expectEqual(r1 + 1, r2);
try std.testing.expect(store.lookup(config_sentinel) == null);
}
test "logstore §C: a delta base offset pointing at a config frame is rejected as corrupt" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var seed = try LogStore.open(std.testing.allocator, file, 0);
seed.deinit(std.testing.allocator); // header only
// A config frame at header_len, then a delta whose (valid) container base
// offset points back at it: resolving the base hits the .config arm.
var cbuf: [max_config_payload]u8 = undefined;
const cfg_payload = encodeConfig(&cbuf, .{ .hash_policy = 4, .default_branch = "trunk", .project_id = "pid" });
try appendCrafted(std.testing.allocator, file, config_sentinel, cfg_payload, .config, 0);
const container = try codec.encode(std.testing.allocator, "delta-ish bytes; only the base offset matters here", .default);
defer std.testing.allocator.free(container);
try appendCrafted(std.testing.allocator, file, "delta-over-config-base-hash-0001", container, .delta, header_len);
var store = try LogStore.open(std.testing.allocator, file, 0);
defer store.deinit(std.testing.allocator);
const rid = store.lookup("delta-over-config-base-hash-0001").?;
try std.testing.expectError(error.Corrupt, store.content(std.testing.allocator, rid));
}
test "logstore §E: parseConfig rejects malformed config payloads" {
var over: [3]u8 = undefined;
over[0] = 0; // hash_policy
std.mem.writeInt(u16, over[1..3], @intCast(max_config_str + 1), .little); // branch_len > cap
const cases = [_][]const u8{
&[_]u8{}, // (a) empty: no hash_policy byte
&[_]u8{ 0x07, 0x05 }, // (b) truncated branch_len (1 of 2 bytes)
&over, // (c) branch_len exceeds max_config_str
&[_]u8{ 0x00, 0x05, 0x00, 'a', 'b' }, // (d) branch_len 5 exceeds remaining
&[_]u8{ 0x00, 0x01, 0x00, 'b', 0x02 }, // (e) truncated pid_len after branch
&[_]u8{ 0x00, 0x01, 0x00, 'b', 0x00, 0x00, 0x09 }, // (f) trailing byte after pid
};
for (cases) |c| try std.testing.expectError(error.Corrupt, parseConfig(c));
}
test "logstore §E/§6.3: parseConfig forward-compat-decodes a legacy pre-credential frame" {
// A frame written before the login credential block existed carries only
// the three string fields (branch, project_id, project_code) and ends
// exactly after project_code — encodeConfig can no longer emit this shape
// (it always writes the two trailing length words), so the bytes are hand-
// built. parseConfig's forward-compat arm must accept it with the credential
// fields defaulting empty, never as corruption.
const legacy = [_]u8{
0x04, // hash_policy
0x05, 0x00, 't', 'r', 'u', 'n', 'k', // branch_len=5, "trunk"
0x03, 0x00, 'p', 'i', 'd', // pid_len=3, "pid"
0x05, 0x00, 'c', 'o', 'd', 'e', '9', // pcode_len=5, "code9"
};
const cfg = try parseConfig(&legacy);
try std.testing.expectEqual(@as(u8, 4), cfg.hash_policy);
try std.testing.expectEqualStrings("trunk", cfg.default_branch);
try std.testing.expectEqualStrings("pid", cfg.project_id);
try std.testing.expectEqualStrings("code9", cfg.project_code);
try std.testing.expectEqual(@as(usize, 0), cfg.login_user.len);
try std.testing.expectEqual(@as(usize, 0), cfg.login_secret_hex.len);
try std.testing.expectEqual(@as(usize, 0), cfg.remote_url.len);
}
test "config logstore §E: encodeConfig/parseConfig round-trips remote_url" {
// sync §4/§6 — the saved remote is the sixth trailing block; encoding a
// full config and decoding it must recover the URL verbatim with every
// other field intact (the fixed §E field order is a lossless round-trip).
var buf: [max_config_payload]u8 = undefined;
const wire = encodeConfig(&buf, .{
.hash_policy = 7,
.default_branch = "trunk",
.project_id = "pid7",
.project_code = "code7",
.login_user = "alice",
.login_secret_hex = "0123456789abcdef0123456789abcdef01234567", // 40 hex
.remote_url = "http://h.example/xfer",
});
const cfg = try parseConfig(wire);
try std.testing.expectEqual(@as(u8, 7), cfg.hash_policy);
try std.testing.expectEqualStrings("trunk", cfg.default_branch);
try std.testing.expectEqualStrings("pid7", cfg.project_id);
try std.testing.expectEqualStrings("code7", cfg.project_code);
try std.testing.expectEqualStrings("alice", cfg.login_user);
try std.testing.expectEqualStrings("0123456789abcdef0123456789abcdef01234567", cfg.login_secret_hex);
try std.testing.expectEqualStrings("http://h.example/xfer", cfg.remote_url);
}
test "config logstore §E: a pre-remote frame decodes with empty remote_url" {
// A frame written before the remote-URL block existed carries the five
// pre-URL fields (branch, pid, pcode, login, secret) and ends exactly after
// the secret — encodeConfig can no longer emit this shape (it always writes
// the URL length word), so the bytes are hand-built. parseConfig's
// forward-compat arm must accept it with remote_url empty, never Corrupt.
const pre_remote = [_]u8{
0x04, // hash_policy
0x05, 0x00, 't', 'r', 'u', 'n', 'k', // branch_len=5
0x03, 0x00, 'p', 'i', 'd', // pid_len=3
0x05, 0x00, 'c', 'o', 'd', 'e', '9', // pcode_len=5
0x03, 0x00, 'b', 'o', 'b', // login_len=3
0x02, 0x00, 'a', 'b', // secret_len=2 (parseConfig caps, does not fix at 40)
};
const cfg = try parseConfig(&pre_remote);
try std.testing.expectEqualStrings("bob", cfg.login_user);
try std.testing.expectEqualStrings("ab", cfg.login_secret_hex);
try std.testing.expectEqual(@as(usize, 0), cfg.remote_url.len);
}
test "config submanifests §6.3/§8.1: the two settings round-trip through encodeConfig/parseConfig" {
var buf: [max_config_payload]u8 = undefined;
const cfg: Config = .{
.hash_policy = 4,
.default_branch = "trunk",
.project_id = "pid",
.project_code = "code9",
.seen_submanifest = true,
.enable_submanifests = true,
.submanifest_k = 3, // the A.5 conformance vector's value
};
const back = try parseConfig(encodeConfig(&buf, cfg));
try std.testing.expect(back.seen_submanifest);
try std.testing.expect(back.enable_submanifests);
try std.testing.expectEqual(@as(u8, 3), back.submanifest_k);
// Every neighbouring field survives the widened trailing block.
try std.testing.expectEqualStrings("trunk", back.default_branch);
try std.testing.expectEqualStrings("code9", back.project_code);
}
test "config submanifests §8.1/§6.4: the defaults are off, unpinned, and manual, and survive a round trip" {
// §8.1 "a project-scope boolean, default false", §6.3's "no pinned
// parameter" state (which §8.1 forbids enabling emission on), and §6.4's
// auto-retune setting default false (erratum
// submanifests-00-s6.4-opt-in-auto-retune). A Config built with none of
// the three fields named must decode back to exactly that.
var buf: [max_config_payload]u8 = undefined;
const cfg: Config = .{ .hash_policy = 4, .default_branch = "trunk", .project_id = "pid" };
try std.testing.expect(!cfg.enable_submanifests);
try std.testing.expectEqual(@as(u8, 0), cfg.submanifest_k);
try std.testing.expect(!cfg.submanifest_auto_retune);
const back = try parseConfig(encodeConfig(&buf, cfg));
try std.testing.expect(!back.enable_submanifests);
try std.testing.expectEqual(@as(u8, 0), back.submanifest_k);
try std.testing.expect(!back.submanifest_auto_retune);
}
test "config submanifests §6.4: auto-retune round-trips through encodeConfig/parseConfig" {
var buf: [max_config_payload]u8 = undefined;
const cfg: Config = .{
.hash_policy = 4,
.default_branch = "trunk",
.project_id = "pid",
.enable_submanifests = true,
.submanifest_k = 3,
.submanifest_auto_retune = true,
};
const back = try parseConfig(encodeConfig(&buf, cfg));
try std.testing.expect(back.submanifest_auto_retune);
// Every neighbouring field survives the widened trailing block.
try std.testing.expect(back.enable_submanifests);
try std.testing.expectEqual(@as(u8, 3), back.submanifest_k);
}
test "config submanifests §6.4: a pre-auto-retune (3-byte-tail) frame decodes with auto-retune off" {
// A frame written by a writer before the auto-retune slice existed ends at
// the 3-byte tail (seen + enable + k); the fourth field must default false
// on decode, never Corrupt, and never a spurious "enabled".
var buf: [max_config_payload]u8 = undefined;
const cfg: Config = .{
.hash_policy = 4,
.default_branch = "trunk",
.project_id = "pid",
.enable_submanifests = true,
.submanifest_k = 5,
};
const full = encodeConfig(&buf, cfg);
const back = try parseConfig(full[0 .. full.len - 1]);
try std.testing.expect(back.enable_submanifests);
try std.testing.expectEqual(@as(u8, 5), back.submanifest_k);
try std.testing.expect(!back.submanifest_auto_retune);
}
test "config submanifests §6.3: an out-of-range k is stored verbatim, never repaired by the storage layer" {
// §6.3 puts the out-of-range fallback on the WRITER ("it MUST fall back to
// emitting flat manifests rather than substitute a parameter of its own").
// Storage clamping or zeroing the value here would take that decision away
// and turn a misconfiguration into a silent, different misconfiguration.
var buf: [max_config_payload]u8 = undefined;
for ([_]u8{ 13, 200, 255 }) |k| {
const cfg: Config = .{
.hash_policy = 4,
.default_branch = "trunk",
.project_id = "pid",
.enable_submanifests = true,
.submanifest_k = k,
};
const back = try parseConfig(encodeConfig(&buf, cfg));
try std.testing.expectEqual(k, back.submanifest_k);
}
}
test "config submanifests §8.1: a seen-indicator-only frame decodes with the settings off" {
// The frame shape the slice that introduced `seen_submanifest` wrote: a
// ONE-byte trailing block. A repository carrying one must still open, with
// the two settings this slice adds reading their defaults — never Corrupt,
// and never a spurious "enabled".
const one_byte_tail = [_]u8{
0x04, // hash_policy
0x05,
0x00,
't',
'r',
'u',
'n',
'k',
0x03,
0x00,
'p',
'i',
'd',
0x05,
0x00,
'c',
'o',
'd',
'e',
'9',
0x03,
0x00,
'b',
'o',
'b',
0x02,
0x00,
'a',
'b',
0x00, 0x00, // url_len = 0
0x01, // seen_submanifest = true, and nothing after it
};
const cfg = try parseConfig(&one_byte_tail);
try std.testing.expect(cfg.seen_submanifest);
try std.testing.expect(!cfg.enable_submanifests);
try std.testing.expectEqual(@as(u8, 0), cfg.submanifest_k);
}
test "config submanifests: legacy settings tails and fixed settled digest are unambiguous" {
// A reader accepts the three prior wire shapes plus exactly one new,
// fixed-width digest shape. Every neighboring width remains fail-closed:
// guessing would either suppress a required scan or treat an unknown
// future field as a settled analysis.
const prefix = [_]u8{
0x04,
0x05,
0x00,
't',
'r',
'u',
'n',
'k',
0x03,
0x00,
'p',
'i',
'd',
0x05,
0x00,
'c',
'o',
'd',
'e',
'9',
0x03,
0x00,
'b',
'o',
'b',
0x02,
0x00,
'a',
'b',
0x00, 0x00, // url_len = 0
};
for ([_]usize{ 2, 5, 6, 35, 37 }) |width| {
var payload: [80]u8 = undefined;
@memcpy(payload[0..prefix.len], &prefix);
@memset(payload[prefix.len..][0..width], 1);
try std.testing.expectError(error.Corrupt, parseConfig(payload[0 .. prefix.len + width]));
}
// …and every legacy legal width still parses.
var ok1: [64]u8 = undefined;
@memcpy(ok1[0..prefix.len], &prefix);
@memset(ok1[prefix.len..][0..1], 1);
_ = try parseConfig(ok1[0 .. prefix.len + 1]);
var ok3: [64]u8 = undefined;
@memcpy(ok3[0..prefix.len], &prefix);
@memset(ok3[prefix.len..][0..3], 1);
_ = try parseConfig(ok3[0 .. prefix.len + 3]);
var ok4: [64]u8 = undefined;
@memcpy(ok4[0..prefix.len], &prefix);
@memset(ok4[prefix.len..][0..4], 1);
const back4 = try parseConfig(ok4[0 .. prefix.len + 4]);
try std.testing.expect(back4.submanifest_auto_retune);
var settled: [80]u8 = undefined;
@memcpy(settled[0..prefix.len], &prefix);
@memset(settled[prefix.len..][0..36], 1);
const parsed = try parseConfig(settled[0 .. prefix.len + 36]);
const digest = parsed.submanifest_settled_cost_shape orelse return error.TestUnexpectedResult;
try std.testing.expectEqual(@as(usize, 32), digest.len);
}
test "config logstore §E: remote_url over max_config_str is rejected" {
// A url_len declaring more than the per-field cap is corruption: the same
// guard every other length field carries (parity, not a special case).
const pre_remote = [_]u8{
0x04, // hash_policy
0x05, 0x00, 't', 'r', 'u', 'n', 'k', // branch
0x03, 0x00, 'p', 'i', 'd', // pid
0x05, 0x00, 'c', 'o', 'd', 'e', '9', // pcode
0x03, 0x00, 'b', 'o', 'b', // login
0x02, 0x00, 'a', 'b', // secret
};
// url_len = max_config_str + 1 = 256 = 0x0100 (LE 0x00, 0x01).
const oversized = pre_remote ++ [_]u8{ 0x00, 0x01 };
try std.testing.expectError(error.Corrupt, parseConfig(&oversized));
}
test "logstore §B: a torn config-frame write reverts to the prior committed config" {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const io = std.testing.io;
var fs = try openTmp(&tmp, io, "log.bin");
defer fs.file.close(io);
const file = fs.storageFile();
var store = try LogStore.open(std.testing.allocator, file, 0);
try store.putConfig(.{ .hash_policy = 9, .default_branch = "trunk", .project_id = "keep" });
try store.commit();
store.deinit(std.testing.allocator);
// Append a torn config B frame (prefix only): scan truncates it away, so
// the surviving latest config is still A.
var pbuf: [max_config_payload]u8 = undefined;
const cfgB = encodeConfig(&pbuf, .{ .hash_policy = 1, .default_branch = "x", .project_id = "y" });
var fbuf: [64]u8 = undefined;
const frameB = encodeFrame(&fbuf, config_sentinel, cfgB, @intFromEnum(FrameKind.config), 0);
try file.append(frameB[0..10]); // torn: short of the fixed header+crc window
var reopened = try LogStore.open(std.testing.allocator, file, 0);
defer reopened.deinit(std.testing.allocator);
var oc = (try reopened.readConfig(std.testing.allocator)).?;
defer oc.deinit(std.testing.allocator);
try std.testing.expectEqual(@as(u8, 9), oc.config.hash_policy);
try std.testing.expectEqualStrings("trunk", oc.config.default_branch);
try std.testing.expectEqualStrings("keep", oc.config.project_id);
}
// Seeded deterministic fuzz: parseConfig on adversarial wire bytes yields a
// value or error.Corrupt, never a crash.
test "logstore §E fuzz: parseConfig never crashes on adversarial payloads" {
var prng = std.Random.DefaultPrng.init(0xC04F16);
const random = prng.random();
var iter: usize = 0;
while (iter < 400) : (iter += 1) {
var garbage: [64]u8 = undefined;
const gn = random.uintAtMost(usize, garbage.len);
random.bytes(garbage[0..gn]);
if (parseConfig(garbage[0..gn])) |_| {} else |err| switch (err) {
error.Corrupt => {},
}
}
}
test "logstore §5: deltaProfitable is the exact <75% integer boundary" {
// 75/100 is 75% exactly → NOT strictly under → reject; 74/100 accepts.
try std.testing.expect(!deltaProfitable(75, 100));
try std.testing.expect(deltaProfitable(74, 100));
// Small-integer edges, where a rounded/float ratio would smear the line:
// 3/4 == 75% rejects; 2/3 < 75% accepts; 1/1 (4 < 3) rejects.
try std.testing.expect(!deltaProfitable(3, 4));
try std.testing.expect(deltaProfitable(2, 3));
try std.testing.expect(!deltaProfitable(1, 1));
// Degenerate ends: a zero-length delta beats any non-empty target; an
// empty target admits no profitable delta.
try std.testing.expect(deltaProfitable(0, 1));
try std.testing.expect(!deltaProfitable(0, 0));
}