//! Receive-side persistence: writing what a sync round received into the
//! content log in one ordered pass (deltas after their bases, sub-manifest
//! references phantomized, shunned names refused), and keeping the cluster
//! index sidecar beside the store current. No CLI: a pull, a clone and a
//! serving push all persist through here.
const std = @import("std");
const PersistError = @import("repo_config.zig").PersistError;
const PersistLockedError = @import("repo_config.zig").PersistLockedError;
const SparseClassification = @import("../repo/sparse.zig").SparseClassification;
const artifact = @import("../artifact/artifact.zig");
const clusterindex = @import("../sync/clusterindex.zig");
const container_mod = @import("../store/container.zig");
const decodeWireName = @import("digest_name.zig").decodeWireName;
const delta = @import("../artifact/delta.zig");
const digest_len = @import("digest_name.zig").digest_len;
const digest_name = @import("digest_name.zig");
const logstore = @import("../store/logstore.zig");
const repocache = @import("../repo/repocache.zig");
const repostate = @import("../repo/repostate.zig");
const storage = @import("../store/storage.zig");
const store_open = @import("store_open.zig");
const sync = @import("../sync/sync.zig");
const updateConfigOpen = @import("repo_config.zig").updateConfigOpen;
/// The disposition of one artifact in `persistStore`'s ordered pass.
const PersistOutcome = union(enum) {
/// A non-phantom frame for this artifact now exists in the log — written
/// here, or already present from a prior cascade / a pull re-persist.
persisted,
/// A reverse-delta whose basis is a content-bearing store entry not YET
/// written to the log. It is deferred on the basis wire-hash and re-driven
/// once that basis lands, so its `delta_of` offset points strictly backward.
deferred: []const u8,
/// Horizon draft section 5.2.3 -- a seed that arrived outside a
/// horizon-seed announcement. Nothing is written. Its bytes stay in the
/// store, so a delta that names it as basis lands in full form at the
/// leftover drain, as one on an elided basis does.
dropped,
};
/// What one `persistStore` sweep did: the artifacts it made durable, and the
/// sub-manifest references (draft-fossil-submanifests-00 §7.1) it phantomized
/// and could not resolve from this repository's own holdings.
pub const PersistSummary = struct {
/// Content-bearing artifacts in the store — the count every caller reports.
written: usize,
/// §7.1 — distinct S references still standing as phantoms when the sweep
/// ended: this repository holds an artifact whose tree closure is open.
/// Zero for every repository that carries no S card, which is every
/// repository predating this draft.
submanifest_gaps: usize,
/// Sparse-cone RFC section 6.3 -- artifacts deliberately not persisted
/// under the clone's cone. Zero for every dense clone.
elided: usize = 0,
/// The part of `elided` that arrived and was dropped here rather than
/// withheld by the origin.
filtered: usize = 0,
};
/// draft-fossil-submanifests-00 §7.1 — the S references one persist sweep saw.
///
/// Rids, not names: `LogStore` entries are append-only, so an interned rid stays
/// valid for the sweep, and whether a reference is still a GAP can only be
/// decided once the sweep has ended — a sub-manifest may be persisted after the
/// tree manifest that names it, and `putZlib` upgrades the phantom minted for it
/// in place. Deciding per S card instead would report a gap for every reference
/// whose artifact happens to sit later in rid order.
pub const SubManifestRefs = struct {
rids: std.AutoArrayHashMapUnmanaged(logstore.Rid, void) = .empty,
/// Delta-dialect memory: this sweep stored a B-card manifest, so the
/// `seen_delta_manifest` config indicator is written alongside `seen`'s
/// on the same false->true edge discipline.
seen_delta: bool = false,
/// submanifests §8.1 — set the first time this sweep stores an artifact
/// inside §7.2's gated set: "When the first S-bearing artifact is stored, an
/// implementation SHOULD record a local seen-submanifest indicator". Both
/// §7.2 shapes count, leaf sub-manifests included — see
/// `phantomizeSubManifestRefs` for why the literal S-card-only reading would
/// leave the gate blind to exactly the artifacts that carry a tree's files.
/// It is also the durable gate on the §7.1 phantom REBUILD (see
/// `persistStore`), which is why it tracks artifacts seen, not gaps left.
seen: bool = false,
pub fn deinit(self: *SubManifestRefs, gpa: std.mem.Allocator) void {
self.rids.deinit(gpa);
self.* = undefined;
}
/// Distinct references that never became a content-bearing frame, split
/// by `sparse` into gaps and elided nodes (all gaps for a dense store).
fn unresolved(
self: *const SubManifestRefs,
log: *const logstore.LogStore,
sparse: ?*const SparseClassification,
) struct { gaps: usize, elided: usize } {
var gaps: usize = 0;
var elided: usize = 0;
for (self.rids.keys()) |rid| {
if (!log.isPhantom(rid)) continue;
if (sparse) |c| {
var name_buf: [2 * digest_len]u8 = undefined;
if (c.elidedNode(digest_name.digestToWireName(log.hashOf(rid), &name_buf))) {
elided += 1;
continue;
}
}
gaps += 1;
}
std.debug.assert(gaps + elided <= self.rids.count());
return .{ .gaps = gaps, .elided = elided };
}
};
/// draft-fossil-submanifests-00 §7.1 — "Storing a structural artifact that
/// carries S cards whose referenced sub-manifests are absent MUST create a
/// phantom for each missing reference, with the referencing artifact's privacy
/// classification, exactly as an absent delta source creates a source phantom."
///
/// This is §7.1's SECOND storing boundary. The receive path
/// (`sync.Client.absorbSubManifestRefs`) covers artifacts a pull absorbs; it
/// cannot cover a clone, which stores every artifact wire-backed and never
/// inflates one (so it has no plaintext to scan), nor a server-side push
/// ingest, which absorbs nothing at all. Both of those reach the durable store
/// only HERE — and here the plaintext is already in hand, reconstructed for the
/// §7.7 verify, so the whole duty costs one `sBearingCandidate` scan of bytes
/// this function was handed anyway. The clone's O(wire) receive bound is
/// untouched: no container is inflated that §7.7 was not already inflating.
///
/// A clone from a peer whose OWN closure is incomplete is exactly the case the
/// receive path cannot see: the cursor delivers every artifact the peer holds,
/// and the peer does not hold the sub-manifests. Without this, such a clone
/// records its gap nowhere and never asks for the leaves.
///
/// Privacy: zit's phantom is an interned entry with no content and carries no
/// per-entry privacy at either boundary, so an S-reference phantom is minted
/// exactly as an absent delta source's is — which is what §7.1 asks for.
fn phantomizeSubManifestRefs(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
content: []const u8,
seen: *SubManifestRefs,
) PersistError!void {
// §8.1's indicator, over BOTH §7.2 shapes rather than over S cards alone.
//
// §8.1 says "when the first S-bearing artifact is stored"; a LEAF
// sub-manifest carries F cards and no S card (§4.4), so the literal reading
// records nothing for a repository holding leaves. That reading cannot be
// right: this indicator is what arms the §7.2 transfer gate, §7.2 gates
// leaves by TYPE, and §8's whole reason for the transfer layer is that a
// pre-S reader can only hold a leaf as "unreadable ... opaque content". A
// repository holding any part of a tree is one such a reader cannot fully
// read, and a repository that holds a tree's leaves is exactly that. So the
// indicator answers `transferGatedArtifact`.
//
// Cheap where it runs: the predicate rejects any byte string whose first two
// bytes are not a card letter followed by SP or LF — every file blob, in two
// comparisons. A structural artifact pays one allocation-free walk of its
// card lines, beside the full-artifact rehash §7.7 already charges here.
if (!seen.seen and repostate.transferGatedArtifact(content)) seen.seen = true;
// The delta-dialect twin: remember that this repository stores delta
// manifests, so `zit commit` keeps the dialect (config `seen_delta`).
if (!seen.seen_delta and repostate.deltaManifestArtifact(content)) seen.seen_delta = true;
var refs: std.ArrayListUnmanaged([]const u8) = .empty;
defer refs.deinit(gpa);
// Validate-then-apply lives in the shared extractor: a malformed card run
// mints nothing, so a half-read artifact never leaves a half-minted phantom
// set. `content` outlives `refs`, which borrows its hash tokens.
if (!try repostate.collectSubManifestRefs(gpa, content, &refs)) return;
for (refs.items) |name| {
var buf: [digest_len]u8 = undefined;
// §3.5 admits a 40- or 64-hex name and `decodeWireName` decodes both; a
// name too wide for this repository's digest cannot key a frame here, so
// it is a corrupt reference rather than a gap.
const digest = decodeWireName(name, &buf) orelse return error.Corrupt;
const rid = log.intern(digest) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
};
try seen.rids.put(gpa, rid, {});
}
}
/// Persist one artifact `rid` into `log`, choosing its stored form, and verify
/// §7.7 integrity (the reconstructed content MUST hash to its wire name) BEFORE
/// any durable write. A retained delta whose basis is already a held log frame
/// under the read depth bound is stored VERBATIM as a `.delta`; a retained delta
/// whose basis is a not-yet-written content-bearing store entry is DEFERRED
/// (when `allow_defer`) so the basis frame is written first (reverse-delta
/// ordering); every other case — no wire form, a full container, or an
/// absent / phantom / over-deep basis — lands as the full form, never a
/// phantom-based or unreadable delta. Idempotent: an artifact already holding a
/// non-phantom frame returns `.persisted` without a re-write or re-verification
/// (§7.7 was proven when it was first persisted).
/// Record a cluster's wire name as it is written (see
/// `persistArtifact`'s `clusters_out`). The dupe is owned by the list.
fn noteCluster(gpa: std.mem.Allocator, clusters_out: ?*std.ArrayListUnmanaged([]const u8), name: []const u8, content: []const u8) error{OutOfMemory}!void {
const list = clusters_out orelse return;
if (!clusterindex.isCluster(content)) return;
const owned = try gpa.dupe(u8, name);
errdefer gpa.free(owned);
try list.append(gpa, owned);
}
/// Horizon draft section 5.2.3 -- whether verified plaintext is a seed: one
/// carrying this store's seed label or a binding card. A seed the origin
/// announced never reaches the store, so any seed found here arrived some
/// other way. Only a control artifact can be one, and control artifacts open
/// with their D card; nothing else is parsed.
fn receivedSeed(gpa: std.mem.Allocator, content: []const u8) error{OutOfMemory}!bool {
if (!std.mem.startsWith(u8, content, "D ")) return false;
var im = artifact.parseForImport(gpa, content) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return false,
};
defer im.deinit(gpa);
return sync.isSeed(&im.manifest);
}
/// Store a received artifact that arrived whole (its §7.7-verified
/// plaintext `content`, and its wire `container` when it came as one).
/// Without `rebase`, the clone shape: the container verbatim, else a zlib
/// frame. With it, as the sender's own commit would have: a delta when one
/// clears the 75% bar, through `putArtifactPreferring` with a check-in's first
/// P-card parent as the hint (a skip base would need the receiver's heights,
/// which the persist does not hold) and pickBase's recent window otherwise.
/// So a manifest received whole, from zit or a stock Fossil sender, lands as
/// a small delta instead of a full copy.
///
/// Cost on a server taking a large push: at most 1 + `max_base_candidates`
/// (6) `delta.create` runs per artifact received whole, each linear in the
/// artifact and its base, plus one zlib encode; a file blob is offered no
/// hint, still pays up to `max_base_candidates` runs against unrelated
/// recent frames, and usually stays whole. The parent lookup is
/// one scan of a structural artifact's bytes; a blob is rejected in two.
fn storeReceivedWhole(
log: *logstore.LogStore,
digest: []const u8,
content: []const u8,
container: ?[]const u8,
rebase: bool,
) error{ OutOfMemory, Storage }!logstore.Rid {
if (!rebase) {
if (container) |c| return log.putContainer(digest, c);
return log.putZlib(digest, content);
}
const hint: ?logstore.Rid = if (primaryParentName(content)) |parent| blk: {
var pbuf: [digest_len]u8 = undefined;
const pdigest = decodeWireName(parent, &pbuf) orelse break :blk null;
break :blk log.lookup(pdigest);
} else null;
return log.putArtifactPreferring(digest, content, hint);
}
/// The store name of `content`'s first P-card parent when that parent arrived
/// in this same exchange and has no frame yet, or null. A received check-in
/// defers on it (the reverse-delta waiting set) so the parent is written
/// first and is there as its delta base: arrival order puts children ahead
/// of parents often enough, and the recent window then holds no manifest.
/// Called before §7.7 and every side effect, so a re-driven artifact repeats
/// nothing; the name is the store's own, which outlives `content`.
fn parentInFlight(store: repostate.ArtifactStore, log: *logstore.LogStore, content: []const u8) ?[]const u8 {
const parent = primaryParentName(content) orelse return null;
var pbuf: [digest_len]u8 = undefined;
const pdigest = decodeWireName(parent, &pbuf) orelse return null;
if (log.lookup(pdigest)) |r| {
if (!log.isPhantom(r)) return null;
}
const srid = store.lookup(parent) orelse return null;
if (!store.held(srid)) return null;
return store.hashOf(srid);
}
/// The first hash on a structural artifact's P card, or null. Cards come in
/// letter order and every card value is encoded, so the P card is the one
/// line that starts "P ". A blob that happens to look like cards yields at
/// worst a wrong hint, which the delta's own profitability check discards.
pub fn primaryParentName(content: []const u8) ?[]const u8 {
if (content.len < 2 or !artifact.isCardLetter(content[0])) return null;
if (content[1] != ' ' and content[1] != '\n') return null;
const at = if (std.mem.startsWith(u8, content, "P "))
2
else
(std.mem.indexOf(u8, content, "\nP ") orelse return null) + 3;
const rest = content[at..];
const end = std.mem.indexOfAny(u8, rest, " \n") orelse return null;
return rest[0..end];
}
/// A delta stored as it arrived, before its plaintext was rebuilt: its
/// basis was not in the memo, and rebuilding it there would walk the basis's
/// chain once per member. `PersistPass.verifyDeferred` rebuilds and checks
/// every one, a chain at a time, before the pass is committed.
const UnverifiedDelta = struct { rid: logstore.Rid, base: logstore.Rid, name: []const u8 };
pub fn persistArtifact(
gpa: std.mem.Allocator,
store: repostate.ArtifactStore,
log: *logstore.LogStore,
rid: repostate.Rid,
allow_defer: bool,
/// Streaming persist: defer even on a basis the store has not
/// SEEN yet — mid-exchange, an unseen basis is usually still in flight, so
/// falling back to a full frame here would write different bytes than the
/// converged pass. The final leftover drain runs with `allow_defer` off,
/// which restores the full-form fallback for a basis that truly never
/// arrived. Always false on the batched paths, whose store IS converged.
defer_unseen: bool,
/// A bounded plaintext memo shared across `persistStore`'s whole sweep. Each
/// artifact reconstructs its own plaintext exactly once here — for the §7.7
/// verify — and primes it under its just-written frame's offset, so a later
/// reverse-delta naming this frame as basis reuses that plaintext instead of
/// re-walking (re-inflating, re-applying) the chain. Bounded (LRU, fixed byte
/// + entry caps): a cache miss just costs a fresh reconstruction, never a
/// wrong answer, so the stored bytes are identical with or without it.
cache: *logstore.PlaintextCache,
/// submanifests §7.1 — the sweep-wide S-reference record. Every plaintext
/// this function reconstructs for the §7.7 verify is offered to it, so the
/// phantom duty attaches to STORING and not to any one arrival path.
srefs: *SubManifestRefs,
/// submanifests §7.1/§8.1 — REBUILD the S-reference phantom set for an
/// artifact that is ALREADY a durable frame. A phantom is an interned entry
/// with no frame (`LogStore.intern`: "Phantoms never reach disk"), so the
/// §7.1 phantoms this repository minted on a previous run are gone the
/// moment the process exits; the gap has to be re-derived from the durable
/// artifacts that name it. Without this an open tree closes itself on the
/// next run: the tree manifest is already persisted, the early return skips
/// its scan, and the repository reports converged while it still cannot
/// enumerate that check-in. Reached by a sweep over the whole store; sync
/// and pull, which persist only what arrived, report gaps through
/// `subManifestGaps` instead.
///
/// Passed as a parameter rather than read here because the decision is the
/// repository's, not this artifact's: §8.1's durable seen-submanifest
/// indicator is false for every repository predating this draft, so the
/// rescan — and its one inflate per already-persisted artifact — is
/// unreachable for all of them and the persist sweep is what it always was.
rescan_persisted: bool,
/// Store an artifact that arrived whole through `storeReceivedWhole`
/// rather than verbatim; false on the clone paths.
rebase_whole: bool,
/// Where to record the wire name of every cluster this call
/// writes, so the persist keeps the cluster index warm without a rescan.
clusters_out: ?*std.ArrayListUnmanaged([]const u8),
/// When set, a delta whose basis plaintext is not memoised is stored as it
/// arrived and recorded here, to be checked before the commit; null
/// rebuilds and checks it now.
unverified: ?*std.ArrayListUnmanaged(UnverifiedDelta),
) PersistError!PersistOutcome {
const name = store.hashOf(rid);
var digest_buf: [digest_len]u8 = undefined;
const digest = decodeWireName(name, &digest_buf) orelse return error.Corrupt;
if (log.lookup(digest)) |ex| {
if (!log.isPhantom(ex)) {
if (rescan_persisted) {
// Read through the sweep's own memo, so an artifact this sweep
// already reconstructed costs nothing to rescan.
const plain = (log.contentCached(gpa, ex, cache) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
}) orelse return .persisted;
defer gpa.free(plain);
try phantomizeSubManifestRefs(gpa, log, plain, srefs);
}
return .persisted;
}
}
// A genuine phantom carries nothing durable — skip it (never a drop of held
// content). `held` classifies without inflating a wire-backed entry.
if (!store.held(rid)) return .persisted;
// §7.7 L1492/L1496-1498 — the repository transaction-commit hook. Recompute
// the artifact hash over the reconstructed bytes and abort BEFORE any frame is
// written on a mismatch (a lying container inflates but does not name its
// content). The verify plaintext is reconstructed against the LOG basis (a
// read bounded by `max_delta_chain`) — NOT the in-memory wire chain, which,
// now a clone stores every artifact wire-backed, is the full reverse-delta
// history and would exceed the read depth bound. `persistStore` writes basis
// frames before the deltas that name them (reverse-delta order), so a delta's
// basis is already a log frame here.
if (store.wire(rid)) |wf| {
if (wf.delta) |d| {
var base_buf: [digest_len]u8 = undefined;
const base_digest = decodeWireName(d.src_hash, &base_buf) orelse return error.Corrupt;
const maybe_base = log.lookup(base_digest);
if (maybe_base == null or log.isPhantom(maybe_base.?)) {
// The basis is not yet a log frame. Defer on it (reverse-delta
// order: the newer basis frame must be written first) when it is a
// held store entry that WILL be persisted.
if (allow_defer) {
if (store.lookup(d.src_hash)) |bs_rid| {
if (store.held(bs_rid)) return .{ .deferred = d.src_hash };
}
if (defer_unseen) return .{ .deferred = d.src_hash };
}
// Basis absent and not deferrable (§3.3): fall back to FULL form
// from the entry's OWN plaintext — a put+retain entry carries it
// directly (never a phantom-based delta). In a converged clone this
// is unreachable (every delta's basis is a written frame, handled
// below); a wire-delta entry whose basis truly never arrived cannot
// reconstruct — a hard error, never a fabricated artifact.
const content = store.content(rid) orelse return error.Corrupt;
if (!artifact.verify(name, content)) return error.Corrupt;
if (try receivedSeed(gpa, content)) return .dropped;
// submanifests §7.1 — the plaintext §7.7 just verified is the same
// bytes the phantom duty is stated over; scan it here rather than
// forcing a second inflate anywhere else.
try phantomizeSubManifestRefs(gpa, log, content, srefs);
try noteCluster(gpa, clusters_out, name, content);
const new_rid = try storeReceivedWhole(log, digest, content, null, rebase_whole);
try cache.put(gpa, log.offsetOf(new_rid), content);
return .persisted;
}
const base_rid = maybe_base.?;
const base_off = log.offsetOf(base_rid);
// Fossil stores a received delta as it arrives and checks the whole
// set once, a delta tree at a time. A basis not in the memo is one
// persisted long before: rebuilding it here reads its chain frame by
// frame, again for every later member, which bound a large clone on
// the receiver's own disk.
if (unverified) |list| {
if (cache.get(base_off) == null and log.depthOf(base_rid) < logstore.max_delta_chain) {
const new_rid = log.putDeltaContainer(digest, base_rid, wf.container) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
try list.append(gpa, .{ .rid = new_rid, .base = base_rid, .name = name });
return .persisted;
}
}
// Reuse the basis plaintext memoised when the basis frame was
// persisted — reverse-delta order writes every basis before the
// delta that names it, so the memo is normally warm. Only a miss
// (first use, or an LRU eviction) reconstructs it by walking the
// chain; either way `base_plain` is owned and freed below.
const base_plain = if (cache.get(base_off)) |hit| blk: {
// §7.7 positive space: a memo entry is the exact plaintext that
// named its wire hash at the basis's own persist, so the reused
// bytes still hash to the basis name (debug-only assertion; the
// hash is never recomputed on the hot path).
std.debug.assert(artifact.verify(d.src_hash, hit));
break :blk gpa.dupe(u8, hit) catch return error.OutOfMemory;
} else blk: {
const p = (log.contentCached(gpa, base_rid, cache) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
}) orelse return error.Corrupt;
// `contentCached` does not memoise a top-level full-form basis;
// prime it here so sibling reverse-deltas on the same basis reuse
// this reconstruction instead of repeating it.
try cache.put(gpa, base_off, p);
break :blk p;
};
defer gpa.free(base_plain);
const raw = container_mod.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Corrupt,
};
defer gpa.free(raw);
const content = delta.apply(gpa, base_plain, raw) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Corrupt,
};
defer gpa.free(content);
if (!artifact.verify(name, content)) return error.Corrupt;
if (try receivedSeed(gpa, content)) return .dropped;
// submanifests §7.1 — the plaintext §7.7 just verified is the same
// bytes the phantom duty is stated over; scan it here rather than
// forcing a second inflate anywhere else.
try phantomizeSubManifestRefs(gpa, log, content, srefs);
try noteCluster(gpa, clusters_out, name, content);
// Store the delta VERBATIM iff its basis is a held frame under the READ
// depth bound; an over-deep basis falls back to full form (never an
// unreadable chain). `delta_of` is the basis's backward offset —
// strictly lower than this new frame, the invariant `putDeltaContainer`
// asserts.
if (log.depthOf(base_rid) < logstore.max_delta_chain) {
const new_rid = log.putDeltaContainer(digest, base_rid, wf.container) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
// Prime the just-verified plaintext so an older reverse-delta
// naming this frame as basis reuses it (no chain re-walk).
try cache.put(gpa, log.offsetOf(new_rid), content);
} else {
const new_rid = log.putZlib(digest, content) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
try cache.put(gpa, log.offsetOf(new_rid), content);
}
return .persisted;
}
// §7.2 full form — decode to verify, then store it (`storeReceivedWhole`).
const content = container_mod.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Corrupt,
};
defer gpa.free(content);
if (rebase_whole and allow_defer) {
if (parentInFlight(store, log, content)) |parent| return .{ .deferred = parent };
}
if (!artifact.verify(name, content)) return error.Corrupt;
if (try receivedSeed(gpa, content)) return .dropped;
// submanifests §7.1 — the plaintext §7.7 just verified is the same
// bytes the phantom duty is stated over; scan it here rather than
// forcing a second inflate anywhere else.
try phantomizeSubManifestRefs(gpa, log, content, srefs);
try noteCluster(gpa, clusters_out, name, content);
const new_rid = try storeReceivedWhole(log, digest, content, wf.container, rebase_whole);
// Prime the just-verified full-form plaintext so a reverse-delta naming
// this frame as basis reuses it instead of re-inflating the container.
try cache.put(gpa, log.offsetOf(new_rid), content);
return .persisted;
}
// §4.1 — no retained wire (a raw pull `put`): the plaintext is memoised in the
// store, no chain to walk. Verify and compress it; the hash names the
// UNCOMPRESSED bytes, so compression is address-transparent.
const content = store.content(rid) orelse return error.Corrupt;
if (rebase_whole and allow_defer) {
if (parentInFlight(store, log, content)) |parent| return .{ .deferred = parent };
}
if (!artifact.verify(name, content)) return error.Corrupt;
if (try receivedSeed(gpa, content)) return .dropped;
// submanifests §7.1 — the plaintext §7.7 just verified is the same
// bytes the phantom duty is stated over; scan it here rather than
// forcing a second inflate anywhere else.
try phantomizeSubManifestRefs(gpa, log, content, srefs);
try noteCluster(gpa, clusters_out, name, content);
const new_rid = try storeReceivedWhole(log, digest, content, null, rebase_whole);
// Prime the plaintext so a reverse-delta naming this frame as basis reuses it.
try cache.put(gpa, log.offsetOf(new_rid), content);
return .persisted;
}
/// Evidence seam: the persist drain's deferred waiting set — the
/// reverse-deltas resident because their basis frame has not been written yet —
/// at its peak, as a frame count and as wire container bytes. This is the
/// number the streaming-persist decision hinges on: it is exactly what a
/// per-round drain would have to keep (or spill) instead of the whole store.
/// Off unless `ZIT_PERSIST_WAITING` asks for it at exit, and it changes
/// nothing else.
pub var persist_waiting_peak: u64 = 0;
pub var persist_waiting_peak_bytes: u64 = 0;
/// Evidence seam: the names the last pull's cluster index held, reported
/// with the sync counters under `ZIT_SYNC_STATS` at exit.
pub var cluster_index_names: u64 = 0;
/// Evidence seam: persists that opened the log by walking every frame header
/// because no scan index described it.
pub var persist_append_scans: u64 = 0;
/// Persists every content-bearing artifact in `store` into an already-open
/// writable log. The caller owns durability, allowing clone to append artifacts
/// and its combined config overlay before one final commit.
pub fn persistStoreOpen(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
store: repostate.ArtifactStore,
keep: ?*const SparseClassification,
start_after: repostate.Rid,
clusters_out: ?*std.ArrayListUnmanaged([]const u8),
dropped_out: ?*std.ArrayListUnmanaged([]const u8),
) PersistError!PersistSummary {
var pass = try PersistPass.init(gpa, log, store, keep, null, start_after, clusters_out);
defer pass.deinit();
pass.dropped_out = dropped_out;
pass.rebase_whole = true;
try pass.drainNew();
return pass.finish();
}
/// The persist drain as a RESUMABLE pass: `drainNew` persists every
/// content-bearing artifact interned since the last call, and `finish` settles
/// the leftovers and the sub-manifest bookkeeping. The batched shape
/// (`persistStoreOpen`) is init + one drainNew + finish; a streaming clone
/// calls drainNew after every round instead, with `mem` set so each persisted
/// artifact's resident bytes are released the moment its frame is durable —
/// git index-pack's shape (stream to disk, defer a delta whose base is unseen,
/// resolve when it lands) applied to fossil's reverse-delta wire.
pub const PersistPass = struct {
gpa: std.mem.Allocator,
log: *logstore.LogStore,
store: repostate.ArtifactStore,
/// Sparse-cone RFC section 6.3 -- the cone's classification of `store`:
/// what persists, and which absent sub-manifest nodes are elided rather
/// than gaps. Null for a dense persist.
keep: ?*const SparseClassification,
/// Set on the streaming path: the concrete store behind `store`, so a
/// persisted rid's bytes can be released. Null keeps every byte resident
/// until the caller drops the store — the batched behaviour.
mem: ?*repostate.MemStore,
/// One bounded plaintext memo for the whole pass: a basis reconstructed for
/// its own §7.7 verify is reused by every reverse-delta that names it, so a
/// delta family reconstructs each basis ~once rather than once per
/// referencing delta. Bounded (LRU byte + entry caps) — never the whole repo.
cache: logstore.PlaintextCache,
/// submanifests §7.1 — one record for the whole pass, so a reference is
/// judged a gap only after every artifact this store carries has landed.
srefs: SubManifestRefs = .{},
rescan: bool,
/// Store an artifact that arrived whole as a delta where one pays
/// (`storeReceivedWhole`). Set by `persistStoreOpen` — sync, pull and a
/// server taking a push; a clone keeps the verbatim shape its deferred
/// verification relies on.
rebase_whole: bool = false,
waiting: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(repostate.Rid)) = .empty,
ready: std.ArrayListUnmanaged(repostate.Rid) = .empty,
written: usize = 0,
elided: usize = 0,
/// The first rid `drainNew` has not seeded yet — the resume cursor.
next_seed: repostate.Rid = 1,
/// See `persistArtifact`; null records nothing.
clusters_out: ?*std.ArrayListUnmanaged([]const u8) = null,
/// Where to record the name of every received seed this pass dropped,
/// as an owned dupe; null records nothing.
dropped_out: ?*std.ArrayListUnmanaged([]const u8) = null,
/// Streaming only: rids the cursor passed while they were
/// phantoms. Fossil interns a delta's unseen DELTASRC as a phantom the
/// moment the delta arrives; the content lands at that SAME rid in a later
/// round, behind the cursor. Each drainNew re-checks these first and
/// drives the ones that have since gained content; whatever is left at
/// `finish` is a true phantom, skipped exactly as the batched pass skips
/// it. Empty on the batched path, whose store is already converged.
revisit: std.ArrayListUnmanaged(repostate.Rid) = .empty,
/// Streaming only, set for the rounds past the clone sequence. What
/// those rounds bring answers the clone's own gimmes -- the basis of a
/// delta an origin sent across its cone, transit -- so it is held back
/// for the clone's cone to judge (`judgeHeldBack`) rather than written as
/// it lands: a frame cannot be taken back once the cone elides it.
hold: bool = false,
held_back: std.ArrayListUnmanaged(repostate.Rid) = .empty,
/// Deltas stored before their plaintext was checked; `finish` checks them.
unverified: std.ArrayListUnmanaged(UnverifiedDelta) = .empty,
// Evidence-seam running totals for the peak counters above.
waiting_count: u64 = 0,
waiting_bytes: u64 = 0,
pub fn init(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
store: repostate.ArtifactStore,
keep: ?*const SparseClassification,
mem: ?*repostate.MemStore,
// Begin the walk AFTER this rid, so a pull persists only the
// artifacts it received (which the holdings store appends beyond its
// log-backed prefix) instead of re-walking every already-durable
// holding. Zero means start at rid 1 (clone/push/serve: the whole
// store IS the new content).
start_after: repostate.Rid,
clusters_out: ?*std.ArrayListUnmanaged([]const u8),
) PersistError!PersistPass {
// submanifests §8.1/§7.1 — THE REBUILD-ON-OPEN INVARIANT, stated once here.
//
// §7.1's phantoms are not durable and are not made durable: a phantom is an
// interned entry with no frame, exactly as an absent delta source's is
// ("exactly as an absent delta source creates a source phantom"), and
// `LogStore` writes none of either kind ("Phantoms never reach disk"). The
// `.zit` on-disk format version is frozen (`logstore.version`), so minting a
// durable phantom record would reject every repository already written.
//
// What makes the in-memory phantoms CORRECT across a restart is this
// invariant: the S-reference gap set is a pure function of the durable
// content-bearing frames. A sweep over the whole store (start_after 0:
// clone, a push ingest) re-derives it here: a repository whose §8.1
// indicator says it has ever stored an S reference rescans its
// already-persisted artifacts (`rescan_persisted`); one that has not —
// every repository predating this draft — rescans nothing and pays
// nothing. Sync and pull persist only what arrived, so they take the same
// function from `subManifestGaps`, which carries it forward over each
// appended tail. The indicator itself IS durable, which is the whole
// trick: one config byte survives the restart instead of a phantom table.
const rescan = blk: {
var oc = log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
};
if (oc) |*c| {
defer c.deinit(gpa);
break :blk c.config.seen_submanifest;
}
break :blk false;
};
return .{
.gpa = gpa,
.log = log,
.store = store,
.keep = keep,
.mem = mem,
.cache = logstore.PlaintextCache.init(),
.rescan = rescan,
.next_seed = start_after + 1,
.clusters_out = clusters_out,
};
}
/// The type-erased `RoundDrain` entry: persist everything the round that
/// just folded in has interned.
pub fn drainErased(ctx: *anyopaque) PersistError!void {
const self: *PersistPass = @ptrCast(@alignCast(ctx));
return self.drainNew();
}
pub fn deinit(self: *PersistPass) void {
self.cache.deinit(self.gpa);
self.srefs.deinit(self.gpa);
var it = self.waiting.valueIterator();
while (it.next()) |list| list.deinit(self.gpa);
self.waiting.deinit(self.gpa);
self.ready.deinit(self.gpa);
self.revisit.deinit(self.gpa);
self.held_back.deinit(self.gpa);
self.unverified.deinit(self.gpa);
}
/// Drive one rid through `persistArtifact`, drain the waiters its frame
/// unblocks, and — on the streaming path — release everything persisted.
fn drive(self: *PersistPass, first: repostate.Rid) PersistError!void {
const gpa = self.gpa;
try self.ready.append(gpa, first);
while (self.ready.pop()) |rid| {
switch (try persistArtifact(gpa, self.store, self.log, rid, true, self.mem != null, &self.cache, &self.srefs, self.rescan, self.rebase_whole, self.clusters_out, &self.unverified)) {
.persisted => {
// `rid`'s frame now exists: re-drive any reverse-deltas that
// deferred on it as their (now-written) basis.
if (self.waiting.fetchRemove(self.store.hashOf(rid))) |kv| {
var list = kv.value;
defer list.deinit(gpa);
for (list.items) |w| {
self.waiting_count -= 1;
self.waiting_bytes -= if (self.store.wire(w)) |wf| wf.container.len else 0;
}
try self.ready.appendSlice(gpa, list.items);
}
if (self.mem) |m| m.releasePersisted(rid);
},
.deferred => |base_hash| {
const gop = try self.waiting.getOrPut(gpa, base_hash);
if (!gop.found_existing) gop.value_ptr.* = .empty;
try gop.value_ptr.append(gpa, rid);
// Evidence seam: a deferred rid is exactly what
// streaming persist keeps resident; meter it.
self.waiting_count += 1;
self.waiting_bytes += if (self.store.wire(rid)) |wf| wf.container.len else 0;
if (self.waiting_count > persist_waiting_peak) persist_waiting_peak = self.waiting_count;
if (self.waiting_bytes > persist_waiting_peak_bytes) persist_waiting_peak_bytes = self.waiting_bytes;
},
// Not received: uncounted, and kept resident so its waiters
// can still be rebuilt in full form at the leftover drain.
.dropped => try self.noteDropped(rid),
}
}
}
fn noteDropped(self: *PersistPass, rid: repostate.Rid) error{OutOfMemory}!void {
self.written -= 1;
const list = self.dropped_out orelse return;
const owned = try self.gpa.dupe(u8, self.store.hashOf(rid));
list.append(self.gpa, owned) catch |e| {
self.gpa.free(owned);
return e;
};
}
/// Persist every content-bearing artifact interned since the last call.
///
/// Reverse-delta persist ordering (§7.2/§8.2). Fossil stores an OLDER
/// artifact as a delta of a NEWER one, so a delta cfile can land in the store
/// at a LOWER rid than the basis it points at, while the log's `delta_of` is a
/// BACKWARD offset that REQUIRES the basis frame to precede the delta. Persist
/// therefore drains a worklist: a delta whose basis is not yet a frame is
/// deferred (keyed by the basis wire-hash) and re-driven the instant that
/// basis lands. Each content-bearing artifact seeds the drain once in rid
/// order; each deferred delta is re-driven exactly once, so the pass is O(n)
/// and terminates — a chain bottoms out at fossil's full newest artifact. A
/// wait can be two-way: a check-in received whole waits for its parent,
/// while that parent, sent as a reverse delta whose basis is the check-in
/// (a peer answering out of rid order), waits for it. `settle` breaks it,
/// in map order, so which of the two lands as the delta is arbitrary; both
/// land.
pub fn drainNew(self: *PersistPass) PersistError!void {
// Revived phantoms first: their frames may be the bases this round's
// deltas are about to defer on.
var i: usize = 0;
while (i < self.revisit.items.len) {
const rid = self.revisit.items[i];
if (self.store.held(rid)) {
_ = self.revisit.swapRemove(i);
if (self.hold) {
try self.held_back.append(self.gpa, rid);
continue;
}
self.written += 1;
try self.drive(rid);
} else i += 1;
}
const n = self.store.count();
while (self.next_seed <= n) : (self.next_seed += 1) {
const seed = self.next_seed;
if (!self.store.held(seed)) {
// Residual phantom: not durable. Mid-exchange it may yet gain
// content at this very rid — remember it for the next drain.
if (self.mem != null) try self.revisit.append(self.gpa, seed);
continue;
}
if (self.keep) |k| {
// Sparse-cone RFC section 6.3: elision writes nothing -- the frame
// simply never lands, and rebuild-on-open re-interns the dangling
// references as phantoms the cone classifies. A kept delta whose
// basis is elided falls to the leftover drain below and stores
// FULL FORM, so elision can never orphan held content.
if (!k.keep.isSet(seed)) {
self.elided += 1;
continue;
}
}
if (self.hold) {
try self.held_back.append(self.gpa, seed);
continue;
}
self.written += 1; // count each content-bearing artifact exactly once, at its rid
try self.drive(seed);
}
}
/// Drop the held-back arrivals the clone's cone does not keep, judged by
/// `cls` over `view`, a store that holds them beside everything written.
/// The rest persist at `finish`; a kept delta whose basis was dropped
/// lands there in full form, as the batched pass stores it.
pub fn judgeHeldBack(self: *PersistPass, view: repostate.ArtifactStore, cls: *const SparseClassification) void {
var i: usize = 0;
while (i < self.held_back.items.len) {
const rid = self.held_back.items[i];
// A name the view cannot place is kept: never less than the cone.
const vr = view.lookup(self.store.hashOf(rid)) orelse {
i += 1;
continue;
};
if (vr < cls.keep.capacity() and cls.keep.isSet(vr)) {
i += 1;
continue;
}
_ = self.held_back.swapRemove(i);
self.elided += 1;
}
}
/// Write the leftovers: every delta still waiting on a basis that never
/// came lands as full form. Idempotent; `finish` calls it.
pub fn settle(self: *PersistPass) PersistError!void {
const gpa = self.gpa;
// A deferred basis is itself content-bearing and is normally persisted
// above, draining its waiters; what remains here is a basis that never
// came, or the two-way wait `drainNew` describes. Its waiters land as full form
// rather than be dropped — every content-bearing artifact stays durable.
var leftover = self.waiting.iterator();
while (leftover.next()) |e| {
for (e.value_ptr.items) |rid| {
switch (try persistArtifact(gpa, self.store, self.log, rid, false, false, &self.cache, &self.srefs, self.rescan, self.rebase_whole, self.clusters_out, &self.unverified)) {
.persisted => if (self.mem) |m| m.releasePersisted(rid),
.deferred => unreachable, // allow_defer=false never defers
.dropped => try self.noteDropped(rid),
}
}
}
var lists = self.waiting.valueIterator();
while (lists.next()) |list| list.deinit(gpa);
self.waiting.clearRetainingCapacity();
}
/// Settle the leftovers and the sub-manifest bookkeeping; the pass is done.
/// §7.7 for the deltas stored unchecked: each is rebuilt and its hash
/// recomputed before the pass is committed, and gets the scans every
/// persisted artifact gets. Fossil's clone does the same once, after the
/// transfer (`rebuild_db`): a delta tree is walked depth first from its
/// root, each member applied against its parent's plaintext, so each
/// frame is read once and no chain twice. A mismatch fails the pass, and
/// nothing it wrote is committed.
fn verifyDeferred(self: *PersistPass) PersistError!void {
if (self.unverified.items.len == 0) return;
const gpa = self.gpa;
const log = self.log;
// Children by basis; a root is a basis that is not itself deferred.
var children: std.AutoHashMapUnmanaged(logstore.Rid, std.ArrayListUnmanaged(u32)) = .empty;
defer {
var it = children.valueIterator();
while (it.next()) |l| l.deinit(gpa);
children.deinit(gpa);
}
var deferred: std.AutoHashMapUnmanaged(logstore.Rid, void) = .empty;
defer deferred.deinit(gpa);
for (self.unverified.items, 0..) |u, i| {
try deferred.put(gpa, u.rid, {});
const gop = try children.getOrPut(gpa, u.base);
if (!gop.found_existing) gop.value_ptr.* = .empty;
try gop.value_ptr.append(gpa, @intCast(i));
}
var stack: std.ArrayListUnmanaged(logstore.Rid) = .empty;
defer stack.deinit(gpa);
var verified: usize = 0;
var roots = children.keyIterator();
while (roots.next()) |root| {
if (deferred.contains(root.*)) continue;
// The root's plaintext once, primed so its children read only
// their own frames.
const off = log.offsetOf(root.*);
if (self.cache.get(off) == null) {
const plain = (try log.contentCached(gpa, root.*, &self.cache)) orelse return error.Corrupt;
defer gpa.free(plain);
try self.cache.put(gpa, off, plain);
}
try stack.append(gpa, root.*);
while (stack.pop()) |parent| {
const kids = children.get(parent) orelse continue;
for (kids.items) |i| {
const u = self.unverified.items[i];
// Re-prime the parent in case a sibling's subtree evicted it.
if (self.cache.get(log.offsetOf(parent)) == null) {
const p = (try log.contentCached(gpa, parent, &self.cache)) orelse return error.Corrupt;
defer gpa.free(p);
try self.cache.put(gpa, log.offsetOf(parent), p);
}
const content = (try log.contentCached(gpa, u.rid, &self.cache)) orelse return error.Corrupt;
defer gpa.free(content);
if (!artifact.verify(u.name, content)) return error.Corrupt;
// A seed is never stored; one that arrives as a delta is a
// malformed transfer.
if (try receivedSeed(gpa, content)) return error.Corrupt;
try phantomizeSubManifestRefs(gpa, log, content, &self.srefs);
try noteCluster(gpa, self.clusters_out, u.name, content);
try self.cache.put(gpa, log.offsetOf(u.rid), content);
try stack.append(gpa, u.rid);
verified += 1;
}
}
}
// Every deferred delta descends from a root, since a basis precedes
// its delta in the log; one the walk did not reach is unchecked.
if (verified != self.unverified.items.len) return error.Corrupt;
self.unverified.clearRetainingCapacity();
}
pub fn finish(self: *PersistPass) PersistError!PersistSummary {
const gpa = self.gpa;
const log = self.log;
for (self.held_back.items) |rid| {
self.written += 1;
try self.drive(rid);
}
self.held_back.clearRetainingCapacity();
try self.settle();
try self.verifyDeferred();
// submanifests §7.1 — decided once, at the end: a reference is a gap only
// if nothing in this whole store turned out to name it. Under a cone, a
// reference its slots place wholly outside it is an elided phantom
// (sparse-cone RFC section 5.1), not a gap.
const refs = self.srefs.unresolved(log, self.keep);
// submanifests §8.1 — "When the first S-bearing artifact is stored, an
// implementation SHOULD record a local seen-submanifest indicator so
// operational surfaces can warn before the setting is enabled or a pre-S
// client is granted access." Written once, on the false->true edge, so a
// steady-state sync appends no config frame; it is also what arms the
// rebuild above on every subsequent run of this repository.
if ((self.srefs.seen or self.srefs.seen_delta) and !self.rescan) {
var oc = log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
};
var cfg: logstore.Config = if (oc) |c| c.config else .{ .hash_policy = 0, .default_branch = "", .project_id = "" };
defer if (oc) |*c| c.deinit(gpa);
const already = (cfg.seen_submanifest or !self.srefs.seen) and (cfg.seen_delta_manifest or !self.srefs.seen_delta);
if (!already) {
if (self.srefs.seen) cfg.seen_submanifest = true;
if (self.srefs.seen_delta) cfg.seen_delta_manifest = true;
log.putConfig(cfg) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
}
}
return .{ .written = self.written, .submanifest_gaps = refs.gaps, .elided = self.elided + refs.elided, .filtered = self.elided };
}
};
/// Opens `dir/.zit` for the legacy persistence callers, preserving their
/// artifact result and commit boundary while routing the write through the
/// already-open-log implementation.
pub fn persistStore(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
store: repostate.ArtifactStore,
keep: ?*const SparseClassification,
start_after: repostate.Rid,
) PersistLockedError!PersistSummary {
return (try persistStoreAdopting(gpa, io, dir, store, keep, start_after, null)).?;
}
/// `persistStore` for a round whose push an adopting server judged against
/// the project code `adopt` (sync §8.5 L1688-1689). The code is written in the
/// same commit as the round's content and ahead of it, so a store holds
/// content exactly when it has stopped adopting. The store's writer lock
/// settles two first pushes racing from two serving processes (one
/// `zit serve` answers one request at a time): the store is judged again
/// under the lock, so only a push that still finds it empty adopts, and one
/// that finds content there is judged against the code stored with it:
/// refused, null, with nothing written, unless it names that code.
pub fn persistStoreAdopting(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
store: repostate.ArtifactStore,
keep: ?*const SparseClassification,
start_after: repostate.Rid,
adopt: ?[]const u8,
) PersistLockedError!?PersistSummary {
// Nothing held past the scoped prefix means nothing to write, and the
// append open below scans every frame header: a no-op exchange skips it
// (and the writer lock) rather than pay O(store) to append nothing.
if (start_after > 0 and !heldAfter(store, start_after)) {
return .{ .written = 0, .submanifest_gaps = 0, .elided = 0, .filtered = 0 };
}
var store_append = store_open.openStoreAppend(io, dir) catch |e| switch (e) {
error.StoreBusy => return error.StoreBusy,
else => return error.Storage,
};
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch return error.Storage;
var fs = storage.FileStorage.init(io, file, st.size);
var log = openForAppend(gpa, io, dir, fs.storageFile()) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
};
defer log.deinit(gpa);
if (adopt) |code| {
var oc = (log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
}) orelse return null;
const same = std.mem.eql(u8, oc.config.project_code, code);
oc.deinit(gpa);
if (!same) {
if (log.entryCount() != 0) return null;
try updateConfigOpen(gpa, &log, .{ .project_code = code });
}
}
// A log with no entries at open is being CREATED by this persist,
// so the clusters it records are the whole index; otherwise they extend one.
const fresh = log.entryCount() == 0;
var clusters: std.ArrayListUnmanaged([]const u8) = .empty;
defer {
for (clusters.items) |n| gpa.free(n);
clusters.deinit(gpa);
}
var dropped: std.ArrayListUnmanaged([]const u8) = .empty;
defer {
for (dropped.items) |n| gpa.free(n);
dropped.deinit(gpa);
}
const summary = try persistStoreOpen(gpa, &log, store, keep, start_after, &clusters, &dropped);
log.commit() catch return error.Storage;
try refreshClusterSidecar(gpa, io, dir, &log, clusters.items, fresh);
try repocache.recordDroppedSeeds(gpa, io, dir, dropped.items);
return summary;
}
/// `store_open.openLogForAppend`, counting the opens that had to scan.
fn openForAppend(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, sf: storage.StorageFile) logstore.OpenError!logstore.LogStore {
var scanned = false;
defer if (scanned) {
persist_append_scans += 1;
};
return store_open.openLogForAppend(gpa, io, dir, sf, &scanned);
}
/// Whether `store` holds content at any rid past `start_after`: what a
/// scoped persist would write. Phantoms past it are not content.
fn heldAfter(store: repostate.ArtifactStore, start_after: repostate.Rid) bool {
var rid = start_after + 1;
while (rid <= store.count()) : (rid += 1) {
if (store.held(rid)) return true;
}
return false;
}
/// Keep the cluster index warm across a persist: adopt the stored
/// index (its key still validates — this persist only appended), append the
/// clusters just written, re-key to the committed log, store. With no valid
/// index to extend, only a persist that created the log writes one; any
/// other store waits for the next sync open, which builds it by one walk.
/// Advisory on storage (a failed write leaves the sidecar as it was); honest
/// on memory, so the persist's OOM contract holds through it.
pub fn refreshClusterSidecar(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
log: *const logstore.LogStore,
written: []const []const u8,
fresh: bool,
) error{OutOfMemory}!void {
var arena = std.heap.ArenaAllocator.init(gpa);
defer arena.deinit();
const a = arena.allocator();
var names: std.ArrayListUnmanaged([]const u8) = .empty;
var adopted = false;
if (try repocache.loadClusterIndex(gpa, io, dir)) |blob| {
defer gpa.free(blob);
if (try clusterindex.parse(a, blob, log)) |p| {
for (p.names) |n| try names.append(a, try a.dupe(u8, n));
adopted = true;
}
}
if (!adopted and !fresh) return;
for (written) |w| try names.append(a, w);
const blob = (try clusterindex.serialize(gpa, log, log.entryCount(), names.items)) orelse return;
defer gpa.free(blob);
repocache.storeClusterIndex(io, dir, blob);
}
/// The cluster index for one sync open: adopt the sidecar when its
/// key validates, walk only the tail it has not seen (or everything, once, on
/// a miss), store the refreshed sidecar, and resolve it against `store`.
/// Nothing here is an error beyond memory: any refusal is a rebuild, and a
/// store the walk cannot read simply stays unindexed — unbounded, never wrong.
pub fn loadClusterIndex(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
log: *logstore.LogStore,
store: repostate.ArtifactStore,
) error{OutOfMemory}!clusterindex.Resolved {
var arena = std.heap.ArenaAllocator.init(gpa);
defer arena.deinit();
const a = arena.allocator();
var names: std.ArrayListUnmanaged([]const u8) = .empty;
var seen: std.StringHashMapUnmanaged(void) = .empty;
defer seen.deinit(gpa);
var after: repostate.Rid = 0;
var adopted = false;
if (try repocache.loadClusterIndex(gpa, io, dir)) |blob| {
defer gpa.free(blob);
if (try clusterindex.parse(a, blob, log)) |p| {
for (p.names) |n| {
const owned = try a.dupe(u8, n);
if ((try seen.getOrPut(gpa, owned)).found_existing) continue;
try names.append(a, owned);
}
after = @intCast(p.n_rids);
adopted = true;
}
}
// Phantoms never reach disk, so an index written after a persist may count
// entries this scan does not see; clamping never skips a real frame the
// writer had not already recorded.
const n: repostate.Rid = @intCast(log.entryCount());
if (after > n) after = n;
if (after < n) {
var walked: std.ArrayListUnmanaged([]const u8) = .empty;
try clusterindex.walkClusters(gpa, a, log, after, &walked);
for (walked.items) |w| {
if ((try seen.getOrPut(gpa, w)).found_existing) continue;
try names.append(a, w);
}
}
if (!adopted or after < n) {
if (try clusterindex.serialize(gpa, log, n, names.items)) |blob| {
defer gpa.free(blob);
repocache.storeClusterIndex(io, dir, blob);
}
}
cluster_index_names = names.items.len;
// Resolving reads every held cluster and looks up every member it names,
// which an unchanged log answers the same way every time: adopt the
// resolved sidecar, re-deriving only what an appended tail can change.
if (try repocache.loadMember(gpa, io, dir, repocache.clusters_resolved_member)) |blob| {
defer gpa.free(blob);
if (try clusterindex.adoptResolved(gpa, blob, log, store, names.items)) |kept| {
var r = kept.resolved;
errdefer r.deinit();
if (!kept.exact) try storeResolved(gpa, io, dir, log, &r);
return r;
}
}
var resolved = try clusterindex.resolve(gpa, store, names.items);
errdefer resolved.deinit();
try storeResolved(gpa, io, dir, log, &resolved);
return resolved;
}
fn storeResolved(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *const logstore.LogStore, r: *const clusterindex.Resolved) error{OutOfMemory}!void {
const blob = (try clusterindex.serializeResolved(gpa, log, r)) orelse return;
defer gpa.free(blob);
repocache.storeClustersResolved(io, dir, blob);
}
test "primaryParentName: a check-in's first P-card hash, null for a root or a blob" {
try std.testing.expectEqualStrings("aa11", primaryParentName("C c\nD 2026-01-01T00:00:00\nF f bb\nP aa11 cc22\nU u\nZ 0\n").?);
try std.testing.expectEqualStrings("aa11", primaryParentName("P aa11\nZ 0\n").?);
try std.testing.expect(primaryParentName("C c\nD 2026-01-01T00:00:00\nU u\nZ 0\n") == null);
try std.testing.expect(primaryParentName("plain text\nP aa11\n") == null);
try std.testing.expect(primaryParentName("") == null);
}