//! The content log seen through the sync responder's store interface: a
//! read-only adapter that indexes a store's frames, masks shunned ones and
//! reads containers on demand, plus the index-adopting read-only log open the
//! verbs share. No CLI: serve, the daemon and the read verbs link it alike.
const std = @import("std");
const SparseClassification = @import("../repo/sparse.zig").SparseClassification;
const artifact = @import("../artifact/artifact.zig");
const checkout = @import("../worktree/checkout.zig");
const cone_mod = @import("../repo/cone.zig");
const coneindex = @import("../repo/coneindex.zig");
const decodeWireName = @import("digest_name.zig").decodeWireName;
const digestToWireName = @import("digest_name.zig").digestToWireName;
const digest_len = @import("digest_name.zig").digest_len;
const logstore = @import("../store/logstore.zig");
const repocache = @import("../repo/repocache.zig");
const repostate = @import("../repo/repostate.zig");
const sparse = @import("../repo/sparse.zig");
const storage = @import("../store/storage.zig");
const structgraph = @import("../repo/structgraph.zig");
const sync = @import("../sync/sync.zig");
const repeat = @import("../repeat.zig").repeat;
const repo_filename = @import("../repo/repository.zig").repo_filename;
/// A serve snapshot that reads container bytes from the log ON DEMAND instead of
/// copying the whole repository into RAM at prime. It owns a read-only
/// `LogStore` — whose open scan already whole-frame-CRC'd every byte, so the
/// single read stays; the second read `loadContentStore` performed does not — and
/// a resident INDEX: the adapter-rid→log-rid map plus the lowercase-hex wire names
/// the responder walks. `held()` answers from that index with zero I/O; `wire()`
/// and `content()` read the one frame they are asked for and memoise it for the
/// round; `releaseTransient` drops those memos between rounds, so a round's
/// resident bytes are bounded by what it emitted, not by the store.
///
/// Heap-allocated because it owns a `storage.FileStorage` whose `storageFile()`
/// vtable points back into the struct — the box must not move once the log borrows
/// it. `openFromFile` takes that ownership (the serve path); `open` borrows a
/// caller-owned `StorageFile` (a test instrumenting read traffic).
///
/// Adapter rids `1..=rids.len` name the log's non-config artifact frames in
/// `ridsInPersistedOrder` — the SAME enumeration `loadContentStore` assigns, so
/// adapter rid r is byte-for-byte the rid the old MemStore snapshot gave, which is
/// what keeps the clone pager's persisted-order cursor (sync §8.2) stable. A frame
/// named in the repository shun table keeps its slot in that enumeration and is
/// MASKED rather than removed (see `masked`), so the shun table never renumbers
/// a rid and §8.2 L1551-1553's "silently skipping gaps" carries the exclusion. Rids
/// above `rids.len` are an appendix `MemStore` holding any hash a caller `intern`s
/// or `put`s that the log does not carry. The serve path never writes the base
/// (every write lands in the per-request `OverlayStore`), so the appendix stays
/// empty in production; it exists so this store honours the whole `ArtifactStore`
/// interface — composed over the existing `MemStore` rather than hand-rolled —
/// instead of trapping on a write it will never see.
pub const LogIndexStore = struct {
gpa: std.mem.Allocator,
/// Owned storage binding on the `openFromFile` path (the log borrows
/// `&self.fs`); `undefined` on the `open` path, where the caller owns the
/// storage and keeps it alive for this store's life.
fs: storage.FileStorage,
log: logstore.LogStore,
/// Adapter rid a (1-based) → `rids[a-1]` = the log rid. Persisted order, config
/// frames removed.
rids: []logstore.Rid,
/// Adapter rid a → `names[a-1]` = its lowercase-hex wire name (sync §3.3),
/// backed by `name_arena`. The single site the binary-log digest becomes the
/// wire name — the boundary `loadContentStore` held, kept to one place.
/// Under `by_log` a slot is empty until `nameAt` first asks for it.
names: [][]const u8,
name_arena: std.heap.ArenaAllocator,
by_name: std.StringHashMapUnmanaged(repostate.Rid) = .empty,
/// Every log entry is a persisted artifact in frame order, so a log rid IS
/// its adapter rid: a name resolves through the log's own index
/// (`nameRid`) and `by_name` stays empty. A freshly opened log always
/// qualifies; building the map and every name up front was most of the
/// cost of opening a large store's holdings for a sync that moves nothing.
by_log: bool = false,
/// Sparse-cone RFC section 6.2 -- single-entry classification memo for
/// cone-scoped requests: the server is stateless per round, so without
/// this every round of a cone-scoped clone would re-parse the store.
/// Keyed by the cone digest; dropped with the snapshot, so a rebuilt
/// snapshot can never serve a stale classification.
cone_memo_digest: ?[32]u8 = null,
cone_memo: ?SparseClassification = null,
/// Delta bases this round reconstructed, so a chain of N deltas costs one
/// inflate per frame rather than N per artifact: the classifier walks rids
/// in persisted order, where a base always precedes its deltas, and the
/// clone walk does the same. Reset with the other per-round memos.
plaintext: logstore.PlaintextCache,
/// The classification view's one-slot plaintext (see `classifyStore`).
classify_last: ?[]u8 = null,
/// The classification view's one-slot wire container and, for a delta
/// frame, its base name; the classifier peeks at one frame at a time.
classify_wire_last: ?[]u8 = null,
classify_wire_hash: [2 * logstore.max_hash_len]u8 = undefined,
classify_wire_hash_len: u8 = 0,
/// Where the cone classification persists between snapshots (the served
/// repository's `.zit-cache`), so a rebuilt snapshot classifies only the
/// frames appended since the last one. Null classifies in memory only.
classify_sidecar: ?struct { io: std.Io, dir: std.Io.Dir, graph: ?*?structgraph.Graph = null } = null,
/// Log rid -> adapter rid, 0 where no adapter rid answers a lookup (a
/// masked frame, a phantom). Built by the first cone-index classification
/// and freed with the snapshot it describes.
log_to_adapter: ?[]u32 = null,
/// sync §9.5 L2170-2174 — adapter rid a → `masked[a-1]`: this frame's name
/// is in the repository shun table, so the enumeration MASKS it. §9.5 asks for
/// "igot for every eligible holding after the Section 3.3 shun … filters", and
/// this array is where that filter lives: a masked entry answers `held` false,
/// `content`/`wire` null, and its name is left out of `by_name`, so the server
/// does not announce it (§3.2's walk gates on `held`), and an inbound `gimme`
/// for it — which keys on `lookup` — lands on §3.3 L569-572's "If absent,
/// silently skip" with no code on that path.
///
/// THE CLONE HALF IS AN ERRATUM CANDIDATE, not a citation. Because the mask
/// lives in the enumeration, a masked holding is also absent from the clone
/// batch — but no draft clause says a clone server withholds one. §9.5's
/// sentence is scoped to the `send-catalog` pragma's igot inventory; §3.4
/// L545-547 hands clone to §8.1-§8.4, and §8.2 L1551-1553 licenses only the
/// MECHANISM ("silently skipping gaps"), never the policy. Applying §3.3's
/// shun set to the clone reply is this implementation's extension, taken
/// because the alternative — a repository under a takedown order handing the
/// content to every new clone — is not a behaviour an operator would read the
/// drafts' silence as licensing. Erratum candidate against sync §8.1/§8.2:
/// state whether a clone server's batch is filtered by the §3.3 shun set.
///
/// SCOPE, stated exactly: this masks what THIS BASE SNAPSHOT enumerates. A
/// served round answers over a per-request `repostate.OverlayStore` wrapping the
/// base, and its `putImpl` keys on `base.lookup` — which the mask answers null
/// for — so a peer that PUSHES `file <masked name>` in the same round would
/// create an overlay-local rid that genuinely holds those bytes and is
/// announced by that round's §3.2 walk. That route is closed, but NOT here: it
/// is closed by receive-side admission (§7.7 L1472-1475: "A non-clone file
/// whose target is shunned is consumed and discarded after its payload and
/// pending private marker; it emits no error … and makes no repository
/// mutation."), which `scanCard` applies before the card can enter `scan.files`
/// and `ingest` re-applies before the `put`, both reading `Server.shun` — the
/// set `serveRound` builds from THIS SAME snapshot's table, so the two can
/// never disagree about a name. Recorded here because the send-side filter
/// genuinely cannot reach that path: what makes the guarantee hold is the pair,
/// and removing either half reopens it.
///
/// A MASK, not a removal, and §8.2 L1551-1553 is why: "SEQNO is a server
/// blob-table RID. Iterate from supplied RID through current maximum,
/// silently skipping gaps; clients MUST NOT infer count or size from range."
/// Dropping the frame from the enumeration would renumber every survivor above
/// it. The server is stateless (§8.9 L1855-1856: "Section 3.7 is authoritative;
/// the stateless server signals no completion") — the client's SEQNO *is* the clone
/// cursor — and a shun row appended between two rounds of one clone grows the
/// log, which is exactly what trips `ServeCtx.baseSnapshot`'s append-only stat
/// gate and rebuilds this index. Renumbering there would make a resuming
/// client's cursor step over a survivor and still be told the sequence was
/// complete: a silently short clone, the failure class `read_fault` exists to
/// prevent. Keeping the slot and answering it as a §8.2 gap makes adapter-rid
/// identity a pure function of the log, independent of the shun table.
///
/// Filled in the SAME loop as `rids`/`names`/`content_memo`/`wire_memo`, off
/// the same index — the FILL alignment is structural (one loop, one counter),
/// not asserted. A mask derived from ANY second walk is the one way adapter rid
/// r could come to name one artifact and mask another, and nothing about the log
/// makes such a walk look wrong on inspection: it would agree on length and on
/// set membership and differ only in which slot carries which answer. The
/// LENGTHS are a separate question and the fill loop cannot speak to them; they
/// are gated by a checked error at the end of `build` (see there).
///
/// AND A PURE FUNCTION OF `shun.digests` — the other half of the same
/// invariant, and the one a later filter is most likely to break. `wireImpl`'s
/// delta-basis check asks the SET (`shun.digests.contains(d.baseHash())`) while
/// `held`/`content`/`wire` by rid ask THIS ARRAY; the two agree only because the
/// fill loop below defines `masked[a]` as exactly that membership. §9.5
/// L1988-1992 names FOUR filters — "igot for every eligible holding after the
/// Section 3.3 shun, phantom, private, and peer-has filters" — and this array is
/// the natural landing site for the other three. The day one of them widens
/// `masked` beyond shun membership, the basis check silently stops following it
/// and the server re-emits a DELTASRC it will not send. Widening this array
/// therefore obliges the same change to resolve the basis THROUGH it (basis
/// digest → adapter rid → `masked[rid - 1]`) rather than through the set.
masked: []bool,
/// Per-round transient memos over the persisted rids: the owned plaintext /
/// wire container that `content()`/`wire()` read on demand, released each round
/// by `releaseTransient`.
content_memo: []?[]u8,
wire_memo: []?WireHold,
/// Hashes not in the log, interned/put by a caller. Empty on the serve path.
appendix: repostate.MemStore,
/// sync §3.3 L553-556 / §8.8 L1839-1841 — the served repository's shun table,
/// re-loaded on every build. That is the freshness mechanism the rest of this
/// index already rides: `ServeCtx.baseSnapshot` rebuilds whenever the log has
/// grown, and a `zit shun` grows it, so a shun row written between two served
/// rounds is honoured by the next round with no restart (§3.8's guarantee
/// applied to the shun table rather than to check-ins).
shun: RepoShunSet,
/// sync §9.3 r28 — the same shun table as ROWS, for the `/shun` config group
/// a peer requests. `shun` above answers "is this artifact suppressed" for
/// the send filter; this carries the operator metadata (when, and why) that
/// a peer needs to reconstruct the tombstone rather than merely obey it.
///
/// The reason text borrows `self.log`'s own row storage (which this
/// snapshot owns and outlives every round served from it), but the NAMES do
/// not: the log keys shun rows by BINARY digest and the wire carries
/// lowercase hex, so the hex is materialised once here rather than per
/// round. `shun_hex` is its backing.
shun_rows: []sync.ShunRow,
shun_hex: []u8,
/// The repository's settings, for the `/interwiki` group a peer
/// requests. Strings borrow `self.log`.
setting_rows: []sync.ConfigRow,
/// sync §7.7 L1463-1467 — the served repository's effective hash-policy, read
/// off the same build as `shun` because it is the other half of the same
/// receive-side admission predicate and must never be a build behind it.
hash_policy: artifact.HashPolicy,
/// Total plaintext this store has read on demand from the log, monotonic
/// across `releaseTransient` — the work counter that makes "answering a round
/// materialises only what it emits" testable (the log-backed twin of
/// `MemStore.inflated_bytes`). Container reads are excluded, as they were
/// there: this measures inflation, not verbatim retransmission.
materialized_bytes: usize = 0,
/// Set true when an on-demand `content()`/`wire()` read this round mapped a
/// real error to null (a post-scan I/O fault or an OOM building the wire
/// form) — NOT when a frame legitimately has no plaintext or no wire form.
/// The vtable cannot signal an error, so the §8.2 clone pager would skip the
/// null rid and still report the sequence complete, shipping a clone missing
/// an artifact; `serveAnswer` reads this after the round and converts a set
/// flag to `error.ServerStorage` so the client retries instead. Cleared with
/// the round's memos by `releaseTransient`, so every round starts clean.
read_fault: bool = false,
/// draft-fossil-submanifests-00 §8.1 — this log's durable seen-submanifest
/// indicator, read from the config frame of the SAME log generation this
/// index describes. It arms the §7.2 send gate (`serveRound`), and it lives
/// here rather than beside the long-lived `sync.Server` so it is re-read
/// whenever the snapshot is: a push that lands the repository's first tree
/// grows the log, the stat gate rebuilds the index, and the very next round
/// serves that tree through an ARMED gate. Cached on a stale snapshot it
/// would stay false for the life of the process.
seen_submanifest: bool = false,
/// draft-fossil-submanifests-00 §8.1 — this log's `enable-submanifests`
/// setting, read from the same config frame and for the same reason: §7.2
/// makes the reply's `avoid-submanifests` pragma a function of it, so a
/// stale copy would advertise the wrong policy for the life of the process.
submanifests_enabled: bool = sync.submanifest_emission_default,
/// draft-fossil-submanifests-00 §6.3 / §6.4 — this log's pinned
/// partition parameter and its auto-retune switch, read off the same config
/// frame as the two above and for the same freshness reason: sync §9.3
/// serves them to a cloning peer, so an operator who re-pins k must be
/// answered by the next round, not by the next process.
submanifest_k: u8 = 0,
submanifest_auto_retune: bool = false,
/// The repository tier of the two push-policy settings,
/// copied (owned) off the SAME per-build config read as the four facts
/// above. Before this rode the snapshot, `serveAnswer` resolved the pair
/// through a fresh `LogStore.openReadOnly` per setting per round -- a
/// full frame-header scan of the log each time, which on a 1.17M-artifact
/// store turned every round into ~11 seconds of dead re-scanning and a
/// clone into half an hour. Freshness is unchanged: a settings
/// write appends a config frame, the log grows, and the (size, mtime)
/// snapshot key rebuilds this copy for the next round -- exactly the
/// contract `seen_submanifest` above already relies on. An empty string
/// is the repository having no opinion, as in `repositorySetting`.
policy_binary_glob: []const u8 = "",
policy_crlf_glob: []const u8 = "",
/// The repository's Git email domain, owned and copied off the same read,
/// served over sync §9.3 beside the sub-manifest settings and fresh for
/// the same reason.
git_email_domain: []const u8 = "",
/// sync §8.5 L1688-1689 — the project code off the same read, so a code
/// an adopting server wrote judges the next round and answers the next
/// clone. Empty when the config carries none (or one too long to be a
/// code), and the server's own then stands.
project_code: sync.AdoptedProject = .{},
/// A memoised wire form for one persisted rid: the owned §4.1 container and,
/// for the delta form, the basis' owned hex name (DELTASRC) and the
/// reconstructed target size (delta §3.2). Freed by `releaseTransient`.
const WireHold = struct {
container: []u8,
delta: ?Delta = null,
const Delta = struct { src_hex: []u8, target_size: u64 };
};
/// Build over a caller-owned `StorageFile` (the log borrows it; `fs` unused).
pub fn open(gpa: std.mem.Allocator, storage_file: storage.StorageFile) logstore.OpenError!*LogIndexStore {
const self = try gpa.create(LogIndexStore);
errdefer gpa.destroy(self);
self.gpa = gpa;
self.fs = undefined;
try self.build(storage_file, null);
return self;
}
/// Build over an owned `FileStorage` bound to `file` (the serve path). `size`
/// must equal `file`'s current length (`FileStorage.init`'s precondition).
pub fn openFromFile(gpa: std.mem.Allocator, io: std.Io, file: anytype, size: u64) logstore.OpenError!*LogIndexStore {
return openFromFileIndexed(gpa, io, file, size, null);
}
/// With `root_dir`, the log is opened through the `.zit-cache` log-index
/// sidecar when it is still a prefix of the file, scanning only the frames
/// appended since, and the sidecar is refreshed when the scan extended it.
/// A served store that only ever grows by pushes re-read every frame
/// header on every rebuild without this (1.25M headers after each push on
/// pkgsrc), and so did every `Repository.open` on the same store.
pub fn openFromFileIndexed(gpa: std.mem.Allocator, io: std.Io, file: anytype, size: u64, root_dir: ?std.Io.Dir) logstore.OpenError!*LogIndexStore {
const self = try gpa.create(LogIndexStore);
errdefer gpa.destroy(self);
self.gpa = gpa;
self.fs = storage.FileStorage.init(io, file, size);
var log: ?logstore.LogStore = null;
if (root_dir) |dir| log = openLogAdoptingIndex(gpa, io, dir, self.fs.storageFile(), .refresh_on_scan) catch null;
try self.build(self.fs.storageFile(), log);
return self;
}
fn build(self: *LogIndexStore, storage_file: storage.StorageFile, opened: ?logstore.LogStore) logstore.OpenError!void {
self.materialized_bytes = 0;
// `self` came from `gpa.create` (undefined memory) and is filled field by
// field, so the struct-declaration default `read_fault = false` never runs.
// `serveAnswer` reads this flag on the FIRST round after a prime/rebuild,
// before any `releaseTransient` clears it — leave it uninitialised and a
// garbage-true value spuriously fails a healthy round. Zero it here.
self.read_fault = false;
// Same undefined-memory rule for the cone memo: garbage here reads as
// a live classification and deinit frees wild pointers.
self.cone_memo = null;
self.cone_memo_digest = null;
self.plaintext = logstore.PlaintextCache.init();
self.classify_last = null;
self.classify_wire_last = null;
self.classify_wire_hash_len = 0;
self.classify_sidecar = null;
self.log_to_adapter = null;
self.name_arena = std.heap.ArenaAllocator.init(self.gpa);
errdefer self.name_arena.deinit();
self.by_name = .empty;
errdefer self.by_name.deinit(self.gpa);
self.appendix = repostate.MemStore.init(self.gpa);
errdefer self.appendix.deinit();
self.log = opened orelse try logstore.LogStore.openReadOnly(self.gpa, storage_file);
errdefer self.log.deinit(self.gpa);
// submanifests §8.1 — one config read per index build, never per round;
// both settings come off the same frame.
self.seen_submanifest = false;
self.submanifests_enabled = sync.submanifest_emission_default;
self.submanifest_k = 0;
self.submanifest_auto_retune = false;
self.policy_binary_glob = "";
self.policy_crlf_glob = "";
self.git_email_domain = "";
self.project_code = .{};
if (try self.log.readConfig(self.gpa)) |*oc_const| {
var oc = oc_const.*;
defer oc.deinit(self.gpa);
const code = oc.config.project_code;
if (code.len > 0 and code.len <= sync.max_projectcode_len) self.project_code = .from(code);
self.seen_submanifest = oc.config.seen_submanifest;
self.submanifests_enabled = oc.config.enable_submanifests;
self.submanifest_k = oc.config.submanifest_k;
self.submanifest_auto_retune = oc.config.submanifest_auto_retune;
self.policy_binary_glob = try self.gpa.dupe(u8, oc.config.binary_glob);
errdefer self.gpa.free(self.policy_binary_glob);
self.policy_crlf_glob = try self.gpa.dupe(u8, oc.config.crlf_glob);
errdefer self.gpa.free(self.policy_crlf_glob);
self.git_email_domain = try self.gpa.dupe(u8, oc.config.git_email_domain);
}
// Function-scope: a failure in ANY later build step must free these
// (a zero-length free of the "" default is a no-op).
errdefer self.gpa.free(self.policy_binary_glob);
errdefer self.gpa.free(self.policy_crlf_glob);
errdefer self.gpa.free(self.git_email_domain);
// sync §3.3 L553-556 — the shun table, off this same build's open. Copied
// rather than borrowed so the set has one shape on both the serve and the
// client path (one hex→digest boundary, one lookup).
self.shun = RepoShunSet.init(self.gpa);
errdefer self.shun.deinit();
try self.shun.load(&self.log);
// §9.3 r28 — the row view of that same table, off the same open, so the
// group a peer requests can never be a build behind the filter.
const rows = self.log.shunList();
var hex_bytes: usize = 0;
for (rows) |r| hex_bytes += 2 * r.hash.len;
self.shun_hex = try self.gpa.alloc(u8, hex_bytes);
errdefer self.gpa.free(self.shun_hex);
self.shun_rows = try self.gpa.alloc(sync.ShunRow, rows.len);
errdefer self.gpa.free(self.shun_rows);
var hex_at: usize = 0;
for (rows, self.shun_rows) |src_row, *dst| {
const span = self.shun_hex[hex_at..][0 .. 2 * src_row.hash.len];
_ = std.fmt.bufPrint(span, "{x}", .{src_row.hash}) catch unreachable;
hex_at += span.len;
dst.* = .{ .hash = span, .epoch = src_row.epoch, .reason = src_row.reason };
}
{
const table = try self.log.settingTable(self.gpa);
defer self.gpa.free(table);
self.setting_rows = try self.gpa.alloc(sync.ConfigRow, table.len);
for (table, self.setting_rows) |src_row, *dst| {
dst.* = .{ .name = src_row.name, .value = src_row.value, .mtime = src_row.mtime };
}
}
errdefer self.gpa.free(self.setting_rows);
// sync §7.7 L1463-1467 — and the effective hash-policy off the same open.
self.hash_policy = try effectiveHashPolicy(self.gpa, &self.log);
const persisted = try self.log.ridsInPersistedOrder(self.gpa);
defer self.gpa.free(persisted);
// The `.config` test is a GUARD, not a live skip: a config frame is never
// interned (`LogStore.scan` records only its offset, and `putConfig` appends
// without touching `entries`), so `ridsInPersistedOrder` — which walks
// `entries` — cannot yield one today. Kept because this counting loop and
// the filling loop below must agree on `n` exactly, and because
// `loadContentStore` states the same guard: the day a config frame becomes
// interned, both stay correct instead of both silently naming it as an
// artifact.
var n: usize = 0;
for (persisted) |lr| {
if (self.log.kindOf(lr) != .config) n += 1;
}
self.rids = try self.gpa.alloc(logstore.Rid, n);
errdefer self.gpa.free(self.rids);
self.names = try self.gpa.alloc([]const u8, n);
errdefer self.gpa.free(self.names);
self.masked = try self.gpa.alloc(bool, n);
errdefer self.gpa.free(self.masked);
self.content_memo = try self.gpa.alloc(?[]u8, n);
errdefer self.gpa.free(self.content_memo);
self.wire_memo = try self.gpa.alloc(?WireHold, n);
errdefer self.gpa.free(self.wire_memo);
self.by_log = n == persisted.len and n == self.log.entryCount() and inFrameOrder(persisted);
var a: usize = 0;
if (self.by_log) {
@memcpy(self.rids, persisted);
@memset(self.names, &.{});
@memset(self.content_memo, null);
@memset(self.wire_memo, null);
// sync §9.5 L2170-2174 — the same mask the loop below derives, asked
// from the shun set's side: a shunned digest the log holds masks
// that entry, which is that entry's digest being in the set.
@memset(self.masked, false);
var shunned = self.shun.digests.keyIterator();
while (shunned.next()) |digest| {
const lr = self.log.lookup(digest.*) orelse continue;
if (!self.log.isPhantom(lr)) self.masked[@intCast(lr - 1)] = true;
}
a = n;
}
if (!self.by_log) try self.by_name.ensureTotalCapacity(self.gpa, @intCast(n));
// Every name in one allocation: a store's worth of small arena
// allocations was a measurable share of opening the holdings.
const name_bytes: []u8 = if (self.by_log) &.{} else try self.name_arena.allocator().alloc(u8, n * 2 * digest_len);
for (persisted) |lr| {
if (self.by_log) break;
if (self.log.kindOf(lr) == .config) continue;
// The one site the binary-log digest becomes the wire name (sync §3.3),
// exactly as `loadContentStore` did — held to one place.
const name_buf = name_bytes[a * 2 * digest_len ..][0 .. 2 * digest_len];
const owned = digestToWireName(self.log.hashOf(lr), name_buf);
self.rids[a] = lr;
self.names[a] = owned;
self.content_memo[a] = null;
self.wire_memo[a] = null;
// sync §9.5 L2170-2174 — the shun filter, applied to the enumeration
// itself and off THIS iteration's index, so every parallel array plus
// the mask derive from one loop and one counter (see `masked`).
const is_masked = self.shun.digests.contains(self.log.hashOf(lr));
self.masked[a] = is_masked;
// Hashes are unique per log entry (content-addressed intern), so no key
// collides; adapter rid is a+1. A masked name is deliberately absent:
// `lookup` answering null is what makes an inbound `gimme` for it hit
// §3.3 L569-572's "If absent, silently skip" with no code of its own.
if (!is_masked) self.by_name.putAssumeCapacity(owned, @intCast(a + 1));
a += 1;
}
// THE PARALLEL-ARRAY LENGTH GATE. `heldImpl`, `contentImpl` and `wireImpl`
// bounds-test an incoming rid against `rids.len` ALONE and then index
// `masked`/`content_memo`/`wire_memo` with it. Their alignment is what makes
// adapter rid r name `rids[r-1]`; a length that disagreed would be an
// out-of-bounds read, and in ReleaseFast/ReleaseSmall the slice bounds check
// is gone too, so it would be a silent one: a garbage `masked` true withholds
// a survivor (the silently short clone `read_fault` exists to prevent), a
// garbage false serves an artifact the operator shunned (the §9.5 L2170-2174
// policy this whole filter exists to enforce). Both must be impossible in a
// RELEASE build, where `std.debug.assert` compiles to nothing — so this is a
// CHECKED error, paid once per build (five comparisons against an O(store)
// walk), not an assertion. It also carries the fill loop's own `a == n`, which
// is the same question asked of the counter rather than of the allocations.
if (a != n or
self.rids.len != n or
self.names.len != n or
self.masked.len != n or
self.content_memo.len != n or
self.wire_memo.len != n) return error.Corrupt;
}
/// Free every per-round transient memo, restoring the store to "index only,
/// no materialised container bytes" — the shape between rounds. Every borrowed
/// `content()`/`wire()` slice handed out this round is invalidated here, so it
/// runs only once every borrow of the snapshot has ended (see
/// `ServeCtx.releaseSnapshotPlaintext`).
pub fn releaseTransient(self: *LogIndexStore) void {
self.plaintext.deinit(self.gpa);
self.plaintext = logstore.PlaintextCache.init();
if (self.classify_last) |b| self.gpa.free(b);
self.classify_last = null;
if (self.classify_wire_last) |b| self.gpa.free(b);
self.classify_wire_last = null;
for (self.content_memo) |*slot| {
if (slot.*) |b| self.gpa.free(b);
slot.* = null;
}
for (self.wire_memo) |*slot| {
if (slot.*) |wh| {
self.gpa.free(wh.container);
if (wh.delta) |d| self.gpa.free(d.src_hex);
}
slot.* = null;
}
// A round's fault is a property of that round's reads; clearing it here
// (once every borrow of the round's memos has ended) lets a retry start
// clean over the same snapshot.
self.read_fault = false;
}
/// Total container/plaintext bytes currently memoised — a test/accounting
/// probe. ~0 right after a prime (the whole point: no whole-store copy) and
/// after `releaseTransient`; rises only as a round materialises what it emits.
pub fn residentBytes(self: *const LogIndexStore) usize {
var total: usize = 0;
for (self.content_memo) |slot| {
if (slot) |b| total += b.len;
}
for (self.wire_memo) |slot| {
if (slot) |wh| total += wh.container.len;
}
return total;
}
pub fn deinit(self: *LogIndexStore) void {
if (self.cone_memo) |*c| c.deinit(self.gpa);
if (self.log_to_adapter) |m| self.gpa.free(m);
self.releaseTransient();
self.plaintext.deinit(self.gpa);
self.gpa.free(self.content_memo);
self.gpa.free(self.wire_memo);
self.gpa.free(self.rids);
self.gpa.free(self.names);
self.gpa.free(self.masked);
self.by_name.deinit(self.gpa);
self.name_arena.deinit();
self.appendix.deinit();
self.shun.deinit();
self.gpa.free(self.shun_rows);
self.gpa.free(self.shun_hex);
self.gpa.free(self.setting_rows);
self.gpa.free(self.policy_binary_glob);
self.gpa.free(self.policy_crlf_glob);
self.gpa.free(self.git_email_domain);
self.log.deinit(self.gpa);
}
/// Free the store's internals and the heap box. `self` is invalid after.
pub fn destroy(self: *LogIndexStore) void {
const gpa = self.gpa;
self.deinit();
gpa.destroy(self);
}
/// The ConeClassifier vtable target (sparse-cone RFC section 6.2). Any
/// failure -- undecodable member, invalid cone, memo OOM -- answers null:
/// the reply degrades to dense, never to an error (section 6.3).
fn classifyCone(
context: *anyopaque,
members: []const []const u8,
not_members: []const []const u8,
parents: bool,
horizon: ?cone_mod.Horizon,
) ?*const SparseClassification {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (members.len == 0 and horizon == null) return null;
if (members.len + not_members.len > cone_mod.max_members) return null;
var arena_state = std.heap.ArenaAllocator.init(self.gpa);
defer arena_state.deinit();
const aa = arena_state.allocator();
var decoded: std.ArrayListUnmanaged([]const u8) = .empty;
for (members) |member| {
const buf = aa.alloc(u8, member.len) catch return null;
const plain = artifact.decodeEscapes(member, buf);
decoded.append(aa, plain) catch return null;
}
var decoded_not: std.ArrayListUnmanaged([]const u8) = .empty;
for (not_members) |member| {
const buf = aa.alloc(u8, member.len) catch return null;
const plain = artifact.decodeEscapes(member, buf);
decoded_not.append(aa, plain) catch return null;
}
var cone = cone_mod.Cone.initWith(self.gpa, decoded.items, decoded_not.items, .{ .horizon = horizon, .parents = parents }) catch return null;
defer cone.deinit(self.gpa);
const digest = cone.digest(self.gpa) catch return null;
if (self.cone_memo_digest) |have| {
if (std.mem.eql(u8, &have, &digest)) return &self.cone_memo.?;
}
// Never intern on the serve path: the snapshot is shared across rounds
// and `count()` includes its appendix, so interned phantoms would sit
// behind the clone cursor and shadow rids a rebuilt snapshot adds.
// The keep set is all a reply needs; phantoms are the client's concern.
// Under a horizon, the tree nodes no slot of a kept tree can reach
// are withheld as a section 6.2 server withholds them, so a cone of
// one directory costs its own path through each tree rather than the
// whole tree: K's trees under a structural horizon, every check-in's
// under a content one. The walk descends only slots that may reach
// the cone, is memoised per node, and lives in the sidecar, so a
// classification reads the trees that arrived since, and a first one
// the manifests plus their in-cone nodes -- never every tree node in
// the store (see `ClassifyState.walkSlots`). A cone with no horizon
// classifies by reading and keeps every node, as before. An origin
// never needs the client's absent-node names.
const classify_opts: sparse.ClassifyOptions = .{ .intern_absent = false, .elide_nodes = false, .slot_held_nodes = horizon != null };
// The graph: read once per served store, then only the frames this
// snapshot added. A cone with path members is classified over it
// too, the walk reading K's trees for their filenames; a cone with
// no horizon has no walk and classifies by reading, as before.
var source: sparse.Source = .parse;
if (self.classify_sidecar) |sc| {
if (sc.graph) |slot| if (horizon != null) {
// A fresh server whose sidecar for this cone was written under
// exactly this log answers from it alone: loading the graph
// (hundreds of megabytes on a large store) would buy nothing.
if (slot.* == null) {
if (sparse.classifyCurrent(self.gpa, sc.io, sc.dir, &self.log, self.classifyStore(), &cone, classify_opts) catch null) |c| {
if (self.cone_memo) |*old_memo| old_memo.deinit(self.gpa);
self.cone_memo = c;
self.cone_memo_digest = digest;
return &self.cone_memo.?;
}
}
if (slot.*) |*g| {
const n_before = g.n;
if (g.extend(self.gpa, self.classifyStore())) |_| {
// Only a graph the snapshot's frames extended is new.
if (g.n != n_before) if (self.log.logKey()) |key| {
if (g.serialize(self.gpa, key, self.log.entryCount()) catch null) |blob| {
defer self.gpa.free(blob);
repocache.storeNamedMember(sc.io, sc.dir, structgraph.Graph.member, blob);
}
} else |_| {};
} else |_| {}
} else {
slot.* = structgraph.Graph.loadOrBuild(self.gpa, sc.io, sc.dir, &self.log, self.classifyStore()) catch null;
}
if (slot.*) |*g| source = .{ .graph = g };
};
}
const classification = self.classify(&cone, classify_opts, source) catch return null;
if (self.cone_memo) |*old_memo| old_memo.deinit(self.gpa);
self.cone_memo = classification;
self.cone_memo_digest = digest;
return &self.cone_memo.?;
}
/// This snapshot's cone classification as an origin serves it. A cone
/// of paths alone is answered from the cone index (`zit.coneindex`) when
/// one can be had: read from `.zit-cache`, folded forward over an
/// appended tail, or built by one sweep and stored. That sweep is the one
/// time the store is read whole; every later cone, and every later
/// snapshot of a grown log, costs the index and the tail. A horizon cone
/// is the horizon classifier's: the index records paths and structure,
/// not the history a horizon cuts. Any refusal -- no cache directory, an
/// unreadable frame, a tree with absent nodes -- classifies by reading,
/// through the cone's sidecar when there is a directory to keep it in.
pub fn classify(self: *LogIndexStore, cone: *const cone_mod.Cone, opts: sparse.ClassifyOptions, source: sparse.Source) error{OutOfMemory}!SparseClassification {
if (cone.horizon == null) {
if (try self.classifyIndexed(cone)) |c| return c;
}
if (self.classify_sidecar) |sc| return sparse.classifyCached(self.gpa, sc.io, sc.dir, &self.log, self.classifyStore(), cone, opts, source);
return sparse.classifyKeepRidsOpts(self.gpa, self.classifyStore(), cone, opts);
}
/// The origin's classification of a path-only `cone` from the cone
/// index, or null when the index cannot answer it exactly. Its sets are
/// the reading classifier's under the origin's options: nothing interned,
/// no absent node elided by name (the index refuses a store that has one).
fn classifyIndexed(self: *LogIndexStore, cone: *const cone_mod.Cone) error{OutOfMemory}!?SparseClassification {
std.debug.assert(cone.horizon == null);
const sc = self.classify_sidecar orelse return null;
if (self.log.entryCount() > coneindex.max_rid) return null;
// The index describes the log; an artifact put since lives only in
// the appendix, which the index has never seen.
if (self.appendix.store().count() != 0) return null;
const blob = (try coneindex.loadOrBuild(self.gpa, sc.io, sc.dir, &self.log)) orelse return null;
defer self.gpa.free(blob);
// Freshly validated against this same log by `loadOrBuild`.
const idx = coneindex.parse(blob, &self.log) orelse return null;
if (!idx.coversAll(&self.log)) return null;
try self.buildAdapterMap();
const rid_map: sparse.RidMap = .{ .ctx = self, .mapFn = mapLogRid };
return sparse.classifyIndexed(self.gpa, self.store(), cone, &idx, rid_map);
}
/// The inverse of `rids`, leaving masked frames out exactly as `by_name`
/// does, so a log rid maps to an adapter rid iff `lookup` of its name would.
pub fn buildAdapterMap(self: *LogIndexStore) error{OutOfMemory}!void {
if (self.log_to_adapter != null) return;
const map = try self.gpa.alloc(u32, self.log.entryCount() + 1);
@memset(map, 0);
for (self.rids, self.masked, 0..) |lr, m, a| {
if (m) continue;
map[@intCast(lr)] = @intCast(a + 1);
}
self.log_to_adapter = map;
}
pub fn mapLogRid(ctx: *const anyopaque, log_rid: logstore.Rid) ?repostate.Rid {
const self: *const LogIndexStore = @ptrCast(@alignCast(ctx));
const map = self.log_to_adapter.?;
if (log_rid >= map.len) return null;
const a = map[@intCast(log_rid)];
return if (a == 0) null else a;
}
pub fn coneClassifier(self: *LogIndexStore) sync.ConeClassifier {
return .{ .context = @ptrCast(self), .vtable = &.{ .classify = classifyCone } };
}
pub fn store(self: *LogIndexStore) repostate.ArtifactStore {
return .{ .context = self, .vtable = &vtable };
}
const vtable = repostate.ArtifactStore.VTable{
.intern = internImpl,
.put = putImpl,
.content = contentImpl,
.hashOf = hashOfImpl,
.lookup = lookupImpl,
.count = countImpl,
.wire = wireImpl,
.retain = retainImpl,
.held = heldImpl,
};
fn wireFrameOf(wh: *const WireHold) repostate.WireFrame {
if (wh.delta) |d| return .{ .container = wh.container, .delta = .{ .src_hash = d.src_hex, .target_size = d.target_size } };
return .{ .container = wh.container };
}
fn countImpl(context: *anyopaque) usize {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
return self.rids.len + self.appendix.store().count();
}
/// The adapter rid of a MASKED entry named `hash`, or null when this
/// enumeration does not carry it under a mask. The mask deliberately keeps the
/// name out of `by_name` (that absence IS §3.3 L569-572's "If absent, silently
/// skip" on the gimme path), so the write doors below need this second, explicit
/// question to avoid MINTING A SECOND IDENTITY for a name the enumeration
/// already holds — see `putImpl`. A scan rather than a second name map, so
/// "masked" has exactly one representation in this struct.
///
/// THE TIME BOUND, in code rather than in prose. The walk is O(`rids.len`) per
/// call and its callers are the vtable's write doors, whose call count belongs
/// to a caller this struct does not bound: interning k names the log does not
/// carry would otherwise pay k full walks plus a `std.mem.eql` per masked slot.
/// An empty shun table is the case in which no slot can match, and it is also
/// the overwhelmingly common one — a repository that has never shunned anything
/// — so answering it in O(1) leaves the linear walk reachable only where a shun
/// table exists to bound it. (Not reached on the serve path either way: every
/// served write lands in the per-request `OverlayStore`.)
fn maskedRidFor(self: *LogIndexStore, hash: []const u8) ?repostate.Rid {
if (self.shun.digests.count() == 0) return null;
for (self.masked, 0..) |m, i| {
if (!m) continue;
if (std.mem.eql(u8, self.nameAt(i), hash)) return @intCast(i + 1);
}
return null;
}
/// Whether `persisted` is rids 1..n in order: log rid and adapter rid agree.
fn inFrameOrder(persisted: []const logstore.Rid) bool {
for (persisted, 1..) |lr, want| {
if (lr != want) return false;
}
return true;
}
/// Adapter slot `i`'s wire name, made on first use under `by_log`. The
/// panic on allocation failure is forced: `ArtifactStore.hashOf`, which
/// reaches here, returns a name and cannot fail, and every caller borrows
/// that name without an error path. The allocation is 64 bytes from an
/// arena that already holds the store's index.
pub fn nameAt(self: *LogIndexStore, i: usize) []const u8 {
if (self.names[i].len != 0) return self.names[i];
std.debug.assert(self.by_log);
const buf = self.name_arena.allocator().alloc(u8, 2 * digest_len) catch
@panic("out of memory naming a held artifact");
self.names[i] = digestToWireName(self.log.hashOf(self.rids[i]), buf[0 .. 2 * digest_len]);
return self.names[i];
}
/// The adapter rid of the unmasked persisted artifact named `hash`, or
/// null: what `by_name` answers, which under `by_log` the log's own index
/// answers. A name must be lowercase hex to match, as a key in `by_name`
/// must.
pub fn nameRid(self: *const LogIndexStore, hash: []const u8) ?repostate.Rid {
if (!self.by_log) return self.by_name.get(hash);
for (hash) |c| switch (c) {
'0'...'9', 'a'...'f' => {},
else => return null,
};
var digest_buf: [digest_len]u8 = undefined;
const digest = decodeWireName(hash, &digest_buf) orelse return null;
const lr = self.log.lookup(digest) orelse return null;
if (lr > self.rids.len or self.log.isPhantom(lr)) return null;
if (self.masked[@intCast(lr - 1)]) return null;
return lr;
}
fn internImpl(context: *anyopaque, hash: []const u8) error{OutOfMemory}!repostate.Rid {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (self.nameRid(hash)) |rid| return rid;
// A MASKED name resolves to ITS OWN slot, not to a fresh appendix phantom:
// the enumeration carries that frame at a fixed adapter rid (§8.2 L1551-1553
// gap), and it is already exactly what §3.3 L537-540 would create — a record
// with the name and no content ("an igot HASH creates a phantom only when
// HASH is absent"; `held` is false and `content`/`wire` are null here). A
// second identity for one name would be the only way this store could hand
// the walk two rids naming the same artifact.
//
// Not the mask's safety argument, which lives one layer up: this store never
// sees a served `intern` at all — `respond` runs over an `OverlayStore` whose
// `internImpl` probes `base.lookup` and, on the mask's null, interns into its
// OWN round-local `added` store, which is discarded at round end. What keeps
// that round's reply from begging for the name is `Server.shun` (§3.3
// L527-530), which `serveRound` sets from this same snapshot's table.
if (self.maskedRidFor(hash)) |rid| return rid;
return self.rids.len + try self.appendix.store().intern(hash);
}
fn lookupImpl(context: *anyopaque, hash: []const u8) ?repostate.Rid {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (self.nameRid(hash)) |rid| return rid;
if (self.appendix.store().lookup(hash)) |l| return self.rids.len + l;
return null;
}
fn putImpl(context: *anyopaque, hash: []const u8, content_bytes: []const u8) error{OutOfMemory}!repostate.Rid {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
// A persisted hash is content-addressed and already carries these bytes,
// so `put` is idempotent and `content(rid)` still reads them from the log
// — the serve path never reaches this arm (writes go to the overlay).
if (self.nameRid(hash)) |rid| {
// Parity with `MemStore.putImpl`: a repeat put must carry identical
// bytes (a differing answer is a hash collision). Verified only under
// runtime safety, and only when the log frame actually inflates — a
// read fault leaves `content` null and the assert is skipped rather
// than firing spuriously. Inflate through a THROWAWAY read (not
// `contentImpl`), so this assertion path never memoises a container or
// bumps `materialized_bytes`/`residentBytes` — the probes the round
// tests assert on stay untouched by an assert that never runs on serve.
if (std.debug.runtime_safety) {
if (self.log.content(self.gpa, self.rids[@intCast(rid - 1)]) catch null) |existing| {
defer self.gpa.free(existing);
std.debug.assert(std.mem.eql(u8, existing, content_bytes));
}
}
return rid;
}
// A MASKED name is not in `by_name`, so without this arm a `put` of one
// would fall through to the appendix and create a SECOND, HELD entry for it:
// `held` true, `content` the pushed bytes, and `lookup` resolving the masked
// name to it — the §3.2 walk would then announce the shunned name and the
// clone pager would emit its `cfile`, undoing the whole §9.5 L2170-2174
// filter through a write. The mask outranks the write: the frame keeps its
// one adapter rid and stays a §8.2 L1551-1553 gap.
//
// A DELIBERATE narrowing of the store contract's "after `put`, `content` is
// readable" — `content(rid)` stays null here. The alternative is a name this
// server has been told never to send becoming servable because a peer pushed
// it back, and admitting a shunned target is a receive-side decision (§7.7
// L1437-1440: "consumed and discarded … makes no repository mutation"), not
// something a store adapter may take by accident.
//
// The narrowing cannot be SIGNALLED — the vtable's error set is
// `error{OutOfMemory}` — so what keeps it safe is that no caller is in a
// position to be surprised by it. On serve that holds because `respond` runs
// over a per-request `repostate.OverlayStore` and every served write lands
// THERE, never on this base. That is a property of the serve path rather than
// of this function, so it is PINNED BY A TEST at this layer ("a served round's
// writes land in the round-local overlay — the base enumeration's write doors
// are never reached") instead of resting on a claim this comment makes about
// code two modules away. A caller that wires this store in as a WRITABLE store
// is outside that proof and must read this arm before relying on `put`.
if (self.maskedRidFor(hash)) |rid| return rid;
return self.rids.len + try self.appendix.store().put(hash, content_bytes);
}
fn hashOfImpl(context: *anyopaque, rid: repostate.Rid) []const u8 {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (rid <= self.rids.len) return self.nameAt(@intCast(rid - 1));
return self.appendix.store().hashOf(rid - self.rids.len);
}
fn heldImpl(context: *anyopaque, rid: repostate.Rid) bool {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
// Every persisted (non-phantom, non-config) frame carries a form to serve
// verbatim, so it is held — answered from the index with no frame read.
// A MASKED frame (sync §9.5 L2170-2174) is the exception: it is a §8.2
// L1504-1506 gap, so it is not held and §3.2's walk never announces it.
// This line is the one place the send side genuinely disagreed with the
// log: `LogStore.content`/`wire` already read a shunned frame as ABSENT,
// while this answered true from the index with no frame read — so the
// inventory walk advertised an artifact no round could then send, and (with
// the name still in `by_name`) `emitCloneReply`'s D3 basis proof could reach
// a resolvable rid whose two forms were both null. `LogStore.shunned`'s doc
// names exactly this disagreement as a SYNC decision to settle; §9.5's
// "every eligible holding after the Section 3.3 shun … filters" settles it.
if (rid <= self.rids.len) return !self.masked[@intCast(rid - 1)];
return self.appendix.store().held(rid - self.rids.len);
}
fn retainImpl(context: *anyopaque, rid: repostate.Rid, ingest: repostate.WireIngest) error{OutOfMemory}!void {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
// Advisory (sync §7.2): a persisted rid already carries a stored form, so
// first-form-wins makes this a no-op — the log is the retained form.
if (rid <= self.rids.len) return;
return self.appendix.store().retain(rid - self.rids.len, ingest);
}
fn contentImpl(context: *anyopaque, rid: repostate.Rid) ?[]const u8 {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (rid > self.rids.len) return self.appendix.store().content(rid - self.rids.len);
const i: usize = @intCast(rid - 1);
// sync §9.5 L2170-2174 — a masked frame has nothing to hand out. Checked
// before the memo so the mask cannot be bypassed by a slot some earlier
// round filled, and it is a plain null: a policy gap, never a read fault.
//
// `LogStore.content` ALREADY answers null for a shunned entry (its own
// rid-keyed read mask), so today this line changes no answer: it is a
// structural guard, not covered behaviour. Measured, not assumed — deleting
// this line leaves the whole suite green, because the read one line below
// returns the same null. What IS covered, by "the adapter owns its content
// mask", is the surrounding fact the guard exists to protect: the frame's
// plaintext is still on disk and still readable through
// `LogStore.contentUnmasked`, so the null is a policy refusal over present
// bytes rather than an absence. That test is what would turn red the day the
// two layers were consolidated into one. It is here
// because the adapter must OWN its mask rather than inherit it. §7.7
// L1444-1449 wants a held-but-shunned artifact to remain usable as a delta
// BASIS — "For an admitted target whose DELTASRC name is in the repository
// shun table, the receiver does not apply admission independently to that
// source: content still held locally MAY serve as the delta basis" — and the
// only reason that works today is that basis resolution walks byte offsets
// and never re-enters `content`. The moment that basis path is given an
// explicitly unmasked door — the repair `LogStore.shunned`'s own doc
// contemplates — the log's mask stops covering this one, and the send-side
// exclusion must not evaporate with it.
if (self.masked[i]) return null;
if (self.content_memo[i]) |b| return b;
// Read the frame on demand and memoise for the round. The vtable cannot
// signal an error, so a read fault is recorded on `read_fault` and
// returned as null; `serveAnswer` turns a set flag into
// `error.ServerStorage` so the client retries rather than accepting a
// silently short clone. A frame that legitimately has no plaintext is a
// plain null and does NOT set the flag.
//
// THIS IS THE PRIMARY CORRUPTION CHANNEL IN SERVE, and it did not used
// to be. `openReadOnly` once CRC-verified every payload while building
// the snapshot, which meant a damaged store failed to open at all and
// this catch saw only genuine post-scan faults -- transient EIO, or rot
// arriving after a clean scan. Both open paths now scan header-only and
// `readFrameAt` CRC-checks every frame it reads instead, so ordinary
// stored corruption reaches serve HERE, on the read, rather than at
// snapshot build.
//
// That is a better shape, not a worse one: one bad frame used to make
// the server refuse to serve anything, and now it fails only the round
// that touches the damage while every healthy artifact still moves. But
// it means this arm is live traffic rather than a rare event, and
// anything reasoning about how often it fires should start from that.
const maybe = self.log.contentCached(self.gpa, self.rids[i], &self.plaintext) catch {
self.read_fault = true;
return null;
};
const bytes = maybe orelse return null;
self.content_memo[i] = bytes;
self.materialized_bytes += bytes.len;
return bytes;
}
/// The store as the CLASSIFIER should see it: identical to `store()` in
/// every answer, but `content` keeps only the most recent plaintext alive,
/// freeing it on the next call, instead of memoizing every artifact for
/// the round. Classification reads every held artifact exactly once and
/// is done with each before asking for the next, so memoizing them all
/// pinned the whole store's inflated plaintext in memory for nothing. The
/// delta-base cache still applies underneath, so chains stay cheap.
pub fn classifyStore(self: *LogIndexStore) repostate.ArtifactStore {
return .{ .context = self, .vtable = &classify_vtable };
}
const classify_vtable = repostate.ArtifactStore.VTable{
.intern = internImpl,
.put = putImpl,
.content = classifyContentImpl,
.hashOf = hashOfImpl,
.lookup = lookupImpl,
.count = countImpl,
.wire = classifyWireImpl,
.retain = retainImpl,
.held = heldImpl,
};
/// `wire` for the classifier: the frame's container read once and held in
/// one slot until the next call, never memoized for the round -- the
/// classifier peeks at a container's first bytes to skip inflating a
/// blob, and memoizing every container would resident the compressed store.
fn classifyWireImpl(context: *anyopaque, rid: repostate.Rid) ?repostate.WireFrame {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (rid > self.rids.len) return self.appendix.store().wire(rid - self.rids.len);
const i: usize = @intCast(rid - 1);
if (self.masked[i]) return null;
if (self.wire_memo[i]) |*wh| return wireFrameOf(wh); // a round already paid for it
if (self.classify_wire_last) |b| self.gpa.free(b);
self.classify_wire_last = null;
const w = (self.log.wire(self.gpa, self.rids[i]) catch {
self.read_fault = true;
return null;
}) orelse return null;
switch (w) {
.full => |c| {
self.classify_wire_last = c;
return .{ .container = c };
},
.delta => |d| {
self.classify_wire_last = d.container;
var hex_len: u8 = 0;
const hex = digestToWireName(d.base_hash_buf[0..d.base_hash_len], &self.classify_wire_hash);
hex_len = @intCast(hex.len);
self.classify_wire_hash_len = hex_len;
return .{ .container = d.container, .delta = .{ .src_hash = self.classify_wire_hash[0..hex_len], .target_size = d.target_size } };
},
}
}
fn classifyContentImpl(context: *anyopaque, rid: repostate.Rid) ?[]const u8 {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (rid > self.rids.len) return self.appendix.store().content(rid - self.rids.len);
const i: usize = @intCast(rid - 1);
if (self.masked[i]) return null;
if (self.content_memo[i]) |b| return b; // a round already paid for it
if (self.classify_last) |b| self.gpa.free(b);
self.classify_last = null;
// Borrowed, not copied: a delta frame's plaintext lives in the base
// cache (which the next read may evict, so the classifier gets one
// artifact at a time, which is all it ever asks for); a top-level full
// frame is a fresh buffer this slot frees on the next call.
const maybe = self.log.contentBorrowed(self.gpa, self.rids[i], &self.plaintext) catch {
self.read_fault = true;
return null;
};
const b = maybe orelse return null;
if (b.owned) self.classify_last = b.bytes;
return b.bytes;
}
fn wireImpl(context: *anyopaque, rid: repostate.Rid) ?repostate.WireFrame {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (rid > self.rids.len) return self.appendix.store().wire(rid - self.rids.len);
const i: usize = @intCast(rid - 1);
// sync §9.5 L2170-2174 — same gap as `contentImpl`, checked ahead of the
// memo, and owned here for the same reason (see there).
if (self.masked[i]) return null;
if (self.wire_memo[i]) |*wh| return wireFrameOf(wh);
// A frame read fault is a real error recorded on `read_fault` so
// `serveAnswer` can fail the round; a `.raw`/phantom frame legitimately
// has no wire form (the responder compresses `content()` instead) and
// yields a plain null that does NOT set the flag. As at `contentImpl`,
// this arm now sees ordinary stored corruption rather than only
// post-scan faults -- no open path CRC-verifies payloads any more.
var w = (self.log.wire(self.gpa, self.rids[i]) catch {
self.read_fault = true;
return null;
}) orelse return null;
switch (w) {
.full => |c| {
self.wire_memo[i] = .{ .container = c };
},
.delta => |d| {
// sync §9.5 L2170-2174 — the delta form is servable only while its
// basis is. `zit shun --defer` records the row without rewriting
// the generation, so a SURVIVING frame here can be a delta rooted
// on a MASKED one; emitting its `cfile` would ship a DELTASRC this
// server never sends, leaving the peer a permanently dangling delta
// whose basis it gimmes and is silently skipped on (§3.3 L569-572)
// — corrupt rather than absent, and `emitCloneReply`'s D3 basis
// lookup would trap under `runtime_safety` on the way. Withhold the
// wire form instead; the responder's §7.2 compress-on-the-fly arm
// then emits the FULL form, and `log.content` reconstructs through
// the frame chain by BYTE OFFSET, which the name-level mask does
// not reach. A plain null, not a read fault: policy, not I/O.
//
// Asked as the SAME predicate the fill loop applied — the raw digest
// against the shun table — so "masked" has one spelling in this
// struct. (`by_name`-absence would answer identically for a log
// delta, whose basis is always a non-config frame in the
// enumeration, but it would make the exclusion mean two things and
// pay a hex encode plus a map lookup to say the second one.)
if (self.shun.digests.contains(d.baseHash())) {
w.deinit(self.gpa);
return null;
}
// The basis precedes this frame in the log, so its wire name is the
// hex of the base digest the wire read carried (delta §3.2 target
// size alongside).
var src_buf: [2 * digest_len]u8 = undefined;
const src_hex = digestToWireName(d.baseHash(), &src_buf);
const owned_src = self.gpa.dupe(u8, src_hex) catch {
// Failing to produce the wire form here would silently drop a
// held delta from the clone; record it so the round errors.
w.deinit(self.gpa);
self.read_fault = true;
return null;
};
self.wire_memo[i] = .{ .container = d.container, .delta = .{ .src_hex = owned_src, .target_size = d.target_size } };
},
}
return wireFrameOf(&self.wire_memo[i].?);
}
};
pub const RepoShunSet = struct {
gpa: std.mem.Allocator,
/// Keys are owned binary digests; the value side carries nothing — this is a
/// set. Empty is the common case (no repository has a shun row until an
/// operator writes one), and an empty map allocates nothing.
digests: std.StringHashMapUnmanaged(void) = .empty,
pub fn init(gpa: std.mem.Allocator) RepoShunSet {
return .{ .gpa = gpa };
}
/// Copy every shun row's digest out of `log`. `shunList` borrows the store's
/// own bytes, which die with the handle they were read through, so each key
/// is duped. Re-loading over a populated set replaces it wholesale, which is
/// what a `LogIndexStore` rebuild wants.
pub fn load(self: *RepoShunSet, log: *const logstore.LogStore) error{OutOfMemory}!void {
self.clear();
const rows = log.shunList();
// State the bound where it is relied on rather than narrowing usize into
// the map's u32 capacity argument by an unchecked cast: a truncated
// reservation would let `getOrPutAssumeCapacity` below write past the
// capacity it assumes, which is silent corruption in ReleaseFast instead
// of the `error.OutOfMemory` a shun table that large has already earned.
// This is the one such count an operator grows by direct action.
const row_count = std.math.cast(u32, rows.len) orelse return error.OutOfMemory;
try self.digests.ensureTotalCapacity(self.gpa, row_count);
for (rows) |row| {
// A pending, cancelled or unverified erasure masks nothing.
if (!log.isShunned(row.hash)) continue;
// Dupe BEFORE inserting. `row.hash` belongs to the store and dies
// with the handle, so the key must be owned — and inserting the
// borrowed slice first and replacing it after leaves a BORROWED key
// in an owned-key map on the one error this function can return.
// Every unwind path then runs `clear`, which frees the store's own
// live digest buffer: a double free at `log.deinit`, and a
// use-after-free of `shun_entries[i].hash` while the store is open.
const owned = try self.gpa.dupe(u8, row.hash);
// Capacity is already reserved for every row, so this cannot fail.
const gop = self.digests.getOrPutAssumeCapacity(owned);
// Two rows naming the same digest collapse to one entry, and the
// second dupe is surplus — say so rather than leaking it.
if (gop.found_existing) self.gpa.free(owned) else gop.key_ptr.* = owned;
}
}
fn clear(self: *RepoShunSet) void {
var it = self.digests.keyIterator();
while (it.next()) |k| self.gpa.free(k.*);
self.digests.clearAndFree(self.gpa);
}
pub fn deinit(self: *RepoShunSet) void {
self.clear();
}
pub fn shunSet(self: *RepoShunSet) sync.ShunSet {
return .{ .context = self, .vtable = &vtable };
}
const vtable: sync.ShunSet.VTable = .{ .contains = containsWireName };
/// Exact-key lookup on the digest a COMPLETE wire name decodes to. A name
/// whose WIDTH is not one of artifact-format §3.2 L347-350's two forms is
/// rejected before the decode, which is the order `isShunned`'s caller
/// contract fixes; a correct-width token that is not hex fails the decode.
/// Neither can be a key any shun row carries (`putShun` rejects a
/// non-complete key with `error.BadKey`, and the open-time scan fails
/// `error.Corrupt` on one), so `false` is the exact answer. Never a prefix
/// match, never a truncate, never a pad: a prefix must not be able to
/// masquerade as a shun-table hit.
///
/// An UPPERCASE-hex spelling of a shunned name decodes to the same digest
/// and therefore answers TRUE. artifact-format §3.3 L370-377 makes such a
/// token invalid but explicitly lets an implementation accept it at name
/// validation and fail it at verification, and this direction is the safe
/// one: it over-suppresses a name that can never verify against content
/// anyway. No spelling of a shunned name can BYPASS the filter, which is the
/// only direction §3.3 L553-556 cares about.
pub fn containsWireName(context: *anyopaque, name: []const u8) bool {
const self: *RepoShunSet = @ptrCast(@alignCast(context));
if (self.digests.count() == 0) return false;
if (name.len != 2 * logstore.sha1_digest_len and name.len != 2 * logstore.sha3_256_digest_len) return false;
var digest_buf: [digest_len]u8 = undefined;
const digest = decodeWireName(name, &digest_buf) orelse return false;
std.debug.assert(logstore.isCompleteDigest(digest.len));
return self.digests.contains(digest);
}
};
/// Opens `dir/.zit` read-only and snapshots its stored artifacts into `ms` — the
/// held/phantom split the pull client announces via `igot` (sync §3.2/§3.3) —
/// and its shun table into `shun` (sync §3.3 L553-556), both off the one open.
/// Built by `loadContentStore`, which is the CLIENT's builder and not `serve`'s:
/// `serve` builds a `LogIndexStore` instead, whose §8.2 slot-preserving mask,
/// masked write doors and `read_fault` channel this snapshot has no use for. The
/// two share the `digestToWireName` encoding and the held/phantom split, and
/// nothing else — a filter added to one does NOT reach the other, which is why
/// the §3.3 masked-basis guard is stated in both. The
/// read-only handle is closed before `persistStore` reopens the log read-write,
/// which is why `shun` owns its keys rather than borrowing the store's.
/// sync §7.7 L1463-1467 — this repository's EFFECTIVE hash-policy value, as the
/// §7.7 target-admission predicate needs it: read from the repository's own
/// stored config, never from peer state or a card field.
///
/// The stored byte is opaque to storage (`logstore.Config.hash_policy`), so two
/// cases have no defined policy: a repository carrying no config frame at all,
/// and a byte naming no artifact-format §7.1 Table 7 ordinal. Neither is
/// shun-sha1 — the ONLY distinction §7.7 draws — so both answer `.auto` and the
/// receive path admits, rather than failing a whole sync over a field this
/// clause reads for one boolean. That is a read-side default, not a repair: the
/// stored byte is left exactly as found.
///
/// A READ FAULT IS NOT ONE OF THOSE TWO CASES. `readConfig` also returns
/// `Storage` (the frame would not read) and `Corrupt` (the frame is not a config
/// frame, or does not parse), and both mean the policy is UNKNOWN rather than
/// undefined. Answering `.auto` there would silently disable the shun-sha1 half
/// of the §7.7 predicate and admit SHA1-named content this repository is
/// configured to refuse, with nothing failing — the same fail-open `serveRound`
/// refuses for the shun-table half ("must fail the round rather than silently
/// serve with no shun table"). So they propagate, exactly as `RepoShunSet.load`
/// does three lines above the serve-side call site.
fn effectiveHashPolicy(gpa: std.mem.Allocator, log: *logstore.LogStore) error{ OutOfMemory, Storage }!artifact.HashPolicy {
var oc = (log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
// A config frame that will not read or will not parse is a storage-level
// failure of this repository, not a policy answer.
error.Storage, error.Corrupt => return error.Storage,
}) orelse return .auto;
defer oc.deinit(gpa);
return std.enums.fromInt(artifact.HashPolicy, oc.config.hash_policy) orelse .auto;
}
/// Serialize and atomically store the content log's scan index (the same
/// best-effort refresh `Repository.logStore` performs): a failure costs the
/// next open one scan, nothing else.
pub fn refreshSettingLogIndex(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *const logstore.LogStore) void {
const idx = log.serializeIndex(gpa) catch return;
defer gpa.free(idx);
repocache.storeLogIndex(io, dir, idx);
}
/// Store a rebuilt log index for the checkout at `root_dir`, beside its
/// STORE: the sink an adopted store calls after repairing a damaged index.
fn storeCheckoutLogIndex(io: std.Io, root_dir: std.Io.Dir, blob: []const u8) void {
var resolved = checkout.storeDirOf(io, root_dir) catch return;
defer if (resolved) |*r| r.close(io);
repocache.storeLogIndex(io, if (resolved) |r| r else root_dir, blob);
}
/// Open a read-only log over `sf`, adopting the persisted scan index when the
/// checkout's store cache holds a fresh one — the CLI pattern, shared so
/// every read verb stops paying the full frame-header scan (diff and
/// status carried an O(store) floor of ~0.2s per invocation on a
/// 148k-artifact store from exactly this). A refused index (absent, stale,
/// corrupt) falls back to the scan and refreshes the sidecar, so the next
/// open is cheap again. The sidecar lives with the STORE, never a worktree,
/// so the cache directory resolves through the checkout's store pointer.
pub fn openLogAdoptingIndex(
gpa: std.mem.Allocator,
io: std.Io,
root_dir: std.Io.Dir,
sf: storage.StorageFile,
/// Whether a scan (or a tail-extended adoption) may persist the refreshed
/// sidecar. `.read_only` exists for verbs PINNED as tree-untouching —
/// extras' B-extras-16 requires that a run create nothing, `.zit-cache`
/// included — which still adopt a sidecar someone else already wrote.
sidecar: enum { refresh_on_scan, read_only },
) logstore.OpenError!logstore.LogStore {
var resolved = checkout.storeDirOf(io, root_dir) catch null;
defer if (resolved) |*r| r.close(io);
const cache_dir = if (resolved) |r| r else root_dir;
indexed: {
const idx = repocache.mapLogIndex(io, cache_dir) orelse break :indexed;
// `adoptIndex` TAKES `idx` on every path: the adopted store reads
// its entries in place and releases it in `deinit`, and every
// refusal releases it there. Nothing to release here.
var extended = false;
const adopted = logstore.LogStore.adoptIndex(gpa, sf, idx, &extended) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => break :indexed,
};
var log = adopted orelse break :indexed;
// The sink re-resolves the store from `root_dir`: `cache_dir` may be
// a handle this frame closes, and the store outlives it.
if (sidecar == .refresh_on_scan) log.setIndexSink(.{ .io = io, .dir = root_dir, .write = storeCheckoutLogIndex });
if (extended and sidecar == .refresh_on_scan) refreshSettingLogIndex(gpa, io, cache_dir, &log);
return log;
}
var log = try logstore.LogStore.openReadOnly(gpa, sf);
// The scan just paid for the product the index persists.
if (sidecar == .refresh_on_scan) refreshSettingLogIndex(gpa, io, cache_dir, &log);
return log;
}
/// The number of content-bearing (non-phantom) artifacts in `store` — the held
/// set (§3.3 r24).
fn contentCount(store: repostate.ArtifactStore) usize {
// `held`, never `content`: on the log-backed client store a
// `content()` probe MATERIALIZES the rid's plaintext — asking it of every
// rid re-inflates the whole repository to produce a number.
return store.heldCount();
}
test "serve §3.8: a per-request store-load on one serve handle sees a later append (no restart)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
const hash_kind: u8 = @backingInt(artifact.defaultPolicy(false, false));
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const append = struct {
fn artifactFrame(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, digest: []const u8, bytes: []const u8) !void {
var f = dir.openFile(i, repo_filename, .{ .mode = .read_write }) catch
try dir.createFile(i, repo_filename, .{ .read = true });
defer f.close(i);
const st = try f.stat(i);
var fs = storage.FileStorage.init(i, f, st.size);
var log = try logstore.LogStore.open(g, fs.storageFile(), hk);
defer log.deinit(g);
_ = try log.put(digest, bytes);
try log.commit();
}
};
// Seed the repo holding exactly one content artifact (a 32-byte raw digest).
const digest_a = repeat("a", digest_len);
try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_a, "content a\n");
// Open ONE read handle — the handle `serve` holds for its whole lifetime.
var handle = try tmp.dir.openFile(io, repo_filename, .{});
defer handle.close(io);
// First per-request read: one content artifact.
var index1 = try LogIndexStore.openFromFile(gpa, io, handle, (try handle.stat(io)).size);
defer index1.destroy();
const n1 = contentCount(index1.store());
try std.testing.expectEqual(@as(usize, 1), n1);
// A SEPARATE writer appends a second artifact while the handle stays open —
// exactly a `zit commit` running alongside a live `zit serve`.
const digest_b = repeat("b", digest_len);
try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_b, "content b\n");
// Second per-request read on the SAME handle sees the new artifact: proof the
// server re-reads the current on-disk log per request (§3.8). A startup
// snapshot would still report n1 — no restart is needed here.
var index2 = try LogIndexStore.openFromFile(gpa, io, handle, (try handle.stat(io)).size);
defer index2.destroy();
try std.testing.expectEqual(n1 + 1, contentCount(index2.store()));
}
// ---------------------------------------------------------------------------
// The cone index. Goal: a cone classification answered from the index is the
// classification reading every held artifact gives -- keep, out_only and
// elided nodes, bit for bit -- across sub-manifest trees whose check-ins
// rename, delete, share an unchanged subtree, move a blob out of the cone and
// back in, exclude a subtree, and carry an orphan node; and it stays so after
// the log grows and the index is folded forward rather than rebuilt. Method:
// a log written frame by frame, an index stored after two check-ins, two more
// check-ins appended, then both classifiers over a fresh snapshot.
// ---------------------------------------------------------------------------
const ConeFixture = struct {
gpa: std.mem.Allocator,
log: *logstore.LogStore,
names: std.ArrayListUnmanaged([]u8) = .empty,
fn deinit(self: *ConeFixture) void {
for (self.names.items) |n| self.gpa.free(n);
self.names.deinit(self.gpa);
}
/// Store `bytes` (unless `held` is false) and return the hex name.
fn add(self: *ConeFixture, bytes: []const u8, held: bool) ![]const u8 {
var buf: [64]u8 = undefined;
const name = try self.gpa.dupe(u8, artifact.artifactName(.sha3_256, bytes, &buf));
errdefer self.gpa.free(name);
try self.names.append(self.gpa, name);
if (held) {
var digest: [32]u8 = undefined;
_ = try std.fmt.hexToBytes(&digest, name);
_ = try self.log.put(&digest, bytes);
}
return name;
}
fn sealed(self: *ConeFixture, body: *std.ArrayListUnmanaged(u8), held: bool) ![]const u8 {
var digest: [16]u8 = undefined;
std.crypto.hash.Md5.hash(body.items, &digest, .{});
try body.print(self.gpa, "Z {x}\n", .{&digest});
return self.add(body.items, held);
}
/// A leaf node: F cards over (path, blob name) in path order.
fn leaf(self: *ConeFixture, files: []const [2][]const u8, held: bool) ![]const u8 {
var body: std.ArrayListUnmanaged(u8) = .empty;
defer body.deinit(self.gpa);
for (files) |f| try body.print(self.gpa, "F {s} {s}\n", .{ f[0], f[1] });
return self.sealed(&body, held);
}
/// An interior node or, with `head`, a tree check-in: S cards over
/// (first path, node name).
fn tree(self: *ConeFixture, head: ?[]const u8, parent: ?[]const u8, slots: []const [2][]const u8) ![]const u8 {
var body: std.ArrayListUnmanaged(u8) = .empty;
defer body.deinit(self.gpa);
if (head) |h| try body.print(self.gpa, "C {s}\nD 2026-10-21T12:00:00.000\n", .{h});
if (parent) |p| try body.print(self.gpa, "P {s}\n", .{p});
for (slots) |s| try body.print(self.gpa, "S {s} {s}\n", .{ s[0], s[1] });
if (head != null) try body.appendSlice(self.gpa, "U u\n");
return self.sealed(&body, true);
}
};
/// The options an origin classifies a path-only cone under (`classifyCone`):
/// what the cone index answers.
const origin_classify_opts: sparse.ClassifyOptions = .{ .intern_absent = false, .elide_nodes = false };
fn expectSameClassification(want: *const SparseClassification, got: *const SparseClassification) !void {
try std.testing.expectEqual(want.keep.capacity(), got.keep.capacity());
var r: usize = 0;
while (r < want.keep.capacity()) : (r += 1) {
try std.testing.expectEqual(want.keep.isSet(r), got.keep.isSet(r));
try std.testing.expectEqual(want.out_only.isSet(r), got.out_only.isSet(r));
}
try std.testing.expectEqual(want.elided_nodes.count(), got.elided_nodes.count());
var it = want.elided_nodes.keyIterator();
while (it.next()) |k| try std.testing.expect(got.elided_nodes.contains(k.*));
}
test "cone index: indexed classification equals the reading one across tree check-ins and a folded tail" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const hash_kind: u8 = @backingInt(artifact.defaultPolicy(false, false));
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer log.deinit(gpa);
var fx: ConeFixture = .{ .gpa = gpa, .log = &log };
defer fx.deinit();
var cone = try cone_mod.Cone.init(gpa, &.{ "lib", "src" }, &.{"src/gen"});
defer cone.deinit(gpa);
const b_docs_a = try fx.add("docs a\n", true);
const b_docs_b = try fx.add("docs b\n", true);
const b_src_x = try fx.add("src x\n", true);
const b_gen = try fx.add("generated\n", true); // only ever under the exclusion
const b_lib = try fx.add("lib a\n", true); // in, then renamed out
const b_docs_b2 = try fx.add("docs b v2\n", true); // only ever out
const b_wander = try fx.add("wanderer\n", true); // out, then moved in
const b_loose = try fx.add("nobody names me\n", true);
const b_orphan = try fx.add("orphan leaf file\n", true);
// C1: three leaves under one root.
const l_docs1 = try fx.leaf(&.{ .{ "docs/a", b_docs_a }, .{ "docs/b", b_docs_b } }, true);
const l_lib1 = try fx.leaf(&.{.{ "lib/a", b_lib }}, true);
const l_src = try fx.leaf(&.{ .{ "src/gen/y", b_gen }, .{ "src/x", b_src_x } }, true);
const c1 = try fx.tree("one", null, &.{ .{ "docs/a", l_docs1 }, .{ "lib/a", l_lib1 }, .{ "src/gen/y", l_src } });
// C2: lib/a renamed out to old/a, docs/b modified, a blob added out of
// cone; the src leaf is the same node as C1's.
const l_docs2 = try fx.leaf(&.{ .{ "docs/a", b_docs_a }, .{ "docs/b", b_docs_b2 }, .{ "old/a", b_lib }, .{ "x/q", b_wander } }, true);
const c2 = try fx.tree("two", c1, &.{ .{ "docs/a", l_docs2 }, .{ "src/gen/y", l_src } });
try log.commit();
const prefix_size = log.size();
// The index is built and stored over the two-check-in prefix.
{
const h = try tmp.dir.openFile(io, repo_filename, .{});
defer h.close(io);
var index = try LogIndexStore.openFromFile(gpa, io, h, prefix_size);
defer index.destroy();
const blob = (try coneindex.loadOrBuild(gpa, io, tmp.dir, &index.log)).?;
gpa.free(blob);
}
// C3, appended: docs/a deleted, x/q moved into the cone as src/q, under a
// two-level tree; and an orphan leaf no check-in names.
const l_docs3 = try fx.leaf(&.{ .{ "docs/b", b_docs_b2 }, .{ "old/a", b_lib } }, true);
const l_src3 = try fx.leaf(&.{ .{ "src/gen/y", b_gen }, .{ "src/q", b_wander }, .{ "src/x", b_src_x } }, true);
const i_src3 = try fx.tree(null, null, &.{.{ "src/gen/y", l_src3 }});
_ = try fx.tree("three", c2, &.{ .{ "docs/b", l_docs3 }, .{ "src/gen/y", i_src3 } });
_ = try fx.leaf(&.{.{ "src/orphan", b_orphan }}, true);
try log.commit();
const grown_size = log.size();
const h = try tmp.dir.openFile(io, repo_filename, .{});
defer h.close(io);
var index = try LogIndexStore.openFromFile(gpa, io, h, grown_size);
defer index.destroy();
const s = index.store();
var reading = try sparse.classifyKeepRidsOpts(gpa, s, &cone, origin_classify_opts);
defer reading.deinit(gpa);
// Folded forward, not rebuilt: the stored prefix index is what grows.
const stored = (try repocache.loadMember(gpa, io, tmp.dir, coneindex.member)).?;
defer gpa.free(stored);
const prefix = coneindex.parse(stored, &index.log).?;
try std.testing.expect(!prefix.coversAll(&index.log));
const blob = (try coneindex.loadOrBuild(gpa, io, tmp.dir, &index.log)).?;
defer gpa.free(blob);
const idx = coneindex.parse(blob, &index.log).?;
try std.testing.expect(idx.coversAll(&index.log));
try index.buildAdapterMap();
var indexed = (try sparse.classifyIndexed(gpa, s, &cone, &idx, .{ .ctx = index, .mapFn = LogIndexStore.mapLogRid })).?;
defer indexed.deinit(gpa);
try expectSameClassification(&reading, &indexed);
// The fixture reaches every rule it claims to.
const elided = struct {
fn of(c: *const SparseClassification, st: repostate.ArtifactStore, name: []const u8) bool {
return c.elided(st.lookup(name).?);
}
}.of;
try std.testing.expect(elided(&reading, s, b_gen)); // excluded
try std.testing.expect(elided(&reading, s, b_docs_a)); // out, then deleted
try std.testing.expect(elided(&reading, s, b_docs_b2)); // only ever out
try std.testing.expect(!elided(&reading, s, b_lib)); // once in the cone
try std.testing.expect(!elided(&reading, s, b_wander)); // came into the cone
try std.testing.expect(!elided(&reading, s, b_src_x));
try std.testing.expect(!elided(&reading, s, b_loose)); // unreferenced stays
try std.testing.expect(!elided(&reading, s, b_orphan)); // named in-cone by the orphan
try std.testing.expect(!elided(&reading, s, l_docs1)); // structural, though out
// Through the snapshot's own entry point, with the cache directory set.
index.classify_sidecar = .{ .io = io, .dir = tmp.dir };
var via = try index.classify(&cone, origin_classify_opts, .parse);
defer via.deinit(gpa);
try expectSameClassification(&reading, &via);
}
test "cone index: a tree naming an absent node is left to the reading classifier" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const hash_kind: u8 = @backingInt(artifact.defaultPolicy(false, false));
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer log.deinit(gpa);
var fx: ConeFixture = .{ .gpa = gpa, .log = &log };
defer fx.deinit();
const b = try fx.add("kept\n", true);
const l_in = try fx.leaf(&.{.{ "src/a", b }}, true);
const l_absent = try fx.leaf(&.{.{ "zzz/a", b }}, false); // never stored
_ = try fx.tree("holed", null, &.{ .{ "src/a", l_in }, .{ "zzz/a", l_absent } });
try log.commit();
var cone = try cone_mod.Cone.init(gpa, &.{"src"}, &.{});
defer cone.deinit(gpa);
const h = try tmp.dir.openFile(io, repo_filename, .{});
defer h.close(io);
var index = try LogIndexStore.openFromFile(gpa, io, h, log.size());
defer index.destroy();
const blob = (try coneindex.loadOrBuild(gpa, io, tmp.dir, &index.log)).?;
defer gpa.free(blob);
const idx = coneindex.parse(blob, &index.log).?;
try index.buildAdapterMap();
try std.testing.expect((try sparse.classifyIndexed(gpa, index.store(), &cone, &idx, .{ .ctx = index, .mapFn = LogIndexStore.mapLogRid })) == null);
// The snapshot falls back and still elides the absent out-of-cone node.
index.classify_sidecar = .{ .io = io, .dir = tmp.dir };
var reading = try sparse.classifyKeepRidsOpts(gpa, index.store(), &cone, origin_classify_opts);
defer reading.deinit(gpa);
var via = try index.classify(&cone, origin_classify_opts, .parse);
defer via.deinit(gpa);
try expectSameClassification(&reading, &via);
try std.testing.expect(!via.elided(index.store().lookup(b).?));
}