//! zit sync / network / store-transfer verbs (serve, clone, open, pack, pull,
//! push, sync, remote) and the helpers and tests private to them.
//!
//! Extracted verbatim from src/main.zig (release step S2). Every declaration
//! below is byte-identical to its main.zig source; only this import prologue is
//! new. Helpers still shared with other verbs stay in main.zig and are aliased
//! back here so the moved bodies call them unqualified. main.zig's verb-dispatch
//! switch calls into this file (`cmd_sync.runServe` ...); its test block imports
//! it so these inline tests keep running under `zig build test`.
const std = @import("std");
const repeat = @import("zit").repeat;
const zit = @import("zit");
const SparseClassification = zit.sparse.SparseClassification;
const classifyKeepRids = zit.sparse.classifyKeepRids;
const cli = @import("zit").cli;
const extras = @import("../extras.zig");
const logstore = @import("zit").logstore;
const storage = @import("zit").storage;
const clusterindex = @import("zit").clusterindex;
const update = @import("zit").update;
const checkout = @import("zit").checkout;
const Repository = @import("zit").Repository;
const serve_mod = @import("../serve.zig");
const transport = @import("../transport.zig");
const sync_driver = @import("../sync_driver.zig");
const Io = std.Io;
const File = Io.File;
const repostate = zit.repostate;
const managed = zit.managed;
const artifact = zit.artifact;
// Shared CLI helpers and test fixtures that stay in main.zig (other verbs use
// them too); aliased back so the moved code and tests reference them by their
// original unqualified names.
const main = @import("../main.zig");
const runInit = main.runInit;
const repo_filename = main.repo_filename;
const renderStorage = main.renderStorage;
const reasonOf = main.reasonOf;
const stateOom = main.stateOom;
pub const not_a_repo_msg = main.not_a_repo_msg;
const repoReadFail = main.repoReadFail;
const repoOpenFail = main.repoOpenFail;
const openRepo = main.openRepo;
const updateApplyFail = main.updateApplyFail;
const digest_len = main.digest_len;
const hexToDigest = main.hexToDigest;
const digestToWireName = main.digestToWireName;
const decodeWireName = main.decodeWireName;
const open_help = main.open_help;
const pack_help = main.pack_help;
const shun_help = main.shun_help;
const serve_help = main.serve_help;
const clone_help = main.clone_help;
const pull_help = main.pull_help;
const push_help = main.push_help;
const sync_help = main.sync_help;
const remote_help = main.remote_help;
const settings_help = main.settings_help;
const packSeedBlob = main.packSeedBlob;
const packSeedCheckin = main.packSeedCheckin;
const packEvolvingBlob = main.packEvolvingBlob;
const packOpenWritable = main.packOpenWritable;
const packOpenReadOnly = main.packOpenReadOnly;
/// Clone phase edges let benchmark code attribute the real command without
/// coupling clocks or counters into production behavior.
pub const ClonePhase = enum { transport, initialize, exchange, persist, config, warm_cache };
pub const ClonePhaseEvent = enum { begin, end };
pub const CloneObserver = struct {
context: *anyopaque,
callback: *const fn (context: *anyopaque, phase: ClonePhase, event: ClonePhaseEvent) void,
};
fn observeClonePhase(observer: ?*const CloneObserver, phase: ClonePhase, event: ClonePhaseEvent) void {
if (observer) |value| value.callback(value.context, phase, event);
}
/// `zit serve [--port N]` — put an HTTP face on the stateless sync responder.
/// Opens `.zit` read-only and keeps it open for the server's lifetime; every
/// request reads a fresh content view of it and is answered independently
/// (sync §3.8). `std.http`/socket I/O lives entirely in `serve.zig`; this only
/// wires the repository and the listen address.
pub fn runServe(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
/// Carried solely so the served rounds can resolve this server's own
/// content policy through the shared settings resolver.
amb: cli.Ambient,
) cli.Exit {
const parsed = switch (cli.parseServeArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "serve", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "serve", .extra_argument, tok),
.bad_port => |tok| {
streams.err.print("zit serve: invalid --port value '{s}'\n", .{tok}) catch {};
return .usage;
},
};
if (parsed.help) {
streams.out.writeAll(serve_help) catch {};
return .ok;
}
const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "serve")) orelse {
streams.err.writeAll(not_a_repo_msg) catch {};
return .failure;
};
var root_dir = root.dir;
// serve answers from the STORE: its open handle, its swap-detect
// stat and its config reads all follow the resolved store root.
if (cli.storeDirOf(io, root_dir) catch {
root_dir.close(io);
return repoReadFail(streams, "serve");
}) |resolved| {
root_dir.close(io);
root_dir = resolved;
}
var file = root_dir.openFile(io, repo_filename, .{}) catch {
root_dir.close(io);
return repoReadFail(streams, "serve");
};
defer file.close(io);
const log_st = file.stat(io) catch {
root_dir.close(io);
return repoReadFail(streams, "serve");
};
// Bring-up order is load-bearing (rationale in `serve_mod.listen`'s
// doc): open the listener FIRST — before ANY O(store) work, which here means
// both the config read below and the prime, each a full-log scan. A client
// that connects during either queues in the accept backlog instead of being
// refused, so the readiness window starts at the bind rather than after the
// scans. Announce "listening" only once the bind has actually succeeded (so
// it never lies about a listener that failed to come up); the accept loop
// then arms each connection's deadline on accept. The backlog property this
// rests on is pinned by serve.zig's "listen opens the accept backlog before
// serve" test.
const address = std.Io.net.IpAddress.parse("127.0.0.1", parsed.port) catch {
streams.err.print("zit serve: invalid --port value '{d}'\n", .{parsed.port}) catch {};
return .usage;
};
var listener = serve_mod.listen(io, address) catch {
streams.err.print("zit serve: cannot bind 127.0.0.1:{d}\n", .{parsed.port}) catch {};
return .failure;
};
defer listener.deinit(io);
streams.err.print("zit serve: listening on http://127.0.0.1:{d}/xfer (Ctrl-C to stop)\n", .{parsed.port}) catch {};
streams.err.flush() catch {};
var fs = storage.FileStorage.init(io, file, log_st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| {
root_dir.close(io);
switch (e) {
error.OutOfMemory => streams.err.writeAll("zit serve: out of memory\n") catch {},
else => streams.err.print("zit serve: cannot read repository: {s}\n", .{reasonOf(e)}) catch {},
}
return .failure;
};
defer log.deinit(gpa);
defer root_dir.close(io);
// §9.1/§8.2 — emit the repository's stored project code in the clone
// reply's `push` card, so a clone learns the shared project identity. A
// repo predating the stored code, or an unreadable config, falls back to
// the fixed placeholder (informational; servercode never gates a clone).
var maybe_cfg: ?logstore.OwnedConfig = log.readConfig(gpa) catch null;
defer if (maybe_cfg) |*oc| oc.deinit(gpa);
const project_code: []const u8 = if (maybe_cfg) |oc|
(if (oc.config.project_code.len > 0) oc.config.project_code else serve_mod.server_projectcode)
else
serve_mod.server_projectcode;
// §9.1 — a real instance identity in the clone reply's `push` card,
// derived from this checkout's absolute path exactly as init derives the
// project code (stable, distinct per instance, no ambient RNG). The
// placeholder constant remains only as the test fixtures' server identity.
const serve_code = zit.sync.deriveServerCode(&.{root.abs});
const server = zit.sync.Server{
.version = serve_mod.server_version,
.credential = null,
.self_servercode = &serve_code,
.project_code = project_code,
};
var ctx = ServeCtx{ .io = io, .root_dir = root_dir, .file = &file, .diag = streams.err, .server = &server, .amb = amb, .fronted = parsed.fronted };
defer ctx.dropSnapshot();
const responder = serve_mod.Responder{ .ctx = &ctx, .answerFn = serveAnswer, .maintainFn = ServeCtx.maintain, .fronted = parsed.fronted };
if (ctx.baseSnapshot(gpa)) |_| {} else |_| return repoReadFail(streams, "serve");
serve_mod.serve(io, gpa, &listener, responder, .{}) catch |e| switch (e) {
error.Accept => {
streams.err.writeAll("zit serve: connection accept failed\n") catch {};
return .failure;
},
};
return .ok;
}
/// The per-request context threaded into `serve.Responder`: the open log to read
/// a fresh store from, the server identity to answer with, and the injected IO +
/// repository directory used to make pushed content durable (a push request
/// ingests new artifacts into the per-request store, which must be persisted into
/// the served repository's on-disk log — sync §3.3).
const ServeCtx = struct {
io: std.Io,
root_dir: std.Io.Dir,
/// The ambient environment, carried only so `resolveSetting` (the one
/// settings resolver) is reachable per round. Reading the config frame
/// directly here instead would be a second reader, and the two drifting is
/// the bug `signingSettings` documents having shipped once already.
amb: ?cli.Ambient = null,
/// Behind a trusted front. Turns on the per-round membership
/// rule and keeps the repository image warm between connections so that
/// rule reads O(image), not O(store), after a push.
fronted: bool = false,
/// The open read handle on the served `.zit`. `respond` re-reads the CURRENT
/// on-disk log from it (§3.8): a fresh stat fixes the committed end-of-log,
/// so a check-in appended by a separate `zit commit` while `serve` runs is
/// advertised on the next request — no restart.
///
/// The handle pins an INODE, not a path. `zit pack` and `zit shun --force`
/// replace `.zit` by rename, which would leave a held-open handle reading
/// the replaced inode forever (its fstat never moves again — no cache key
/// over the handle can detect the swap). `baseSnapshot` therefore stats
/// the PATH beside the handle each round and REOPENS on an inode change:
/// the swap is noticed at the next round boundary, so a shun that
/// rode a generation rewrite stops being served with no restart, while an
/// in-flight round keeps its consistent pre-swap view. A pointer into
/// `runServe`'s frame so the reopen and the shutdown close agree on one
/// current handle.
file: *File,
/// Where `serveAnswer` names a failed request, or null to stay silent. The
/// tests and `driveHttp` leave it unset; `runServe` points it at stderr.
diag: ?*std.Io.Writer = null,
server: *const zit.sync.Server,
/// The most recent snapshot of the on-disk log, reused across requests. A
/// log-backed INDEX (not a full-container copy): it reads the containers it
/// serves on demand and holds only the index between rounds. Built from
/// the long-lived allocator `serve` threads into every answer, so it outlives
/// the request that built it, and owned here — heap-allocated because it owns
/// the `FileStorage` its log borrows; `dropSnapshot` frees it. Never handed to
/// `respond` directly — see `serveAnswer`.
snapshot: ?*LogIndexStore = null,
/// The snapshot's cluster index, rebuilt with it.
clusters: ?clusterindex.Resolved = null,
/// The log size `snapshot` was built from — one half of the cache key that
/// `baseSnapshot` stats per request (see there for why size alone is not it).
snapshot_log_size: u64 = 0,
/// The log's last-modification time (ns) when `snapshot` was built. Together
/// with the size it is the cache key: a write that does not change the length
/// still moves `mtime`. A `std.Io.Timestamp`, unwrapped to its `nanoseconds`
/// here so the key is two scalar comparisons.
snapshot_log_mtime_ns: i96 = 0,
/// How many snapshots this server has built. Not used by the server; it is
/// the observable that makes "snapshot builds do not grow with the round
/// count" a testable property rather than a wall-clock claim.
loads: u64 = 0,
/// Free the cached snapshot, leaving the context usable: the next
/// `baseSnapshot` rebuilds. Both a caller finishing with the context and the
/// rebuild path itself want exactly this.
fn dropSnapshot(self: *ServeCtx) void {
if (self.clusters) |*c| c.deinit();
self.clusters = null;
if (self.snapshot) |s| s.destroy();
self.snapshot = null;
self.snapshot_log_size = 0;
self.snapshot_log_mtime_ns = 0;
}
/// Drop plaintext the cached snapshot memoised while answering, keeping the
/// retained containers it serves from. Called once per round, after every
/// borrow of the snapshot has ended.
///
/// Inflating is how `content()` answers a wire-backed entry, and the
/// responder's own §7.2 size proofs call `content()` on every artifact they
/// emit. Against a throwaway per-request store that plaintext died with the
/// request; against a store that outlives the request it would accumulate
/// until the server held the whole repository's uncompressed history. The
/// containers stay, so the next round answers identically.
///
/// The memo is given up rather than capped because in an optimized build
/// there is almost nothing to give up: the size proofs are `runtime_safety`
/// only, so a clone round inflates nothing and the memo would stay empty
/// regardless. Measured over four clones of a 3001-artifact repository,
/// ReleaseFast: 12 MB resident and 0.09 s of server CPU with this release,
/// against 26 MB and 0.15 s without it. A safety-checked build does re-prove
/// every emitted artifact each round and so pays to re-inflate; that cost
/// belongs to the proofs, not to the cache.
fn releaseSnapshotPlaintext(self: *ServeCtx) void {
if (self.snapshot) |s| s.releaseTransient();
}
/// §3.8 — the store describing the CURRENT on-disk log, rebuilt only when the
/// log has actually grown. The §3.8 guarantee is that a check-in appended by
/// a separate `zit commit` is advertised on the NEXT request with no restart;
/// that needs a fresh `stat` per request, not a fresh rebuild per request.
/// Rebuilding regardless made every round cost the whole repository, which is
/// what turned a clone of a large store into O(rounds x store) server work.
///
/// A torn tail from an append caught mid-write is rejected by the read-only
/// scanner exactly as before; the previous snapshot is dropped only once a
/// replacement is fully built, so a failed rebuild leaves the cache coherent.
///
/// The rebuild itself is O(store) and must therefore not fall inside a
/// request whose connection carries a deadline. `runServe` primes the cache
/// before it accepts anything. A round that appends to the log does NOT
/// rebuild before it returns — it deliberately leaves the snapshot stale
/// (describing the pre-append log) rather than pay the O(store) rebuild inside
/// its own reply deadline. The append-only stat gate above makes that
/// stale snapshot self-correcting: the size no longer matches, so the next
/// `baseSnapshot` rebuilds. That next call is the accept loop's `maintain`
/// hook (see `ServeCtx.maintain`), run between connections with no connection
/// open — so the deferred rebuild still lands outside every connection
/// deadline, never on an innocent later request. Nothing here can bound the
/// startup build itself; that is bounded by the store, and always was.
///
/// THE CACHE KEY IS NOT THE SIZE ALONE, and since this snapshot began carrying
/// the §3.3 shun table it cannot be. A stale snapshot used to cost stale CONTENT
/// — a peer misses a newly committed artifact and the next round supplies it —
/// which is self-correcting. It now also decides whether the SUPPRESSION table is
/// re-read, and a missed re-read serves an artifact the operator shunned, with
/// nothing failing and nothing logged. "Every writer only appends, so the length
/// always grows" is true of `zit shun --defer` and `zit commit` today, but it is
/// an assumption about all present and future writers rather than a checked
/// property: a generation rewrite landing on the same byte length, or any
/// out-of-band mutation, would keep the old mask forever. So the key is size AND
/// mtime — an in-place rewrite that does not change the length still moves the
/// mtime. Two scalar comparisons against one `stat` this function already
/// performs, and they can only over-invalidate (which costs a rebuild), never
/// under-invalidate on a write that reaches THIS INODE.
///
/// WHAT THE CACHE KEY CANNOT COVER, THE PATH STAT DOES. `st` comes
/// from an `fstat` on the held handle, so it detects every write reaching
/// that inode and no rename ABOVE it: a `zit shun --force` (or `zit pack`)
/// generation swap leaves the old inode's size and mtime frozen forever,
/// with the shun row in a file this handle would never see. So the path is
/// stat'ed beside the handle each round, and an inode mismatch reopens the
/// handle (dropping the snapshot) before anything is answered — the round
/// then serves the live generation. Failure on the path stat or the reopen
/// keeps the old handle and answers from it: availability first, and the
/// next round retries the swap detection.
fn baseSnapshot(self: *ServeCtx, gpa: std.mem.Allocator) serve_mod.AnswerError!repostate.ArtifactStore {
var st = self.file.stat(self.io) catch return error.ServerStorage;
if (self.root_dir.statFile(self.io, repo_filename, .{})) |path_st| {
if (path_st.inode != st.inode) {
if (self.root_dir.openFile(self.io, repo_filename, .{})) |fresh_handle| {
self.file.close(self.io);
self.file.* = fresh_handle;
self.dropSnapshot();
st = self.file.stat(self.io) catch return error.ServerStorage;
} else |_| {}
}
} else |_| {}
if (self.snapshot) |s| {
if (st.size == self.snapshot_log_size and
st.mtime.nanoseconds == self.snapshot_log_mtime_ns) return s.store();
}
// Build the fresh index BEFORE dropping the old one: a failed rebuild
// leaves the previous snapshot (and its size) intact, so the stat gate
// keeps mismatching until a rebuild succeeds and stale content is never
// served in the meantime. `openFromFile` fully frees itself on any error.
const fresh = LogIndexStore.openFromFile(gpa, self.io, self.file.*, st.size) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage, error.Corrupt => return error.ServerStorage,
};
self.dropSnapshot();
self.snapshot = fresh;
// Resolve the cluster index over the new snapshot, then drop
// the cluster plaintext it memoised: the index keeps only names and bits.
self.clusters = loadClusterIndex(gpa, self.io, self.root_dir, &fresh.log, fresh.store()) catch null;
fresh.releaseTransient();
self.snapshot_log_size = st.size;
self.snapshot_log_mtime_ns = st.mtime.nanoseconds;
self.loads += 1;
return fresh.store();
}
/// The accept loop's between-connections maintenance hook (see
/// `serve.Responder.maintainFn`). Runs after a connection has closed and
/// before the next accept, so no connection is open to charge its O(store)
/// work to. It re-primes the cache via the same stat-gated `baseSnapshot`:
/// after a pure read round the stat is unchanged and this is one `stat`; after
/// a round that appended (a push) the log grew, so this rebuilds the snapshot
/// the pushing round deliberately left stale rather than pay inside its
/// deadline. A failed rebuild is swallowed for the same reason
/// `baseSnapshot` leaves the cache coherent on failure: `baseSnapshot` drops
/// the old snapshot only after a fresh one is fully built and updates
/// `snapshot_log_size` only on success, so a failed rebuild RETAINS the
/// previous (stale) snapshot with its old size — the stat gate keeps
/// mismatching until a rebuild succeeds, and stale content is never served in
/// the meantime. The next round (or the next `maintain`) retries, still
/// outside any earlier round's budget. Type-erased to match the `maintainFn`
/// seam.
fn maintain(ctx_ptr: *anyopaque, gpa: std.mem.Allocator) void {
const self: *ServeCtx = @ptrCast(@alignCast(ctx_ptr));
const before = self.snapshot_log_size;
if (self.baseSnapshot(gpa)) |_| {} else |_| {}
// A push left the zero-parse image stale, and the next
// protected push would re-derive it inside its own deadline. Derive it
// here instead, between connections, where O(store) is allowed — and
// only when the log actually grew, so a pull-heavy fronted server does
// not pay an image load per connection. Only a fronted server has a
// resolver to warm; a failure is retried by the round itself, which
// derives on demand and fails closed on a fault.
if (self.fronted and self.snapshot_log_size != before) {
var repo: Repository = undefined;
if (Repository.open(&repo, gpa, self.io, self.root_dir, .{ .mode = .fast_read })) |_| {
repo.deinit(gpa);
} else |_| {}
}
}
};
/// The `serve.Responder` seam: snapshot the repository into an in-memory store,
/// answer the framed request over it, and — when the request pushed new content —
/// make that content durable in the served repository's on-disk log (sync §3.3).
///
/// `respond` stays sans-IO: it only `put`s pushed artifacts into the snapshot
/// store it is given. The durable write is this serve-layer's job and reuses the
/// idempotent `persistStore` (a re-push of an already-stored hash appends nothing
/// — content is addressed by hash), the very path clone/pull persist through. A
/// pure gimme/pull request makes nothing new servable, so the overlay stays
/// empty and the log is never reopened; only a genuine push touches disk.
///
/// `respond` is answered over a per-request WRITE OVERLAY of the cached snapshot
/// (`repostate.OverlayStore`), never over the snapshot itself: the responder
/// interns phantoms and upgrades entries as it works, and letting those survive
/// into the next round changes what that round serves. Reads fall through to the
/// snapshot, so reuse costs nothing; writes are confined to the overlay, and the
/// persist pass — which is idempotent over everything already in the log — can
/// only append what the overlay holds.
///
/// The pushed bytes were size-validated by `respond`'s request scan before they
/// entered the store, so `persistStore` only ever writes well-formed content, and
/// a mid-write fault surfaces before `commit`, never as a partial log append.
fn serveAnswer(ctx_ptr: *anyopaque, gpa: std.mem.Allocator, request_container: []const u8, cookie: ?[]const u8, front: serve_mod.Front, request_compressed: bool) serve_mod.AnswerError!serve_mod.XferReply {
const ctx: *ServeCtx = @ptrCast(@alignCast(ctx_ptr));
return serveAnswerInner(ctx, gpa, request_container, cookie, front, request_compressed) catch |e| {
// NAME THE FAILURE. `serve.zig` collapses every AnswerError into one
// bare status -- `ServerStorage`, `OutOfMemory` and `ResponseTooLarge`
// all become 500 with no body and no log line -- so an operator (or a
// bench harness) sees "HTTP 500" and has no way to learn which of three
// unrelated faults produced it. This is the server-side half of naming
// the status the client now reports; without both ends a 500 is a dead
// end for anyone diagnosing from the outside.
//
// Single funnel on purpose: every `AnswerError` this responder can
// produce passes through here, so a future cause is reported without a
// second edit. Diagnostics only -- the error is re-raised unchanged and
// the status the peer sees does not move.
if (ctx.diag) |w| {
w.print("zit serve: request failed: {s}\n", .{@errorName(e)}) catch {};
w.flush() catch {};
}
return e;
};
}
fn serveAnswerInner(ctx: *ServeCtx, gpa: std.mem.Allocator, request_container: []const u8, cookie: ?[]const u8, front: serve_mod.Front, request_compressed: bool) serve_mod.AnswerError!serve_mod.XferReply {
const reply = try serveRound(ctx, gpa, request_container, cookie, front, request_compressed);
// Past this point nothing borrows the snapshot: the overlay is gone with
// `serveRound`'s frame and the reply owns its own bytes. Dropping the
// plaintext memo is bounded (it frees only what this round inflated), so it
// is safe inside the deadline.
//
// A round that appended (a push) has left the snapshot stale — it still
// describes the pre-push log. It is NOT rebuilt here: an O(store) rebuild
// inside the reply's connection deadline is exactly the failure this avoids. The
// rebuild is the accept loop's job, via `ServeCtx.maintain`, run after this
// connection closes and before the next accept — outside every deadline. The
// append-only stat gate in `baseSnapshot` makes the stale snapshot self-
// correcting: the next `baseSnapshot` (in `maintain` or the next round) sees
// the grown size and rebuilds, so no `appended` signal needs to be threaded
// out of the round.
//
// A post-scan read fault this round mapped a held artifact to null. The §8.2
// clone pager skips such a rid and still reports the sequence complete
// (NEXT 0), silently finishing a clone MISSING an artifact — a failure the
// copy-based prime could not produce (a fault surfaced at prime, and a
// retained-but-uninflatable container was still served verbatim). Read the
// flag BEFORE releasing the round's memos (which clears it), then surface it
// as a storage error so the client retries; the next round re-reads.
const faulted = if (ctx.snapshot) |s| s.read_fault else false;
ctx.releaseSnapshotPlaintext();
if (faulted) {
gpa.free(reply.body);
if (reply.diverted) |d| gpa.free(d);
return error.ServerStorage;
}
return reply;
}
/// One round over a per-request write overlay. Split out of `serveAnswer` so the
/// overlay — which borrows the cached snapshot — is provably destroyed before
/// `serveAnswer` touches that snapshot's storage. A round that pushes new content
/// persists it durably here; the snapshot it read from is left describing the
/// pre-push log, and `baseSnapshot`'s append-only stat gate rebuilds it on the
/// next call (in the accept loop's `maintain`, never inside this round's
/// deadline), so this round signals nothing back to its caller.
fn serveRound(
ctx: *ServeCtx,
gpa: std.mem.Allocator,
request_container: []const u8,
cookie: ?[]const u8,
front: serve_mod.Front,
request_compressed: bool,
) serve_mod.AnswerError!serve_mod.XferReply {
var overlay = repostate.OverlayStore.init(gpa, try ctx.baseSnapshot(gpa));
defer overlay.deinit();
// sync §3.3 L553-556 / §8.8 L1839-1841 — answer this round with THIS
// snapshot's shun table. `ctx.server` is the caller's long-lived config and
// stays const; the per-round copy carries the fields that are properties of
// the snapshot rather than of the server, so the reply's phantom scan never
// begs for an artifact this repository has shunned (a client `igot` for a
// shunned hash still interns its phantom — §3.3 r24(1) L537-540 — it is just
// never requested).
// `ctx.baseSnapshot` above published it — the rebuild path assigns
// `self.snapshot`, the cached path requires it non-null — so the null arm is
// unreachable today. It is a CHECKED error rather than a fail-open `.none`
// because "the filter is not skipped" must hold in a release build: a future
// ctx that primes a store without publishing the snapshot, or an early-out
// added above, must fail the round rather than silently serve with no shun
// table and re-beg every shunned hash with nothing failing.
//
// submanifests §8.1/§7.2 — the same per-round freshness arms the send gate.
// The long-lived `ctx.server` is a build identity (version, servercode,
// project code) and outlives every snapshot; the seen-submanifest indicator,
// the §8.1 emission setting, and the two settings this server only REPORTS
// are facts about the CURRENT log and are re-read whenever the snapshot is.
// Copying per round is what keeps them from drifting: a push that stores this
// repository's first tree arms the gate for the next round rather than for
// the next process, and a `zit submanifest enable`/`retune` against a served
// repository is obeyed by the next round rather than the next process. These
// read the snapshot unconditionally for the same reason the shun table does —
// serving a stale default is the failure, not a fallback.
var round_server = ctx.server.*;
// sync §3.2/§6.4 — the round's time for the reply's timestamp comment;
// the sans-IO server has no clock of its own.
var reply_time_buf: [19]u8 = undefined;
const now_s: i64 = @intCast(@divFloor(std.Io.Timestamp.now(ctx.io, .real).nanoseconds, std.time.ns_per_s));
round_server.reply_time = cli.formatWriterTimestamp(now_s, &reply_time_buf);
const snapshot = ctx.snapshot orelse return error.ServerStorage;
round_server.shun = snapshot.shun.shunSet();
// §9.3 r28 — and the row view, for a `reqconfig /shun`.
round_server.shun_rows = snapshot.shun_rows;
// Sparse-cone RFC section 6.2 -- cone-scoped requests answer filtered.
round_server.cone_classifier = snapshot.coneClassifier();
// Bounded announcement: the clusters carry what they name.
if (ctx.clusters) |*c| {
round_server.clustered = &c.clustered;
round_server.cluster_names = c.cluster_names;
}
// This server's own content policy, re-read per round like every
// other setting here, so `zit settings binary-glob ...` against a served
// repository is obeyed by the next round rather than the next process.
//
// Resolved from the SERVER's settings, never from the artifacts arriving in
// the push: a push carrying its own policy would otherwise arrive bearing
// permission to land. The versioned tier is still consulted, and that is
// deliberate -- it reads the server's own working tree, which a push does
// not touch, so adopting a project's policy stays an act the operator takes
// by updating, exactly as `zit trust` is for before-commit consent.
var policy_arena = std.heap.ArenaAllocator.init(gpa);
defer policy_arena.deinit();
if (ctx.amb) |amb| {
// The repository tier comes from the snapshot's per-build
// config read, NOT from `resolveSetting`, whose repository read opens
// the store and re-scans every frame header. That scan per setting
// per round is what made a 1.17M-artifact clone take half an hour.
// The snapshot's (size, mtime) key preserves the per-round freshness
// contract: a settings write grows the log and rebuilds the copy.
const pa = policy_arena.allocator();
const repo_bin: ?[]const u8 = if (snapshot.policy_binary_glob.len != 0) snapshot.policy_binary_glob else null;
const repo_crlf: ?[]const u8 = if (snapshot.policy_crlf_glob.len != 0) snapshot.policy_crlf_glob else null;
const bin = resolveSettingWith(ctx.io, pa, ctx.root_dir, amb, .binary_glob, repo_bin) catch
zit.settings.Resolved{ .value = "", .source = .default };
const crlf = resolveSettingWith(ctx.io, pa, ctx.root_dir, amb, .crlf_glob, repo_crlf) catch
zit.settings.Resolved{ .value = "", .source = .default };
round_server.push_policy = .{ .binary_glob = bin.value, .crlf_glob = crlf.value };
}
// Sparse-cone RFC section 5.3 -- a sparse origin refuses clones it does
// not cover. Read per round like every other snapshot-freshness fact: an
// `update --cone` against the served tree is obeyed by the next round.
var origin_cone = zit.repocache.readStoreCone(gpa, ctx.io, ctx.root_dir);
defer if (origin_cone) |*c| c.deinit(gpa);
round_server.origin_cone = if (origin_cone) |*c| c else null;
// sync §7.7 L1463-1467 — the other half of the same admission predicate, from
// the same snapshot, so `ingest` never pairs this round's shun table with a
// stale policy.
round_server.hash_policy = snapshot.hash_policy;
round_server.seen_submanifest = snapshot.seen_submanifest;
round_server.submanifests_enabled = snapshot.submanifests_enabled;
round_server.submanifest_k = snapshot.submanifest_k;
round_server.submanifest_auto_retune = snapshot.submanifest_auto_retune;
// The front's membership rule for THIS push, and the resolver it
// needs. Both set together: a rule with no resolver fails closed in
// `pushProtectVerdict`, which is the right answer to a misconfiguration but
// not a state this code should be able to produce. The resolver opens the
// repository lazily -- a round that pushes nothing protected never pays --
// and the view it opens reads the on-disk log, which every earlier round
// already persisted, so a parent pushed last round resolves; a parent in
// this round is resolved in-round by the rule itself.
var resolver = RoundBranchResolver{ .ctx = ctx, .gpa = gpa };
defer resolver.deinit();
if (front.protect) |globs| if (globs.len != 0) {
round_server.push_policy.protect_globs = globs;
round_server.branch_of = resolver.resolver();
};
// A divert means nothing without a rule to divert from, so the front's
// login is read only under one.
var ledger: ?RefusalLedger = null;
defer if (ledger) |*l| l.deinit();
if (front.divert_login) |login| if (round_server.push_policy.protect_globs.len != 0) {
const now_ms: u64 = @intCast(@divFloor(std.Io.Timestamp.now(ctx.io, .real).nanoseconds, std.time.ns_per_ms));
ledger = try RefusalLedger.load(gpa, ctx.io, ctx.root_dir, login, now_ms);
round_server.divert = .{ .login = login, .now_ms = now_ms, .ledger = ledger.?.ledger() };
};
var reply = try serve_mod.handleXfer(gpa, &round_server, overlay.store(), request_container, cookie, request_compressed);
errdefer gpa.free(reply.body);
if (ledger) |*l| {
l.save();
reply.diverted = try l.header(gpa);
}
errdefer if (reply.diverted) |d| gpa.free(d);
// sync §9.3 r33 — an AUTHORIZED pushed config card's settings, applied to
// the served repository here because this is where the config lives; the
// responder is sans-IO and only reports the decision. Unreachable today by
// construction: `runServe` configures no credential, so `Server.hasAdmin` is
// false for every requester and every push is rejected (the e2e pins that).
// It is wired anyway because the alternative — a server that can AUTHORIZE a
// push and then silently drop it — is the failure mode worth ruling out.
if (reply.config_apply.any()) {
updateConfig(gpa, ctx.io, ctx.root_dir, .{
.enable_submanifests = reply.config_apply.enable,
.submanifest_k = reply.config_apply.k,
.submanifest_auto_retune = reply.config_apply.auto_retune,
}) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.StoreBusy => return error.ServerBusy,
error.Storage, error.Corrupt => return error.ServerStorage,
};
}
if (overlay.writtenCount() > 0) {
// submanifests §7.1 — the durable half of the push-side duty; the
// reply-visible half is `sync.ingest`, which phantomizes into the very
// store the §3.2 walk reads. The gap COUNT is discarded on purpose: a
// stateless server (§3.8/§8.9) has no operator to report to, and §7.1
// asks for a phantom, not a diagnostic.
_ = persistStore(gpa, ctx.io, ctx.root_dir, overlay.store(), null, 0) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.StoreBusy => return error.ServerBusy,
error.Storage, error.Corrupt => return error.ServerStorage,
};
}
return reply;
}
/// The served repository's memory of divert refusals (`sync.DivertLedger`),
/// one line per refusal: `MS LOGIN HASH`, the login fossil-encoded. Loaded
/// for one request's login, written back only when the round added to it,
/// with lines past `refusal_ttl_ms` dropped on the way.
const RefusalLedger = struct {
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
now_ms: u64,
/// The request's login, encoded as it is stored.
login: []u8,
/// Every unexpired line, every login's, ready to write back.
lines: std.ArrayListUnmanaged(u8) = .empty,
/// This login's refused check-ins.
mine: std.StringHashMapUnmanaged(void) = .empty,
added: bool = false,
moves: std.ArrayListUnmanaged(zit.sync.DivertMove) = .empty,
arena: std.heap.ArenaAllocator,
/// How long a refusal counts as the pusher having been told.
const refusal_ttl_ms: u64 = 7 * 24 * 60 * 60 * 1000;
/// How far ahead of now a line may be stamped and still count: a clock
/// stepped back must not make a refusal last forever.
const refusal_skew_ms: u64 = 24 * 60 * 60 * 1000;
/// The most lines kept; the oldest go first. A pusher refused this often
/// in a week is warned again, which is the only cost.
const refusal_lines_max: usize = 10_000;
fn load(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, login: []const u8, now_ms: u64) error{OutOfMemory}!RefusalLedger {
var self: RefusalLedger = .{ .gpa = gpa, .io = io, .dir = dir, .now_ms = now_ms, .login = undefined, .arena = std.heap.ArenaAllocator.init(gpa) };
errdefer self.arena.deinit();
const a = self.arena.allocator();
const enc = try a.alloc(u8, 2 * login.len);
self.login = @constCast(zit.artifact.encodeEscapes(login, enc));
const blob = (try zit.repocache.loadMember(gpa, io, dir, zit.repocache.refusals_member)) orelse return self;
defer gpa.free(blob);
// Newest lines are at the end; keep at most the last `refusal_lines_max`.
var total: usize = std.mem.count(u8, blob, "\n");
var it = std.mem.splitScalar(u8, blob, '\n');
while (it.next()) |line| {
if (total > refusal_lines_max) {
total -= 1;
continue;
}
var f = std.mem.splitScalar(u8, line, ' ');
const ms = std.fmt.parseInt(u64, f.next() orelse continue, 10) catch continue;
const who = f.next() orelse continue;
const hash = f.next() orelse continue;
if (f.next() != null) continue;
if (now_ms -| ms > refusal_ttl_ms or ms -| now_ms > refusal_skew_ms) continue;
try self.lines.appendSlice(a, line);
try self.lines.append(a, '\n');
if (std.mem.eql(u8, who, self.login)) try self.mine.put(a, try a.dupe(u8, hash), {});
}
return self;
}
fn deinit(self: *RefusalLedger) void {
self.arena.deinit();
self.* = undefined;
}
fn ledger(self: *RefusalLedger) zit.sync.DivertLedger {
return .{ .ctx = self, .refusedFn = refused, .recordFn = record, .divertedFn = diverted };
}
fn refused(ctx: *anyopaque, hash: []const u8) bool {
const self: *RefusalLedger = @ptrCast(@alignCast(ctx));
return self.mine.contains(hash);
}
fn record(ctx: *anyopaque, hash: []const u8) error{OutOfMemory}!void {
const self: *RefusalLedger = @ptrCast(@alignCast(ctx));
const a = self.arena.allocator();
try self.lines.print(a, "{d} {s} {s}\n", .{ self.now_ms, self.login, hash });
try self.mine.put(a, try a.dupe(u8, hash), {});
self.added = true;
}
fn diverted(ctx: *anyopaque, mv: zit.sync.DivertMove) error{OutOfMemory}!void {
const self: *RefusalLedger = @ptrCast(@alignCast(ctx));
const a = self.arena.allocator();
try self.moves.append(a, .{ .hash = try a.dupe(u8, mv.hash), .branch = try a.dupe(u8, mv.branch), .protected = try a.dupe(u8, mv.protected) });
}
/// Write back what this round added. Advisory: a lost write costs the
/// pusher one more refusal.
fn save(self: *RefusalLedger) void {
if (self.added) zit.repocache.storeRefusals(self.io, self.dir, self.lines.items);
}
/// The `x-zit-diverted` values for this round's moves, one per line
/// (`serve` sends one header per line), or null. Each field is
/// fossil-encoded, as on the sync wire, so a branch name carrying a space,
/// a comma or a newline can neither split a field nor reach the front as
/// a header of its own.
fn header(self: *RefusalLedger, gpa: std.mem.Allocator) error{OutOfMemory}!?[]u8 {
if (self.moves.items.len == 0) return null;
var out: std.ArrayListUnmanaged(u8) = .empty;
errdefer out.deinit(gpa);
for (self.moves.items, 0..) |mv, i| {
if (i != 0) try out.append(gpa, '\n');
for ([_][]const u8{ mv.hash, mv.branch, mv.protected }, 0..) |field, k| {
if (k != 0) try out.append(gpa, ' ');
const enc = try gpa.alloc(u8, 2 * field.len);
defer gpa.free(enc);
try out.appendSlice(gpa, zit.artifact.encodeEscapes(field, enc));
}
}
return try out.toOwnedSlice(gpa);
}
};
test "serve: the x-zit-diverted value is one fossil-encoded line per move, so no branch name can split it" {
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var ledger = try RefusalLedger.load(gpa, std.testing.io, tmp.dir, "jan", 1000);
defer ledger.deinit();
const l = ledger.ledger();
try l.diverted(.{ .hash = "aa", .branch = "attempt/jan/aa", .protected = "release, x\nx-evil: 1" });
try l.diverted(.{ .hash = "bb", .branch = "attempt/jan/bb", .protected = "trunk" });
const value = (try ledger.header(gpa)).?;
defer gpa.free(value);
try std.testing.expectEqualStrings("aa attempt/jan/aa release,\\sx\\nx-evil:\\s1\nbb attempt/jan/bb trunk", value);
}
test "serve: a refusal record expires after a week, or when stamped too far ahead" {
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const day: u64 = 24 * 60 * 60 * 1000;
const now: u64 = 100 * day;
{
var ledger = try RefusalLedger.load(gpa, std.testing.io, tmp.dir, "jan", now);
defer ledger.deinit();
try ledger.ledger().record("fresh");
ledger.save();
}
// Rewrite with a stale line and a future one around the fresh one.
const blob = try std.fmt.allocPrint(gpa, "{d} jan stale\n{d} jan fresh\n{d} jan future\n", .{ now - 8 * day, now, now + 2 * day });
defer gpa.free(blob);
zit.repocache.storeRefusals(std.testing.io, tmp.dir, blob);
var ledger = try RefusalLedger.load(gpa, std.testing.io, tmp.dir, "jan", now);
defer ledger.deinit();
const l = ledger.ledger();
try std.testing.expect(l.refused("fresh"));
try std.testing.expect(!l.refused("stale"));
try std.testing.expect(!l.refused("future"));
}
/// `sync.BranchResolver` over the served repository's own view,
/// opened on first use and only for the round. `.fast_read` takes the
/// zero-parse image when it is fresh (`ServeCtx.maintain` keeps it so) and
/// derives when it is not, so the answer is crosslink's own in either case --
/// which is the point: "which branch does this check-in resolve to" has one
/// authority, and this is it, not a re-implementation over raw tags.
const RoundBranchResolver = struct {
ctx: *ServeCtx,
gpa: std.mem.Allocator,
repo: Repository = undefined,
opened: bool = false,
failed: bool = false,
fn resolver(self: *RoundBranchResolver) zit.sync.BranchResolver {
return .{ .ctx = self, .branchOfFn = branchOf, .mainBranchFn = mainBranch, .tagsOfFn = tagsOf };
}
fn deinit(self: *RoundBranchResolver) void {
if (self.opened) self.repo.deinit(self.gpa);
self.* = undefined;
}
/// Open once; a failure is sticky for the round so the rule fails closed
/// on every question rather than retrying a broken read per artifact.
fn open(self: *RoundBranchResolver) error{Unavailable}!*Repository {
if (self.failed) return error.Unavailable;
if (!self.opened) {
Repository.open(&self.repo, self.gpa, self.ctx.io, self.ctx.root_dir, .{ .mode = .fast_read }) catch {
self.failed = true;
return error.Unavailable;
};
self.opened = true;
}
return &self.repo;
}
fn branchOf(ctx_ptr: *anyopaque, hash: []const u8) error{Unavailable}!?[]const u8 {
const self: *RoundBranchResolver = @ptrCast(@alignCast(ctx_ptr));
const repo = try self.open();
const rid = repo.lookup(hash) orelse return null;
// `branchOf` is defined on check-ins; a P card naming a blob (or a
// phantom) resolves to nothing rather than tripping its precondition.
if (!repo.view.state.hasEvent(rid)) return null;
return repo.branchOf(rid);
}
fn mainBranch(ctx_ptr: *anyopaque) error{Unavailable}![]const u8 {
const self: *RoundBranchResolver = @ptrCast(@alignCast(ctx_ptr));
const repo = try self.open();
return repo.view.main_branch;
}
fn tagsOf(ctx_ptr: *anyopaque, gpa: std.mem.Allocator, hash: []const u8) error{ Unavailable, OutOfMemory }!?[]repostate.RepoState.TagEntry {
const self: *RoundBranchResolver = @ptrCast(@alignCast(ctx_ptr));
const repo = try self.open();
const rid = repo.lookup(hash) orelse return null;
if (!repo.view.state.hasEvent(rid)) return null;
return try repo.view.state.tagsOf(gpa, rid);
}
};
/// A serve snapshot that reads container bytes from the log ON DEMAND instead of
/// copying the whole repository into RAM at prime. It owns a read-only
/// `LogStore` — whose open scan already whole-frame-CRC'd every byte, so the
/// single read stays; the second read `loadContentStore` performed does not — and
/// a resident INDEX: the adapter-rid→log-rid map plus the lowercase-hex wire names
/// the responder walks. `held()` answers from that index with zero I/O; `wire()`
/// and `content()` read the one frame they are asked for and memoise it for the
/// round; `releaseTransient` drops those memos between rounds, so a round's
/// resident bytes are bounded by what it emitted, not by the store.
///
/// Heap-allocated because it owns a `storage.FileStorage` whose `storageFile()`
/// vtable points back into the struct — the box must not move once the log borrows
/// it. `openFromFile` takes that ownership (the serve path); `open` borrows a
/// caller-owned `StorageFile` (a test instrumenting read traffic).
///
/// Adapter rids `1..=rids.len` name the log's non-config artifact frames in
/// `ridsInPersistedOrder` — the SAME enumeration `loadContentStore` assigns, so
/// adapter rid r is byte-for-byte the rid the old MemStore snapshot gave, which is
/// what keeps the clone pager's persisted-order cursor (sync §8.2) stable. A frame
/// named in the repository shun table keeps its slot in that enumeration and is
/// MASKED rather than removed (see `masked`), so the shun table never renumbers
/// a rid and §8.2 L1551-1553's "silently skipping gaps" carries the exclusion. Rids
/// above `rids.len` are an appendix `MemStore` holding any hash a caller `intern`s
/// or `put`s that the log does not carry. The serve path never writes the base
/// (every write lands in the per-request `OverlayStore`), so the appendix stays
/// empty in production; it exists so this store honours the whole `ArtifactStore`
/// interface — composed over the existing `MemStore` rather than hand-rolled —
/// instead of trapping on a write it will never see.
const LogIndexStore = struct {
gpa: std.mem.Allocator,
/// Owned storage binding on the `openFromFile` path (the log borrows
/// `&self.fs`); `undefined` on the `open` path, where the caller owns the
/// storage and keeps it alive for this store's life.
fs: storage.FileStorage,
log: logstore.LogStore,
/// Adapter rid a (1-based) → `rids[a-1]` = the log rid. Persisted order, config
/// frames removed.
rids: []logstore.Rid,
/// Adapter rid a → `names[a-1]` = its lowercase-hex wire name (sync §3.3),
/// backed by `name_arena`. The single site the binary-log digest becomes the
/// wire name — the boundary `loadContentStore` held, kept to one place.
names: [][]const u8,
name_arena: std.heap.ArenaAllocator,
by_name: std.StringHashMapUnmanaged(repostate.Rid) = .empty,
/// Sparse-cone RFC section 6.2 -- single-entry classification memo for
/// cone-scoped requests: the server is stateless per round, so without
/// this every round of a cone-scoped clone would re-parse the store.
/// Keyed by the cone digest; dropped with the snapshot, so a rebuilt
/// snapshot can never serve a stale classification.
cone_memo_digest: ?[32]u8 = null,
cone_memo: ?SparseClassification = null,
/// sync §9.5 L2042-2046 — adapter rid a → `masked[a-1]`: this frame's name
/// is in the repository shun table, so the enumeration MASKS it. §9.5 asks for
/// "igot for every eligible holding after the Section 3.3 shun … filters", and
/// this array is where that filter lives: a masked entry answers `held` false,
/// `content`/`wire` null, and its name is left out of `by_name`, so the server
/// does not announce it (§3.2's walk gates on `held`), and an inbound `gimme`
/// for it — which keys on `lookup` — lands on §3.3 L569-572's "If absent,
/// silently skip" with no code on that path.
///
/// THE CLONE HALF IS AN ERRATUM CANDIDATE, not a citation. Because the mask
/// lives in the enumeration, a masked holding is also absent from the clone
/// batch — but no draft clause says a clone server withholds one. §9.5's
/// sentence is scoped to the `send-catalog` pragma's igot inventory; §3.4
/// L545-547 hands clone to §8.1-§8.4, and §8.2 L1551-1553 licenses only the
/// MECHANISM ("silently skipping gaps"), never the policy. Applying §3.3's
/// shun set to the clone reply is this implementation's extension, taken
/// because the alternative — a repository under a takedown order handing the
/// content to every new clone — is not a behaviour an operator would read the
/// drafts' silence as licensing. Erratum candidate against sync §8.1/§8.2:
/// state whether a clone server's batch is filtered by the §3.3 shun set.
///
/// SCOPE, stated exactly: this masks what THIS BASE SNAPSHOT enumerates. A
/// served round answers over a per-request `repostate.OverlayStore` wrapping the
/// base, and its `putImpl` keys on `base.lookup` — which the mask answers null
/// for — so a peer that PUSHES `file <masked name>` in the same round would
/// create an overlay-local rid that genuinely holds those bytes and is
/// announced by that round's §3.2 walk. That route is closed, but NOT here: it
/// is closed by receive-side admission (§7.7 L1472-1475: "A non-clone file
/// whose target is shunned is consumed and discarded after its payload and
/// pending private marker; it emits no error … and makes no repository
/// mutation."), which `scanCard` applies before the card can enter `scan.files`
/// and `ingest` re-applies before the `put`, both reading `Server.shun` — the
/// set `serveRound` builds from THIS SAME snapshot's table, so the two can
/// never disagree about a name. Recorded here because the send-side filter
/// genuinely cannot reach that path: what makes the guarantee hold is the pair,
/// and removing either half reopens it.
///
/// A MASK, not a removal, and §8.2 L1551-1553 is why: "SEQNO is a server
/// blob-table RID. Iterate from supplied RID through current maximum,
/// silently skipping gaps; clients MUST NOT infer count or size from range."
/// Dropping the frame from the enumeration would renumber every survivor above
/// it. The server is stateless (§8.9 L1855-1856: "Section 3.7 is authoritative;
/// the stateless server signals no completion") — the client's SEQNO *is* the clone
/// cursor — and a shun row appended between two rounds of one clone grows the
/// log, which is exactly what trips `ServeCtx.baseSnapshot`'s append-only stat
/// gate and rebuilds this index. Renumbering there would make a resuming
/// client's cursor step over a survivor and still be told the sequence was
/// complete: a silently short clone, the failure class `read_fault` exists to
/// prevent. Keeping the slot and answering it as a §8.2 gap makes adapter-rid
/// identity a pure function of the log, independent of the shun table.
///
/// Filled in the SAME loop as `rids`/`names`/`content_memo`/`wire_memo`, off
/// the same index — the FILL alignment is structural (one loop, one counter),
/// not asserted. A mask derived from ANY second walk is the one way adapter rid
/// r could come to name one artifact and mask another, and nothing about the log
/// makes such a walk look wrong on inspection: it would agree on length and on
/// set membership and differ only in which slot carries which answer. The
/// LENGTHS are a separate question and the fill loop cannot speak to them; they
/// are gated by a checked error at the end of `build` (see there).
///
/// AND A PURE FUNCTION OF `shun.digests` — the other half of the same
/// invariant, and the one a later filter is most likely to break. `wireImpl`'s
/// delta-basis check asks the SET (`shun.digests.contains(d.baseHash())`) while
/// `held`/`content`/`wire` by rid ask THIS ARRAY; the two agree only because the
/// fill loop below defines `masked[a]` as exactly that membership. §9.5
/// L1988-1992 names FOUR filters — "igot for every eligible holding after the
/// Section 3.3 shun, phantom, private, and peer-has filters" — and this array is
/// the natural landing site for the other three. The day one of them widens
/// `masked` beyond shun membership, the basis check silently stops following it
/// and the server re-emits a DELTASRC it will not send. Widening this array
/// therefore obliges the same change to resolve the basis THROUGH it (basis
/// digest → adapter rid → `masked[rid - 1]`) rather than through the set.
masked: []bool,
/// Per-round transient memos over the persisted rids: the owned plaintext /
/// wire container that `content()`/`wire()` read on demand, released each round
/// by `releaseTransient`.
content_memo: []?[]u8,
wire_memo: []?WireHold,
/// Hashes not in the log, interned/put by a caller. Empty on the serve path.
appendix: repostate.MemStore,
/// sync §3.3 L553-556 / §8.8 L1839-1841 — the served repository's shun table,
/// re-loaded on every build. That is the freshness mechanism the rest of this
/// index already rides: `ServeCtx.baseSnapshot` rebuilds whenever the log has
/// grown, and a `zit shun` grows it, so a shun row written between two served
/// rounds is honoured by the next round with no restart (§3.8's guarantee
/// applied to the shun table rather than to check-ins).
shun: RepoShunSet,
/// sync §9.3 r28 — the same shun table as ROWS, for the `/shun` config group
/// a peer requests. `shun` above answers "is this artifact suppressed" for
/// the send filter; this carries the operator metadata (when, and why) that
/// a peer needs to reconstruct the tombstone rather than merely obey it.
///
/// The reason text borrows `self.log`'s own row storage (which this
/// snapshot owns and outlives every round served from it), but the NAMES do
/// not: the log keys shun rows by BINARY digest and the wire carries
/// lowercase hex, so the hex is materialised once here rather than per
/// round. `shun_hex` is its backing.
shun_rows: []zit.sync.ShunRow,
shun_hex: []u8,
/// sync §7.7 L1463-1467 — the served repository's effective hash-policy, read
/// off the same build as `shun` because it is the other half of the same
/// receive-side admission predicate and must never be a build behind it.
hash_policy: artifact.HashPolicy,
/// Total plaintext this store has read on demand from the log, monotonic
/// across `releaseTransient` — the work counter that makes "answering a round
/// materialises only what it emits" testable (the log-backed twin of
/// `MemStore.inflated_bytes`). Container reads are excluded, as they were
/// there: this measures inflation, not verbatim retransmission.
materialized_bytes: usize = 0,
/// Set true when an on-demand `content()`/`wire()` read this round mapped a
/// real error to null (a post-scan I/O fault or an OOM building the wire
/// form) — NOT when a frame legitimately has no plaintext or no wire form.
/// The vtable cannot signal an error, so the §8.2 clone pager would skip the
/// null rid and still report the sequence complete, shipping a clone missing
/// an artifact; `serveAnswer` reads this after the round and converts a set
/// flag to `error.ServerStorage` so the client retries instead. Cleared with
/// the round's memos by `releaseTransient`, so every round starts clean.
read_fault: bool = false,
/// draft-fossil-submanifests-00 §8.1 — this log's durable seen-submanifest
/// indicator, read from the config frame of the SAME log generation this
/// index describes. It arms the §7.2 send gate (`serveRound`), and it lives
/// here rather than beside the long-lived `sync.Server` so it is re-read
/// whenever the snapshot is: a push that lands the repository's first tree
/// grows the log, the stat gate rebuilds the index, and the very next round
/// serves that tree through an ARMED gate. Cached on a stale snapshot it
/// would stay false for the life of the process.
seen_submanifest: bool = false,
/// draft-fossil-submanifests-00 §8.1 — this log's `enable-submanifests`
/// setting, read from the same config frame and for the same reason: §7.2
/// makes the reply's `avoid-submanifests` pragma a function of it, so a
/// stale copy would advertise the wrong policy for the life of the process.
submanifests_enabled: bool = zit.sync.submanifest_emission_default,
/// draft-fossil-submanifests-00 §6.3 / §6.4 — this log's pinned
/// partition parameter and its auto-retune switch, read off the same config
/// frame as the two above and for the same freshness reason: sync §9.3
/// serves them to a cloning peer, so an operator who re-pins k must be
/// answered by the next round, not by the next process.
submanifest_k: u8 = 0,
submanifest_auto_retune: bool = false,
/// The repository tier of the two push-policy settings,
/// copied (owned) off the SAME per-build config read as the four facts
/// above. Before this rode the snapshot, `serveAnswer` resolved the pair
/// through a fresh `LogStore.openReadOnly` per setting per round -- a
/// full frame-header scan of the log each time, which on a 1.17M-artifact
/// store turned every round into ~11 seconds of dead re-scanning and a
/// clone into half an hour. Freshness is unchanged: a settings
/// write appends a config frame, the log grows, and the (size, mtime)
/// snapshot key rebuilds this copy for the next round -- exactly the
/// contract `seen_submanifest` above already relies on. An empty string
/// is the repository having no opinion, as in `repositorySetting`.
policy_binary_glob: []const u8 = "",
policy_crlf_glob: []const u8 = "",
/// A memoised wire form for one persisted rid: the owned §4.1 container and,
/// for the delta form, the basis' owned hex name (DELTASRC) and the
/// reconstructed target size (delta §3.2). Freed by `releaseTransient`.
const WireHold = struct {
container: []u8,
delta: ?Delta = null,
const Delta = struct { src_hex: []u8, target_size: u64 };
};
/// Build over a caller-owned `StorageFile` (the log borrows it; `fs` unused).
fn open(gpa: std.mem.Allocator, storage_file: storage.StorageFile) logstore.OpenError!*LogIndexStore {
const self = try gpa.create(LogIndexStore);
errdefer gpa.destroy(self);
self.gpa = gpa;
self.fs = undefined;
try self.build(storage_file);
return self;
}
/// Build over an owned `FileStorage` bound to `file` (the serve path). `size`
/// must equal `file`'s current length (`FileStorage.init`'s precondition).
fn openFromFile(gpa: std.mem.Allocator, io: std.Io, file: File, size: u64) logstore.OpenError!*LogIndexStore {
const self = try gpa.create(LogIndexStore);
errdefer gpa.destroy(self);
self.gpa = gpa;
self.fs = storage.FileStorage.init(io, file, size);
try self.build(self.fs.storageFile());
return self;
}
fn build(self: *LogIndexStore, storage_file: storage.StorageFile) logstore.OpenError!void {
self.materialized_bytes = 0;
// `self` came from `gpa.create` (undefined memory) and is filled field by
// field, so the struct-declaration default `read_fault = false` never runs.
// `serveAnswer` reads this flag on the FIRST round after a prime/rebuild,
// before any `releaseTransient` clears it — leave it uninitialised and a
// garbage-true value spuriously fails a healthy round. Zero it here.
self.read_fault = false;
// Same undefined-memory rule for the cone memo: garbage here reads as
// a live classification and deinit frees wild pointers.
self.cone_memo = null;
self.cone_memo_digest = null;
self.name_arena = std.heap.ArenaAllocator.init(self.gpa);
errdefer self.name_arena.deinit();
self.by_name = .empty;
errdefer self.by_name.deinit(self.gpa);
self.appendix = repostate.MemStore.init(self.gpa);
errdefer self.appendix.deinit();
self.log = try logstore.LogStore.openReadOnly(self.gpa, storage_file);
errdefer self.log.deinit(self.gpa);
// submanifests §8.1 — one config read per index build, never per round;
// both settings come off the same frame.
self.seen_submanifest = false;
self.submanifests_enabled = zit.sync.submanifest_emission_default;
self.submanifest_k = 0;
self.submanifest_auto_retune = false;
self.policy_binary_glob = "";
self.policy_crlf_glob = "";
if (try self.log.readConfig(self.gpa)) |*oc_const| {
var oc = oc_const.*;
defer oc.deinit(self.gpa);
self.seen_submanifest = oc.config.seen_submanifest;
self.submanifests_enabled = oc.config.enable_submanifests;
self.submanifest_k = oc.config.submanifest_k;
self.submanifest_auto_retune = oc.config.submanifest_auto_retune;
self.policy_binary_glob = try self.gpa.dupe(u8, oc.config.binary_glob);
errdefer self.gpa.free(self.policy_binary_glob);
self.policy_crlf_glob = try self.gpa.dupe(u8, oc.config.crlf_glob);
}
// Function-scope: a failure in ANY later build step must free these
// (a zero-length free of the "" default is a no-op).
errdefer self.gpa.free(self.policy_binary_glob);
errdefer self.gpa.free(self.policy_crlf_glob);
// sync §3.3 L553-556 — the shun table, off this same build's open. Copied
// rather than borrowed so the set has one shape on both the serve and the
// client path (one hex→digest boundary, one lookup).
self.shun = RepoShunSet.init(self.gpa);
errdefer self.shun.deinit();
try self.shun.load(&self.log);
// §9.3 r28 — the row view of that same table, off the same open, so the
// group a peer requests can never be a build behind the filter.
const rows = self.log.shunList();
var hex_bytes: usize = 0;
for (rows) |r| hex_bytes += 2 * r.hash.len;
self.shun_hex = try self.gpa.alloc(u8, hex_bytes);
errdefer self.gpa.free(self.shun_hex);
self.shun_rows = try self.gpa.alloc(zit.sync.ShunRow, rows.len);
errdefer self.gpa.free(self.shun_rows);
var hex_at: usize = 0;
for (rows, self.shun_rows) |src_row, *dst| {
const span = self.shun_hex[hex_at..][0 .. 2 * src_row.hash.len];
_ = std.fmt.bufPrint(span, "{x}", .{src_row.hash}) catch unreachable;
hex_at += span.len;
dst.* = .{ .hash = span, .epoch = src_row.epoch, .reason = src_row.reason };
}
// sync §7.7 L1463-1467 — and the effective hash-policy off the same open.
self.hash_policy = try effectiveHashPolicy(self.gpa, &self.log);
const persisted = try self.log.ridsInPersistedOrder(self.gpa);
defer self.gpa.free(persisted);
// The `.config` test is a GUARD, not a live skip: a config frame is never
// interned (`LogStore.scan` records only its offset, and `putConfig` appends
// without touching `entries`), so `ridsInPersistedOrder` — which walks
// `entries` — cannot yield one today. Kept because this counting loop and
// the filling loop below must agree on `n` exactly, and because
// `loadContentStore` states the same guard: the day a config frame becomes
// interned, both stay correct instead of both silently naming it as an
// artifact.
var n: usize = 0;
for (persisted) |lr| {
if (self.log.kindOf(lr) != .config) n += 1;
}
self.rids = try self.gpa.alloc(logstore.Rid, n);
errdefer self.gpa.free(self.rids);
self.names = try self.gpa.alloc([]const u8, n);
errdefer self.gpa.free(self.names);
self.masked = try self.gpa.alloc(bool, n);
errdefer self.gpa.free(self.masked);
self.content_memo = try self.gpa.alloc(?[]u8, n);
errdefer self.gpa.free(self.content_memo);
self.wire_memo = try self.gpa.alloc(?WireHold, n);
errdefer self.gpa.free(self.wire_memo);
try self.by_name.ensureTotalCapacity(self.gpa, @intCast(n));
var a: usize = 0;
for (persisted) |lr| {
if (self.log.kindOf(lr) == .config) continue;
// The one site the binary-log digest becomes the wire name (sync §3.3),
// exactly as `loadContentStore` did — held to one place.
var name_buf: [2 * digest_len]u8 = undefined;
const wire_name = digestToWireName(self.log.hashOf(lr), &name_buf);
const owned = try self.name_arena.allocator().dupe(u8, wire_name);
self.rids[a] = lr;
self.names[a] = owned;
self.content_memo[a] = null;
self.wire_memo[a] = null;
// sync §9.5 L2042-2046 — the shun filter, applied to the enumeration
// itself and off THIS iteration's index, so every parallel array plus
// the mask derive from one loop and one counter (see `masked`).
const is_masked = self.shun.digests.contains(self.log.hashOf(lr));
self.masked[a] = is_masked;
// Hashes are unique per log entry (content-addressed intern), so no key
// collides; adapter rid is a+1. A masked name is deliberately absent:
// `lookup` answering null is what makes an inbound `gimme` for it hit
// §3.3 L569-572's "If absent, silently skip" with no code of its own.
if (!is_masked) self.by_name.putAssumeCapacity(owned, @intCast(a + 1));
a += 1;
}
// THE PARALLEL-ARRAY LENGTH GATE. `heldImpl`, `contentImpl` and `wireImpl`
// bounds-test an incoming rid against `rids.len` ALONE and then index
// `masked`/`content_memo`/`wire_memo` with it. Their alignment is what makes
// adapter rid r name `rids[r-1]`; a length that disagreed would be an
// out-of-bounds read, and in ReleaseFast/ReleaseSmall the slice bounds check
// is gone too, so it would be a silent one: a garbage `masked` true withholds
// a survivor (the silently short clone `read_fault` exists to prevent), a
// garbage false serves an artifact the operator shunned (the §9.5 L2042-2046
// policy this whole filter exists to enforce). Both must be impossible in a
// RELEASE build, where `std.debug.assert` compiles to nothing — so this is a
// CHECKED error, paid once per build (five comparisons against an O(store)
// walk), not an assertion. It also carries the fill loop's own `a == n`, which
// is the same question asked of the counter rather than of the allocations.
if (a != n or
self.rids.len != n or
self.names.len != n or
self.masked.len != n or
self.content_memo.len != n or
self.wire_memo.len != n) return error.Corrupt;
}
/// Free every per-round transient memo, restoring the store to "index only,
/// no materialised container bytes" — the shape between rounds. Every borrowed
/// `content()`/`wire()` slice handed out this round is invalidated here, so it
/// runs only once every borrow of the snapshot has ended (see
/// `ServeCtx.releaseSnapshotPlaintext`).
fn releaseTransient(self: *LogIndexStore) void {
for (self.content_memo) |*slot| {
if (slot.*) |b| self.gpa.free(b);
slot.* = null;
}
for (self.wire_memo) |*slot| {
if (slot.*) |wh| {
self.gpa.free(wh.container);
if (wh.delta) |d| self.gpa.free(d.src_hex);
}
slot.* = null;
}
// A round's fault is a property of that round's reads; clearing it here
// (once every borrow of the round's memos has ended) lets a retry start
// clean over the same snapshot.
self.read_fault = false;
}
/// Total container/plaintext bytes currently memoised — a test/accounting
/// probe. ~0 right after a prime (the whole point: no whole-store copy) and
/// after `releaseTransient`; rises only as a round materialises what it emits.
fn residentBytes(self: *const LogIndexStore) usize {
var total: usize = 0;
for (self.content_memo) |slot| {
if (slot) |b| total += b.len;
}
for (self.wire_memo) |slot| {
if (slot) |wh| total += wh.container.len;
}
return total;
}
fn deinit(self: *LogIndexStore) void {
if (self.cone_memo) |*c| c.deinit(self.gpa);
self.releaseTransient();
self.gpa.free(self.content_memo);
self.gpa.free(self.wire_memo);
self.gpa.free(self.rids);
self.gpa.free(self.names);
self.gpa.free(self.masked);
self.by_name.deinit(self.gpa);
self.name_arena.deinit();
self.appendix.deinit();
self.shun.deinit();
self.gpa.free(self.shun_rows);
self.gpa.free(self.shun_hex);
self.gpa.free(self.policy_binary_glob);
self.gpa.free(self.policy_crlf_glob);
self.log.deinit(self.gpa);
}
/// Free the store's internals and the heap box. `self` is invalid after.
fn destroy(self: *LogIndexStore) void {
const gpa = self.gpa;
self.deinit();
gpa.destroy(self);
}
/// The ConeClassifier vtable target (sparse-cone RFC section 6.2). Any
/// failure -- undecodable member, invalid cone, memo OOM -- answers null:
/// the reply degrades to dense, never to an error (section 6.3).
fn classifyCone(
context: *anyopaque,
members: []const []const u8,
not_members: []const []const u8,
) ?*const std.DynamicBitSetUnmanaged {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (members.len == 0) return null;
if (members.len + not_members.len > zit.cone.max_members) return null;
var arena_state = std.heap.ArenaAllocator.init(self.gpa);
defer arena_state.deinit();
const aa = arena_state.allocator();
var decoded: std.ArrayListUnmanaged([]const u8) = .empty;
for (members) |member| {
const buf = aa.alloc(u8, member.len) catch return null;
const plain = artifact.decodeEscapes(member, buf);
decoded.append(aa, plain) catch return null;
}
var decoded_not: std.ArrayListUnmanaged([]const u8) = .empty;
for (not_members) |member| {
const buf = aa.alloc(u8, member.len) catch return null;
const plain = artifact.decodeEscapes(member, buf);
decoded_not.append(aa, plain) catch return null;
}
var cone = zit.cone.Cone.init(self.gpa, decoded.items, decoded_not.items) catch return null;
defer cone.deinit(self.gpa);
const digest = cone.digest(self.gpa) catch return null;
if (self.cone_memo_digest) |have| {
if (std.mem.eql(u8, &have, &digest)) return &self.cone_memo.?.keep;
}
const classification = classifyKeepRids(self.gpa, self.store(), &cone) catch return null;
if (self.cone_memo) |*old_memo| old_memo.deinit(self.gpa);
self.cone_memo = classification;
self.cone_memo_digest = digest;
return &self.cone_memo.?.keep;
}
fn coneClassifier(self: *LogIndexStore) zit.sync.ConeClassifier {
return .{ .context = @ptrCast(self), .vtable = &.{ .classify = classifyCone } };
}
fn store(self: *LogIndexStore) repostate.ArtifactStore {
return .{ .context = self, .vtable = &vtable };
}
const vtable = repostate.ArtifactStore.VTable{
.intern = internImpl,
.put = putImpl,
.content = contentImpl,
.hashOf = hashOfImpl,
.lookup = lookupImpl,
.count = countImpl,
.wire = wireImpl,
.retain = retainImpl,
.held = heldImpl,
};
fn wireFrameOf(wh: *const WireHold) repostate.WireFrame {
if (wh.delta) |d| return .{ .container = wh.container, .delta = .{ .src_hash = d.src_hex, .target_size = d.target_size } };
return .{ .container = wh.container };
}
fn countImpl(context: *anyopaque) usize {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
return self.rids.len + self.appendix.store().count();
}
/// The adapter rid of a MASKED entry named `hash`, or null when this
/// enumeration does not carry it under a mask. The mask deliberately keeps the
/// name out of `by_name` (that absence IS §3.3 L569-572's "If absent, silently
/// skip" on the gimme path), so the write doors below need this second, explicit
/// question to avoid MINTING A SECOND IDENTITY for a name the enumeration
/// already holds — see `putImpl`. A scan rather than a second name map, so
/// "masked" has exactly one representation in this struct.
///
/// THE TIME BOUND, in code rather than in prose. The walk is O(`rids.len`) per
/// call and its callers are the vtable's write doors, whose call count belongs
/// to a caller this struct does not bound: interning k names the log does not
/// carry would otherwise pay k full walks plus a `std.mem.eql` per masked slot.
/// An empty shun table is the case in which no slot can match, and it is also
/// the overwhelmingly common one — a repository that has never shunned anything
/// — so answering it in O(1) leaves the linear walk reachable only where a shun
/// table exists to bound it. (Not reached on the serve path either way: every
/// served write lands in the per-request `OverlayStore`.)
fn maskedRidFor(self: *const LogIndexStore, hash: []const u8) ?repostate.Rid {
if (self.shun.digests.count() == 0) return null;
for (self.masked, 0..) |m, i| {
if (!m) continue;
if (std.mem.eql(u8, self.names[i], hash)) return @intCast(i + 1);
}
return null;
}
fn internImpl(context: *anyopaque, hash: []const u8) error{OutOfMemory}!repostate.Rid {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (self.by_name.get(hash)) |rid| return rid;
// A MASKED name resolves to ITS OWN slot, not to a fresh appendix phantom:
// the enumeration carries that frame at a fixed adapter rid (§8.2 L1551-1553
// gap), and it is already exactly what §3.3 L537-540 would create — a record
// with the name and no content ("an igot HASH creates a phantom only when
// HASH is absent"; `held` is false and `content`/`wire` are null here). A
// second identity for one name would be the only way this store could hand
// the walk two rids naming the same artifact.
//
// Not the mask's safety argument, which lives one layer up: this store never
// sees a served `intern` at all — `respond` runs over an `OverlayStore` whose
// `internImpl` probes `base.lookup` and, on the mask's null, interns into its
// OWN round-local `added` store, which is discarded at round end. What keeps
// that round's reply from begging for the name is `Server.shun` (§3.3
// L527-530), which `serveRound` sets from this same snapshot's table.
if (self.maskedRidFor(hash)) |rid| return rid;
return self.rids.len + try self.appendix.store().intern(hash);
}
fn lookupImpl(context: *anyopaque, hash: []const u8) ?repostate.Rid {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (self.by_name.get(hash)) |rid| return rid;
if (self.appendix.store().lookup(hash)) |l| return self.rids.len + l;
return null;
}
fn putImpl(context: *anyopaque, hash: []const u8, content_bytes: []const u8) error{OutOfMemory}!repostate.Rid {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
// A persisted hash is content-addressed and already carries these bytes,
// so `put` is idempotent and `content(rid)` still reads them from the log
// — the serve path never reaches this arm (writes go to the overlay).
if (self.by_name.get(hash)) |rid| {
// Parity with `MemStore.putImpl`: a repeat put must carry identical
// bytes (a differing answer is a hash collision). Verified only under
// runtime safety, and only when the log frame actually inflates — a
// read fault leaves `content` null and the assert is skipped rather
// than firing spuriously. Inflate through a THROWAWAY read (not
// `contentImpl`), so this assertion path never memoises a container or
// bumps `materialized_bytes`/`residentBytes` — the probes the round
// tests assert on stay untouched by an assert that never runs on serve.
if (std.debug.runtime_safety) {
if (self.log.content(self.gpa, self.rids[@intCast(rid - 1)]) catch null) |existing| {
defer self.gpa.free(existing);
std.debug.assert(std.mem.eql(u8, existing, content_bytes));
}
}
return rid;
}
// A MASKED name is not in `by_name`, so without this arm a `put` of one
// would fall through to the appendix and create a SECOND, HELD entry for it:
// `held` true, `content` the pushed bytes, and `lookup` resolving the masked
// name to it — the §3.2 walk would then announce the shunned name and the
// clone pager would emit its `cfile`, undoing the whole §9.5 L2042-2046
// filter through a write. The mask outranks the write: the frame keeps its
// one adapter rid and stays a §8.2 L1551-1553 gap.
//
// A DELIBERATE narrowing of the store contract's "after `put`, `content` is
// readable" — `content(rid)` stays null here. The alternative is a name this
// server has been told never to send becoming servable because a peer pushed
// it back, and admitting a shunned target is a receive-side decision (§7.7
// L1437-1440: "consumed and discarded … makes no repository mutation"), not
// something a store adapter may take by accident.
//
// The narrowing cannot be SIGNALLED — the vtable's error set is
// `error{OutOfMemory}` — so what keeps it safe is that no caller is in a
// position to be surprised by it. On serve that holds because `respond` runs
// over a per-request `repostate.OverlayStore` and every served write lands
// THERE, never on this base. That is a property of the serve path rather than
// of this function, so it is PINNED BY A TEST at this layer ("a served round's
// writes land in the round-local overlay — the base enumeration's write doors
// are never reached") instead of resting on a claim this comment makes about
// code two modules away. A caller that wires this store in as a WRITABLE store
// is outside that proof and must read this arm before relying on `put`.
if (self.maskedRidFor(hash)) |rid| return rid;
return self.rids.len + try self.appendix.store().put(hash, content_bytes);
}
fn hashOfImpl(context: *anyopaque, rid: repostate.Rid) []const u8 {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (rid <= self.rids.len) return self.names[@intCast(rid - 1)];
return self.appendix.store().hashOf(rid - self.rids.len);
}
fn heldImpl(context: *anyopaque, rid: repostate.Rid) bool {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
// Every persisted (non-phantom, non-config) frame carries a form to serve
// verbatim, so it is held — answered from the index with no frame read.
// A MASKED frame (sync §9.5 L2042-2046) is the exception: it is a §8.2
// L1504-1506 gap, so it is not held and §3.2's walk never announces it.
// This line is the one place the send side genuinely disagreed with the
// log: `LogStore.content`/`wire` already read a shunned frame as ABSENT,
// while this answered true from the index with no frame read — so the
// inventory walk advertised an artifact no round could then send, and (with
// the name still in `by_name`) `emitCloneReply`'s D3 basis proof could reach
// a resolvable rid whose two forms were both null. `LogStore.shunned`'s doc
// names exactly this disagreement as a SYNC decision to settle; §9.5's
// "every eligible holding after the Section 3.3 shun … filters" settles it.
if (rid <= self.rids.len) return !self.masked[@intCast(rid - 1)];
return self.appendix.store().held(rid - self.rids.len);
}
fn retainImpl(context: *anyopaque, rid: repostate.Rid, ingest: repostate.WireIngest) error{OutOfMemory}!void {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
// Advisory (sync §7.2): a persisted rid already carries a stored form, so
// first-form-wins makes this a no-op — the log is the retained form.
if (rid <= self.rids.len) return;
return self.appendix.store().retain(rid - self.rids.len, ingest);
}
fn contentImpl(context: *anyopaque, rid: repostate.Rid) ?[]const u8 {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (rid > self.rids.len) return self.appendix.store().content(rid - self.rids.len);
const i: usize = @intCast(rid - 1);
// sync §9.5 L2042-2046 — a masked frame has nothing to hand out. Checked
// before the memo so the mask cannot be bypassed by a slot some earlier
// round filled, and it is a plain null: a policy gap, never a read fault.
//
// `LogStore.content` ALREADY answers null for a shunned entry (its own
// rid-keyed read mask), so today this line changes no answer: it is a
// structural guard, not covered behaviour. Measured, not assumed — deleting
// this line leaves the whole suite green, because the read one line below
// returns the same null. What IS covered, by "the adapter owns its content
// mask", is the surrounding fact the guard exists to protect: the frame's
// plaintext is still on disk and still readable through
// `LogStore.contentUnmasked`, so the null is a policy refusal over present
// bytes rather than an absence. That test is what would turn red the day the
// two layers were consolidated into one. It is here
// because the adapter must OWN its mask rather than inherit it. §7.7
// L1444-1449 wants a held-but-shunned artifact to remain usable as a delta
// BASIS — "For an admitted target whose DELTASRC name is in the repository
// shun table, the receiver does not apply admission independently to that
// source: content still held locally MAY serve as the delta basis" — and the
// only reason that works today is that basis resolution walks byte offsets
// and never re-enters `content`. The moment that basis path is given an
// explicitly unmasked door — the repair `LogStore.shunned`'s own doc
// contemplates — the log's mask stops covering this one, and the send-side
// exclusion must not evaporate with it.
if (self.masked[i]) return null;
if (self.content_memo[i]) |b| return b;
// Read the frame on demand and memoise for the round. The vtable cannot
// signal an error, so a read fault is recorded on `read_fault` and
// returned as null; `serveAnswer` turns a set flag into
// `error.ServerStorage` so the client retries rather than accepting a
// silently short clone. A frame that legitimately has no plaintext is a
// plain null and does NOT set the flag.
//
// THIS IS THE PRIMARY CORRUPTION CHANNEL IN SERVE, and it did not used
// to be. `openReadOnly` once CRC-verified every payload while building
// the snapshot, which meant a damaged store failed to open at all and
// this catch saw only genuine post-scan faults -- transient EIO, or rot
// arriving after a clean scan. Both open paths now scan header-only and
// `readFrameAt` CRC-checks every frame it reads instead, so ordinary
// stored corruption reaches serve HERE, on the read, rather than at
// snapshot build.
//
// That is a better shape, not a worse one: one bad frame used to make
// the server refuse to serve anything, and now it fails only the round
// that touches the damage while every healthy artifact still moves. But
// it means this arm is live traffic rather than a rare event, and
// anything reasoning about how often it fires should start from that.
const maybe = self.log.content(self.gpa, self.rids[i]) catch {
self.read_fault = true;
return null;
};
const bytes = maybe orelse return null;
self.content_memo[i] = bytes;
self.materialized_bytes += bytes.len;
return bytes;
}
fn wireImpl(context: *anyopaque, rid: repostate.Rid) ?repostate.WireFrame {
const self: *LogIndexStore = @ptrCast(@alignCast(context));
if (rid > self.rids.len) return self.appendix.store().wire(rid - self.rids.len);
const i: usize = @intCast(rid - 1);
// sync §9.5 L2042-2046 — same gap as `contentImpl`, checked ahead of the
// memo, and owned here for the same reason (see there).
if (self.masked[i]) return null;
if (self.wire_memo[i]) |*wh| return wireFrameOf(wh);
// A frame read fault is a real error recorded on `read_fault` so
// `serveAnswer` can fail the round; a `.raw`/phantom frame legitimately
// has no wire form (the responder compresses `content()` instead) and
// yields a plain null that does NOT set the flag. As at `contentImpl`,
// this arm now sees ordinary stored corruption rather than only
// post-scan faults -- no open path CRC-verifies payloads any more.
var w = (self.log.wire(self.gpa, self.rids[i]) catch {
self.read_fault = true;
return null;
}) orelse return null;
switch (w) {
.full => |c| {
self.wire_memo[i] = .{ .container = c };
},
.delta => |d| {
// sync §9.5 L2042-2046 — the delta form is servable only while its
// basis is. `zit shun --defer` records the row without rewriting
// the generation, so a SURVIVING frame here can be a delta rooted
// on a MASKED one; emitting its `cfile` would ship a DELTASRC this
// server never sends, leaving the peer a permanently dangling delta
// whose basis it gimmes and is silently skipped on (§3.3 L569-572)
// — corrupt rather than absent, and `emitCloneReply`'s D3 basis
// lookup would trap under `runtime_safety` on the way. Withhold the
// wire form instead; the responder's §7.2 compress-on-the-fly arm
// then emits the FULL form, and `log.content` reconstructs through
// the frame chain by BYTE OFFSET, which the name-level mask does
// not reach. A plain null, not a read fault: policy, not I/O.
//
// Asked as the SAME predicate the fill loop applied — the raw digest
// against the shun table — so "masked" has one spelling in this
// struct. (`by_name`-absence would answer identically for a log
// delta, whose basis is always a non-config frame in the
// enumeration, but it would make the exclusion mean two things and
// pay a hex encode plus a map lookup to say the second one.)
if (self.shun.digests.contains(d.baseHash())) {
w.deinit(self.gpa);
return null;
}
// The basis precedes this frame in the log, so its wire name is the
// hex of the base digest the wire read carried (delta §3.2 target
// size alongside).
var src_buf: [2 * digest_len]u8 = undefined;
const src_hex = digestToWireName(d.baseHash(), &src_buf);
const owned_src = self.gpa.dupe(u8, src_hex) catch {
// Failing to produce the wire form here would silently drop a
// held delta from the clone; record it so the round errors.
w.deinit(self.gpa);
self.read_fault = true;
return null;
};
self.wire_memo[i] = .{ .container = d.container, .delta = .{ .src_hex = owned_src, .target_size = d.target_size } };
},
}
return wireFrameOf(&self.wire_memo[i].?);
}
};
/// Snapshot every stored artifact from the persisted log into `ms`, skipping
/// config frames (not artifacts). A content-bearing frame becomes a held entry;
/// a content-absent frame becomes a phantom — exactly the held/phantom split
/// `respond` walks (sync §3.2/§3.3).
///
/// The log keys frames by the RAW binary digest, but the sync wire names an
/// artifact by that digest's lowercase hex (sync §3.3 — a `file`/`igot`/`gimme`
/// HASH is 40/64 hex characters). The store the responder walks must therefore
/// hold the hex name, or `respond` would emit non-hex bytes that break the
/// peer's card tokenizer. Every hash crossing this boundary is hex-encoded once,
/// here — the single place binary-log identity becomes wire identity.
fn loadContentStore(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
ms: *repostate.MemStore,
) serve_mod.AnswerError!void {
const rids = log.ridsInPersistedOrder(gpa) catch return error.OutOfMemory;
defer gpa.free(rids);
for (rids) |rid| {
if (log.kindOf(rid) == .config) continue;
var name_buf: [2 * digest_len]u8 = undefined;
const name = digestToWireName(log.hashOf(rid), &name_buf);
// sync §4.1/§7.2 — copy the stored container into the snapshot verbatim
// so the clone serves it without inflating here and re-compressing at
// serve. `wire()` is null only for a `.raw` frame or a phantom; those
// fall back to the plaintext path below. A `.zlib` frame carries the
// full form; a `.delta` frame carries the stored delta container plus
// its basis name and reconstructed size (the §7.2 USIZE, delta §3.2).
var maybe_wire = log.wire(gpa, rid) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage, error.Corrupt => return error.ServerStorage,
};
if (maybe_wire) |*w| {
defer w.deinit(gpa);
const recorded: bool = switch (w.*) {
.full => |container| blk: {
_ = ms.putWire(name, container) catch return error.OutOfMemory;
break :blk true;
},
.delta => |d| blk: {
// sync §3.3 L553-556 — the delta form is usable only while its
// basis is. `zit shun --defer` records the row without
// rewriting the generation, so a SURVIVING frame here can be a
// delta rooted on a shunned one; that basis reads null through
// this loop's own masked doors and interns as a PHANTOM, so
// recording the delta would leave an entry the store calls held
// (`wire != null`) whose `content()` can never resolve. §3.2's
// igot walk would then announce a name no `gimme` can ever be
// answered with, and the push would spin the round with nothing
// to send. Fall through to the plaintext path instead: the
// artifact itself is NOT shunned, so `log.content` reads it
// through the frame chain by BYTE OFFSET — which the name-level
// mask does not reach — and it is snapshotted as a held full
// form. Exactly `LogIndexStore.wireImpl`'s masked-basis guard,
// stated for the client's own holdings snapshot.
const base = d.baseHash();
if (logstore.isCompleteDigest(base.len) and log.isShunned(base)) break :blk false;
// The basis frame precedes this one in the log (delta_of <
// offset), so it was already inserted at an earlier rid; its
// wire name is the hex of the base's binary digest.
var src_name_buf: [2 * digest_len]u8 = undefined;
const src_name = digestToWireName(base, &src_name_buf);
_ = ms.putWireDelta(name, d.container, src_name, d.target_size) catch return error.OutOfMemory;
break :blk true;
},
};
if (recorded) continue;
}
const maybe = log.content(gpa, rid) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage, error.Corrupt => return error.ServerStorage,
};
if (maybe) |bytes| {
defer gpa.free(bytes);
_ = ms.store().put(name, bytes) catch return error.OutOfMemory;
} else {
_ = ms.store().intern(name) catch return error.OutOfMemory;
}
}
}
// ---------------------------------------------------------------------------
// `zit clone URL DIR` — the interop milestone: a real client converging against
// a real `zit serve` over HTTP. Create a fresh repo at DIR, open an HTTP
// transport to URL, pump the sans-IO clone client (`sync.Client`) to §3.7
// convergence through `sync_driver.runSync`, then persist the received
// artifacts into DIR's on-disk log so the clone is durable. Anonymous only.
// ---------------------------------------------------------------------------
/// §9.1 — a fixed placeholder repository identity emitted in the client's
/// pull/push operation cards. Neither a fresh clone nor a pulled repository
/// carries a persistent SERVERCODE/PROJECTCODE of its own yet (real per-repo
/// sync identity is future work); the stateless server echoes the peer's codes
/// and never gates on a configured project code (sync §9.1), so any well-formed
/// 16-hex pair carries a content-only exchange. Shared by clone and pull.
const sync_servercode = "0000000000000000";
const sync_projectcode = "0000000000000000";
/// §10.1 — the release integer the sync client announces as `pragma
/// client-version`. A fixed build identity, matching `serve`'s own version
/// constant style. (`announcedVersion` raises what actually goes on the wire —
/// submanifests §7.2.)
const sync_client_version: u32 = 22700;
/// §10.1 r11/r12 — the DATED client-version pragma every zit run announces.
///
/// The DATE and TIME are not decoration: §10.1 r11/r12 make the server's
/// `server-version` reply conditional on them ("at least five tokens total"),
/// and submanifests §7.2 gates every upload on "the remote peer's announced
/// version". An undated pragma therefore leaves `Client.peer.version` at 0 for
/// the whole session — below every threshold — so an armed client would refuse
/// its own trees to a server that would have announced support if only it had
/// been asked. Soliciting the reply is what makes the gate a peer-version gate
/// rather than a permanent refusal. Fossil sends the dated form too (§A.4).
const sync_client_version_info: zit.sync.VersionInfo = .{
.version = sync_client_version,
.datetime = .{ .date = 20260715, .time = 0 },
};
/// An explicit bound on CONSECUTIVE ROUNDS THAT MOVE NO BYTES (TigerStyle:
/// every loop is bounded). See `sync_driver.runSync` for why the bound counts
/// stalls rather than rounds: a total-round cap multiplies out against the
/// per-round batch size into an accidental ceiling on repository size, which a
/// large healthy repository trips while converging perfectly.
///
/// This value bounds only unproductive looping, so it is a statement about how
/// long a peer may accomplish nothing — not about how much there is to
/// transfer. §3.7's longest by-design stall is the unversioned three-round
/// grace window ((d)(2)); config exchange (f) and private-content discovery (c)
/// are each shorter. 16 clears every one of them several times over while still
/// failing a stuck peer in well under a second of round trips.
const max_stalled_sync_rounds: usize = 16;
/// The canonical form of a clone URL for persistence: rebuilt from parsed
/// components — scheme://[user@]host[:port]/path[?query] — so a userinfo
/// password is excluded BY CONSTRUCTION: a component the rebuild never names
/// cannot survive into the stored value, even if a future parser change adds
/// fields (fossil url.c's canonical form works the same way). The username is
/// kept, matching fossil. The REQUESTED URL is what canonicalizes, never the
/// redirect-resolved one: an alias in front of a moving host is load-bearing,
/// and the redirect chain re-resolves on every sync anyway.
pub fn canonicalRemoteUrl(gpa: std.mem.Allocator, url: []const u8) error{ OutOfMemory, InvalidUrl }![]u8 {
const uri = std.Uri.parse(url) catch return error.InvalidUrl;
const bytesOf = struct {
fn f(c: std.Uri.Component) []const u8 {
return switch (c) {
.raw => |r| r,
.percent_encoded => |p| p,
};
}
}.f;
var out: std.ArrayList(u8) = .empty;
errdefer out.deinit(gpa);
try out.appendSlice(gpa, uri.scheme);
try out.appendSlice(gpa, "://");
if (uri.user) |u| {
try out.appendSlice(gpa, bytesOf(u));
try out.append(gpa, '@');
}
const host = uri.host orelse return error.InvalidUrl;
try out.appendSlice(gpa, bytesOf(host));
if (uri.port) |p| try out.print(gpa, ":{d}", .{p});
const path = bytesOf(uri.path);
if (path.len == 0) try out.append(gpa, '/') else try out.appendSlice(gpa, path);
if (uri.query) |q| {
try out.append(gpa, '?');
try out.appendSlice(gpa, bytesOf(q));
}
return out.toOwnedSlice(gpa);
}
test "canonicalRemoteUrl: password excluded by construction, username kept" {
const gpa = std.testing.allocator;
const cases = [_]struct { in: []const u8, want: []const u8 }{
.{ .in = "http://user:secret@host/repo", .want = "http://user@host/repo" },
.{ .in = "http://user@host:8080/repo", .want = "http://user@host:8080/repo" },
.{ .in = "https://host", .want = "https://host/" },
.{ .in = "https://host/a/b?x=1#frag", .want = "https://host/a/b?x=1" },
};
for (cases) |c| {
const got = try canonicalRemoteUrl(gpa, c.in);
defer gpa.free(got);
try std.testing.expectEqualStrings(c.want, got);
try std.testing.expect(std.mem.indexOf(u8, got, "secret") == null);
}
try std.testing.expectError(error.InvalidUrl, canonicalRemoteUrl(gpa, "not a url"));
}
/// `zit clone URL DIR` — thin wiring over three deep modules: `transport`
/// (HTTP/IO), `sync.Client` + `sync_driver.runSync` (sans-IO §3.7 loop), and
/// `LogStore` (persistence). The URL is used verbatim (sync §4 r1 — the client
/// never appends /xfer). All-or-nothing failures surface an honest exit-1
/// diagnostic; a bad URL scheme is rejected before any repo is created.
pub fn runClone(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
return runCloneObserved(gpa, io, arena, verb_args, streams, amb, null);
}
/// `runClone` with benchmark-owned phase attribution. Sync §8.2's project-code
/// adoption remains in the observed config phase; a null observer is the
/// ordinary CLI path, which reads no clock and owns no counters.
pub fn runCloneObserved(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
observer: ?*const CloneObserver,
) cli.Exit {
const exit = cloneObserved(gpa, io, arena, verb_args, streams, amb, observer);
if (exit != .ok) return exit;
// A clone is followed by the `open` the operator would type next
// (Fossil 2.x), run inside DIR by this same binary so the two verbs stay
// one implementation; --no-open keeps the bare store. It runs only here,
// after cloneObserved returned and released its exclusive store lock —
// the child takes that same lock. The child opens the same cone the clone
// elided to: members from the flags and from --cone-file (read here,
// relative to this cwd) are passed explicitly.
const parsed = switch (cli.parseCloneArgs(verb_args)) {
.ok => |p| p,
else => return exit,
};
if (parsed.help or !parsed.open) return exit;
var members = parsed.cone_members;
var len = parsed.cone_len;
var not_members = parsed.cone_not_members;
var not_len = parsed.cone_not_len;
if (parsed.cone_file) |cf| {
if (cli.appendConeFile(arena, io, cf, &members, len, ¬_members, not_len)) |lens| {
len = lens.members;
not_len = lens.not_members;
}
}
return autoOpen(gpa, io, arena, streams, parsed.dir, members[0..len], not_members[0..not_len]);
}
/// Spawn this same binary as `zit open` inside the freshly cloned DIR.
fn autoOpen(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
streams: cli.Streams,
dir: []const u8,
cone: []const []const u8,
cone_not: []const []const u8,
) cli.Exit {
streams.out.flush() catch {};
streams.err.flush() catch {};
const exe = std.process.executablePathAlloc(io, gpa) catch {
streams.err.writeAll("zit clone: cloned, but cannot locate zit to open the working tree; run 'zit open' inside the directory\n") catch {};
return .failure;
};
defer gpa.free(exe);
var argv = std.ArrayList([]const u8).empty;
argv.append(arena, exe) catch return .failure;
argv.append(arena, "open") catch return .failure;
for (cone) |m| argv.append(arena, std.fmt.allocPrint(arena, "--cone={s}", .{m}) catch return .failure) catch return .failure;
for (cone_not) |m| argv.append(arena, std.fmt.allocPrint(arena, "--cone-not={s}", .{m}) catch return .failure) catch return .failure;
var child = std.process.spawn(io, .{
.argv = argv.items,
.cwd = .{ .path = dir },
}) catch {
streams.err.print("zit clone: cloned, but 'zit open' could not be started; run it inside '{s}'\n", .{dir}) catch {};
return .failure;
};
const term = child.wait(io) catch return .failure;
return switch (term) {
.exited => |code| if (code == 0) .ok else .failure,
else => .failure,
};
}
fn cloneObserved(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
observer: ?*const CloneObserver,
) cli.Exit {
const parsed = switch (cli.parseCloneArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "clone", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "clone", .extra_argument, tok),
.no_url => return cli.usageError(streams, "clone", .no_url, ""),
.no_dir => return cli.usageError(streams, "clone", .no_dir, ""),
.empty_value => |tok| return cli.usageError(streams, "clone", .empty_value, tok),
.too_many_cones => |tok| return cli.usageError(streams, "clone", .too_many_cones, tok),
};
if (parsed.help) {
streams.out.writeAll(clone_help) catch {};
return .ok;
}
// Open the transport first: a malformed URL fails before any repo lands on
// disk (nothing half-made to roll back). `open` performs no network I/O.
observeClonePhase(observer, .transport, .begin);
// sync §6 — a private repository is unreadable anonymously. The login is
// --user or the URL's user@; the secret cannot be derived until the round-0
// reply names the project code, so the client holds name and password and
// signs from round 1, exactly as fossil does. The plaintext lives only in
// pw_buf/argv/env and is scrubbed on every exit.
var pw_buf: [max_password_len]u8 = undefined;
defer std.crypto.secureZero(u8, &pw_buf);
const login_user: ?[]const u8 = blk: {
const u = parsed.user orelse urlUser(parsed.url) orelse break :blk null;
if (u.len == 0) break :blk null;
if (zit.sync.containsFossilWhitespace(u)) {
streams.err.writeAll("zit clone: --user must not contain whitespace\n") catch {};
return .usage;
}
break :blk u;
};
const login: ?zit.sync.PendingLogin = if (login_user) |u| blk: {
const password = parsed.password orelse amb.push_password orelse (readPasswordNoEcho(streams, u, &pw_buf) orelse {
if (parsed.user != null) {
streams.err.print("zit clone: no password for '{s}' (give --password, set ZIT_PASSWORD, or run on a terminal)\n", .{u}) catch {};
return .failure;
}
// A user named only by the URL is a hint: clone anonymously, and say so.
streams.err.print("zit clone: no password for '{s}' from the URL (set ZIT_PASSWORD or pass --user/--password); cloning anonymously\n", .{u}) catch {};
break :blk null;
});
break :blk .{ .user = u, .password = password };
} else null;
var tr = transport.Transport.open(io, gpa, parsed.url) catch |e| {
observeClonePhase(observer, .transport, .end);
return syncOpenFail(streams, "clone", parsed.url, e);
};
observeClonePhase(observer, .transport, .end);
defer tr.close();
notePlaintext(streams, "clone", parsed.url);
if (login) |l| tr.setBasicAuth(l.user, l.password) catch return stateOom(streams, "clone");
// Create a fresh `.zit` at DIR by reusing the init sequence, quietly (its
// own success line is suppressed; clone prints its own). A pre-existing repo
// there is init's "already exists" error, propagated verbatim.
observeClonePhase(observer, .initialize, .begin);
const init_rc = runInit(gpa, io, arena, &.{ "-q", "--", parsed.dir }, streams);
observeClonePhase(observer, .initialize, .end);
if (init_rc != .ok) return init_rc;
// Converge an in-memory store against the peer, then persist it into DIR.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
// §9.1 — the clone client's own instance identity, derived from the
// destination it is creating (informational; a fresh instance is exactly
// what a clone is).
const clone_code = blk: {
const cwd = (cli.currentPathAlloc(arena, io) catch return stateOom(streams, "clone")) orelse
break :blk zit.sync.deriveServerCode(&.{parsed.dir});
const abs = std.fs.path.resolve(arena, &.{ cwd, parsed.dir }) catch return stateOom(streams, "clone");
break :blk zit.sync.deriveServerCode(&.{abs});
};
// Live per-round progress, on stderr and ONLY when stderr is a
// terminal: an operator watching a slow clone sees each round land instead
// of silence, while piped stderr (scripts, tests) stays byte-identical to
// before. The line is \r-overwritten in place — never fossil's
// percent-flood — and finished with one accounting line when the exchange
// ends, either way it ends.
const stderr_tty = std.Io.File.stderr().isTty(io) catch false;
var progress: CloneProgress = .{ .err = streams.err };
const sink: ?sync_driver.RoundSink = if (stderr_tty)
.{ .ctx = &progress, .emit = CloneProgress.emit }
else
null;
observeClonePhase(observer, .exchange, .begin);
var clone_members = parsed.cone_members;
var clone_len = parsed.cone_len;
var clone_not_members = parsed.cone_not_members;
var clone_not_len = parsed.cone_not_len;
if (parsed.cone_file) |cf| {
const lens = cli.appendConeFile(arena, io, cf, &clone_members, clone_len, &clone_not_members, clone_not_len) orelse {
streams.err.print("zit clone: cannot read --cone-file '{s}' (or it overflows the {d}-member cap)\n", .{ cf, zit.cone.max_members }) catch {};
return .failure;
};
clone_len = lens.members;
clone_not_len = lens.not_members;
}
// The destination log, open BEFORE the exchange: a dense clone
// persists each round as it converges instead of holding the repository
// resident, so the log has to exist while rounds are still arriving. A
// failed exchange leaves DIR holding a partial, uncommitted `.zit` the
// operator deletes with the directory — the same recovery as before, with
// different debris.
var target = std.Io.Dir.cwd().openDir(io, parsed.dir, .{}) catch
return renderStorage(streams, parsed.dir, "cannot reopen the clone target directory");
defer target.close(io);
var store_append = cli.openStoreAppend(io, target) catch
return renderStorage(streams, parsed.dir, "cannot write the cloned repository");
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch
return renderStorage(streams, parsed.dir, "cannot write the cloned repository");
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"),
};
defer log.deinit(gpa);
// Stream the persist unless a cone is requested. A cone-scoped
// clone must classify its keep set over the CONVERGED store (sparse-cone
// RFC section 6.3), so it keeps the batched path; every dense clone drains
// per round and releases what it wrote, bounding peak memory by the round
// batch plus the deltas whose basis has not arrived (git index-pack's
// shape, applied to fossil's reverse-delta wire).
const cone_scoped = clone_len + clone_not_len != 0;
// Every cluster this clone writes, for the index it creates.
var clone_clusters: std.ArrayListUnmanaged([]const u8) = .empty;
defer {
for (clone_clusters.items) |n| gpa.free(n);
clone_clusters.deinit(gpa);
}
var stream_pass: ?PersistPass = if (cone_scoped) null else PersistPass.init(gpa, &log, ms.store(), null, &ms, 0, &clone_clusters) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"),
};
defer if (stream_pass) |*pass| pass.deinit();
const drain: ?sync_driver.RoundDrain = if (stream_pass) |*pass|
.{ .ctx = pass, .run = PersistPass.drainErased }
else
null;
var peer_error: PeerErrorCopy = .{};
var truncated: u64 = 0;
var outcome = cloneRun(gpa, &tr, &ms, &clone_code, login, sink, drain, clone_members[0..clone_len], clone_not_members[0..clone_not_len], &peer_error, &truncated) catch |e| {
progress.finishLine();
observeClonePhase(observer, .exchange, .end);
reportTruncatedPayloads(streams, "clone", truncated);
// A peer that stated its refusal is quoted; anything else keeps the
// generic mapping (sparse-cone RFC section 5.3 diagnostic path).
if (e == error.FatalProtocolError) {
if (peer_error.text()) |text| {
streams.err.print("zit clone: the peer refused: {s}\n", .{text}) catch {};
return .failure;
}
}
return syncRunFail(streams, "clone", parsed.url, e, tr.last_status, basicStateOf(&tr));
};
// §9.3 r28 — the latched shun rows are the one thing this outcome owns.
defer outcome.deinit(gpa);
progress.finishLine();
// Fossil fails a clone on any handler error. A cut-off reply also loses
// the clone_seqno card that would have asked for the next batch, so the
// exchange can end looking complete when it is not. It fails here, before
// anything is committed, leaving the same partial `.zit` as any other
// failed exchange.
if (truncated > 0) {
observeClonePhase(observer, .exchange, .end);
reportTruncatedPayloads(streams, "clone", truncated);
streams.err.print("zit clone: the clone of {s} is incomplete; remove {s} and clone again\n", .{ parsed.url, parsed.dir }) catch {};
return .failure;
}
if (stderr_tty) {
streams.err.print(
"clone exchange: {d} rounds, {d} protocol bytes received, {d} sent\n",
.{ outcome.rounds, outcome.received_bytes, outcome.sent_bytes },
) catch {};
}
observeClonePhase(observer, .exchange, .end);
// Sparse-cone RFC section 6.3: a cone-scoped clone filters LOCALLY --
// correct against every peer, including cone-ignorant ones. The keep set
// is classified over the converged in-memory store before any frame is
// written; nothing is discarded silently (the count prints below).
var clone_cone: ?zit.cone.Cone = null;
defer if (clone_cone) |*c| c.deinit(gpa);
var keep_bits: ?SparseClassification = null;
defer if (keep_bits) |*k| k.deinit(gpa);
if (cone_scoped) {
clone_cone = zit.cone.Cone.init(gpa, clone_members[0..clone_len], clone_not_members[0..clone_not_len]) catch |e|
return cloneConeFail(streams, e);
keep_bits = classifyKeepRids(gpa, ms.store(), &clone_cone.?) catch
return stateOom(streams, "clone");
}
const clone_keep: ?*const std.DynamicBitSetUnmanaged = if (keep_bits) |*k| &k.keep else null;
observeClonePhase(observer, .persist, .begin);
const persisted = blk: {
if (stream_pass) |*pass| {
// Streaming: the rounds already drained; settle any straggler the
// last round interned, then the leftovers and the bookkeeping.
const summary = res: {
pass.drainNew() catch |e| break :res @as(PersistError!PersistSummary, e);
break :res pass.finish();
};
break :blk summary;
}
break :blk persistStoreOpen(gpa, &log, ms.store(), clone_keep, 0, &clone_clusters);
} catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"),
};
const written = persisted.written;
observeClonePhase(observer, .persist, .end);
// §8.2 and submanifests §6.3/§8.1 — adopt every origin-owned setting in
// one field-preserving overlay. Missing fields leave `init`'s values
// untouched, and one combined frame avoids intermediate clone config state.
observeClonePhase(observer, .config, .begin);
const project_code = outcome.projectCode();
// Save the clone URL as the default remote (fossil's behavior), unless
// --once. The canonical rebuild strips a userinfo password by construction;
// `zit remote` remains the explicit override and the audit view. The
// InvalidUrl arm is unreachable in practice — `transport.open` parsed this
// same URL — and degrades to not-saving rather than failing a completed
// clone.
const saved_remote: ?[]const u8 = if (parsed.once) null else canonicalRemoteUrl(arena, parsed.url) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.InvalidUrl => null,
};
const cloned_patch: ConfigPatch = .{
.project_code = if (project_code.len > 0) project_code else null,
// The login the peer accepted from round 1 on, stored the way a push
// stores its own, so the next pull or push authenticates unasked.
.login = outcome.login,
.remote_url = saved_remote,
.enable_submanifests = outcome.submanifest.enable,
.submanifest_k = outcome.submanifest.k,
.submanifest_auto_retune = outcome.submanifest.auto_retune,
};
if (cloned_patch.any()) {
updateConfigOpen(gpa, &log, cloned_patch) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"),
};
}
// §9.3 r28 — adopt the origin's shun table. Written BEFORE the commit
// below, so a clone is durable with its tombstones or not at all: a
// repository that came up holding the content but not the record of its
// suppression is the exact state this carriage exists to prevent.
//
// The wire carries hex; the log keys by binary digest. A name that will not
// decode is skipped rather than fatal — the artifacts are already down, and
// failing a finished clone over one unparseable row helps nobody. The count
// is reported so a skip is never silent.
var shun_adopted: usize = 0;
var shun_skipped: usize = 0;
for (outcome.shun_rows) |row| {
var digest_buf: [32]u8 = undefined;
if (row.hash.len != 40 and row.hash.len != 64) {
shun_skipped += 1;
continue;
}
const digest = digest_buf[0 .. row.hash.len / 2];
_ = std.fmt.hexToBytes(digest, row.hash) catch {
shun_skipped += 1;
continue;
};
log.putShun(digest, row.reason, row.epoch) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
else => {
shun_skipped += 1;
continue;
},
};
shun_adopted += 1;
}
if (shun_adopted != 0 or shun_skipped != 0) {
streams.out.print("adopted {d} shun row{s} from the origin", .{ shun_adopted, plural(shun_adopted) }) catch {};
if (shun_skipped != 0) streams.out.print("; {d} unreadable and skipped", .{shun_skipped}) catch {};
streams.out.writeAll("\n") catch {};
}
// Every persisted artifact and the optional combined config frame must be
// durable before the cache observes this log or clone reports success.
log.commit() catch return renderStorage(streams, parsed.dir, "cannot write the cloned repository");
// A store carrying a credential is owner-only (best-effort, as for push).
if (outcome.login != null) file.setPermissions(io, File.Permissions.fromMode(0o600)) catch {};
refreshClusterSidecar(gpa, io, target, &log, clone_clusters.items, true) catch return stateOom(streams, "clone");
observeClonePhase(observer, .config, .end);
// The committed writable log is still open, so reuse its derivation path
// directly instead of reopening `.zit` solely to prime the disposable cache.
// Say so first: on a large store this phase is minutes of derive, and an
// operator watching a silent clone cannot tell transfer from derivation.
streams.out.print("clone from {s}: transferred {d} artifact{s} in {d} round{s}; deriving caches\n", .{ parsed.url, written, plural(written), outcome.rounds, plural(outcome.rounds) }) catch {};
observeClonePhase(observer, .warm_cache, .begin);
Repository.warmCacheFromLog(gpa, io, target, &log);
observeClonePhase(observer, .warm_cache, .end);
if (clone_cone) |*c| {
// Policy lands with the disposable tier (RFC section 5.2 loss rule).
zit.repocache.writeStoreCone(gpa, io, target, c);
streams.out.print(
"cloned {d} artifacts into {s} in {d} rounds ({d} elided by the cone)\n",
.{ written, parsed.dir, outcome.rounds, persisted.elided },
) catch {};
} else {
streams.out.print("cloned {d} artifact{s} into {s} in {d} round{s}\n", .{ written, plural(written), parsed.dir, outcome.rounds, plural(outcome.rounds) }) catch {};
}
if (project_code.len > 0) streams.out.print("project-code: {s}\n", .{project_code}) catch {};
reportUvSkipped(streams, "clone", outcome.uv_skipped);
// submanifests §7.1/§7.3 — a clone converges on the cfile cursor, so it
// copies the peer's holdings faithfully, INCLUDING an incomplete tree
// closure the peer itself was carrying. The phantoms the persist hook minted
// say so; report them, because a check-in whose file set cannot be produced
// (§5.2) is not a converged clone, and "exit 0 with nothing said" is exactly
// how a repository stays quietly broken. Reported AFTER the success line and
// AFTER the repository is fully written: a partial clone is still progress,
// and this is the same shape `pull` already uses for artifacts it did not
// obtain.
if (persisted.submanifest_gaps > 0) {
streams.err.print(
"zit clone: {d} sub-manifest references are missing; the cloned repository cannot enumerate every check-in\n",
.{persisted.submanifest_gaps},
) catch {};
return .failure;
}
return .ok;
}
/// Render an `open` diagnostic to stderr and fail. Best-effort on the write: a
/// diagnostic that cannot be printed must not change the exit code.
fn openDiag(s: cli.Streams, msg: []const u8) cli.Exit {
s.err.print("zit open: {s}\n", .{msg}) catch {};
return .failure;
}
/// Map a materialization fault to its `open` diagnostic, one arm per
/// `update.ApplyError` variant and NO `else`: a new variant must fail this
/// compile and force a decision here, exactly as it does in `updateApplyFail`.
///
/// Only `InvalidCheckinId` and `CheckinNotFound` are genuinely unreachable —
/// `open` hands the engine a hash it just resolved out of this repository's own
/// derived state, never a caller-supplied name — and even those get honest
/// prose rather than a corruption claim. Everything else is reachable on an
/// intact repository: the content/manifest arms describe a repository that
/// really is damaged, while `SymlinkUnsupported` / `InvalidPath` /
/// `ReservedPath` are raised while planning the TARGET manifest, so a clone of
/// any repository containing a symlink or a path this platform cannot represent
/// reaches them with nothing corrupt at all. Reporting those as corruption
/// would be a false accusation against a healthy repository; the prose mirrors
/// `updateApplyFail`'s.
/// Decode a lowercase-hex artifact name to its raw digest for a presence probe.
/// Null for anything that cannot be a stored name, which the caller reads as
/// "not a name this store could hold" and therefore not a missing-content claim.
fn openDecodeName(name: []const u8, out: *[logstore.max_hash_len]u8) ?[]const u8 {
if (name.len == 0 or name.len % 2 != 0) return null;
const bytes = name.len / 2;
if (bytes > logstore.max_hash_len) return null;
_ = std.fmt.hexToBytes(out[0..bytes], name) catch return null;
return out[0..bytes];
}
/// The tip resolved to a check-in whose file content this repository does not
/// hold. Named as a CLASS rather than as a missing file, because the missing
/// file is a symptom: the usual cause is that the store holds artifacts which
/// parse as check-ins of a different repository, and no amount of looking at the
/// named path explains that.
///
/// This refuses. It never falls back to another check-in — picking a different
/// tip because the real one is unusable would be inventing history, and the
/// operator would have no way to know it happened.
fn openForeignCheckin(
s: cli.Streams,
target: []const u8,
branch: []const u8,
missing: usize,
first: []const u8,
) cli.Exit {
s.err.print(
"zit open: the tip of '{s}' is check-in {s}, whose file content this repository does not hold\n",
.{ branch, target },
) catch {};
s.err.print(
"zit open: {d} file(s) are missing, starting with '{s}'\n",
.{ missing, first },
) catch {};
s.err.writeAll(
"zit open: this happens when a repository stores artifacts that parse as check-ins of\n" ++
"zit open: ANOTHER repository -- imported mirror metadata is the common source -- so they\n" ++
"zit open: join the branch as check-ins whose blobs were never here. The working tree was\n" ++
"zit open: not modified. 'zit leaves' shows the branch's check-ins; 'zit update <check-in>'\n" ++
"zit open: materializes a specific one.\n",
) catch {};
return .failure;
}
fn openApplyFail(s: cli.Streams, err: update.ApplyError) cli.Exit {
return switch (err) {
// applyCheckin never emits this: only applyMerge's cross-cone guard
// does, and this switch maps a checkin materialization. The unreachable
// IS the assertion; a future emitter trips it in tests immediately.
error.SparseMergeUnsupported => unreachable,
error.FileContentMissing => blk: {
var b: [512]u8 = undefined;
break :blk openDiag(s, update.stageFaultReason(&b));
},
error.OutOfMemory => openDiag(s, "out of memory"),
error.InvalidCheckinId => openDiag(s, "cannot read repository: the resolved check-in ID is not a valid artifact name"),
error.CheckinNotFound => openDiag(s, "cannot read repository: the resolved check-in is not present"),
error.CheckinContentMissing => openDiag(s, "cannot read repository: check-in content is missing"),
error.AggregateChecksumMismatch => openDiag(s, "the check-in's R checksum disagrees with its content"),
error.CheckinNameMismatch => openDiag(s, "cannot read repository: check-in content does not match its name"),
error.TargetNotManifest => openDiag(s, "cannot read repository: the check-in is not a valid manifest"),
error.BaselineNotFound => openDiag(s, "cannot read repository: the check-in baseline is not found"),
error.BaselineContentMissing => openDiag(s, "cannot read repository: the check-in baseline content is missing"),
error.BaselineNameMismatch => openDiag(s, "cannot read repository: the check-in baseline content does not match its name"),
error.BaselineNotManifest => openDiag(s, "cannot read repository: the check-in baseline is not a valid manifest"),
error.StateRecordInvalid => openDiag(s, "cannot read repository: " ++ update.state_record_invalid_reason),
error.Corrupt => openDiag(s, "cannot read repository: repository is corrupt"),
error.SymlinkUnsupported => openDiag(s, "the check-in contains an unsupported symlink entry"),
error.InvalidPath => openDiag(s, "the check-in names an invalid file path"),
error.ReservedPath => openDiag(s, "the check-in names a reserved file path"),
error.WorktreeNotClean => openDiag(s, "working tree is not clean"),
error.UnmanagedCollision => openDiag(s, "an untracked file or directory is in the way"),
error.CurrentStateInvalid => openDiag(s, "current working-tree state is invalid"),
error.Storage => openDiag(s, "cannot update working tree: input/output error"),
error.TransactionUnavailable => openDiag(s, "cannot create a transaction directory"),
error.RollbackFailed => openDiag(s, "open failed and could not be rolled back; the previous working-tree state is retained for recovery"),
error.CleanupFailed => openDiag(s, "open committed, but the transaction directory could not be removed"),
};
}
/// `zit open` — the second half of `zit clone URL DIR`: turn a repository whose
/// checkout is still UNBORN into a materialized working tree. Fossil's own UX is
/// the same two steps (`fossil clone` then `fossil open`), and splitting them
/// keeps `runClone` strictly repo-only.
///
/// Thin wiring over the shipped machinery, deliberately: probe the checkout
/// record, resolve the tip of the main branch out of the derived repo state, and
/// hand materialization to the SAME `update.applyCheckin` engine every other
/// working-tree move rides (journal-armed, transactional, rollback-safe). Only
/// the tip resolution is open's own work, and it is load-bearing: `update`
/// accepts nothing but a literal 40/64-hex check-in ID, so a freshly cloned tree
/// has no way to name its own tip.
///
/// It writes NO checkout record and NO managed set of its own. `init` — which
/// `clone` delegates repo creation to — already wrote the unborn record; an
/// absent managed set already reads as the empty set; and the update engine is
/// the single writer of the born record and the managed set. Adding a second
/// writer of that bootstrap state would only create two things to keep in sync.
pub fn runOpen(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
// One demand-fetch retry, statically bounded (TigerStyle: no unbounded
// recursion -- the second attempt runs with fetching disabled).
// `zit open URL` is `zit clone URL` — cloned into a directory
// named from the URL and opened there.
for (verb_args) |tok| {
if (std.mem.startsWith(u8, tok, "http://") or std.mem.startsWith(u8, tok, "https://")) {
return runClone(gpa, io, arena, verb_args, streams, amb);
}
if (tok.len > 0 and tok[0] != '-') break;
}
return openAttempt(gpa, io, arena, verb_args, streams, amb, true);
}
/// The `--store` bootstrap: validate both ends, then write the container.
/// The current directory must not already be a checkout (either control
/// file), and PATH must hold a readable `.zit` -- checked here so the
/// operator hears "no store at PATH" instead of a discovery miss. The
/// container write is `checkout.write`'s atomic splice plus one
/// `store_location` section carrying the store root's ABSOLUTE path: a
/// worktree is addressed from anywhere, so a relative pointer would go
/// stale with every `cd`.
const Bootstrap = union(enum) { created, repaired, failed: cli.Exit };
fn bootstrapDetached(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
streams: cli.Streams,
store_arg: []const u8,
) Bootstrap {
var cwd = std.Io.Dir.cwd();
if (cwd.statFile(io, repo_filename, .{})) |_| {
streams.err.writeAll("zit open: this directory holds the repository itself; --store points a DETACHED tree elsewhere\n") catch {};
return .{ .failed = .failure };
} else |_| {}
const repairing = if (cwd.statFile(io, checkout.filename, .{})) |_| true else |_| false;
// PATH may name the store root or the `.zit` file itself; either way the
// pointer stored is the DIRECTORY, absolute.
const named_file = std.mem.endsWith(u8, store_arg, "/.zit") or std.mem.eql(u8, store_arg, ".zit");
const store_root_rel = if (named_file) (std.fs.path.dirname(store_arg) orelse ".") else store_arg;
var store_root = cwd.openDir(io, store_root_rel, .{}) catch {
streams.err.print("zit open: no directory at '{s}'\n", .{store_root_rel}) catch {};
return .{ .failed = .failure };
};
defer store_root.close(io);
_ = store_root.statFile(io, repo_filename, .{}) catch {
streams.err.print("zit open: no repository ('.zit') under '{s}'\n", .{store_root_rel}) catch {};
return .{ .failed = .failure };
};
const abs = store_root.realPathFileAlloc(io, ".", arena) catch {
streams.err.print("zit open: cannot resolve '{s}'\n", .{store_root_rel}) catch {};
return .{ .failed = .failure };
};
if (repairing) {
// Re-open: the checkout exists and its pointer (or the store it
// pointed at) went stale — a moved store strands its satellites, and
// a stranded tree can hold uncommitted work, so the pointer must be
// repairable in place. The one gate that matters: the NEW store must
// contain this tree's current check-in, or the recorded tree state
// would be a lie against the store it now names. "Same store at a
// new address" passes; "some other repository" is refused.
var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined;
const current = checkout.read(io, cwd, &ck_buf) catch {
streams.err.writeAll("zit open: cannot read the checkout state\n") catch {};
return .{ .failed = .failure };
};
if (current) |state| if (state.version.len != 0) {
var digest_buf: [digest_len]u8 = undefined;
const digest = hexToDigest(state.version, &digest_buf) orelse {
streams.err.writeAll("zit open: the checkout state is corrupt\n") catch {};
return .{ .failed = .failure };
};
var store_file = store_root.openFile(io, repo_filename, .{}) catch {
streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {};
return .{ .failed = .failure };
};
defer store_file.close(io);
const store_st = store_file.stat(io) catch {
streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {};
return .{ .failed = .failure };
};
var store_fs = storage.FileStorage.init(io, store_file, store_st.size);
var store_log = logstore.LogStore.openReadOnly(gpa, store_fs.storageFile()) catch {
streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {};
return .{ .failed = .failure };
};
defer store_log.deinit(gpa);
if (store_log.lookup(digest) == null) {
streams.err.print(
"zit open: the repository under '{s}' does not contain this tree's check-in '{s}'; refusing to re-point at a different repository\n",
.{ store_root_rel, state.version },
) catch {};
return .{ .failed = .failure };
}
};
checkout.updateSections(io, cwd, &.{.{ .tag = .store_location, .payload = abs }}) catch {
streams.err.writeAll("zit open: cannot write the checkout state\n") catch {};
return .{ .failed = .failure };
};
streams.out.print("checkout now points at '{s}'\n", .{abs}) catch {};
return .repaired;
}
checkout.write(io, cwd, .{ .version = "", .branch = "trunk", .merge_parent = "", .special_op = "", .manifest_flags = "" }) catch {
streams.err.writeAll("zit open: cannot write the checkout state\n") catch {};
return .{ .failed = .failure };
};
checkout.updateSections(io, cwd, &.{.{ .tag = .store_location, .payload = abs }}) catch {
cwd.deleteFile(io, checkout.filename) catch {};
streams.err.writeAll("zit open: cannot write the checkout state\n") catch {};
return .{ .failed = .failure };
};
return .created;
}
fn openAttempt(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
allow_fetch: bool,
) cli.Exit {
const parsed = switch (cli.parseOpenArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "open", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "open", .extra_argument, tok),
.repeated_option => |tok| return cli.usageError(streams, "open", .repeated_option, tok),
.empty_value => |tok| return cli.usageError(streams, "open", .empty_value, tok),
.bad_manifest => |tok| return cli.usageError(streams, "open", .bad_manifest, tok),
.too_many_cones => |tok| return cli.usageError(streams, "open", .too_many_cones, tok),
};
if (parsed.help) {
streams.out.writeAll(open_help) catch {};
return .ok;
}
// `--store PATH` bootstraps a DETACHED checkout in the current
// directory: an unborn checkout record (exactly what `init` writes) plus
// the store pointer, and nothing else. Everything after this block is the
// ordinary open — discovery finds the fresh container, `Repository.open`
// follows the pointer, and the materialization below neither knows nor
// cares that the store lives elsewhere.
if (parsed.store) |store_arg| {
switch (bootstrapDetached(gpa, io, arena, streams, store_arg)) {
.created => {},
// A repair is complete in itself: the tree is already open at its
// check-in, so the materialization below has nothing to do.
.repaired => return .ok,
.failed => |rc| return rc,
}
}
// The full derive: resolving the main branch reads the tag tables, which the
// `.fast_read` image path does not carry.
var repo: Repository = undefined;
var root_dir: std.Io.Dir = undefined;
const rc = openRepo(gpa, io, arena, streams, "open", null, false, false, &repo, &root_dir);
if (rc != .ok) return rc;
defer root_dir.close(io);
defer repo.deinit(gpa);
// BORN-ness, not presence, is the "already open" signal. Every fresh
// repository carries a checkout record — `init` writes an unborn one
// (`version == ""`, branch `trunk`) as its step 8, and `clone` delegates to
// `init` — so a presence-based gate would reject `zit clone && zit open` on
// every clone, the primary path this verb exists to serve. Only a non-empty
// version means a tree has actually been materialized against a check-in.
var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined;
const current = checkout.read(io, root_dir, &ck_buf) catch |e| switch (e) {
error.Storage => return openDiag(streams, "cannot read the checkout state: input/output error"),
error.Corrupt => return openDiag(streams, "the checkout state is corrupt"),
};
if (current) |state| if (state.version.len != 0) {
streams.err.print(
"zit open: this working tree is already open at check-in '{s}'; use 'zit update' to change check-ins\n",
.{state.version},
) catch {};
return .failure;
};
// The target: an explicit CHECKIN resolved like `zit info` (§8.3), or the
// tip of the main branch (repo-state §9 current-trunk). Everything below —
// preflight, materialization, the manifest opt-in — is target-generic.
const main_branch = repo.mainBranch();
const tip = if (parsed.checkin) |name|
repo.resolve(name, .checkin) catch |e| {
switch (e) {
error.NotFound => streams.err.print("zit open: check-in '{s}' not found\n", .{name}) catch {},
error.Ambiguous => streams.err.print("zit open: name '{s}' is ambiguous\n", .{name}) catch {},
error.OutOfMemory => return openDiag(streams, "out of memory"),
}
return .failure;
}
else
// Switched, not an unbound `catch`: the success message below is an
// exit-0 claim that the tree is already correct, and it is only honest
// for `NotFound`. Naming the arm means a later Storage/OOM arm on
// `currentTrunk` fails this compile instead of silently reporting a
// genuine failure as success on a materialization verb.
repo.state().currentTrunk(main_branch) catch |e| switch (e) {
// A repository with no check-in on its main branch is ALREADY in its
// correct end state — the unborn record and an empty tree. There is
// nothing to materialize and nothing wrong, so this is reported, not
// failed, and the record is left exactly as it was found. An
// explicit CHECKIN never lands here: in an empty repository it
// resolves to NotFound above, a failure, because the operator asked
// for something specific and did not get it.
error.NotFound => {
streams.out.print(
"open: no check-in on '{s}' yet; the working tree is already in its correct (empty) state\n",
.{main_branch},
) catch {};
return .ok;
},
// Settling an equal-mtime tie by ancestry allocates (repo-state
// `MtimeMaxPick`). A failure here is a genuine failure — reporting
// it as the exit-0 "already correct" message would be the lie the
// switch above exists to prevent.
error.OutOfMemory => return openDiag(streams, "out of memory"),
};
// Both borrow the derived view, which outlives the update below.
const target = repo.hashOf(tip);
const branch = repo.branchOf(tip);
// An assertion, not a check: unlike `update`, whose target is user-supplied
// and therefore validated, open's target comes out of this repository's own
// derived state. A malformed name here is a defect in the derivation, not
// bad input, so it belongs in the negative space rather than in a
// diagnostic path no correct run can reach.
std.debug.assert(artifact.validateName(target));
// Distinct from the open above, which shares `repoOpenFail`'s wording: this
// one has already derived the view successfully and is opening the log a
// SECOND time, read-only, for materialization. Reporting both identically
// made a failure here indistinguishable from a failure to derive — and they
// are different repositories' worth of problem.
const log = repo.logStore(gpa) catch |e| {
switch (e) {
error.Corrupt => streams.err.print(
"zit open: the repository derived cleanly but its log will not reopen for reading: corrupt\n",
.{},
) catch {},
else => return repoOpenFail(streams, "open", e),
}
return .failure;
};
// Sparse-cone RFC section 4.1: `open --cone` materializes the first
// checkout already scoped. Absent flag = dense, exactly the shipped
// behavior; `open` has no --dense because a fresh checkout IS dense.
var open_members = parsed.cone_members;
var open_len = parsed.cone_len;
var open_not_members = parsed.cone_not_members;
var open_not_len = parsed.cone_not_len;
if (parsed.cone_file) |cf| {
const lens = cli.appendConeFile(arena, io, cf, &open_members, open_len, &open_not_members, open_not_len) orelse {
streams.err.print("zit open: cannot read --cone-file '{s}' (or it overflows the {d}-member cap)\n", .{ cf, zit.cone.max_members }) catch {};
return .failure;
};
open_len = lens.members;
open_not_len = lens.not_members;
}
var open_cone: ?zit.cone.Cone = null;
defer if (open_cone) |*c| c.deinit(gpa);
const sparse: update.SparseOpt = if (open_len + open_not_len != 0) blk: {
open_cone = zit.cone.Cone.init(gpa, open_members[0..open_len], open_not_members[0..open_not_len]) catch |e|
return openConeFail(streams, e);
break :blk .{ .set = &open_cone.? };
} else .inherit;
// PREFLIGHT: does this repository actually hold the content of the check-in
// it just resolved as the tip?
//
// The format is content-addressed and types artifacts by STRUCTURE, so a
// stored blob whose bytes happen to be a valid check-in manifest IS a
// check-in — repo-state §4.4 grants opacity only to artifacts a conformant
// parser DECLINES to classify, and a foreign manifest classifies fine. A
// repository that tracks a file containing one (a Fossil->Git mirror's
// `manifest`, a test fixture, a documentation example) therefore holds
// check-ins of a DIFFERENT repository, whose file content it never had. That
// is conformant derivation, not corruption, and Fossil behaves the same way.
//
// What is not acceptable is spending several minutes materializing before
// discovering it. `stagePlan` already refuses safely — nothing is written
// until every byte is verified and staged — so this changes no outcome; it
// moves the refusal earlier and lets it NAME the situation, which a bare
// missing-content error cannot.
//
// Presence-only: `lookup` asks whether a name is held, reconstructing
// nothing. Cost is one probe per file in the tip, against a derive that
// already walked every artifact in the store.
if (repo.files(gpa, tip)) |maybe_list| {
if (maybe_list) |list_const| {
var list = list_const;
defer list.deinit(gpa);
// Sparse-cone RFC section 5.2: absence has three classes, and
// this preflight must not read a deliberately sparse store as the
// foreign-check-in class. A missing blob whose path the STORE
// cone does not cover is elided -- healthy. Elided files that the
// requested materialization would need refuse by the elision
// class (section 6.4; demand fetch is a later wave), never as
// foreign, and never as corruption.
var store_cone = zit.repocache.readStoreCone(gpa, io, root_dir);
defer if (store_cone) |*c| c.deinit(gpa);
var missing: usize = 0;
var first_missing: ?[]const u8 = null;
var elided: usize = 0;
var first_elided: ?[]const u8 = null;
var it = list.iterator();
while (it.next()) |f| {
const h = f.hash orelse continue;
var digest_buf: [logstore.max_hash_len]u8 = undefined;
const raw = openDecodeName(h, &digest_buf) orelse continue;
if (log.lookup(raw) == null) {
if (store_cone) |*c| {
if (!c.covers(f.filename)) {
elided += 1;
if (first_elided == null) first_elided = f.filename;
continue;
}
}
missing += 1;
if (first_missing == null) first_missing = f.filename;
}
}
if (missing != 0) return openForeignCheckin(streams, target, branch, missing, first_missing.?);
if (elided != 0) {
// The materialization cone decides whether elision matters:
// an open scoped inside the held content proceeds; one that
// needs elided bytes refuses, naming the class and the count.
const wants_elided = blk: {
switch (sparse) {
.set => |c| {
if (c) |request| {
if (!request.covers(first_elided.?)) {
var again = list.iterator();
var any = false;
while (again.next()) |f| {
const h2 = f.hash orelse continue;
var db2: [logstore.max_hash_len]u8 = undefined;
const raw2 = openDecodeName(h2, &db2) orelse continue;
if (log.lookup(raw2) == null and request.covers(f.filename)) {
any = true;
break;
}
}
break :blk any;
}
break :blk true;
}
break :blk true; // dense request needs everything
},
.inherit => break :blk true, // no worktree cone yet on open
}
};
if (wants_elided) {
// Sparse-cone RFC section 6.4, policy form: the requested
// materialization IS the new coverage intent. Write it as
// the store cone (null = dense), run an ordinary pull --
// whose suppression and receive filtering read the NEW
// policy -- and reprobe. Anything the peer lacked stays a
// transit phantom for later synchronization to heal.
var url_buf: [logstore.max_config_str]u8 = undefined;
const fetchable = allow_fetch and blk: {
_ = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch break :blk false;
break :blk true;
};
if (!fetchable) {
streams.err.print(
"zit open: {d} file(s) of this check-in are elided by the store's sparse cone, starting with '{s}'\n" ++
"zit open: no saved remote to fetch them from; open inside the cone\n" ++
"zit open: ('zit open --cone=...'), set a remote ('zit remote URL'), or re-clone\n",
.{ elided, first_elided.? },
) catch {};
return .failure;
}
streams.err.print(
"zit open: {d} file(s) are elided by the store's sparse cone; fetching from the saved remote\n",
.{elided},
) catch {};
const requested: ?*const zit.cone.Cone = switch (sparse) {
.set => |c| c,
.inherit => null, // a dense open drops the cone
};
zit.repocache.writeStoreCone(gpa, io, root_dir, requested);
const pull_exit = runPull(gpa, io, arena, &.{}, streams, .{ .now_epoch = 0, .user_env = null });
if (pull_exit != .ok) return pull_exit;
// Everything resolved so far (tip, branch, this log
// handle) predates the fetch; re-attempt from scratch with
// fetching disabled, so a peer that also lacked the
// content produces the elided refusal rather than a loop.
return openAttempt(gpa, io, arena, verb_args, streams, amb, false);
}
}
}
} else |_| {
// A files() fault is not this preflight's to report: the engine below
// reaches the same data and has the diagnostics for it. Falling through
// keeps this advisory, never a new way to fail.
}
// `open` is the ONE verb that SETS the manifest opt-in. `--manifest`
// is the operator's explicit choice and always wins; `--manifest=off` is
// remembered as the refusal. Absent, the PROJECT's
// `manifest` setting supplies it, which is fossil's model: a clone gets
// build stamps because the project asked for them, not because each
// developer remembered to. The tree is not materialized yet, so the
// versioned tier is read out of the check-in itself, as fossil's
// `db_get_versioned` does through `historical_blob`.
// The verb's own `log` is already open, so the project setting is
// resolved from the config it holds rather than through a second open.
var open_config: ?logstore.OwnedConfig = log.readConfig(gpa) catch null;
defer if (open_config) |*c| c.deinit(gpa);
const open_cfg: ?*const logstore.Config = if (open_config) |*c| &c.config else null;
const manifest: update.ManifestOpt = if (parsed.manifest) |m| .{ .set = m } else .{
.inherit_or = blk: {
const r = checkinSetting(gpa, io, arena, &repo, tip, amb, open_cfg, .manifest) catch break :blk "";
break :blk r.value;
},
};
// The check-in is named on failure because `open` RESOLVES its own target
// (the tip of the configured main branch) rather than being handed one. A
// diagnostic that omits it cannot distinguish "materializing the expected
// check-in went wrong" from "resolved a different check-in than the operator
// believes is the tip" — and the operator has no other way to see which.
update.applyCheckinWith(gpa, io, root_dir, log, target, manifest, sparse, &repo, null) catch |e| {
streams.err.print("zit open: materializing check-in {s} on {s}\n", .{ target, branch }) catch {};
return openApplyFail(streams, e);
};
// The `empty-dirs` shape, laid over the tree the check-in
// just produced. Read AFTER the materialization because a versioned
// `.fossil-settings/empty-dirs` does not exist on disk before it; fossil's
// checkout reads the same value out of the check-in it is installing, so
// the two see the same list.
{
const dirs: []const u8 = blk: {
const r = localSetting(io, arena, root_dir, amb, open_cfg, .empty_dirs) catch break :blk "";
break :blk r.value;
};
update.ensureEmptyDirs(io, root_dir, dirs, streams.err);
}
// A successful open stamps this tree into the store's worktree
// registry (best-effort; discovery data never gates the open).
if (std.process.currentPathAlloc(io, gpa)) |own_abs| {
defer gpa.free(own_abs);
cli.stampWorktreeRegistry(gpa, io, root_dir, own_abs);
} else |_| {}
streams.out.print("opened {s} on {s}\n", .{ target, branch }) catch {};
return .ok;
}
/// Map a Cone.init refusal for `open`. Same wording discipline as update's
/// mapper: states the rule, never says "missing" or "corrupt".
fn openConeFail(s: cli.Streams, err: zit.cone.Error) cli.Exit {
switch (err) {
error.OutOfMemory => s.err.writeAll("zit open: out of memory\n") catch {},
error.InvalidMember => s.err.writeAll(
"zit open: invalid --cone member: members are repository-relative paths using '/', with no '.', '..', or empty components\n",
) catch {},
error.ReservedMember => s.err.writeAll("zit open: a --cone member names a reserved control path\n") catch {},
error.Bounds => s.err.writeAll("zit open: --cone needs between 1 and 255 members (exclusions included), at least one inclusion\n") catch {},
error.DeadExclusion => s.err.writeAll("zit open: a --cone-not member no --cone member covers is a dead rule\n") catch {},
error.Contradiction => s.err.writeAll("zit open: one path given as both --cone and --cone-not\n") catch {},
}
return .failure;
}
/// The fields a `updateConfig` overlay may change; a null field is left exactly
/// as the stored config has it. Naming a single change site keeps the "copy the
/// whole config, overlay one field" discipline in one place — a caller that
/// means to change only the project code can never silently drop the credential
/// or the saved remote (the class of bug the pre-`updateConfig` re-puts had).
pub const ConfigPatch = struct {
project_code: ?[]const u8 = null,
login: ?zit.sync.Credential = null,
remote_url: ?[]const u8 = null,
/// draft-fossil-submanifests-00 §6.3/§8.1 and §6.4 — the three
/// project-scope settings sync §9.3's scalar channel carries. Optional for
/// the same reason every field here is: a peer that named only one of them
/// must leave the other two exactly as stored.
enable_submanifests: ?bool = null,
submanifest_k: ?u8 = null,
submanifest_auto_retune: ?bool = null,
/// The autosync setting, stored inverted (true = off). Local, like
/// `remote_url`; only the settings verb writes it.
autosync_off: ?bool = null,
ignore_glob: ?[]const u8 = null,
/// The two signing settings, both LOCAL and both deliberately
/// outside every config group: fossil classifies `pgp-command` sensitive
/// (it is a command line a peer would otherwise get to choose) and puts
/// `clearsign` in no group at all.
clearsign: ?bool = null,
pgp_command: ?[]const u8 = null,
/// The rest of fossil's checkout settings at the repository tier.
crnl_glob: ?[]const u8 = null,
clean_glob: ?[]const u8 = null,
empty_dirs: ?[]const u8 = null,
encoding_glob: ?[]const u8 = null,
dotfiles: ?[]const u8 = null,
mimetypes: ?[]const u8 = null,
attachment_size_limit: ?[]const u8 = null,
/// `zit git export`'s remembered mirror.
git_mirror: ?[]const u8 = null,
git_autopush: ?[]const u8 = null,
git_email_domain: ?[]const u8 = null,
git_push_branch: ?[]const u8 = null,
/// `main-branch`.
default_branch: ?[]const u8 = null,
/// The explicit-set bits, which are what make "this repository has no
/// opinion" representable and therefore let the global tier be reached.
autosync_set: ?bool = null,
clearsign_set: ?bool = null,
keep_glob: ?[]const u8 = null,
manifest: ?[]const u8 = null,
before_commit: ?[]const u8 = null,
/// The two content-policy globs. Data settings, so a tracked
/// `.zit-settings/` file outranks whatever is stored here.
binary_glob: ?[]const u8 = null,
crlf_glob: ?[]const u8 = null,
/// The consent record. Empty string is a real value here (revoke),
/// so it is written whenever the field is present, not when non-empty.
before_commit_trust: ?[]const u8 = null,
editor: ?[]const u8 = null,
autosync_mode: ?[]const u8 = null,
fn any(self: ConfigPatch) bool {
return self.project_code != null or
self.login != null or
self.remote_url != null or
self.enable_submanifests != null or
self.submanifest_k != null or
self.submanifest_auto_retune != null or
self.autosync_off != null or
self.ignore_glob != null or
self.clearsign != null or
self.pgp_command != null or
self.autosync_set != null or
self.clearsign_set != null or
self.keep_glob != null or
self.manifest != null or
self.before_commit != null or
self.before_commit_trust != null or
self.binary_glob != null or
self.crlf_glob != null or
self.editor != null or
self.autosync_mode != null;
}
};
/// Read the live config from an already-open log, overlay `patch`, and append
/// the result while preserving every unnamed field. The caller owns durability,
/// so a clone can combine its origin settings in one config frame and commit
/// its complete result once.
fn updateConfigOpen(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
patch: ConfigPatch,
) PersistError!void {
var oc = (log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
}) orelse return error.Corrupt; // init always writes a config frame
defer oc.deinit(gpa);
// Copy every field, then overlay only the named ones. `next`'s string slices
// borrow `oc.backing` / `patch`; both outlive the `putConfig` memcpy below.
var next = oc.config;
if (patch.project_code) |c| next.project_code = c;
if (patch.login) |cred| {
next.login_user = cred.user;
// Point at the function-scoped `patch` storage, not the block-local
// capture `cred`: `next` is read by `putConfig` after this block closes.
next.login_secret_hex = &patch.login.?.secret_hex;
}
if (patch.remote_url) |u| next.remote_url = u;
if (patch.enable_submanifests) |v| next.enable_submanifests = v;
if (patch.submanifest_k) |v| next.submanifest_k = v;
if (patch.submanifest_auto_retune) |v| next.submanifest_auto_retune = v;
if (patch.autosync_off) |v| next.autosync_off = v;
if (patch.ignore_glob) |v| next.ignore_glob = v;
if (patch.clearsign) |v| next.clearsign = v;
if (patch.pgp_command) |v| next.pgp_command = v;
if (patch.autosync_set) |v| next.autosync_set = v;
if (patch.clearsign_set) |v| next.clearsign_set = v;
if (patch.keep_glob) |v| next.keep_glob = v;
if (patch.manifest) |v| next.manifest = v;
if (patch.before_commit) |v| next.before_commit = v;
if (patch.before_commit_trust) |v| next.before_commit_trust = v;
if (patch.binary_glob) |v| next.binary_glob = v;
if (patch.crlf_glob) |v| next.crlf_glob = v;
if (patch.editor) |v| next.editor = v;
if (patch.autosync_mode) |v| next.autosync_mode = v;
if (patch.crnl_glob) |v| next.crnl_glob = v;
if (patch.clean_glob) |v| next.clean_glob = v;
if (patch.empty_dirs) |v| next.empty_dirs = v;
if (patch.encoding_glob) |v| next.encoding_glob = v;
if (patch.dotfiles) |v| next.dotfiles = v;
if (patch.mimetypes) |v| next.mimetypes = v;
if (patch.attachment_size_limit) |v| next.attachment_size_limit = v;
if (patch.git_mirror) |v| next.git_mirror = v;
if (patch.git_autopush) |v| next.git_autopush = v;
if (patch.git_email_domain) |v| next.git_email_domain = v;
if (patch.git_push_branch) |v| next.git_push_branch = v;
if (patch.default_branch) |v| next.default_branch = v;
log.putConfig(next) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
}
/// Open `dir/.zit` for a field-preserving config update, retaining the legacy
/// wrapper for pull, sync, serve, and remote callers that do not already own a
/// writable log.
pub fn updateConfig(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
patch: ConfigPatch,
) PersistLockedError!void {
var store_append = cli.openStoreAppend(io, dir) catch |e| switch (e) {
error.StoreBusy => return error.StoreBusy,
else => return error.Storage,
};
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch return error.Storage;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
};
defer log.deinit(gpa);
try updateConfigOpen(gpa, &log, patch);
log.commit() catch return error.Storage;
}
/// §8.2 — persist the origin's project code into the freshly cloned repo at
/// `dir/.zit`, preserving every other field `init` wrote. A no-op when the peer
/// sent no code. Thin wrapper over `updateConfig` (the field-preserving RMW).
fn storeClonedProjectCode(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
project_code: []const u8,
) PersistLockedError!void {
if (project_code.len == 0) return; // peer sent none; keep the local init code
return updateConfig(gpa, io, dir, .{ .project_code = project_code });
}
/// draft-fossil-submanifests-00 §6.3/§8.1 (and §6.4) over sync §9.3
/// — persist the origin's three sub-manifest settings into the freshly cloned
/// repo at `dir/.zit`, preserving every other field `init` wrote. A no-op when
/// the peer named none. Thin wrapper over `updateConfig`, the same shape
/// `storeClonedProjectCode` has and for the same reason.
///
/// One-way and clone-only, deliberately. A fresh clone has nothing of its own
/// to conflict with, so adoption needs no merge; an EXISTING repository would,
/// and §9.3's channel resolves conflicts by MTIME, which zit's config frame does
/// not store (see `sync.submanifest_config_mtime`). Refreshing settings on a
/// later pull is therefore a real conflict-resolution feature with a config
/// schema change behind it, not plumbing this wrapper can be widened into —
/// `runPull`/`runPush` deliberately have no seam here.
fn storeClonedSubmanifestSettings(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
cfg: zit.sync.SubmanifestConfig,
) PersistLockedError!void {
if (!cfg.any()) return; // peer sent none; keep the local init defaults
return updateConfig(gpa, io, dir, .{
.enable_submanifests = cfg.enable,
.submanifest_k = cfg.k,
.submanifest_auto_retune = cfg.auto_retune,
});
}
/// Drives a modern-clone client (§3.8 CLONE_ROUND0 → EXCHANGE) to convergence
/// over `tr`, filling `ms` with the peer's artifacts; returns the round count.
/// Sans-IO on this side of `tr.exchange`: the state machine only moves
/// `[]const u8` bytes. Its closed error set is the driver's own causes unioned
/// with the transport's, so `runClone` switches the whole surface exhaustively.
/// The result of a clone run: the round count plus the origin's project code
/// (§8.2), copied out of the sans-IO client so the caller can persist the
/// shared identity after the client is torn down.
const CloneOutcome = struct {
rounds: usize,
/// The login the peer accepted from round 1 on: the derived secret, copied
/// out so the CLI can store it in the new repository once it exists. The
/// user borrows argv.
login: ?zit.sync.Credential = null,
uv_skipped: u64 = 0,
/// Uncompressed protocol bytes moved (`RunStats` totals), for the
/// end-of-clone accounting line.
sent_bytes: usize,
received_bytes: usize,
project_code_buf: [zit.sync.max_projectcode_len]u8,
project_code_len: usize,
/// submanifests §6.3/§8.1 over sync §9.3 — the origin's three project-scope
/// settings, learned from the cleanup round's config card. Copied out of the
/// sans-IO client for the same reason the project code is: the client is
/// torn down before the caller persists anything. Owns no memory (three
/// optional scalars), so no buffer twin is needed.
submanifest: zit.sync.SubmanifestConfig,
/// §9.3 r28 — the origin's shun table, learned from the same cleanup round.
/// Unlike the settings above this OWNS memory (a row count the wire
/// decides), so `deinit` is not optional: the client that parsed these rows
/// is torn down before the caller writes anything durable.
///
/// Carrying the shun table is not a nicety. fossil's shun table is its
/// takedown mechanism; a clone that drops it will, on its next sync with
/// any third party, re-accept exactly the content an operator was ordered
/// to remove.
shun_rows: []zit.sync.OwnedShunRow = &.{},
fn projectCode(self: *const CloneOutcome) []const u8 {
return self.project_code_buf[0..self.project_code_len];
}
fn deinit(self: *CloneOutcome, gpa: std.mem.Allocator) void {
for (self.shun_rows) |row| {
gpa.free(row.hash);
gpa.free(row.reason);
}
gpa.free(self.shun_rows);
self.shun_rows = &.{};
}
};
/// The stderr live-progress sink a TTY clone runs with. Each completed
/// round overwrites one line in place (\r, no newline); `finishLine` closes it
/// with a newline exactly once, so whatever stderr carries next starts clean.
/// Flushed per round — a progress line that sits in a buffer until the clone
/// ends reports nothing.
const CloneProgress = struct {
err: *std.Io.Writer,
emitted: bool = false,
fn emit(ctx: *anyopaque, info: sync_driver.RoundInfo) void {
const self: *CloneProgress = @ptrCast(@alignCast(ctx));
self.emitted = true;
self.err.print(
"\rround {d}: {d} bytes received",
.{ info.round, info.total_received },
) catch return;
self.err.flush() catch {};
}
fn finishLine(self: *CloneProgress) void {
if (!self.emitted) return;
self.emitted = false;
self.err.writeAll("\n") catch return;
self.err.flush() catch {};
}
};
/// Map a Cone.init refusal for clone. Same wording discipline as the other
/// verbs' mappers.
fn cloneConeFail(s: cli.Streams, err: zit.cone.Error) cli.Exit {
switch (err) {
error.OutOfMemory => s.err.writeAll("zit clone: out of memory\n") catch {},
error.InvalidMember => s.err.writeAll(
"zit clone: invalid --cone member: members are repository-relative paths using '/', with no '.', '..', or empty components\n",
) catch {},
error.ReservedMember => s.err.writeAll("zit clone: a --cone member names a reserved control path\n") catch {},
error.Bounds => s.err.writeAll("zit clone: --cone needs between 1 and 255 members (exclusions included), at least one inclusion\n") catch {},
error.DeadExclusion => s.err.writeAll("zit clone: a --cone-not member no --cone member covers is a dead rule\n") catch {},
error.Contradiction => s.err.writeAll("zit clone: one path given as both --cone and --cone-not\n") catch {},
}
return .failure;
}
/// A bounded copy of a peer error-card text, taken before the aborted sync
/// client goes out of scope so the CLI can quote the refusal.
const PeerErrorCopy = struct {
buf: [zit.sync.peer_error_cap]u8 = undefined,
len: usize = 0,
fn set(self: *PeerErrorCopy, source: ?[]const u8) void {
const text_in = source orelse return;
std.debug.assert(text_in.len <= self.buf.len);
@memcpy(self.buf[0..text_in.len], text_in);
self.len = text_in.len;
}
fn text(self: *const PeerErrorCopy) ?[]const u8 {
if (self.len == 0) return null;
return self.buf[0..self.len];
}
};
/// What the server said beside the exchange, copied out of the client before
/// it is deinit'd: its `message` cards, and whether it kept a pushed check-in
/// on an attempt branch (`pragma zit-diverted`).
const ServerNotes = struct {
messages: std.ArrayListUnmanaged([]u8) = .empty,
diverted: bool = false,
fn adopt(self: *ServerNotes, client: *zit.sync.Client) void {
self.messages = client.server_messages;
client.server_messages = .empty;
self.diverted = client.diverted;
}
fn deinit(self: *ServerNotes, gpa: std.mem.Allocator) void {
for (self.messages.items) |m| gpa.free(m);
self.messages.deinit(gpa);
}
/// Each message on its own line, the way fossil prints them, so a person
/// reads what the server had to say.
fn show(self: *const ServerNotes, streams: cli.Streams) void {
for (self.messages.items) |m| streams.err.print("server: {s}\n", .{m}) catch {};
}
};
fn cloneRun(
gpa: std.mem.Allocator,
tr: *transport.Transport,
ms: *repostate.MemStore,
self_servercode: []const u8,
login: ?zit.sync.PendingLogin,
sink: ?sync_driver.RoundSink,
drain: ?sync_driver.RoundDrain,
cone_members: []const []const u8,
cone_not_members: []const []const u8,
peer_error_out: *PeerErrorCopy,
truncated_out: *u64,
) sync_driver.RunError(*transport.Transport)!CloneOutcome {
var client = zit.sync.Client{
.state = zit.sync.initialState(.clone),
.operation = .clone,
.self_servercode = self_servercode,
.project_code = sync_projectcode,
.version = sync_client_version_info,
.pending_login = login,
// Sparse-cone RFC section 6.1: request server-side filtering; the
// local persist filter stays as the section 6.3 belt regardless.
.cone_members = cone_members,
.cone_not_members = cone_not_members,
};
defer client.deinit(gpa);
// Read on every exit, the error ones included: the cut-off replies are
// reported whether or not the run went on to fail.
defer truncated_out.* = client.payloads_truncated;
const stats = sync_driver.runSyncProgress(gpa, &client, ms.store(), tr, max_stalled_sync_rounds, sink, drain) catch |e| {
// A refusing peer's words outlive the aborted client (bounded copy)
// so the caller renders them instead of a generic protocol error.
peer_error_out.set(client.peerErrorText());
return e;
};
// §8.2 — the peer's project code, learned from the clone reply's `push`
// card, copied out before the client is deinitialized.
var outcome = CloneOutcome{
.rounds = stats.rounds,
.uv_skipped = client.uv_skipped,
.sent_bytes = stats.sent_bytes,
.received_bytes = stats.received_bytes,
.project_code_buf = undefined,
.project_code_len = 0,
// §9.3 — the origin's sub-manifest settings, latched from the config
// card the §8.4 cleanup round asked for.
.submanifest = client.peerSubmanifestConfig(),
};
const pc = client.peerProjectCode();
@memcpy(outcome.project_code_buf[0..pc.len], pc);
outcome.project_code_len = pc.len;
outcome.login = client.credential;
// §9.3 r28 — copy the latched shun rows out of the client, which the
// `defer` above tears down on return.
const latched = client.peerShunRows();
if (latched.len != 0) {
var rows = try gpa.alloc(zit.sync.OwnedShunRow, latched.len);
var filled: usize = 0;
errdefer {
for (rows[0..filled]) |r| {
gpa.free(r.hash);
gpa.free(r.reason);
}
gpa.free(rows);
}
for (latched, rows) |src_row, *dst| {
const h = try gpa.dupe(u8, src_row.hash);
errdefer gpa.free(h);
const r = try gpa.dupe(u8, src_row.reason);
dst.* = .{ .hash = h, .epoch = src_row.epoch, .reason = r };
filled += 1;
}
outcome.shun_rows = rows;
}
return outcome;
}
/// The closed error set of `persistStore`: memory, a storage write fault, or a
/// corrupt/oversized peer artifact name that will not decode to a log key.
const PersistError = error{ OutOfMemory, Storage, Corrupt };
/// The persist paths that take the writer lock themselves. `StoreBusy`:
/// another writer held it past the wait; stderr already names it, so a
/// caller reports nothing more.
const PersistLockedError = PersistError || error{StoreBusy};
/// The disposition of one artifact in `persistStore`'s ordered pass.
const PersistOutcome = union(enum) {
/// A non-phantom frame for this artifact now exists in the log — written
/// here, or already present from a prior cascade / a pull re-persist.
persisted,
/// A reverse-delta whose basis is a content-bearing store entry not YET
/// written to the log. It is deferred on the basis wire-hash and re-driven
/// once that basis lands, so its `delta_of` offset points strictly backward.
deferred: []const u8,
};
/// What one `persistStore` sweep did: the artifacts it made durable, and the
/// sub-manifest references (draft-fossil-submanifests-00 §7.1) it phantomized
/// and could not resolve from this repository's own holdings.
const PersistSummary = struct {
/// Content-bearing artifacts in the store — the count every caller reports.
written: usize,
/// §7.1 — distinct S references still standing as phantoms when the sweep
/// ended: this repository holds an artifact whose tree closure is open.
/// Zero for every repository that carries no S card, which is every
/// repository predating this draft.
submanifest_gaps: usize,
/// Sparse-cone RFC section 6.3 -- artifacts deliberately not persisted
/// under the clone's cone. Zero for every dense clone.
elided: usize = 0,
};
/// draft-fossil-submanifests-00 §7.1 — the S references one persist sweep saw.
///
/// Rids, not names: `LogStore` entries are append-only, so an interned rid stays
/// valid for the sweep, and whether a reference is still a GAP can only be
/// decided once the sweep has ended — a sub-manifest may be persisted after the
/// tree manifest that names it, and `putZlib` upgrades the phantom minted for it
/// in place. Deciding per S card instead would report a gap for every reference
/// whose artifact happens to sit later in rid order.
const SubManifestRefs = struct {
rids: std.AutoArrayHashMapUnmanaged(logstore.Rid, void) = .empty,
/// Delta-dialect memory: this sweep stored a B-card manifest, so the
/// `seen_delta_manifest` config indicator is written alongside `seen`'s
/// on the same false->true edge discipline.
seen_delta: bool = false,
/// submanifests §8.1 — set the first time this sweep stores an artifact
/// inside §7.2's gated set: "When the first S-bearing artifact is stored, an
/// implementation SHOULD record a local seen-submanifest indicator". Both
/// §7.2 shapes count, leaf sub-manifests included — see
/// `phantomizeSubManifestRefs` for why the literal S-card-only reading would
/// leave the gate blind to exactly the artifacts that carry a tree's files.
/// It is also the durable gate on the §7.1 phantom REBUILD (see
/// `persistStore`), which is why it tracks artifacts seen, not gaps left.
seen: bool = false,
fn deinit(self: *SubManifestRefs, gpa: std.mem.Allocator) void {
self.rids.deinit(gpa);
self.* = undefined;
}
/// Distinct references that never became a content-bearing frame.
fn unresolved(self: *const SubManifestRefs, log: *const logstore.LogStore) usize {
var n: usize = 0;
for (self.rids.keys()) |rid| {
if (log.isPhantom(rid)) n += 1;
}
std.debug.assert(n <= self.rids.count());
return n;
}
};
/// draft-fossil-submanifests-00 §7.1 — "Storing a structural artifact that
/// carries S cards whose referenced sub-manifests are absent MUST create a
/// phantom for each missing reference, with the referencing artifact's privacy
/// classification, exactly as an absent delta source creates a source phantom."
///
/// This is §7.1's SECOND storing boundary. The receive path
/// (`sync.Client.absorbSubManifestRefs`) covers artifacts a pull absorbs; it
/// cannot cover a clone, which stores every artifact wire-backed and never
/// inflates one (so it has no plaintext to scan), nor a server-side push
/// ingest, which absorbs nothing at all. Both of those reach the durable store
/// only HERE — and here the plaintext is already in hand, reconstructed for the
/// §7.7 verify, so the whole duty costs one `sBearingCandidate` scan of bytes
/// this function was handed anyway. The clone's O(wire) receive bound is
/// untouched: no container is inflated that §7.7 was not already inflating.
///
/// A clone from a peer whose OWN closure is incomplete is exactly the case the
/// receive path cannot see: the cursor delivers every artifact the peer holds,
/// and the peer does not hold the sub-manifests. Without this, such a clone
/// records its gap nowhere and never asks for the leaves.
///
/// Privacy: zit's phantom is an interned entry with no content and carries no
/// per-entry privacy at either boundary, so an S-reference phantom is minted
/// exactly as an absent delta source's is — which is what §7.1 asks for.
fn phantomizeSubManifestRefs(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
content: []const u8,
seen: *SubManifestRefs,
) PersistError!void {
// §8.1's indicator, over BOTH §7.2 shapes rather than over S cards alone.
//
// §8.1 says "when the first S-bearing artifact is stored"; a LEAF
// sub-manifest carries F cards and no S card (§4.4), so the literal reading
// records nothing for a repository holding leaves. That reading cannot be
// right: this indicator is what arms the §7.2 transfer gate, §7.2 gates
// leaves by TYPE, and §8's whole reason for the transfer layer is that a
// pre-S reader can only hold a leaf as "unreadable ... opaque content". §8.2
// reads the same way — "a repository containing an S-bearing artifact" is a
// repository holding any part of a tree, and a repository that holds a tree's
// leaves is exactly that. So the indicator answers `transferGatedArtifact`.
//
// Cheap where it runs: the predicate rejects any byte string whose first two
// bytes are not a card letter followed by SP or LF — every file blob, in two
// comparisons. A structural artifact pays one allocation-free walk of its
// card lines, beside the full-artifact rehash §7.7 already charges here.
if (!seen.seen and repostate.transferGatedArtifact(content)) seen.seen = true;
// The delta-dialect twin: remember that this repository stores delta
// manifests, so `zit commit` keeps the dialect (config `seen_delta`).
if (!seen.seen_delta and repostate.deltaManifestArtifact(content)) seen.seen_delta = true;
var refs: std.ArrayListUnmanaged([]const u8) = .empty;
defer refs.deinit(gpa);
// Validate-then-apply lives in the shared extractor: a malformed card run
// mints nothing, so a half-read artifact never leaves a half-minted phantom
// set. `content` outlives `refs`, which borrows its hash tokens.
if (!try repostate.collectSubManifestRefs(gpa, content, &refs)) return;
for (refs.items) |name| {
var buf: [digest_len]u8 = undefined;
// §3.5 admits a 40- or 64-hex name and `decodeWireName` decodes both; a
// name too wide for this repository's digest cannot key a frame here, so
// it is a corrupt reference rather than a gap.
const digest = decodeWireName(name, &buf) orelse return error.Corrupt;
const rid = log.intern(digest) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
};
try seen.rids.put(gpa, rid, {});
}
}
/// Persist one artifact `rid` into `log`, choosing its stored form, and verify
/// §7.7 integrity (the reconstructed content MUST hash to its wire name) BEFORE
/// any durable write. A retained delta whose basis is already a held log frame
/// under the read depth bound is stored VERBATIM as a `.delta`; a retained delta
/// whose basis is a not-yet-written content-bearing store entry is DEFERRED
/// (when `allow_defer`) so the basis frame is written first (reverse-delta
/// ordering); every other case — no wire form, a full container, or an
/// absent / phantom / over-deep basis — lands as the full form, never a
/// phantom-based or unreadable delta. Idempotent: an artifact already holding a
/// non-phantom frame returns `.persisted` without a re-write or re-verification
/// (§7.7 was proven when it was first persisted).
/// Record a cluster's wire name as it is written (see
/// `persistArtifact`'s `clusters_out`). The dupe is owned by the list.
fn noteCluster(gpa: std.mem.Allocator, clusters_out: ?*std.ArrayListUnmanaged([]const u8), name: []const u8, content: []const u8) error{OutOfMemory}!void {
const list = clusters_out orelse return;
if (!clusterindex.isCluster(content)) return;
const owned = try gpa.dupe(u8, name);
errdefer gpa.free(owned);
try list.append(gpa, owned);
}
fn persistArtifact(
gpa: std.mem.Allocator,
store: repostate.ArtifactStore,
log: *logstore.LogStore,
rid: repostate.Rid,
allow_defer: bool,
/// Streaming persist: defer even on a basis the store has not
/// SEEN yet — mid-exchange, an unseen basis is usually still in flight, so
/// falling back to a full frame here would write different bytes than the
/// converged pass. The final leftover drain runs with `allow_defer` off,
/// which restores the full-form fallback for a basis that truly never
/// arrived. Always false on the batched paths, whose store IS converged.
defer_unseen: bool,
/// A bounded plaintext memo shared across `persistStore`'s whole sweep. Each
/// artifact reconstructs its own plaintext exactly once here — for the §7.7
/// verify — and primes it under its just-written frame's offset, so a later
/// reverse-delta naming this frame as basis reuses that plaintext instead of
/// re-walking (re-inflating, re-applying) the chain. Bounded (LRU, fixed byte
/// + entry caps): a cache miss just costs a fresh reconstruction, never a
/// wrong answer, so the stored bytes are identical with or without it.
cache: *logstore.PlaintextCache,
/// submanifests §7.1 — the sweep-wide S-reference record. Every plaintext
/// this function reconstructs for the §7.7 verify is offered to it, so the
/// phantom duty attaches to STORING and not to any one arrival path.
srefs: *SubManifestRefs,
/// submanifests §7.1/§8.1 — REBUILD the S-reference phantom set for an
/// artifact that is ALREADY a durable frame. A phantom is an interned entry
/// with no frame (`LogStore.intern`: "Phantoms never reach disk"), so the
/// §7.1 phantoms this repository minted on a previous run are gone the
/// moment the process exits; the gap has to be re-derived from the durable
/// artifacts that name it. Without this an open tree closes itself on the
/// next run: the tree manifest is already persisted, the early return skips
/// its scan, and the repository reports converged while it still cannot
/// enumerate that check-in.
///
/// Passed as a parameter rather than read here because the decision is the
/// repository's, not this artifact's: §8.1's durable seen-submanifest
/// indicator is false for every repository predating this draft, so the
/// rescan — and its one inflate per already-persisted artifact — is
/// unreachable for all of them and the persist sweep is what it always was.
rescan_persisted: bool,
/// Where to record the wire name of every cluster this call
/// writes, so the persist keeps the cluster index warm without a rescan.
clusters_out: ?*std.ArrayListUnmanaged([]const u8),
) PersistError!PersistOutcome {
const name = store.hashOf(rid);
var digest_buf: [digest_len]u8 = undefined;
const digest = decodeWireName(name, &digest_buf) orelse return error.Corrupt;
if (log.lookup(digest)) |ex| {
if (!log.isPhantom(ex)) {
if (rescan_persisted) {
// Read through the sweep's own memo, so an artifact this sweep
// already reconstructed costs nothing to rescan.
const plain = (log.contentCached(gpa, ex, cache) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
}) orelse return .persisted;
defer gpa.free(plain);
try phantomizeSubManifestRefs(gpa, log, plain, srefs);
}
return .persisted;
}
}
// A genuine phantom carries nothing durable — skip it (never a drop of held
// content). `held` classifies without inflating a wire-backed entry.
if (!store.held(rid)) return .persisted;
// §7.7 L1492/L1496-1498 — the repository transaction-commit hook. Recompute
// the artifact hash over the reconstructed bytes and abort BEFORE any frame is
// written on a mismatch (a lying container inflates but does not name its
// content). The verify plaintext is reconstructed against the LOG basis (a
// read bounded by `max_delta_chain`) — NOT the in-memory wire chain, which,
// now a clone stores every artifact wire-backed, is the full reverse-delta
// history and would exceed the read depth bound. `persistStore` writes basis
// frames before the deltas that name them (reverse-delta order), so a delta's
// basis is already a log frame here.
if (store.wire(rid)) |wf| {
if (wf.delta) |d| {
var base_buf: [digest_len]u8 = undefined;
const base_digest = decodeWireName(d.src_hash, &base_buf) orelse return error.Corrupt;
const maybe_base = log.lookup(base_digest);
if (maybe_base == null or log.isPhantom(maybe_base.?)) {
// The basis is not yet a log frame. Defer on it (reverse-delta
// order: the newer basis frame must be written first) when it is a
// held store entry that WILL be persisted.
if (allow_defer) {
if (store.lookup(d.src_hash)) |bs_rid| {
if (store.held(bs_rid)) return .{ .deferred = d.src_hash };
}
if (defer_unseen) return .{ .deferred = d.src_hash };
}
// Basis absent and not deferrable (§3.3): fall back to FULL form
// from the entry's OWN plaintext — a put+retain entry carries it
// directly (never a phantom-based delta). In a converged clone this
// is unreachable (every delta's basis is a written frame, handled
// below); a wire-delta entry whose basis truly never arrived cannot
// reconstruct — a hard error, never a fabricated artifact.
const content = store.content(rid) orelse return error.Corrupt;
if (!zit.artifact.verify(name, content)) return error.Corrupt;
// submanifests §7.1 — the plaintext §7.7 just verified is the same
// bytes the phantom duty is stated over; scan it here rather than
// forcing a second inflate anywhere else.
try phantomizeSubManifestRefs(gpa, log, content, srefs);
try noteCluster(gpa, clusters_out, name, content);
const new_rid = log.putZlib(digest, content) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
try cache.put(gpa, log.offsetOf(new_rid), content);
return .persisted;
}
const base_rid = maybe_base.?;
const base_off = log.offsetOf(base_rid);
// Reuse the basis plaintext memoised when the basis frame was
// persisted — reverse-delta order writes every basis before the
// delta that names it, so the memo is normally warm. Only a miss
// (first use, or an LRU eviction) reconstructs it by walking the
// chain; either way `base_plain` is owned and freed below.
const base_plain = if (cache.get(base_off)) |hit| blk: {
// §7.7 positive space: a memo entry is the exact plaintext that
// named its wire hash at the basis's own persist, so the reused
// bytes still hash to the basis name (debug-only assertion; the
// hash is never recomputed on the hot path).
std.debug.assert(zit.artifact.verify(d.src_hash, hit));
break :blk gpa.dupe(u8, hit) catch return error.OutOfMemory;
} else blk: {
const p = (log.contentCached(gpa, base_rid, cache) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
}) orelse return error.Corrupt;
// `contentCached` does not memoise a top-level full-form basis;
// prime it here so sibling reverse-deltas on the same basis reuse
// this reconstruction instead of repeating it.
try cache.put(gpa, base_off, p);
break :blk p;
};
defer gpa.free(base_plain);
const raw = zit.container.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Corrupt,
};
defer gpa.free(raw);
const content = zit.delta.apply(gpa, base_plain, raw) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Corrupt,
};
defer gpa.free(content);
if (!zit.artifact.verify(name, content)) return error.Corrupt;
// submanifests §7.1 — the plaintext §7.7 just verified is the same
// bytes the phantom duty is stated over; scan it here rather than
// forcing a second inflate anywhere else.
try phantomizeSubManifestRefs(gpa, log, content, srefs);
try noteCluster(gpa, clusters_out, name, content);
// Store the delta VERBATIM iff its basis is a held frame under the READ
// depth bound; an over-deep basis falls back to full form (never an
// unreadable chain). `delta_of` is the basis's backward offset —
// strictly lower than this new frame, the invariant `putDeltaContainer`
// asserts.
if (log.depthOf(base_rid) < logstore.max_delta_chain) {
const new_rid = log.putDeltaContainer(digest, base_rid, wf.container) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
// Prime the just-verified plaintext so an older reverse-delta
// naming this frame as basis reuses it (no chain re-walk).
try cache.put(gpa, log.offsetOf(new_rid), content);
} else {
const new_rid = log.putZlib(digest, content) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
try cache.put(gpa, log.offsetOf(new_rid), content);
}
return .persisted;
}
// §7.2 full form — decode to verify, then store the container verbatim.
const content = zit.container.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Corrupt,
};
defer gpa.free(content);
if (!zit.artifact.verify(name, content)) return error.Corrupt;
// submanifests §7.1 — the plaintext §7.7 just verified is the same
// bytes the phantom duty is stated over; scan it here rather than
// forcing a second inflate anywhere else.
try phantomizeSubManifestRefs(gpa, log, content, srefs);
try noteCluster(gpa, clusters_out, name, content);
const new_rid = log.putContainer(digest, wf.container) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
// Prime the just-verified full-form plaintext so a reverse-delta naming
// this frame as basis reuses it instead of re-inflating the container.
try cache.put(gpa, log.offsetOf(new_rid), content);
return .persisted;
}
// §4.1 — no retained wire (a raw pull `put`): the plaintext is memoised in the
// store, no chain to walk. Verify and compress it; the hash names the
// UNCOMPRESSED bytes, so compression is address-transparent.
const content = store.content(rid) orelse return error.Corrupt;
if (!zit.artifact.verify(name, content)) return error.Corrupt;
// submanifests §7.1 — the plaintext §7.7 just verified is the same
// bytes the phantom duty is stated over; scan it here rather than
// forcing a second inflate anywhere else.
try phantomizeSubManifestRefs(gpa, log, content, srefs);
try noteCluster(gpa, clusters_out, name, content);
const new_rid = log.putZlib(digest, content) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
// Prime the plaintext so a reverse-delta naming this frame as basis reuses it.
try cache.put(gpa, log.offsetOf(new_rid), content);
return .persisted;
}
/// Evidence seam: the persist drain's deferred waiting set — the
/// reverse-deltas resident because their basis frame has not been written yet —
/// at its peak, as a frame count and as wire container bytes. This is the
/// number the streaming-persist decision hinges on: it is exactly what a
/// per-round drain would have to keep (or spill) instead of the whole store.
/// Off unless `ZIT_PERSIST_WAITING` asks for it at exit, and it changes
/// nothing else.
pub var persist_waiting_peak: u64 = 0;
/// Evidence seam: what the last sync run moved over the wire and
/// what its cluster index cost — bounded announcement is a claim about bytes
/// per round, and this is where it is measured. `ZIT_SYNC_STATS` at exit.
pub var sync_rounds: u64 = 0;
pub var sync_sent_bytes: u64 = 0;
pub var sync_received_bytes: u64 = 0;
pub var cluster_index_names: u64 = 0;
fn recordSyncStats(stats: sync_driver.RunStats) void {
sync_rounds = stats.rounds;
sync_sent_bytes = stats.sent_bytes;
sync_received_bytes = stats.received_bytes;
}
pub var persist_waiting_peak_bytes: u64 = 0;
/// Persists every content-bearing artifact in `store` into an already-open
/// writable log. The caller owns durability, allowing clone to append artifacts
/// and its combined config overlay before one final commit.
fn persistStoreOpen(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
store: repostate.ArtifactStore,
keep: ?*const std.DynamicBitSetUnmanaged,
start_after: repostate.Rid,
clusters_out: ?*std.ArrayListUnmanaged([]const u8),
) PersistError!PersistSummary {
var pass = try PersistPass.init(gpa, log, store, keep, null, start_after, clusters_out);
defer pass.deinit();
try pass.drainNew();
return pass.finish();
}
/// The persist drain as a RESUMABLE pass: `drainNew` persists every
/// content-bearing artifact interned since the last call, and `finish` settles
/// the leftovers and the sub-manifest bookkeeping. The batched shape
/// (`persistStoreOpen`) is init + one drainNew + finish; a streaming clone
/// calls drainNew after every round instead, with `mem` set so each persisted
/// artifact's resident bytes are released the moment its frame is durable —
/// git index-pack's shape (stream to disk, defer a delta whose base is unseen,
/// resolve when it lands) applied to fossil's reverse-delta wire.
const PersistPass = struct {
gpa: std.mem.Allocator,
log: *logstore.LogStore,
store: repostate.ArtifactStore,
keep: ?*const std.DynamicBitSetUnmanaged,
/// Set on the streaming path: the concrete store behind `store`, so a
/// persisted rid's bytes can be released. Null keeps every byte resident
/// until the caller drops the store — the batched behaviour.
mem: ?*repostate.MemStore,
/// One bounded plaintext memo for the whole pass: a basis reconstructed for
/// its own §7.7 verify is reused by every reverse-delta that names it, so a
/// delta family reconstructs each basis ~once rather than once per
/// referencing delta. Bounded (LRU byte + entry caps) — never the whole repo.
cache: logstore.PlaintextCache,
/// submanifests §7.1 — one record for the whole pass, so a reference is
/// judged a gap only after every artifact this store carries has landed.
srefs: SubManifestRefs = .{},
rescan: bool,
waiting: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(repostate.Rid)) = .empty,
ready: std.ArrayListUnmanaged(repostate.Rid) = .empty,
written: usize = 0,
elided: usize = 0,
/// The first rid `drainNew` has not seeded yet — the resume cursor.
next_seed: repostate.Rid = 1,
/// See `persistArtifact`; null records nothing.
clusters_out: ?*std.ArrayListUnmanaged([]const u8) = null,
/// Streaming only: rids the cursor passed while they were
/// phantoms. Fossil interns a delta's unseen DELTASRC as a phantom the
/// moment the delta arrives; the content lands at that SAME rid in a later
/// round, behind the cursor. Each drainNew re-checks these first and
/// drives the ones that have since gained content; whatever is left at
/// `finish` is a true phantom, skipped exactly as the batched pass skips
/// it. Empty on the batched path, whose store is already converged.
revisit: std.ArrayListUnmanaged(repostate.Rid) = .empty,
// Evidence-seam running totals for the peak counters above.
waiting_count: u64 = 0,
waiting_bytes: u64 = 0,
fn init(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
store: repostate.ArtifactStore,
keep: ?*const std.DynamicBitSetUnmanaged,
mem: ?*repostate.MemStore,
// Begin the walk AFTER this rid, so a pull persists only the
// artifacts it received (which the holdings store appends beyond its
// log-backed prefix) instead of re-walking every already-durable
// holding. Zero means start at rid 1 (clone/push/serve: the whole
// store IS the new content).
start_after: repostate.Rid,
clusters_out: ?*std.ArrayListUnmanaged([]const u8),
) PersistError!PersistPass {
// submanifests §8.1/§7.1 — THE REBUILD-ON-OPEN INVARIANT, stated once here.
//
// §7.1's phantoms are not durable and are not made durable: a phantom is an
// interned entry with no frame, exactly as an absent delta source's is
// ("exactly as an absent delta source creates a source phantom"), and
// `LogStore` writes none of either kind ("Phantoms never reach disk"). The
// `.zit` on-disk format version is frozen (`logstore.version`), so minting a
// durable phantom record would reject every repository already written.
//
// What makes the in-memory phantoms CORRECT across a restart is this
// invariant: the S-reference gap set is a pure function of the durable
// content-bearing frames, and this sweep re-derives it in full on every run.
// A repository whose §8.1 indicator says it has ever stored an S reference
// rescans its already-persisted artifacts (`rescan_persisted`); one that has
// not — every repository predating this draft — rescans nothing and pays
// nothing. The indicator itself IS durable, which is the whole trick: one
// config byte survives the restart instead of a phantom table.
const rescan = blk: {
var oc = log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
};
if (oc) |*c| {
defer c.deinit(gpa);
break :blk c.config.seen_submanifest;
}
break :blk false;
};
return .{
.gpa = gpa,
.log = log,
.store = store,
.keep = keep,
.mem = mem,
.cache = logstore.PlaintextCache.init(),
.rescan = rescan,
.next_seed = start_after + 1,
.clusters_out = clusters_out,
};
}
/// The type-erased `RoundDrain` entry: persist everything the round that
/// just folded in has interned.
fn drainErased(ctx: *anyopaque) sync_driver.DrainError!void {
const self: *PersistPass = @ptrCast(@alignCast(ctx));
return self.drainNew();
}
fn deinit(self: *PersistPass) void {
self.cache.deinit(self.gpa);
self.srefs.deinit(self.gpa);
var it = self.waiting.valueIterator();
while (it.next()) |list| list.deinit(self.gpa);
self.waiting.deinit(self.gpa);
self.ready.deinit(self.gpa);
self.revisit.deinit(self.gpa);
}
/// Drive one rid through `persistArtifact`, drain the waiters its frame
/// unblocks, and — on the streaming path — release everything persisted.
fn drive(self: *PersistPass, first: repostate.Rid) PersistError!void {
const gpa = self.gpa;
try self.ready.append(gpa, first);
while (self.ready.pop()) |rid| {
switch (try persistArtifact(gpa, self.store, self.log, rid, true, self.mem != null, &self.cache, &self.srefs, self.rescan, self.clusters_out)) {
.persisted => {
// `rid`'s frame now exists: re-drive any reverse-deltas that
// deferred on it as their (now-written) basis.
if (self.waiting.fetchRemove(self.store.hashOf(rid))) |kv| {
var list = kv.value;
defer list.deinit(gpa);
for (list.items) |w| {
self.waiting_count -= 1;
self.waiting_bytes -= if (self.store.wire(w)) |wf| wf.container.len else 0;
}
try self.ready.appendSlice(gpa, list.items);
}
if (self.mem) |m| m.releasePersisted(rid);
},
.deferred => |base_hash| {
const gop = try self.waiting.getOrPut(gpa, base_hash);
if (!gop.found_existing) gop.value_ptr.* = .empty;
try gop.value_ptr.append(gpa, rid);
// Evidence seam: a deferred rid is exactly what
// streaming persist keeps resident; meter it.
self.waiting_count += 1;
self.waiting_bytes += if (self.store.wire(rid)) |wf| wf.container.len else 0;
if (self.waiting_count > persist_waiting_peak) persist_waiting_peak = self.waiting_count;
if (self.waiting_bytes > persist_waiting_peak_bytes) persist_waiting_peak_bytes = self.waiting_bytes;
},
}
}
}
/// Persist every content-bearing artifact interned since the last call.
///
/// Reverse-delta persist ordering (§7.2/§8.2). Fossil stores an OLDER
/// artifact as a delta of a NEWER one, so a delta cfile can land in the store
/// at a LOWER rid than the basis it points at, while the log's `delta_of` is a
/// BACKWARD offset that REQUIRES the basis frame to precede the delta. Persist
/// therefore drains a worklist: a delta whose basis is not yet a frame is
/// deferred (keyed by the basis wire-hash) and re-driven the instant that
/// basis lands. Each content-bearing artifact seeds the drain once in rid
/// order; each deferred delta is re-driven exactly once, so the pass is O(n)
/// and terminates — a chain bottoms out at fossil's full newest artifact, and
/// content addressing forbids a cycle.
fn drainNew(self: *PersistPass) PersistError!void {
// Revived phantoms first: their frames may be the bases this round's
// deltas are about to defer on.
var i: usize = 0;
while (i < self.revisit.items.len) {
const rid = self.revisit.items[i];
if (self.store.held(rid)) {
_ = self.revisit.swapRemove(i);
self.written += 1;
try self.drive(rid);
} else i += 1;
}
const n = self.store.count();
while (self.next_seed <= n) : (self.next_seed += 1) {
const seed = self.next_seed;
if (!self.store.held(seed)) {
// Residual phantom: not durable. Mid-exchange it may yet gain
// content at this very rid — remember it for the next drain.
if (self.mem != null) try self.revisit.append(self.gpa, seed);
continue;
}
if (self.keep) |k| {
// Sparse-cone RFC section 6.3: elision writes nothing -- the frame
// simply never lands, and rebuild-on-open re-interns the dangling
// references as phantoms the cone classifies. A kept delta whose
// basis is elided falls to the leftover drain below and stores
// FULL FORM, so elision can never orphan held content.
if (!k.isSet(seed)) {
self.elided += 1;
continue;
}
}
self.written += 1; // count each content-bearing artifact exactly once, at its rid
try self.drive(seed);
}
}
/// Settle the leftovers and the sub-manifest bookkeeping; the pass is done.
fn finish(self: *PersistPass) PersistError!PersistSummary {
const gpa = self.gpa;
const log = self.log;
// Defensive drain: a deferred basis is itself content-bearing and therefore
// persisted above (draining its waiters), so `waiting` is normally empty
// here. Should a basis nonetheless be missing, its waiters land as full form
// rather than be dropped — every content-bearing artifact stays durable.
var leftover = self.waiting.iterator();
while (leftover.next()) |e| {
for (e.value_ptr.items) |rid| {
switch (try persistArtifact(gpa, self.store, log, rid, false, false, &self.cache, &self.srefs, self.rescan, self.clusters_out)) {
.persisted => if (self.mem) |m| m.releasePersisted(rid),
.deferred => unreachable, // allow_defer=false never defers
}
}
}
// submanifests §7.1 — decided once, at the end: a reference is a gap only
// if nothing in this whole store turned out to name it.
const gaps = self.srefs.unresolved(log);
// submanifests §8.1 — "When the first S-bearing artifact is stored, an
// implementation SHOULD record a local seen-submanifest indicator so
// operational surfaces can warn before the setting is enabled or a pre-S
// client is granted access." Written once, on the false->true edge, so a
// steady-state sync appends no config frame; it is also what arms the
// rebuild above on every subsequent run of this repository.
if ((self.srefs.seen or self.srefs.seen_delta) and !self.rescan) {
var oc = log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
};
var cfg: logstore.Config = if (oc) |c| c.config else .{ .hash_policy = 0, .default_branch = "", .project_id = "" };
defer if (oc) |*c| c.deinit(gpa);
const already = (cfg.seen_submanifest or !self.srefs.seen) and (cfg.seen_delta_manifest or !self.srefs.seen_delta);
if (!already) {
if (self.srefs.seen) cfg.seen_submanifest = true;
if (self.srefs.seen_delta) cfg.seen_delta_manifest = true;
log.putConfig(cfg) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
};
}
}
return .{ .written = self.written, .submanifest_gaps = gaps, .elided = self.elided };
}
};
/// Opens `dir/.zit` for the legacy persistence callers, preserving their
/// artifact result and commit boundary while routing the write through the
/// already-open-log implementation.
fn persistStore(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
store: repostate.ArtifactStore,
keep: ?*const std.DynamicBitSetUnmanaged,
start_after: repostate.Rid,
) PersistLockedError!PersistSummary {
var store_append = cli.openStoreAppend(io, dir) catch |e| switch (e) {
error.StoreBusy => return error.StoreBusy,
else => return error.Storage,
};
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch return error.Storage;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.Storage => return error.Storage,
error.Corrupt => return error.Corrupt,
};
defer log.deinit(gpa);
// A log with no entries at open is being CREATED by this persist,
// so the clusters it records are the whole index; otherwise they extend one.
const fresh = log.entryCount() == 0;
var clusters: std.ArrayListUnmanaged([]const u8) = .empty;
defer {
for (clusters.items) |n| gpa.free(n);
clusters.deinit(gpa);
}
const summary = try persistStoreOpen(gpa, &log, store, keep, start_after, &clusters);
log.commit() catch return error.Storage;
try refreshClusterSidecar(gpa, io, dir, &log, clusters.items, fresh);
return summary;
}
/// Keep the cluster index warm across a persist: adopt the stored
/// index (its key still validates — this persist only appended), append the
/// clusters just written, re-key to the committed log, store. With no valid
/// index to extend, only a persist that created the log writes one; any
/// other store waits for the next sync open, which builds it by one walk.
/// Advisory on storage (a failed write leaves the sidecar as it was); honest
/// on memory, so the persist's OOM contract holds through it.
fn refreshClusterSidecar(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
log: *const logstore.LogStore,
written: []const []const u8,
fresh: bool,
) error{OutOfMemory}!void {
var arena = std.heap.ArenaAllocator.init(gpa);
defer arena.deinit();
const a = arena.allocator();
var names: std.ArrayListUnmanaged([]const u8) = .empty;
var adopted = false;
if (try zit.repocache.loadClusterIndex(gpa, io, dir)) |blob| {
defer gpa.free(blob);
if (try clusterindex.parse(a, blob, log)) |p| {
for (p.names) |n| try names.append(a, try a.dupe(u8, n));
adopted = true;
}
}
if (!adopted and !fresh) return;
for (written) |w| try names.append(a, w);
const blob = (try clusterindex.serialize(gpa, log, log.entryCount(), names.items)) orelse return;
defer gpa.free(blob);
zit.repocache.storeClusterIndex(io, dir, blob);
}
/// The cluster index for one sync open: adopt the sidecar when its
/// key validates, walk only the tail it has not seen (or everything, once, on
/// a miss), store the refreshed sidecar, and resolve it against `store`.
/// Nothing here is an error beyond memory: any refusal is a rebuild, and a
/// store the walk cannot read simply stays unindexed — unbounded, never wrong.
fn loadClusterIndex(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
log: *logstore.LogStore,
store: repostate.ArtifactStore,
) error{OutOfMemory}!clusterindex.Resolved {
var arena = std.heap.ArenaAllocator.init(gpa);
defer arena.deinit();
const a = arena.allocator();
var names: std.ArrayListUnmanaged([]const u8) = .empty;
var seen: std.StringHashMapUnmanaged(void) = .empty;
defer seen.deinit(gpa);
var after: repostate.Rid = 0;
var adopted = false;
if (try zit.repocache.loadClusterIndex(gpa, io, dir)) |blob| {
defer gpa.free(blob);
if (try clusterindex.parse(a, blob, log)) |p| {
for (p.names) |n| {
const owned = try a.dupe(u8, n);
if ((try seen.getOrPut(gpa, owned)).found_existing) continue;
try names.append(a, owned);
}
after = @intCast(p.n_rids);
adopted = true;
}
}
// Phantoms never reach disk, so an index written after a persist may count
// entries this scan does not see; clamping never skips a real frame the
// writer had not already recorded.
const n: repostate.Rid = @intCast(log.entryCount());
if (after > n) after = n;
if (after < n) {
var walked: std.ArrayListUnmanaged([]const u8) = .empty;
try clusterindex.walkClusters(gpa, a, log, after, &walked);
for (walked.items) |w| {
if ((try seen.getOrPut(gpa, w)).found_existing) continue;
try names.append(a, w);
}
}
if (!adopted or after < n) {
if (try clusterindex.serialize(gpa, log, n, names.items)) |blob| {
defer gpa.free(blob);
zit.repocache.storeClusterIndex(io, dir, blob);
}
}
cluster_index_names = names.items.len;
return clusterindex.resolve(gpa, store, names.items);
}
// ---------------------------------------------------------------------------
// `zit pack` — repack the store into a compact new generation.
//
// On-disk storage is UNSPECIFIED by the RFCs (they fix the wire format and
// content-addressed naming, §4.1 L889-892/L918-919 and artifact §3.1 L318-326), so pack is free to reorganize
// the log as long as every artifact's exact content and name survive and the
// §7.2 verbatim wire bytes are still servable. It re-ingests every artifact,
// in the source's offset order, into a FRESH log through `putArtifact` with a
// lineage base hint — a check-in against its parent manifest, a file blob
// against the same path's blob in the parent check-in (the commit path's own
// §4.1/§7.2 compaction heuristic, applied offline). Offset order plus the
// strictly-backward `delta_of` invariant means every base is already written
// before the delta that names it, so no worklist is needed (unlike
// `persistStore`, whose input is rid-ordered).
//
// The new log is written to a temporary file beside `.zit`, fsynced, and then
// atomically renamed over `.zit`: a crash anywhere before the rename leaves the
// original byte-identical (the temp is discarded next run). The `.zit-cache`
// invalidates itself on the tail change and rebuilds on the next read verb.
// ---------------------------------------------------------------------------
/// pack's core store-to-store transform lives in the `zit` module (pack.zig)
/// so the DST/fuzz harnesses drive the REAL pass in-process; the CLI wiring
/// below (temp file, atomic rename, stats rendering) stays here. These aliases
/// keep every call site and test reading
/// `packLog`/`PackError`/`PackStats`/`Filter`/`shunFilter`.
const PackError = zit.pack.PackError;
const PackStats = zit.pack.PackStats;
const packLog = zit.pack.packLog;
const Filter = zit.pack.Filter;
const shunFilter = zit.pack.shunFilter;
/// The name of the transient new-generation file the rewriting verbs share,
/// beside `.zit`. A fixed name (never a random one) so a temp left by a crashed
/// run is reused rather than accumulating, and so `extras.zig` can name it in
/// the single never-tracked predicate. Because it IS fixed and TWO verbs now
/// reach for it, `rewriteAndSwap` holds an exclusive advisory lock on it for the
/// whole rewrite — see there.
const pack_tmp_filename = extras.pack_tmp_filename;
/// The generation-rewrite lock: `pack` and `shun --force` exclude each other
/// with it, never waiting, since a second rewrite of the same store only
/// repeats the first. Its busy notice is the caller's to render.
const rewrite_lock_name = pack_tmp_filename ++ ".lock";
fn lockRewrite(io: std.Io, dir: std.Io.Dir) error{ RewriteBusy, Storage }!zit.storelock.Lock {
var sink = std.Io.Writer.Discarding.init(&.{});
return zit.storelock.acquire(io, dir, rewrite_lock_name, 0, &sink.writer) catch |e| switch (e) {
error.Busy => error.RewriteBusy,
error.HeldByThisProcess, error.Io => error.Storage,
};
}
/// `zit pack` — repack the current repository's store into a compact new
/// generation and atomically swap it in. Thin CLI wiring over `packLog`: parse
/// (no positional), discover the checkout root, open `.zit` read-only and
/// capture its tail key, stream the packed log into a temp file, fsync, re-check
/// the source tail (the no-repo-lock concurrency guard), then rename the temp
/// over `.zit`. On any failure the temp is deleted and the original store is
/// intact.
///
/// The integrity guarantee this rewrite depends on is UNCHANGED, but it is no
/// longer collected at open. `openReadOnly` used to whole-frame-CRC-verify every
/// frame while scanning, so a corrupt source aborted before a byte was written —
/// an eager pre-flight. It scans header-only now. The check that matters happens
/// on the reads instead: `packLog` pulls every artifact through `srcContent`,
/// which propagates `error.Corrupt` rather than skipping the frame, and
/// `readFrameAt` CRC-checks every frame it reads. So a rotted payload still
/// aborts the pack with the temp deleted and `.zit` untouched — it is caught in
/// the copy loop rather than at open, and the operator sees the same outcome.
/// This ordering is load-bearing for a verb that RENAMES OVER the original: if
/// `srcContent` were ever changed to answer null on a corrupt read, the rewrite
/// would silently omit the damaged artifact and then delete the only copy of it.
/// The packed-generation memo's byte shape: magic, then the four
/// `LogKey` fields little-endian — 32 bytes exactly. No self-checksum: a
/// damaged memo can only fail the equality below, and the cost of a miss is
/// one real pack.
const packed_key_magic = "zPK1";
const packed_key_len = 4 + 8 + 8 + 8 + 4;
fn serializePackedKey(buf: *[packed_key_len]u8, key: logstore.LogStore.LogKey) []const u8 {
@memcpy(buf[0..4], packed_key_magic);
std.mem.writeInt(u64, buf[4..12], key.end, .little);
std.mem.writeInt(u64, buf[12..20], key.frame_count, .little);
std.mem.writeInt(u64, buf[20..28], key.last_frame_offset, .little);
std.mem.writeInt(u32, buf[28..32], key.last_header_crc, .little);
return buf[0..packed_key_len];
}
fn parsePackedKey(blob: []const u8) ?logstore.LogStore.LogKey {
if (blob.len != packed_key_len or !std.mem.eql(u8, blob[0..4], packed_key_magic)) return null;
return .{
.end = std.mem.readInt(u64, blob[4..12], .little),
.frame_count = std.mem.readInt(u64, blob[12..20], .little),
.last_frame_offset = std.mem.readInt(u64, blob[20..28], .little),
.last_header_crc = std.mem.readInt(u32, blob[28..32], .little),
};
}
/// Record the just-swapped generation's key so the next identity-filter
/// pack over an unchanged log is a no-op, and refresh the scan-index sidecar
/// the rename invalidated — the open below pays the scan either way, so the
/// next read verb should not pay it again. Best-effort throughout: any failure
/// leaves the memo absent, which costs a real pack, never an answer.
fn recordPackedGeneration(gpa: std.mem.Allocator, io: std.Io, store_dir: std.Io.Dir) void {
var file = store_dir.openFile(io, repo_filename, .{}) catch return;
defer file.close(io);
const st = file.stat(io) catch return;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch return;
defer log.deinit(gpa);
const key = log.logKey() catch return;
var buf: [packed_key_len]u8 = undefined;
zit.repocache.storePackedKey(io, store_dir, serializePackedKey(&buf, key));
refreshSettingLogIndex(gpa, io, store_dir, &log);
}
pub fn runPack(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
) cli.Exit {
var drop_elided = false;
var pack_args: std.ArrayListUnmanaged([]const u8) = .empty;
defer pack_args.deinit(gpa);
for (verb_args) |tok| {
if (std.mem.eql(u8, tok, "--drop-elided")) {
drop_elided = true;
} else {
pack_args.append(gpa, tok) catch return stateOom(streams, "pack");
}
}
const help = switch (cli.parseNoPositionalArgs(pack_args.items)) {
.ok => |a| a.help,
.unknown_option => |tok| return cli.usageError(streams, "pack", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "pack", .extra_argument, tok),
};
if (help) {
streams.out.writeAll(pack_help) catch {};
return .ok;
}
const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "pack")) orelse {
streams.err.writeAll(not_a_repo_msg) catch {};
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
// pack is store-scoped end to end: the reads, the pack-tmp lock
// and the final rename must all land beside the STORE, so a detached
// worktree resolves its pointer once here and the whole flow inherits it.
if (cli.storeDirOf(io, root_dir) catch return repoReadFail(streams, "pack")) |resolved| {
root_dir.close(io);
root_dir = resolved;
}
// Open the source read-only: `openReadOnly` scans and whole-frame-CRC-checks
// every frame, so a corrupt source aborts here (no temp written). The tail
// key pins the log we packed for the pre-rename concurrency re-check.
//
// LIMIT OF THE RECLAIM'S DAMAGE TOLERANCE, stated where it is imposed: this
// pre-flight runs before `packLog` sees the filter, so it covers the shunned
// frames too. The reclaim contract's filter-before-reconstruction ordering
// rescues a shunned artifact whose CONTENT no longer hashes to its name (and
// one that will not decode) — but NOT one whose frame CHECKSUM is damaged,
// which fails here with the reclaim never attempted. `pack_help` says so in
// those terms; narrowing the pre-flight to kept frames would trade a
// whole-store integrity gate for a rarer recovery path and is not this
// slice's call to make.
var src_file = root_dir.openFile(io, repo_filename, .{}) catch return repoReadFail(streams, "pack");
defer src_file.close(io);
const src_st = src_file.stat(io) catch return repoReadFail(streams, "pack");
var src_fs = storage.FileStorage.init(io, src_file, src_st.size);
var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| {
switch (e) {
error.OutOfMemory => streams.err.writeAll("zit pack: out of memory\n") catch {},
else => streams.err.print("zit pack: cannot read repository: {s}\n", .{reasonOf(e)}) catch {},
}
return .failure;
};
defer src.deinit(gpa);
const src_key = src.logKey() catch return packFail(streams, error.Corrupt);
// The no-op gate: a log whose key equals the one the last pack
// recorded has had NO append since (a commit, a shun row, and a pull all
// grow the log), so there is nothing to compact and nothing to reclaim
// that the last pack did not already handle. Gated to the identity
// shape: a `--drop-elided` run composes a cone filter the memo knows
// nothing about, so it always runs.
if (!drop_elided) {
if (zit.repocache.tryLoadPackedKey(gpa, io, root_dir)) |blob| {
defer gpa.free(blob);
if (parsePackedKey(blob)) |recorded| {
if (std.meta.eql(recorded, src_key)) {
streams.out.writeAll("already packed; nothing to do\n") catch {};
return .ok;
}
}
}
}
// `zit pack` is where the repository's deferred shuns are RECLAIMED: every
// artifact the source's shun table names is dropped from the new generation
// instead of relocated into it (the pack filter). The tombstones themselves
// carry forward, so the name keeps reading as absent and a later sync will
// not re-fetch what the operator removed. pack names no NEW tombstone — that
// is `zit shun`'s job — so `extra_shuns` is empty.
// Sparse-cone RFC section 5.4 -- narrow-then-compact, the spec's ONLY
// deletion: an explicit `pack --drop-elided` drops what the store cone
// elides (blobs referenced exclusively out-of-cone), composed with the
// ordinary shun reclaim. Refused without a store cone: with no policy on
// record there is nothing "elided" to drop, and a bare pack must never
// guess one.
var elided_dropper: ?ElidedDropper = null;
defer if (elided_dropper) |*d| d.deinit(gpa);
if (drop_elided) {
var cone = zit.repocache.readStoreCone(gpa, io, root_dir) orelse {
streams.err.writeAll(
"zit pack: --drop-elided needs a store cone and this store has none\n" ++
"zit pack: narrow first ('zit update --cone=...'), or drop the flag\n",
) catch {};
return .failure;
};
elided_dropper = ElidedDropper.build(gpa, src_fs.storageFile(), &cone) orelse {
cone.deinit(gpa);
streams.err.writeAll("zit pack: cannot classify the store for --drop-elided\n") catch {};
return .failure;
};
cone.deinit(gpa);
}
const pack_filter: Filter = if (elided_dropper) |*d|
d.composed(shunFilter(&src))
else
shunFilter(&src);
const stats = rewriteAndSwap(
gpa,
io,
root_dir,
&src,
src_key,
pack_filter,
&.{},
) catch |e| switch (e) {
error.TailChanged => return tailChangedFail(streams, "pack"),
error.RewriteBusy => return rewriteBusyFail(streams, "pack"),
else => |pe| return packFail(streams, pe),
};
renderPackStats(streams.out, stats);
// Only the identity-filter shape records the memo: a --drop-elided pack's
// outcome depends on a cone the memo does not capture, so the next plain
// pack must run for real.
if (!drop_elided) recordPackedGeneration(gpa, io, root_dir);
return .ok;
}
/// `rewriteAndSwap`'s causes: everything `packLog` can fail with, plus the two
/// aborts that are not failures of the rewrite at all and must not be reported as
/// if they were. `TailChanged` means the source log is intact and NEWER than the
/// snapshot we packed; `RewriteBusy` means another rewrite already owns the temp.
/// In both the original store is untouched and the operator's next move is
/// obvious, whereas every `PackError` arm means the new generation could not be
/// built.
pub const RewriteError = PackError || error{ TailChanged, RewriteBusy };
/// The new-generation rewrite `pack` and `shun` share: create the fixed-name
/// readable temp beside `.zit`, stream `packLog(src → temp)` through `filter`
/// (recording `extra_shuns` in the new generation only), fsync, re-check the
/// source tail against `src_key`, then atomically rename the temp over `.zit`.
/// Returns the pack's own stats; on every failing path the temp is deleted and
/// the original store is byte-identical.
///
/// ONE durable transition — the rename — and that is why the two verbs share
/// this body rather than each having its own: the property is a property of this
/// sequence, so a second copy would be a second place for it to stop holding.
/// `src` is already open (both callers must read it before they can decide what
/// to filter); the pre-rename check deliberately reads NEITHER it nor the
/// caller's file handle.
///
/// THAT IS THE GUARD, and it takes a fresh HANDLE, not merely a fresh stat.
/// `storage.FileStorage` caches the append watermark it was constructed with and
/// `size()` returns that cached value without a stat (src/storage.zig
/// `sizeImpl`), so re-checking the tail through the caller's existing view
/// compares `key.end` against the very number the key was derived from — it
/// cannot fail, and the concurrency guard is then a no-op that reads as a guard.
/// The caller's HANDLE fails the same way against the other concurrent writer:
/// once a competing rewrite has renamed its generation over `.zit`, that handle
/// names an ORPHAN inode whose size and tail are frozen, so it agrees with the
/// key forever — and this run would then rename a snapshot taken before the
/// winner existed over the winner, silently un-shunning what it just shunned.
/// Reopening `.zit` BY NAME under the lock is what makes both an append (`zit
/// commit`) and a rename (a racing `pack`/`shun --force`) detectable, and — for
/// `shun` — is what makes "an aborted run leaves NO tombstone" something more
/// than an assertion about code that never runs. Pinned by the two "a run the
/// concurrency guard aborts leaves .zit untouched and records NO tombstone"
/// tests, one per concurrent-writer shape. Same argument, same shape, as
/// `shunAppendRows`.
///
/// MUTUAL EXCLUSION, and why it lives here rather than in either verb. The temp
/// has a FIXED name, so two overlapping rewrites would each truncate the other's
/// in-flight new generation and then fail the §7.7 reconstruct-and-rehash pass on
/// frames that had become holes — reported as `error.Corrupt`, i.e. two healthy
/// processes telling the operator their repository is corrupt. A cron'd `zit pack`
/// overlapping an operator's `zit shun --force` is an entirely ordinary way to hit
/// that, and the shun is the run that must not silently not happen. So the temp is
/// written only under the rewrite lock, `.zit-tmp-pack.lock`, taken by exclusive
/// create without waiting: the second arrival gets `error.RewriteBusy` and says
/// so, instead of a corruption claim. A crashed run's lock names a process that
/// no longer exists and is broken by the next one (`storelock`).
///
/// The final check and the rename run under the store's WRITER lock too. An
/// append that landed between the check and the rename would go into the
/// generation the rename discards; holding the writer lock across both makes the
/// check final.
///
/// Parameterised by DATA only — a drop predicate and a tombstone list — never by
/// a policy callback. Which names those are, what to tell the operator, and
/// whether to refuse at all stay entirely in the callers; nothing here knows
/// which verb it is running, which is why it needs no verb label.
///
/// `pub` for one reason: `src/sim/shun_sweep.zig` drives THIS FUNCTION under a
/// fault-injecting `std.Io`. The property the sweep exists to pin — one durable
/// transition, so a crash leaves either the old store or the new one — is a
/// property of this sequence and of nothing else, so the sweep has to reach the
/// sequence. It deliberately does NOT drive `runPack`/`runShun`: both begin at
/// `cli.discoverRoot`, which reads the process's current directory, and a sweep
/// that depends on ambient cwd is a sweep whose failures do not replay.
pub fn rewriteAndSwap(
gpa: std.mem.Allocator,
io: std.Io,
root_dir: std.Io.Dir,
src: *logstore.LogStore,
src_key: logstore.LogStore.LogKey,
filter: Filter,
extra_shuns: []const logstore.ShunRecord,
) RewriteError!PackStats {
// The rewrite lock first; the busy path must NOT touch the temp, which
// belongs to the run that holds it.
var rewrite_lock = try lockRewrite(io, root_dir);
defer rewrite_lock.release(io);
// New-generation file: a readable temp beside `.zit` (createFileAtomic opens
// write-only, but pack reads its own delta bases back, so we roll our own
// readable temp and rename it). Truncated on open: whatever a prior crashed
// run left is ours to discard now that the lock is held, and `LogStore.open`
// below writes from offset 0, so trailing bytes of a longer stale temp would
// otherwise survive past the new generation's end. Removed on every path but
// the successful rename, before the lock is released.
var tmp_file = root_dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = true }) catch
return error.Storage;
var renamed = false;
defer {
if (!renamed) root_dir.deleteFile(io, pack_tmp_filename) catch {};
tmp_file.close(io);
}
var dst_fs = storage.FileStorage.init(io, tmp_file, 0);
var dst = try logstore.LogStore.open(gpa, dst_fs.storageFile(), src.hash_kind);
const stats = packLog(gpa, src, &dst, filter, extra_shuns) catch |e| {
dst.deinit(gpa);
return e;
};
dst.deinit(gpa); // `packLog` already fsynced; deinit only frees memory
// No repo lock exists: if the source log changed under us since we scanned
// it, abort rather than clobber a concurrent write with our stale snapshot.
// O(1) (one stat + one header read); the original stays intact. For `shun`
// this is also what makes an aborted run leave NO tombstone behind: the rows
// were written into the temp this path is about to delete.
//
// A FRESH HANDLE, a fresh stat and a fresh storage view, for the reason given
// in the doc above: the caller's view answers `size()` from a cached
// watermark, so asking it would always agree with the key — and the caller's
// HANDLE would agree just as unconditionally once a competing rewrite has
// renamed its generation over `.zit`, since that leaves the handle on an
// orphan inode whose size and tail can never move again. Reopening by NAME is
// what makes that case an abort; the caller's handle only ever catches an
// append to the same inode.
var writer_lock = cli.lockStoreWriter(io, root_dir) catch return error.Storage;
defer writer_lock.release(io);
var check_file = root_dir.openFile(io, repo_filename, .{}) catch return error.Storage;
defer check_file.close(io);
const now_st = check_file.stat(io) catch return error.Storage;
var check_fs = storage.FileStorage.init(io, check_file, now_st.size);
if (!logstore.LogStore.validateTail(check_fs.storageFile(), src_key)) return error.TailChanged;
// The single durable transition: atomically replace `.zit` with the packed
// log. A crash before this leaves the original untouched; after it, the
// complete new log. There is no window where `.zit` is partial.
root_dir.rename(pack_tmp_filename, root_dir, repo_filename, io) catch return error.Storage;
renamed = true;
// The rename is durable only once the directory is. Without this a power
// loss can undo it after the verb has reported success, and a shunned
// artifact comes back.
storage.syncDirectory(io, root_dir) catch return error.Storage;
return stats;
}
/// The no-repo-lock concurrency guard's diagnostic, general over `verb` because
/// both rewriting verbs abort on it identically. Not a crash and not corruption:
/// the wording leads with what the operator most needs to know.
fn tailChangedFail(s: cli.Streams, verb: []const u8) cli.Exit {
s.err.print(
"zit {s}: repository changed during {s}; aborted (original is intact)\n",
.{ verb, verb },
) catch {};
return .failure;
}
/// The temp-lock diagnostic, general over `verb` for the same reason: both
/// rewriting verbs contend for one fixed-name temp. Says what is happening and
/// what to do, and — critically — does NOT say "corrupt", which is what an
/// unguarded collision used to produce.
fn rewriteBusyFail(s: cli.Streams, verb: []const u8) cli.Exit {
s.err.print(
"zit {s}: another repository rewrite is in progress; try again when it finishes\n",
.{verb},
) catch {};
return .failure;
}
/// Render pack's failure `err` as `zit pack: <reason>` on stderr, exit 1. The
/// original store is always intact when this runs (every write went to the
/// temp, which the caller's `defer` deletes).
fn packFail(s: cli.Streams, err: PackError) cli.Exit {
s.err.print("zit pack: {s}\n", .{reasonOf(err)}) catch {};
return .failure;
}
/// One human line summarizing a repack. "already packed" only when the run
/// changed NOTHING an operator would care about — the log did not shrink AND
/// nothing was reclaimed (the store was already compact; pack still verified every
/// frame, its integrity-scrub value). A reclaim is never described as "already
/// packed" even when the log grew, which it legitimately can: dropping an artifact
/// that others were deltified against moves its content into one of them, so a
/// byte count is exactly the wrong signal for whether work happened.
///
/// The reclaim clause appears only when the run actually dropped something: a
/// repository with no deferred shuns is the common case and "0 shunned reclaimed"
/// on every pack would be noise, while a run that DID remove bytes must say so —
/// dropping content silently is the one outcome an operator must never have to
/// infer from a byte count. `pack_help` states the clause the same conditional way.
///
/// `artifacts` is labelled "walked" rather than "packed" because `PackStats`
/// counts SOURCE frames: on a reclaiming run `artifacts - dropped` is what the new
/// generation holds, and an unlabelled first number would read as the packed
/// count. Split from `runPack` so the exact wording is unit tested without a
/// filesystem.
fn renderPackStats(out: *std.Io.Writer, stats: PackStats) void {
const unchanged = stats.bytes_after >= stats.bytes_before and stats.dropped == 0;
const verb = if (unchanged) "already packed" else "packed";
out.print("{s}: {d} artifacts walked, {d} re-deltaed, ", .{ verb, stats.artifacts, stats.deltas }) catch {};
if (stats.dropped > 0) out.print("{d} shunned reclaimed, ", .{stats.dropped}) catch {};
out.print("{d} -> {d} bytes\n", .{ stats.bytes_before, stats.bytes_after }) catch {};
}
// ---------------------------------------------------------------------------
// `zit shun` — record artifacts in this repository's shun table and, by default,
// remove their bytes in the same operation.
//
// NO DRAFT SPECIFIES THIS VERB. sync §3.3 (docs/rfcs/
// draft-fossil-sync-protocol-00.txt L518-519) defines the repository shun table
// as "a persistent, repository-local set keyed by complete artifact name",
// L520-521 grants that "Local administration MAY add or remove names", and
// L527-530 makes the set mask the phantom scan. That is the whole of it: no
// draft names a command, an option, a report, a refusal, or the operation that
// removes a shunned artifact's bytes. Everything below — the refuse-by-default
// policy, the `--force` gate, the unconditional current-checkout stop, the
// per-type diagnostics — is this implementation's decision, and the choices are
// stated at the point that makes them rather than attributed to a section.
//
// The one advisory the plan of record cites for sub-manifests cannot be
// implemented or quoted here: there is no `submanifest` variant in
// `artifact.ArtifactType` on this branch (it runs manifest=1 .. forum=8) and no
// sub-manifest draft in docs/rfcs/. `shunConsequence`'s switch is total over
// that enum, so the day a ninth type lands the compiler demands its diagnostic
// — including the wider-blast-radius warning a sub-manifest deserves.
// ---------------------------------------------------------------------------
const NameSet = zit.pack.NameSet;
/// What a `zit shun` run will do about one hash the operator named. Split into
/// three groups by consequence, because the runner acts on the group, not the
/// member: the three `blocking` values stop the whole invocation before anything
/// durable happens; `remove`/`absent` each need a new tombstone; `reclaim` needs
/// no row but still has bytes to remove; `already` needs nothing at all.
const ShunDisposition = enum {
/// Held here: the rewrite drops its bytes and records its tombstone.
remove,
/// Already in the set from an earlier `--defer`, but the BYTES ARE STILL HERE:
/// no new row to write, and yet the removal the operator just asked for has
/// genuinely not happened. Grouping this with `already` is what would make
/// `zit shun --force H` after `zit shun --defer --force H` print "nothing to
/// do" and exit 0 with the content still readable — the worst sentence this
/// verb could produce, on the one path where an operator is acting under an
/// order. So it is its own disposition: no tombstone, but a rewrite.
reclaim,
/// Not held here: the tombstone is recorded and nothing is reclaimed. A
/// pre-emptive shun is meaningful and free — sync §3.3 L553-556 filters the
/// phantom scan on the set, so a name recorded before the artifact ever
/// arrives is never requested.
absent,
/// Already in the set AND already gone: a true no-op. `putShun` is idempotent
/// as set membership, so the durable row keeps the epoch and reason it was
/// written with.
already,
/// The manifest of the current checkout. REFUSED, with no `--force`
/// override — the one hard stop.
current_checkout,
/// Not a lowercase-hex artifact-name prefix at all.
not_a_name,
/// An abbreviated prefix matching no stored artifact. Refused rather than
/// recorded: the shun table is keyed by COMPLETE name, so a prefix cannot be
/// completed and a row keyed on one would name nothing.
unresolved,
/// A prefix matching more than one stored artifact.
ambiguous,
/// Whether this disposition stops the whole invocation before any durable
/// write. A total switch so a value added later cannot silently join the
/// permissive group.
fn blocks(self: ShunDisposition) bool {
return switch (self) {
.current_checkout, .not_a_name, .unresolved, .ambiguous => true,
.remove, .reclaim, .absent, .already => false,
};
}
/// Whether this disposition needs a new tombstone row.
fn records(self: ShunDisposition) bool {
return switch (self) {
.remove, .absent => true,
.reclaim, .already, .current_checkout, .not_a_name, .unresolved, .ambiguous => false,
};
}
/// Whether this disposition has BYTES for the rewrite to drop. Separate from
/// `records` precisely because the two are not the same question: `absent`
/// records without removing and `reclaim` removes without recording.
fn reclaims(self: ShunDisposition) bool {
return switch (self) {
.remove, .reclaim => true,
.absent, .already, .current_checkout, .not_a_name, .unresolved, .ambiguous => false,
};
}
};
/// One hash the operator named, resolved. `name` is the full lowercase-hex
/// artifact name — or, for the three unresolvable dispositions, the operator's
/// own token, which is all there is to name. `digest` is the raw binary key the
/// shun table and the pack filter both use, empty when there is none. `kind` is
/// null for an opaque blob (an artifact that does not parse as a structural one),
/// which is a first-class answer here exactly as it is for `zit artifact`.
const ShunTarget = struct {
name: []const u8,
digest: []const u8 = &.{},
kind: ?artifact.ArtifactType = null,
disposition: ShunDisposition,
};
/// The report label for a target's type. Structural artifacts use §5.1 Table 3's
/// external name (`artifact.ArtifactType.externalName`), the same vocabulary
/// `zit artifact` prints, so an operator sees one word per type across the CLI;
/// a blob is "file", matching `zit artifact`'s "type: file".
fn shunTypeLabel(kind: ?artifact.ArtifactType) []const u8 {
const t = kind orelse return "file";
return t.externalName();
}
/// The consequence of removing an artifact of this type, in one clause.
///
/// A TOTAL switch on `artifact.ArtifactType`, deliberately: the alternative is a
/// default arm, and a type added later would then get no diagnostic at all while
/// the operator was told the removal was safe. The six types that all lose the
/// same thing share one arm rather than repeating one sentence six times.
///
/// NO COUNTS, and that is a deliberate reduction from the intended report shape.
/// "N descendant check-ins become unresolvable" needs a public descendants
/// accessor over `repostate`'s child map plus a BFS, and "src/secret.key, in 6
/// check-ins" needs a reverse F-card index that exists nowhere in this tree —
/// producing either means decoding every manifest in the repository on the
/// refusal path, which is supposed to be the cheap path. The clause without the
/// count states the same consequence; the count is worth revisiting only
/// alongside the index that would make it cheap.
fn shunConsequence(kind: ?artifact.ArtifactType) []const u8 {
const t = kind orelse return "file content; every check-in listing it loses that file";
return switch (t) {
.manifest => "a check-in; its descendants become unresolvable",
.cluster => "sync housekeeping only; regenerable",
.submanifest => "part of a check-in's file list; every check-in whose tree names it stops enumerating",
.control, .wiki, .ticket, .attachment, .event, .forum => "structural content; what it renders stops rendering",
};
}
/// English plural suffix for `n`. Used only in these reports.
fn plural(n: usize) []const u8 {
return if (n == 1) "" else "s";
}
/// The `-es` plural suffix, for the one noun these reports pluralize that does
/// not take a bare `-s`: "hash" / "hashes". A separate function rather than a
/// caller-supplied string so no call site can spell it "hashs".
fn pluralEs(n: usize) []const u8 {
return if (n == 1) "" else "es";
}
/// Render the hard stops: one line per blocking target, then nothing happens.
/// Every arm of the switch prints, including the non-blocking ones — a renderer
/// with an `unreachable` arm would be undefined behavior in `ReleaseFast`, which
/// is a poor trade for a line of output nobody should ever see.
fn renderShunBlockers(err: *std.Io.Writer, targets: []const ShunTarget) void {
for (targets) |t| switch (t.disposition) {
.current_checkout => {
err.print("zit shun: {s} is the check-in this checkout is on; refusing\n", .{t.name}) catch {};
err.writeAll("run 'zit update' to a different check-in first\n") catch {};
},
.not_a_name => err.print("zit shun: '{s}' is not an artifact name\n", .{t.name}) catch {},
.unresolved => err.print(
"zit shun: no artifact matches '{s}'; give the complete name to shun a hash this repository does not hold\n",
.{t.name},
) catch {},
.ambiguous => err.print("zit shun: hash '{s}' is ambiguous\n", .{t.name}) catch {},
.remove, .reclaim, .absent, .already => {},
};
}
/// Whether `src` still holds reclaimable BYTES under `digest` — an interned,
/// non-phantom frame. This is exactly `packLog`'s own `dropped` predicate (it
/// counts non-phantom rids the filter drops), so "would be removed" here and "was
/// removed" there cannot disagree. No decode, so the refusal path stays cheap.
fn holdsBytes(src: *const logstore.LogStore, digest: []const u8) bool {
const rid = src.lookup(digest) orelse return false;
return !src.isPhantom(rid);
}
/// How many rows already in `src`'s shun table still hold bytes that this run's
/// rewrite would drop, EXCLUDING the ones `targets` names (those are counted as
/// `.reclaim` targets and listed by name, so counting them here too would double
/// them). This is the clause the refusal — the consent gate — would otherwise be
/// silent about: `renderShunStats` is honest that `dropped` can exceed `recorded`,
/// but that honesty arrives after the deletion.
fn deferredStillHeld(src: *const logstore.LogStore, targets: []const ShunTarget) usize {
var n: usize = 0;
rows: for (src.shunList()) |row| {
if (!holdsBytes(src, row.hash)) continue;
for (targets) |t| {
if (std.mem.eql(u8, t.digest, row.hash)) continue :rows;
}
n += 1;
}
return n;
}
/// Render the default refusal: what this run would do, then stop. The register is
/// `clean`'s (`cleanRefuse` in cmd/tracking.zig) — the counts and the word
/// "refusing" in ONE line, so an operator learns nothing happened before reading
/// six lines of inventory, then the flag that proceeds, and never a prompt.
///
/// THREE counts, because they answer three different questions and only the first
/// used to be stated. How many rows become permanent (`recording` — including the
/// pre-emptive ones, which remove nothing). How much content this invocation's own
/// targets destroy (`removing` — which is NOT a subset of `recording`: a hash
/// shunned earlier with `--defer` records nothing and still has bytes to lose).
/// And `deferred_held`, the content the same rewrite reclaims from rows an EARLIER
/// run recorded — collateral the operator never named, which the success line
/// already admits to and the consent gate must not hide.
///
/// `defer_reclaim` IS A PARAMETER BECAUSE THE TWO RUNS ARE DIFFERENT OPERATIONS,
/// and this block's whole job is to state the operation accurately before an
/// irreversible act. A `--defer` run removes nothing NOW: describing it as "would
/// be permanently removed" overstates it, the `deferred_held` collateral clause is
/// about a rewrite that is not going to happen, and — worst — a suggestion of
/// `zit shun --force` would have an operator who copies it escalate a deferred
/// tombstone into a full-store reclaim of the whole union set, including collateral
/// the run never named. The suggested command is therefore always the one the
/// operator actually asked for, plus `--force`.
fn renderShunRefusal(
err: *std.Io.Writer,
targets: []const ShunTarget,
deferred_held: usize,
defer_reclaim: bool,
) void {
var recording: usize = 0;
var removing: usize = 0;
for (targets) |t| {
if (t.disposition.records()) recording += 1;
if (t.disposition.reclaims()) removing += 1;
}
err.writeAll("zit shun: ") catch {};
if (recording == 0) {
err.writeAll("no new hash would be recorded") catch {};
} else {
err.print("{d} hash{s} would be shunned in this repository", .{ recording, pluralEs(recording) }) catch {};
}
if (removing == 0) {
err.writeAll("; nothing you named holds content to remove") catch {};
} else if (defer_reclaim) {
// Named, still held, and deliberately NOT reclaimed by this run. "stays
// reconstructible" rather than merely "unreclaimed": the mask is on the
// name, so the plaintext is still materialisable through any surviving
// delta, which is precisely what an operator under a takedown order would
// otherwise assume they had already prevented.
err.print(
"; the bytes of {d} artifact{s} stay on disk and stay reconstructible until 'zit pack'",
.{ removing, plural(removing) },
) catch {};
} else {
err.print("; {d} artifact{s} would be permanently removed", .{ removing, plural(removing) }) catch {};
}
// Collateral of the REWRITE, so it is silent on the deferred path: there is no
// rewrite in a `--defer` run and those rows keep their bytes untouched.
if (deferred_held != 0 and !defer_reclaim) {
err.print(
"; the same rewrite also removes {d} artifact{s} shunned earlier and not yet reclaimed",
.{ deferred_held, plural(deferred_held) },
) catch {};
}
err.writeAll("; refusing without --force:\n\n") catch {};
for (targets) |t| switch (t.disposition) {
.remove => err.print(" {s} {s} {s}\n", .{
t.name,
shunTypeLabel(t.kind),
shunConsequence(t.kind),
}) catch {},
.reclaim => if (defer_reclaim) {
err.print(" {s} {s} already shunned; content still here and stays\n", .{
t.name,
shunTypeLabel(t.kind),
}) catch {};
} else {
err.print(" {s} {s} already shunned; content still here and would be removed\n", .{
t.name,
shunTypeLabel(t.kind),
}) catch {};
},
.absent => err.print(" {s} recorded only (not present in this repository)\n", .{t.name}) catch {},
.already => err.print(" {s} already shunned; content already gone\n", .{t.name}) catch {},
// Unreachable in practice (`renderShunBlockers` ran and the caller
// exited), but printed rather than asserted: see `renderShunBlockers`.
.current_checkout, .not_a_name, .unresolved, .ambiguous => err.print(
" {s} cannot be shunned\n",
.{t.name},
) catch {},
};
err.writeAll(
\\
\\This is irreversible: zit undo does not recover a shun, and the content
\\cannot be re-fetched from a peer.
\\
) catch {};
err.print("run 'zit shun {s}--force <HASH>...' to proceed\n", .{
if (defer_reclaim) "--defer " else "",
}) catch {};
}
/// One human line summarizing a completed reclaiming shun, in `renderPackStats`'s
/// register. `dropped` can EXCEED `shuns_recorded`, because the same rewrite also
/// reclaims any shun that was deferred earlier — an operator must be told that
/// rather than left to infer it from a byte count. Split out so the exact wording
/// is unit tested without a filesystem.
///
/// EVERY NUMBER HERE COMES FROM THE PACK, none from the caller's intent — which is
/// the whole reason `PackStats.shuns_recorded` exists. The rows this run meant to
/// write and the rows the new generation actually holds are two different facts,
/// and the one an operator is owed after a takedown is the second. `packLog`'s
/// row-count gate makes them equal or fails the pack; if that gate were ever
/// weakened, this line would under-report rather than over-report.
fn renderShunStats(out: *std.Io.Writer, stats: PackStats) void {
// `dropped` is counted among the source frames walked, so this subtraction
// cannot underflow. Asserted anyway, and precisely because it is a tautology:
// it is the only unchecked arithmetic in this verb's reporting, and an
// underflow would print an absurd remaining count rather than trap.
std.debug.assert(stats.dropped <= stats.artifacts);
out.print("shunned: {d} hash{s} recorded, {d} artifact{s} removed, {d} -> {d} remaining, {d} -> {d} bytes\n", .{
stats.shuns_recorded,
pluralEs(stats.shuns_recorded),
stats.dropped,
plural(stats.dropped),
stats.artifacts,
stats.artifacts - stats.dropped,
stats.bytes_before,
stats.bytes_after,
}) catch {};
}
/// `--defer`'s success line. It does NOT promise a byte count: nothing on this
/// path measured one, and the only honest number would require doing the rewrite
/// the operator just declined.
///
/// `recorded == 0` with `holding != 0` is the re-`--defer` of a hash already
/// deferred: no row to add, and yet content still to lose. Saying "0 hashes
/// recorded" there would read as a no-op on a repository that still holds the
/// bytes, so that case gets its own clause naming what is left.
fn renderShunDeferred(out: *std.Io.Writer, recorded: usize, holding: usize) void {
if (recorded == 0) {
out.print(
"shunned: no new hash to record; {d} artifact{s} still hold{s} content; run 'zit pack' to reclaim the space\n",
.{ holding, plural(holding), if (holding == 1) "s" else "" },
) catch {};
return;
}
out.print(
"shunned: {d} hash{s} recorded; run 'zit pack' to reclaim the space\n",
.{ recorded, pluralEs(recorded) },
) catch {};
}
/// The success line for a run with tombstones to write and NOTHING to reclaim —
/// the purely pre-emptive shun. It takes the append path rather than the rewrite
/// (see `runShun`), so there is no byte count to report and, more to the point,
/// nothing was removed: a line ending in a store that had GROWN would be the
/// wrong thing to print under the word "shunned".
fn renderShunRecordedOnly(out: *std.Io.Writer, recorded: usize) void {
out.print(
"shunned: {d} hash{s} recorded; none is held here, so nothing was removed\n",
.{ recorded, pluralEs(recorded) },
) catch {};
}
/// `--list`: the shun table in persisted order, one row per line — full artifact
/// name, the UTC date it was recorded, and the reason when one was given. The
/// date comes from `cli.formatWriterTimestamp`'s first ten bytes, so this shares
/// the CLI's one epoch formatter rather than growing a second.
fn renderShunList(out: *std.Io.Writer, rows: []const logstore.ShunRecord) void {
if (rows.len == 0) {
out.writeAll("the shun table is empty\n") catch {};
return;
}
for (rows) |row| {
var hex: [2 * digest_len]u8 = undefined;
var ts: [19]u8 = undefined;
const date = cli.formatWriterTimestamp(row.epoch, &ts)[0..10];
out.print("{s} {s}", .{ digestToWireName(row.hash, &hex), date }) catch {};
if (row.reason.len != 0) out.print(" {s}", .{row.reason}) catch {};
out.writeAll("\n") catch {};
}
}
/// Render shun's failure `err` as `zit shun: <reason>` on stderr, exit 1. On
/// every path that reaches this the store is byte-identical: either nothing was
/// written yet, or every write went to the temp the rewrite's `defer` deletes.
fn shunFail(s: cli.Streams, err: anyerror) cli.Exit {
s.err.print("zit shun: {s}\n", .{reasonOf(err)}) catch {};
return .failure;
}
/// The two shun-row rejections (`BadKey`, `ReasonTooLong`) that are unreachable by
/// construction here — every digest passed `isCompleteDigest` and every reason
/// passed the length gate — given a REAL phrase rather than left to `reasonOf`,
/// whose `else` arm renders both as "input/output error". Being unreachable is
/// exactly why the phrase matters: if one ever fires it is a key-width or
/// reason-length bug, and telling the operator their disk failed would send them
/// looking at hardware. Nothing durable has happened on either path.
fn shunRowBug(s: cli.Streams) cli.Exit {
s.err.writeAll("zit shun: internal error: a shun row was rejected as malformed; nothing was changed\n") catch {};
return .failure;
}
/// `shunAppendRows`' failure, with its two unreachable-by-construction arms routed
/// to `shunRowBug` instead of through `reasonOf`'s "input/output error" catch-all.
/// A torn append is truncated by the log's own recovery, so a failure here leaves
/// the shun simply absent, never half present.
///
/// `TailChanged` and `RewriteBusy` join the set for the reason `RewriteError`
/// carries them: neither is a failure of the append at all, both leave the store
/// exactly as it was, and both have the same operator-facing meaning here as they
/// do on the rewrite path — which is why they route to the SAME two diagnostics
/// rather than growing a second pair of sentences for the append.
const ShunAppendError = logstore.OpenError || logstore.ShunError || logstore.CommitError ||
error{ TailChanged, RewriteBusy, StoreBusy };
/// A config write that could not take the writer lock. A busy lock already
/// said who holds it; anything else is the unreadable-repository failure.
fn storeAppendFail(s: cli.Streams, verb: []const u8, err: anyerror) cli.Exit {
if (err == error.StoreBusy) return .failure;
return repoReadFail(s, verb);
}
fn shunAppendFail(s: cli.Streams, err: ShunAppendError) cli.Exit {
return switch (err) {
error.OutOfMemory => stateOom(s, "shun"),
error.BadKey, error.ReasonTooLong => shunRowBug(s),
error.TailChanged => tailChangedFail(s, "shun"),
error.RewriteBusy => rewriteBusyFail(s, "shun"),
error.StoreBusy => .failure, // the lock notice already said it
else => shunFail(s, err),
};
}
/// Canonicalize a `zit shun` positional to a lowercase-hex artifact-name prefix,
/// or null when it is not one. The same band `zit artifact` accepts (4..64 hex),
/// deliberately: an operator pastes the same token into both verbs.
fn canonicalizeShunHex(arg: []const u8, out: *[2 * logstore.max_hash_len]u8) ?[]const u8 {
if (arg.len < 4 or arg.len > out.len) return null;
for (arg, 0..) |c, i| {
const lc = std.ascii.toLower(c);
if (!((lc >= '0' and lc <= '9') or (lc >= 'a' and lc <= 'f'))) return null;
out[i] = lc;
}
return out[0..arg.len];
}
/// Decode a COMPLETE lowercase-hex artifact name (40 or 64 hex, the two forms
/// artifact-format §3.2 defines) into `out`, returning the used prefix, or null
/// when `hex` is not one of those two widths. The width, not `hash_kind`, decides
/// — the same rule `logstore.isCompleteDigest` enforces on the write end, so
/// nothing here interprets a digest.
fn shunHexToDigest(hex: []const u8, out: *[logstore.max_hash_len]u8) ?[]const u8 {
if (hex.len != 40 and hex.len != 64) return null;
const n = hex.len / 2;
std.debug.assert(n <= out.len);
_ = std.fmt.hexToBytes(out[0..n], hex) catch return null;
return out[0..n];
}
/// Append `rows` to the live store and fsync — `--defer`'s whole effect, and the
/// path a `--force` run with nothing to reclaim takes too. One frame per row plus
/// one `commit`, so a crash mid-append leaves a torn tail the log's existing
/// recovery truncates: the shun is then simply absent, never half present.
/// Deliberately NOT the rewrite path: nothing here is atomic across rows, and
/// nothing needs to be, because each row is independently meaningful.
///
/// IT CARRIES BOTH OF `rewriteAndSwap`'S CONCURRENCY PROTECTIONS, for reasons that
/// are this path's own and not a symmetry argument.
///
/// The LOCK — the same exclusive non-blocking lock on the same fixed
/// `pack_tmp_filename`, so an append and a rewrite exclude each other. Without it a
/// `zit pack` (or another `zit shun --force`) that renames its new generation over
/// `.zit` after our `openFile` leaves us appending to an ORPHANED INODE: the row
/// lands nowhere anyone will ever read, and this run still reports success. On the
/// `--defer` path that means the deferred shun silently does not exist while the
/// help text has just told the operator to run `zit pack` to finish it. A cron'd
/// pack overlapping an operator's shun is an entirely ordinary way to hit that.
///
/// The TAIL RE-VALIDATION against `src_key` — because the caller classified its
/// targets against a snapshot taken before the lock existed, and both success
/// lines this path can print (`renderShunRecordedOnly`'s "none is held here, so
/// nothing was removed", `renderShunDeferred`'s count of what still holds content)
/// are claims about STORE STATE, not about the argument list. A `zit commit`
/// landing a named hash in that window makes the first of them false: the
/// tombstone would be recorded, exit 0, and an operator under a takedown order
/// told nothing was held while the bytes were live. Re-checking the tail on the
/// very handle the append will go through — under the lock, so no rename can have
/// intervened — turns that into an abort. `error.TailChanged` here means nothing
/// was written: the check precedes the first `putShun`.
///
/// The check reads the REOPENED handle rather than the caller's, and that is the
/// stronger of the two: a rewrite that completed just before we took the lock has
/// left the caller's handle pointing at an orphan whose tail still matches its own
/// key, while `.zit` by name is now a different generation. Validating the handle
/// we are about to WRITE is what makes that case an abort instead of an append
/// driven by a stale snapshot.
fn shunAppendRows(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
src_key: logstore.LogStore.LogKey,
rows: []const logstore.ShunRecord,
) ShunAppendError!void {
// The rewrite lock, so no `pack` or `shun --force` runs underneath this
// append, then the writer lock inside it, in the order every rewrite takes
// them.
var rewrite_lock = try lockRewrite(io, dir);
defer rewrite_lock.release(io);
var store_append = cli.openStoreAppend(io, dir) catch |e| switch (e) {
error.StoreBusy => return error.StoreBusy,
else => return error.Storage,
};
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch return error.Storage;
var fs = storage.FileStorage.init(io, file, st.size);
// A FRESH handle and a FRESH stat, for the same reason `rewriteAndSwap` takes
// them: the caller's view answers `size()` from the cached watermark its key
// was derived from, so asking it could never disagree.
if (!logstore.LogStore.validateTail(fs.storageFile(), src_key)) return error.TailChanged;
// The header of the existing repo carries the hash kind; `open` reads it from
// the non-empty file, so the passed value is a don't-care (0).
var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0);
defer log.deinit(gpa);
for (rows) |row| try log.putShun(row.hash, row.reason, row.epoch);
try log.commit();
}
/// `zit shun` — record one or more artifacts in this repository's shun table and,
/// unless `--defer`, remove their bytes in the same operation.
///
/// POLICY, decided here and not by any draft: refuse by default and name the
/// consequence; require `--force` for BOTH paths, because `--defer` also appends
/// an irreversible tombstone and the read mask hides the content the instant it
/// lands; never prompt; refuse the current checkout's own check-in with no
/// override, because that is the one target whose removal breaks the tree the
/// operator is standing in and `zit update` is a cheap way out.
///
/// CRASH SAFETY. `--force` performs exactly ONE durable transition. When there is
/// content to remove that is the rename inside `rewriteAndSwap`, and the new
/// tombstones are written into the NEW generation (`packLog`'s `extra_shuns`),
/// never appended to `.zit` first — two durable transitions would expose a state
/// where the tombstone exists and the content still does. It also means an aborted
/// run leaves no tombstone at all: the rows were in the temp that the abort
/// deletes. Pinned by "runShun: a --force run the temp lock turns away records NO
/// tombstone".
///
/// When NOTHING is held — every named hash is pre-emptive — the single durable
/// transition is instead the append plus commit, and the rewrite is skipped
/// altogether. It is not load-bearing there: the rule exists to stop a tombstone
/// outliving its content, and with no content to remove the append is already
/// crash-safe under the log's torn-tail recovery. Running a whole-store rewrite to
/// reclaim nothing would cost O(store) plus transient room for both generations and
/// end with a report whose byte count had GROWN.
///
/// The rename is visible to a running `zit serve` DESPITE its held handle:
/// serve's `baseSnapshot` stats the path beside the handle each round and
/// reopens on an inode change, so the round after this swap serves the
/// rewritten generation — shunned content gone — with no restart.
pub fn runShun(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
const parsed = switch (cli.parseShunArgs(arena, verb_args) catch return stateOom(streams, "shun")) {
.ok => |a| a,
.unknown_option => |tok| return cli.usageError(streams, "shun", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "shun", .extra_argument, tok),
};
if (parsed.help) {
streams.out.writeAll(shun_help) catch {};
return .ok;
}
if (!parsed.list and parsed.hashes.len == 0) {
return cli.usageError(streams, "shun", .no_hash, "");
}
// A reason the durable row cannot hold is refused BEFORE anything is opened,
// and refused rather than truncated: the note on a shun is often the legal
// reference the operator will be asked for, and silently keeping a prefix of
// it is worse than making them shorten it themselves.
//
// A CONTROL BYTE IS REFUSED THE SAME WAY, and this is the reason `--reason` is
// validated at all rather than merely bounded. `--list` is the only view of a
// durable, irreversible, un-removable record, and `renderShunList` prints one
// row per line: a newline inside a reason forges a row there that is
// indistinguishable from a real one. Reasons arrive from scripts and from pasted
// takedown text as often as they are typed, so this is not a hand-typed-only
// hazard. Rejecting at input rather than escaping at render keeps the fix to one
// place and keeps the stored bytes exactly what the operator supplied. NO DRAFT
// SPEAKS TO THIS and `logstore.ShunError` has no arm for it — a local decision,
// taken the same way the length gate was: refuse rather than mangle.
if (parsed.reason) |r| {
if (r.len > logstore.max_shun_reason) {
streams.err.print(
"zit shun: --reason is {d} bytes; the recorded note is at most {d}\n",
.{ r.len, logstore.max_shun_reason },
) catch {};
return .failure;
}
for (r) |c| {
if (c >= 0x20 and c != 0x7f) continue;
streams.err.writeAll(
"zit shun: --reason contains a control byte; the recorded note must be a single line of text\n",
) catch {};
return .failure;
}
}
const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "shun")) orelse {
streams.err.writeAll(not_a_repo_msg) catch {};
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
// shun is store-scoped end to end: the reads, the pack-tmp lock
// and the final rename must all land beside the STORE, so a detached
// worktree resolves its pointer once here and the whole flow inherits it.
if (cli.storeDirOf(io, root_dir) catch return repoReadFail(streams, "shun")) |resolved| {
root_dir.close(io);
root_dir = resolved;
}
// Open the source read-only, exactly as `pack` does, and for the same
// reasons — including the same caveat: this open no longer CRC-verifies
// payloads (it scans header-only), so a corrupt source aborts during the
// rewrite's own reads rather than here. Nothing is written either way. The
// tail key pins the log for the pre-rename re-check. See `runPack`'s doc
// comment for why that ordering matters to a verb that renames over `.zit`.
var src_file = root_dir.openFile(io, repo_filename, .{}) catch return repoReadFail(streams, "shun");
defer src_file.close(io);
const src_st = src_file.stat(io) catch return repoReadFail(streams, "shun");
var src_fs = storage.FileStorage.init(io, src_file, src_st.size);
var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "shun"),
else => |re| return shunFail(streams, re),
};
defer src.deinit(gpa);
if (parsed.list) {
renderShunList(streams.out, src.shunList());
return .ok;
}
const src_key = src.logKey() catch return shunFail(streams, error.Corrupt);
// The current checkout's own check-in, for the one refusal with no override.
// A `.zit-checkout` we cannot parse is a hard failure, not an empty answer:
// the guard would silently become vacuous and the operator would be allowed
// to delete the check-in they are standing on.
var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined;
const ck_state = checkout.read(io, root_dir, &ck_buf) catch |e|
return main.checkoutFail(streams, "shun", e);
const ck_version: []const u8 = if (ck_state) |s| s.version else "";
// Classified, then DEDUPLICATED BY RESOLVED DIGEST — which is why it cannot be
// done in the parser. `zit shun --force H <prefix-of-H>` is one artifact named
// twice, and an operator pasting a takedown list beside an abbreviation from
// `zit log` cannot see that it is. Left folded in, the duplicate builds two
// identical rows, `putShun` collapses them, and `packLog`'s row-count gate —
// whose documented precondition is that `extra_shuns` rows are DISTINCT — fires
// `error.Corrupt`: the verb would tell the operator their repository is corrupt,
// on the one occasion they can least afford to doubt it. Deduping here rather
// than in `packLog` keeps that gate a gate. O(n^2) over an argument list.
const target_buf = arena.alloc(ShunTarget, parsed.hashes.len) catch return stateOom(streams, "shun");
var target_count: usize = 0;
tokens: for (parsed.hashes) |tok| {
const t = classifyShunTarget(gpa, arena, &src, ck_version, tok) catch |e|
return shunFail(streams, e);
// Only a resolved target can be a duplicate: the unresolvable dispositions
// carry the operator's own token and no digest, and two different bad tokens
// both deserve their own diagnostic line.
if (t.digest.len != 0) {
for (target_buf[0..target_count]) |prev| {
if (std.mem.eql(u8, prev.digest, t.digest)) continue :tokens;
}
}
target_buf[target_count] = t;
target_count += 1;
}
const targets = target_buf[0..target_count];
// THE GUARD RUNS BEFORE ANY DURABLE WRITE — the ordering is the point, not an
// implementation detail. A refused run must leave the store byte-identical,
// which it cannot do if a tombstone was recorded while deciding.
for (targets) |t| {
if (t.disposition.blocks()) {
renderShunBlockers(streams.err, targets);
return .failure;
}
}
// Two counts, and they are independent: `absent` records without removing,
// `reclaim` removes without recording. Either one alone is work to do.
var recorded: usize = 0;
var reclaiming: usize = 0;
for (targets) |t| {
if (t.disposition.records()) recorded += 1;
if (t.disposition.reclaims()) reclaiming += 1;
}
if (recorded == 0 and reclaiming == 0) {
// Every named hash was already in the set AND its content already gone: a
// genuine no-op, so it needs no `--force` and it is not a failure. Reaching
// this on a hash whose bytes SURVIVED is the defect `.reclaim` exists to
// prevent — it would report "nothing to do" over content the operator had
// just been ordered to remove.
streams.out.print(
"shunned: nothing to do; {d} hash{s} already shunned\n",
.{ targets.len, pluralEs(targets.len) },
) catch {};
return .ok;
}
if (!parsed.force) {
renderShunRefusal(streams.err, targets, deferredStillHeld(&src, targets), parsed.defer_reclaim);
return .failure;
}
// ONE owned name set drives both the drop predicate and the tombstone list,
// so a name cannot be recorded without also being dropped. It is the UNION of
// the source's already-deferred shuns and the hashes named now: `shunFilter`
// covers only the first half, and half a drop set would relocate into the new
// generation exactly the bytes this run reports as removed.
var set: NameSet = .{};
defer set.deinit(gpa);
for (src.shunList()) |row| set.add(gpa, row.hash) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "shun"),
// A persisted row whose key is not a complete digest is a corrupt store,
// and that is the one place `BadKey` here can come from.
error.BadKey => return shunFail(streams, error.Corrupt),
};
const row_buf = arena.alloc(logstore.ShunRecord, recorded) catch return stateOom(streams, "shun");
var n: usize = 0;
for (targets) |t| {
if (!t.disposition.records()) continue;
set.add(gpa, t.digest) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "shun"),
error.BadKey => return shunRowBug(streams),
};
row_buf[n] = .{ .hash = t.digest, .epoch = amb.now_epoch, .reason = parsed.reason orelse "" };
n += 1;
}
// SLICED, not asserted-and-passed-whole. `recorded` and `n` count the same pure
// predicate over the same unmutated `targets`, so they cannot differ today — but
// `row_buf` is arena memory and the FULL buffer used to be what went to
// `packLog` and `shunAppendRows`. Had they ever differed, `row_buf[n..]` would
// be undefined `ShunRecord`s whose `hash` is a garbage slice, handed to
// `putShun` on the way to a durable irreversible write — in `ReleaseFast`, where
// the assert below does not exist. Taking the slice makes the mismatch
// unrepresentable and leaves the assert as documentation of the tautology.
const rows = row_buf[0..n];
std.debug.assert(rows.len == recorded);
// Nothing to reclaim: take the append path even without `--defer`. See the
// crash-safety note in the doc above — the rewrite is not what makes this safe,
// and running it would cost a whole-store copy to remove nothing.
if (reclaiming == 0) {
shunAppendRows(gpa, io, root_dir, src_key, rows) catch |e| return shunAppendFail(streams, e);
renderShunRecordedOnly(streams.out, rows.len);
return .ok;
}
if (parsed.defer_reclaim) {
// `rows` is empty when every target was already tombstoned and only its
// bytes remain; skip the append rather than fsync for nothing.
if (rows.len != 0) {
shunAppendRows(gpa, io, root_dir, src_key, rows) catch |e| return shunAppendFail(streams, e);
}
renderShunDeferred(streams.out, rows.len, reclaiming);
return .ok;
}
// THE CURRENT-CHECKOUT STOP, RE-ASKED OF THE DROP SET ITSELF — one hashmap
// lookup, at the point of consequence. `classifyShunTarget` enforces it over the
// NAMED TOKENS, but `set` is the union of those with every row already in the
// shun table, and the rewrite deletes the union. A table that already contained
// this check-in's manifest would therefore have `--force` delete the check-in the
// operator is standing in while the per-token guard never fired — the refusal
// line would count it only as an anonymous integer of earlier deferred
// collateral. That state is not constructible through zit today (`--defer` goes
// through the same per-token guard, and S1's mask makes `zit update` to a shunned
// check-in fail), which is exactly why the invariant belongs here as well: the
// property `shun_help` promises is about what gets deleted, not about the
// argument list.
if (ck_version.len != 0) {
var ck_raw: [logstore.max_hash_len]u8 = undefined;
if (shunHexToDigest(ck_version, &ck_raw)) |ck_digest| {
if (set.contains(ck_digest)) {
renderShunBlockers(streams.err, &.{.{
.name = ck_version,
.digest = ck_digest,
.disposition = .current_checkout,
}});
return .failure;
}
}
}
const stats = rewriteAndSwap(
gpa,
io,
root_dir,
&src,
src_key,
set.filter(),
rows,
) catch |e| switch (e) {
error.TailChanged => return tailChangedFail(streams, "shun"),
error.RewriteBusy => return rewriteBusyFail(streams, "shun"),
else => |pe| return shunFail(streams, pe),
};
// Reported from the PACK's observation of the new generation, not from
// `rows.len`: see `renderShunStats`. They agree or `packLog`'s row-count gate
// failed the pack, which is why this is an assert rather than a branch.
std.debug.assert(stats.shuns_recorded == rows.len);
renderShunStats(streams.out, stats);
return .ok;
}
/// Resolve one operator token against `src` into a `ShunTarget`. Impure only in
/// that it reads the store: no policy, no output, no durable write, so the
/// runner's whole decision is a fold over the results.
///
/// RESOLUTION RULE. Every token is offered to prefix resolution FIRST, at every
/// width in `canonicalizeShunHex`'s band — the same band and the same resolver
/// `zit artifact` uses (cmd/history.zig), which is what makes "paste the same
/// token into both verbs" true rather than merely intended. Only when nothing
/// matches does a COMPLETE name (40 or 64 hex — artifact-format §3.2's two forms)
/// fall back to being used literally, because a hash this repository does not hold
/// is precisely the pre-emptive case worth recording; an ABBREVIATED prefix that
/// matches nothing is refused, since the shun table is keyed by complete name
/// (sync §3.3 L544-545), a prefix cannot be completed, and a row keyed on one would
/// read as a removal while naming nothing.
///
/// TRYING RESOLUTION FIRST IS THE LOAD-BEARING PART, not an optimisation. Treating
/// a complete-WIDTH token as a complete NAME without asking the store means a
/// 40-hex abbreviation of a 64-hex artifact (and a 64-hex abbreviation in a sha1
/// repository) records a tombstone naming nothing, reports success, and leaves the
/// content the operator was ordered to remove still readable — the `unresolved`
/// refusal exists to stop exactly that, and a width-first rule walks past it.
/// It also un-hides the current-checkout guard, which cannot fire on a token that
/// never resolved to the check-in's name.
fn classifyShunTarget(
gpa: std.mem.Allocator,
arena: std.mem.Allocator,
src: *logstore.LogStore,
ck_version: []const u8,
tok: []const u8,
) (logstore.ContentError || error{OutOfMemory})!ShunTarget {
var canon_buf: [2 * logstore.max_hash_len]u8 = undefined;
const canon = canonicalizeShunHex(tok, &canon_buf) orelse
return .{ .name = tok, .disposition = .not_a_name };
var raw_buf: [logstore.max_hash_len]u8 = undefined;
var digest: []const u8 = undefined;
if (src.resolveHexPrefix(canon) catch |e| switch (e) {
error.Ambiguous => return .{ .name = tok, .disposition = .ambiguous },
}) |rid| {
digest = src.hashOf(rid);
} else if (shunHexToDigest(canon, &raw_buf)) |d| {
digest = d;
} else {
return .{ .name = tok, .disposition = .unresolved };
}
std.debug.assert(logstore.isCompleteDigest(digest.len));
const owned_digest = try arena.dupe(u8, digest);
var hex_buf: [2 * digest_len]u8 = undefined;
const name = try arena.dupe(u8, digestToWireName(owned_digest, &hex_buf));
// THE CURRENT-CHECKOUT STOP IS CHECKED FIRST, ahead of the shun table. An
// already-tombstoned check-in whose bytes survive is otherwise `.reclaim`, and
// reclaiming it would delete the check-in the operator is standing on — the one
// thing this verb refuses outright. `--defer` goes through this same guard, so
// the state cannot arise through zit today; the ordering is what keeps that
// true if it ever does.
if (std.mem.eql(u8, name, ck_version)) return .{
.name = name,
.digest = owned_digest,
.disposition = .current_checkout,
};
const already = src.isShunned(owned_digest);
const rid = src.lookup(owned_digest) orelse return .{
.name = name,
.digest = owned_digest,
.disposition = if (already) .already else .absent,
};
// THROUGH THE MAINTENANCE READER, deliberately. `content` applies the shun
// mask, so an already-shunned artifact whose bytes are still on disk would read
// as absent and be reported as nothing left to do — which is the whole defect
// `.reclaim` exists to prevent. Shunning is maintenance, not an artifact read.
//
// A phantom is an announced name with no frame: there are no bytes to reclaim,
// so it is the `absent` case with a rid attached.
const content = (try src.contentUnmasked(gpa, rid, null)) orelse return .{
.name = name,
.digest = owned_digest,
.disposition = if (already) .already else .absent,
};
defer gpa.free(content);
// The type verdict, through the same door `zit artifact` uses: a structural
// artifact classifies (`parseWindow` runs §5's `classify` internally), and
// anything that does not parse is an opaque blob — a first-class answer, not
// an error. A blob is by far the common shun target.
const held: ShunDisposition = if (already) .reclaim else .remove;
var m = artifact.parseWindow(gpa, content) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return .{ .name = name, .digest = owned_digest, .kind = null, .disposition = held },
};
defer m.deinit();
return .{ .name = name, .digest = owned_digest, .kind = m.kind, .disposition = held };
}
/// Maps a transport-open failure (bad URL) to the `verb` diagnostic + exit 1.
/// Shared by clone and pull — the only difference is the verb label.
fn syncOpenFail(s: cli.Streams, verb: []const u8, url: []const u8, err: transport.TransportError) cli.Exit {
switch (err) {
error.InvalidUrl => s.err.print("zit {s}: '{s}' is not a valid URL\n", .{ verb, url }) catch {},
error.UnsupportedScheme => s.err.print("zit {s}: unsupported URL scheme in '{s}' (use http or https)\n", .{ verb, url }) catch {},
error.OutOfMemory => s.err.print("zit {s}: out of memory\n", .{verb}) catch {},
else => s.err.print("zit {s}: cannot open '{s}'\n", .{ verb, url }) catch {},
}
return .failure;
}
/// Maps a sync run (transport + driver) failure to the `verb` diagnostic + exit
/// 1. Exhaustive over the driver's own causes and the transport's, so the whole
/// closed error set is accounted for (no `else`). Shared by clone and pull.
/// `status` is the transport's `last_status` -- the HTTP status that produced an
/// `error.ProtocolError`, or null for every other cause. Named in the message
/// because "an unexpected HTTP status" sends an investigation nowhere: 500, 413,
/// 400 and 417 are four different bugs and the operator could not tell which one
/// they had hit.
/// Tolerate-and-report — one stderr line naming what the session
/// declined: unversioned files a fossil peer offered that zit has no store
/// for. Zero is silent; the count makes the data non-transfer VISIBLE, which
/// is the entire posture (the alternative was an abort nobody chose).
fn reportUvSkipped(s: cli.Streams, verb: []const u8, count: u64) void {
if (count == 0) return;
s.err.print(
"zit {s}: {d} unversioned file(s) not transferred (unsupported)\n",
.{ verb, count },
) catch {};
}
/// sync §5.1/§10.1 — a reply that ended inside a file, cfile or uvfile
/// payload: the cards before it were kept and the exchange went on, so a
/// later round can ask again for what was cut off. A warning for pull, push,
/// sync, config and uv; clone also fails, as Fossil's does.
fn reportTruncatedPayloads(s: cli.Streams, verb: []const u8, count: u64) void {
if (count == 0) return;
s.err.print(
"zit {s}: warning: {d} reply payload(s) cut short by the end of the reply; the cards before each were kept\n",
.{ verb, count },
) catch {};
}
/// What the transport did about an HTTP Basic challenge, for the 401 sentence.
const BasicState = enum { none, challenged, sent };
fn basicStateOf(tr: *const transport.Transport) BasicState {
if (tr.basic_armed) return .sent;
if (tr.basic_challenged) return .challenged;
return .none;
}
fn syncRunFail(s: cli.Streams, verb: []const u8, url: []const u8, err: sync_driver.RunError(*transport.Transport), status: ?std.http.Status, basic: BasicState) cli.Exit {
const reason: []const u8 = switch (err) {
error.OutOfMemory => "out of memory",
// Not "did not converge": the guard counts rounds that moved no bytes,
// so this fires only when the peer kept the exchange open while
// delivering nothing. A slow or very large transfer never reaches it.
error.RoundLimitExceeded => "the peer kept the exchange open without transferring anything",
error.FatalProtocolError => "the peer sent a fatal protocol error",
error.RequestTooLarge => "the sync request exceeded the transport limit",
error.ReplyTruncated, error.ReplyMalformed => "the peer sent a malformed reply",
// The streaming drain's own faults, surfaced through the
// driver: writing the destination log failed, or a received artifact
// did not verify against its name.
error.Storage => "cannot write the cloned repository",
error.Corrupt => "a received artifact failed verification",
error.ReplyTooLarge => "the peer's reply exceeded the size limit",
error.RedirectLimit => "too many HTTP redirects",
error.InsecureRedirect => "the peer redirected HTTPS to plaintext HTTP",
error.RedirectNotAllowed => "the peer redirected on a transport that forbids it",
error.MissingLocation => "the peer sent a redirect with no location",
error.InvalidUrl => "a redirect target was not a valid URL",
error.UnsupportedScheme => "a redirect named an unsupported URL scheme",
error.ProtocolError => blk: {
if (status) |st| {
if (st == .unauthorized) {
// The one status that means "ask again, properly". Which
// half is missing decides the sentence.
if (basic == .sent) {
s.err.print("zit {s}: the peer refused the login (HTTP 401)\n", .{verb}) catch {};
} else if (basic == .challenged) {
s.err.print("zit {s}: the peer wants a login (HTTP 401): give --user and --password, or set ZIT_PASSWORD\n", .{verb}) catch {};
} else {
s.err.print("zit {s}: the peer answered HTTP 401 (Unauthorized) without saying how to log in\n", .{verb}) catch {};
}
return .failure;
}
if (st == .service_unavailable) {
// What a zit server answers when another writer kept its
// repository locked; nothing was lost, so say to retry.
s.err.print("zit {s}: the peer's repository is busy with another writer (HTTP 503); try again\n", .{verb}) catch {};
return .failure;
}
s.err.print("zit {s}: the peer answered HTTP {d} ({s})\n", .{ verb, @backingInt(st), st.phrase() orelse "unrecognized" }) catch {};
return .failure;
}
break :blk "the peer answered with an unexpected HTTP status";
},
error.ConnectionFailed => {
s.err.print("zit {s}: cannot connect to {s}\n", .{ verb, url }) catch {};
return .failure;
},
error.InsecureCredential => {
// Name the fix, not just the refusal: the same URL over https.
const rest = if (std.ascii.startsWithIgnoreCase(url, "http://")) url["http://".len..] else url;
s.err.print(
"zit {s}: refusing to send credentials over unencrypted http://{s}; use https://{s}\n",
.{ verb, rest, rest },
) catch {};
return .failure;
},
};
s.err.print("zit {s}: {s}\n", .{ verb, reason }) catch {};
return .failure;
}
/// One stderr note when a sync crosses plaintext http to a host that is not
/// this machine. Credentials are refused there outright (the transport's
/// `InsecureCredential`), but the exchange itself still crosses readable: a
/// private repository's content included. Loopback is silent, which is how a
/// forge reaches its own zit child.
fn notePlaintext(s: cli.Streams, verb: []const u8, url: []const u8) void {
if (!transport.plaintextToRemote(url)) return;
s.err.print("zit {s}: note: {s} is unencrypted http; what this exchange carries is readable in transit\n", .{ verb, url }) catch {};
}
/// The causes `resolveRemoteUrl` can surface: a store read failure, OOM, or the
/// repository having no URL to use (`NoRemote` — neither an explicit arg nor a
/// saved remote). Closed set so the caller switches exhaustively.
const RemoteResolveError = error{ OutOfMemory, Storage, NoRemote };
/// sync §4/§6 — resolve the peer URL for a sync-family verb: an explicit `arg`
/// (non-empty) wins; otherwise the saved remote (`config.remote_url`) is used;
/// a repository with neither is a usage error (`NoRemote`). The saved URL is
/// copied into `buf` so it outlives the read-only config frame. `buf` need only
/// live until `transport.Transport.open` (which dupes the URL). An out-of-spec
/// stored URL longer than the config-field cap is treated as none.
fn resolveRemoteUrl(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
arg: []const u8,
buf: *[logstore.max_config_str]u8,
) RemoteResolveError![]const u8 {
if (arg.len > 0) return arg;
var file = cli.openStoreRead(io, dir) catch return error.Storage;
defer file.close(io);
const st = file.stat(io) catch return error.Storage;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
};
defer log.deinit(gpa);
var oc = (log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
}) orelse return error.NoRemote;
defer oc.deinit(gpa);
const saved = oc.config.remote_url;
if (saved.len == 0 or saved.len > buf.len) return error.NoRemote;
@memcpy(buf[0..saved.len], saved);
return buf[0..saved.len];
}
/// sync §4/§6 — the usage error (exit 2) when a sync-family `verb` is given no
/// URL and the repository has no saved remote to fall back to. Names the fix.
/// Shared by pull, push, and sync so the wording lives in one place.
fn noRemoteError(s: cli.Streams, verb: []const u8) cli.Exit {
s.err.print("zit {s}: no repository URL given and no saved remote\n", .{verb}) catch {};
s.err.writeAll("set one with 'zit remote URL', or pass the URL explicitly\n") catch {};
return .usage;
}
// ---------------------------------------------------------------------------
// `zit pull URL` — incremental sync into an EXISTING repository. The sibling of
// clone: it reuses the same transport + `runSync` + persist machinery, but opens
// the cwd's `.zit` (never creating one), announces what it already holds so the
// peer sends only the missing artifacts (sync §3.2/§3.3 igot/gimme), and drives
// to §3.7 termination. Receive-only (§3.3 r24: pull obtains but never uploads).
// ---------------------------------------------------------------------------
/// `zit pull URL` — open the current repository, converge it toward the peer at
/// URL, and persist the newly received artifacts. Thin wiring over the same deep
/// modules clone uses: `transport` (HTTP/IO), `sync.Client` + `runSync` (sans-IO
/// §3.7 loop), and `LogStore` (persistence). With URL omitted the saved remote
/// (`zit remote`) is used; a repo with neither is a usage error. Anonymous only.
pub fn runPull(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
const parsed = switch (cli.parsePullArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "pull", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "pull", .extra_argument, tok),
// An omitted URL is not an error: it falls back to the saved remote below.
.no_url => cli.PullArgs{ .url = "" },
};
if (parsed.help) {
streams.out.writeAll(pull_help) catch {};
return .ok;
}
if (parsed.user) |u| {
if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) {
streams.err.writeAll("zit pull: --user must not contain whitespace\n") catch {};
return .usage;
}
}
// Locate the current repository before any network I/O: a pull outside a repo
// is a usage failure, not a wasted connection.
const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "pull")) orelse {
streams.err.writeAll(not_a_repo_msg) catch {};
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
// sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved
// remote; a repo with neither is a usage error. `url_buf` outlives `open`.
var url_buf: [logstore.max_config_str]u8 = undefined;
const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "pull"),
error.Storage => return repoReadFail(streams, "pull"),
error.NoRemote => return noRemoteError(streams, "pull"),
};
var tr = transport.Transport.open(io, gpa, url) catch |e|
return syncOpenFail(streams, "pull", url, e);
defer tr.close();
notePlaintext(streams, "pull", url);
// The client's holdings view: igot names, held/phantom, shun mask, and
// hash-policy off the log's frame index, with content lazy (the
// old whole-store `MemStore` snapshot held the repository's plaintext
// resident for the run).
var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "pull"),
error.Storage => return repoReadFail(streams, "pull"),
};
defer holdings.deinit(io);
var admission = holdings.admission();
// Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once
// and suppress spontaneous requests for elided phantoms. Both the
// classification and the wrapper outlive the run below.
var sync_cone: ?zit.cone.Cone = null;
defer if (sync_cone) |*c| c.deinit(gpa);
var sync_classification: ?SparseClassification = null;
defer if (sync_classification) |*c| c.deinit(gpa);
var sync_elided: RepoElidedSet = undefined;
if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| {
sync_cone = c;
sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch
return stateOom(streams, "pull");
sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() };
admission.elided = sync_elided.shunSet();
admission.cone_members = sync_cone.?.members;
admission.cone_not_members = sync_cone.?.not_members;
}
// §9.1 — announce this repository's adopted project code in the pull card;
// a real peer refuses a PROJECTCODE mismatch, so the placeholder cannot be
// sent against fossil. The buffer outlives the run (the client borrows it).
// The cluster index: what the held clusters already announce
// (never announced again individually) and what they promised that this
// repository never collected (seeded as phantoms, so the run asks for it).
// The clusters' plaintext is memoised by the holdings adapter while it
// resolves; drop it — the index keeps only names and bits.
var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "pull");
defer clusters.deinit();
holdings.index.releaseTransient();
var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined;
const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "pull"),
error.Storage => return repoReadFail(streams, "pull"),
};
// sync §6 — a read of a private repository is as gated as a write. The
// login comes from --user, the URL's user@, or the login a previous
// authenticated push or pull stored here; with none the pull is anonymous.
var pw_buf: [max_password_len]u8 = undefined;
defer std.crypto.secureZero(u8, &pw_buf);
var user_buf: [logstore.max_config_str]u8 = undefined;
const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf, &user_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "pull"),
error.Storage => return repoReadFail(streams, "pull"),
error.NoPassword => {
streams.err.print("zit pull: no password for '{s}' (give --password, set ZIT_PASSWORD, or run on a terminal)\n", .{parsed.user orelse urlUser(url) orelse ""}) catch {};
return .failure;
},
};
// Live per-round progress on a terminal, the same sink clone uses.
const stderr_tty = std.Io.File.stderr().isTty(io) catch false;
var progress: CloneProgress = .{ .err = streams.err };
const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null;
defer progress.finishLine();
if (resolved) |r| armBasicAuth(&tr, r.credential.user, r.plaintext) catch return stateOom(streams, "pull");
var truncated: u64 = 0;
defer reportTruncatedPayloads(streams, "pull", truncated);
const outcome = pullRun(gpa, &tr, holdings.store(), identity, if (resolved) |r| r.credential else null, admission, sink, &clusters, &truncated) catch |e|
return syncRunFail(streams, "pull", url, e, tr.last_status, basicStateOf(&tr));
// Persist a freshly derived secret only after the peer accepted the
// authenticated session (mirrors push and sync).
if (resolved) |r| {
if (r.persist) {
storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "pull"),
else => return repoReadFail(streams, "pull"),
};
}
}
// Sparse-cone RFC section 6.3 applies to EVERY receive: a dense peer
// volunteers content the client never announced, so the filter runs at
// persist. Reclassified over the post-run store, so a brand-new commit's
// blobs classify by their own F cards -- new in-cone content persists,
// new out-of-cone content elides with the rest.
var post_classification: ?SparseClassification = null;
defer if (post_classification) |*c| c.deinit(gpa);
if (sync_cone) |*c| {
post_classification = classifyKeepRids(gpa, holdings.store(), c) catch
return stateOom(streams, "pull");
}
// Persist only what this pull received. The holdings store appends
// received artifacts beyond its log-backed prefix, so the walk begins at that
// boundary and never re-visits the already-durable holdings. A pull that
// received nothing persists nothing: a no-op pull was O(store) here (every
// holding read once from disk to be early-returned), 196 s on a 905k store.
const persisted = persistStore(gpa, io, root_dir, holdings.store(), if (post_classification) |*c| &c.keep else null, holdings.logBackedCount()) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "pull"),
error.Storage => return repoReadFail(streams, "pull"),
error.StoreBusy => return .failure, // the lock notice already said it
error.Corrupt => return renderStorage(streams, root.abs, "the peer sent a malformed artifact name"),
};
// The walk started past the holdings, so `written` is exactly the received
// count (each received artifact is persisted once).
const received = persisted.written;
// The peer named artifacts this run never obtained. Everything is persisted
// — a partial pull is still progress — but the exit status says so, because
// "exit 0 with nothing said" is indistinguishable from a converged sync and
// leaves a repository quietly out of date. This holds whether `received` is
// zero or not: the invariant is convergence, not motion.
// A promise a held cluster made that this peer could not keep is
// reported, not failed on: the run asked on its own initiative (no peer
// igot set the expectation) and the repository IS converged against this
// peer. Fossil asks for these every round, silently; zit says so once.
if (outcome.cluster_gaps > 0) {
streams.err.print(
"zit pull: {d} artifact{s} named by held clusters could not be obtained from this peer\n",
.{ outcome.cluster_gaps, plural(outcome.cluster_gaps) },
) catch {};
}
if (outcome.unresolved > 0) {
streams.err.print(
"zit pull: the peer holds {d} artifacts this pull did not obtain\n",
.{outcome.unresolved},
) catch {};
return .failure;
}
// submanifests §7.1/§7.3 — the same reasoning one level down. A pull can
// converge against the peer (nothing left to gimme) while the tree closure
// this repository now holds is still open, because the peer's own closure
// was open: §7.1's "at most d+1 pull rounds" bounds a tree whose levels are
// ARRIVING, not one whose leaves nobody has. The phantoms the persist hook
// minted are that distinction, so report them rather than call it converged.
//
// §7.3/§9.4 — two counts, because they answer different questions and the
// draft names both: `submanifest_gaps` is the durable store's missing-hash
// count, `deferred_trees` is the session's DEPENDENT-keyed record ("enough
// to rediscover that manifest"). A run reporting the first without the
// second tells the operator something is missing but not what it breaks.
if (persisted.submanifest_gaps > 0 or outcome.deferred_trees > 0) {
// Two sentences, not one, because the two counts have different
// LIFETIMES and a single sentence reads as one fact. `submanifest_gaps`
// is a property of the durable store and survives a restart;
// `deferred_trees` is this SESSION's §9.4 record and is zero for a run
// that received no S card — including every re-run against an already
// open closure. Printed together they said "N references are missing; 0
// check-ins cannot be enumerated", which is exactly backwards.
if (persisted.submanifest_gaps > 0) {
streams.err.print(
"zit pull: {d} sub-manifest references are missing; the check-ins that reference them cannot be enumerated\n",
.{persisted.submanifest_gaps},
) catch {};
}
if (outcome.deferred_trees > 0) {
streams.err.print(
"zit pull: {d} check-ins received this session are deferred until they arrive\n",
.{outcome.deferred_trees},
) catch {};
}
return .failure;
}
if (received == 0) {
streams.out.print("pull from {s}: already in sync\n", .{url}) catch {};
} else {
streams.out.print("pull from {s}: received {d} artifact{s} in {d} round{s}\n", .{ url, received, plural(received), outcome.rounds, plural(outcome.rounds) }) catch {};
reportUvSkipped(streams, "pull", outcome.uv_skipped);
}
return .ok;
}
/// Drives a pull client (§3.2/§3.3) to convergence over `tr`, filling `ms` with
/// the artifacts the peer holds and this side lacked; returns the round count.
/// The client announces `ms`'s holdings via igot and requests the rest via
/// gimme (receive-only — §3.3 r24: pull obtains content but never uploads).
/// Sans-IO on this side of `tr.exchange`. Its closed error set is the driver's
/// own causes unioned with the transport's, so `runPull` switches it exhaustively.
fn pullRun(
gpa: std.mem.Allocator,
tr: *transport.Transport,
holdings: repostate.ArtifactStore,
identity: RepoSyncIdentity,
credential: ?zit.sync.Credential,
admission: RepoAdmission,
sink: ?sync_driver.RoundSink,
clusters: ?*const clusterindex.Resolved,
truncated_out: *u64,
) sync_driver.RunError(*transport.Transport)!PullOutcome {
var client = syncClient(.pull, &identity, credential, admission);
defer client.deinit(gpa);
defer truncated_out.* = client.payloads_truncated;
if (clusters) |c| {
client.clustered = &c.clustered;
try client.seedClusterPhantoms(gpa, holdings, c.missing);
}
const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null);
recordSyncStats(stats);
return .{
.rounds = stats.rounds,
.unresolved = client.unresolvedPhantoms(holdings),
.cluster_gaps = client.clusterGaps(holdings),
// submanifests §7.3/§9.4 — the DEPENDENT-keyed half of the verdict: how
// many received check-ins still have a recorded S-reference gap.
.deferred_trees = client.deferredTrees(),
.uv_skipped = client.uv_skipped,
};
}
/// What one pull run reports: the round count, and how many artifacts the peer
/// announced (§3.3 r24 — by igot, or as a member of a cluster it sent) that this
/// side still lacks when §3.7 reached its clean terminal. A nonzero `unresolved`
/// is a pull that stopped short of what it was told existed; the caller must not
/// report success for it, however well-formed every card on the wire was.
/// submanifests §7.3/§9.4 — `deferred_trees` is the §9.4 record read back: the
/// count of DEPENDENTS (check-ins) whose processing this session deferred on a
/// sub-manifest that never arrived. `unresolved` counts missing HASHES; a
/// dependent-keyed count is what §9.4 requires be recorded and is what an
/// operator can act on ("which check-in is broken", not "how many blobs").
const PullOutcome = struct { rounds: usize, unresolved: usize, deferred_trees: usize = 0, uv_skipped: u64 = 0, cluster_gaps: usize = 0 };
/// sync §3.3 L553-556 / §8.8 L1839-1841 — this repository's shun table as the
/// `zit.sync.ShunSet` the phantom scan asks. Owns a copy of every shunned
/// DIGEST, taken while the caller already has the log open, so the set outlives
/// the handle it was read through.
///
/// FRESHNESS, stated exactly. This is a SNAPSHOT, not a live view: `load` reads
/// `shunList` once and the set answers from that copy thereafter. On the SERVE
/// path that is per-round freshness, because `LogIndexStore` rebuilds — and so
/// re-loads this — whenever the stat gate sees the log grow, which a `zit shun`
/// append does; a shun written between two served rounds is honoured by the next
/// one. On the CLIENT path it is a PROCESS snapshot: `ClientHoldings` builds
/// its index once at run start and never rebuilds mid-run, so a shun applied
/// mid-run lands on the next process, not the next round. That is a
/// deliberate limit, recorded here rather than implied away — the protocol layer
/// re-probes every scan (`sync.ShunSet`), so nothing else has to change if this
/// ever grows a re-read.
///
/// This is the ONE place a wire NAME becomes a log DIGEST for shun purposes —
/// the boundary `LogIndexStore.names` already declares is held to one place. The
/// width/hex validation is a CHECKED rejection rather than an assert, because an
/// assert is nothing in ReleaseFast: `logstore.LogStore.isShunned`'s caller
/// contract (see its doc) binds this boundary to reject a non-complete name
/// BEFORE decoding it, so the width is gated on the NAME, ahead of the decode.
/// The sync-facing elided-set: answers RFC section 5.2 rule 3 over a
/// classification computed once at run start. Same total-function contract
/// as `RepoShunSet`; borrows both the classification and the store, so both
/// must outlive the run.
/// `pack --drop-elided`'s filter half (sparse-cone RFC section 5.4): built
/// once over the SOURCE log, drops exactly the classification's elide set.
/// A digest that does not resolve in the index is kept -- unknown provenance
/// is never deleted.
const ElidedDropper = struct {
index: *LogIndexStore,
classification: SparseClassification,
inner: zit.pack.Filter,
fn build(gpa: std.mem.Allocator, src: storage.StorageFile, cone: *const zit.cone.Cone) ?ElidedDropper {
const index = LogIndexStore.open(gpa, src) catch return null;
const classification = classifyKeepRids(gpa, index.store(), cone) catch {
index.deinit();
return null;
};
return .{ .index = index, .classification = classification, .inner = zit.pack.Filter.keep_all };
}
fn drop(ctx: *anyopaque, digest: []const u8) bool {
const self: *ElidedDropper = @ptrCast(@alignCast(ctx));
if (self.inner.dropFn(self.inner.ctx, digest)) return true; // shun reclaim first
var name_buf: [2 * logstore.max_hash_len]u8 = undefined;
const name = digestToWireName(digest[0..digest.len], &name_buf);
const rid = self.index.store().lookup(name) orelse return false;
return self.classification.elided(rid);
}
fn composed(self: *ElidedDropper, inner: zit.pack.Filter) zit.pack.Filter {
self.inner = inner;
return .{ .ctx = @ptrCast(self), .dropFn = drop };
}
fn deinit(self: *ElidedDropper, gpa: std.mem.Allocator) void {
self.classification.deinit(gpa);
self.index.destroy();
self.* = undefined;
}
};
const RepoElidedSet = struct {
classification: *const SparseClassification,
store: repostate.ArtifactStore,
fn contains(context: *anyopaque, name: []const u8) bool {
const self: *const RepoElidedSet = @ptrCast(@alignCast(context));
const rid = self.store.lookup(name) orelse return false;
return self.classification.elided(rid);
}
fn shunSet(self: *RepoElidedSet) zit.sync.ShunSet {
return .{ .context = @ptrCast(self), .vtable = &.{ .contains = contains } };
}
};
const RepoShunSet = struct {
gpa: std.mem.Allocator,
/// Keys are owned binary digests; the value side carries nothing — this is a
/// set. Empty is the common case (no repository has a shun row until an
/// operator writes one), and an empty map allocates nothing.
digests: std.StringHashMapUnmanaged(void) = .empty,
fn init(gpa: std.mem.Allocator) RepoShunSet {
return .{ .gpa = gpa };
}
/// Copy every shun row's digest out of `log`. `shunList` borrows the store's
/// own bytes, which die with the handle they were read through, so each key
/// is duped. Re-loading over a populated set replaces it wholesale, which is
/// what a `LogIndexStore` rebuild wants.
fn load(self: *RepoShunSet, log: *const logstore.LogStore) error{OutOfMemory}!void {
self.clear();
const rows = log.shunList();
// State the bound where it is relied on rather than narrowing usize into
// the map's u32 capacity argument by an unchecked cast: a truncated
// reservation would let `getOrPutAssumeCapacity` below write past the
// capacity it assumes, which is silent corruption in ReleaseFast instead
// of the `error.OutOfMemory` a shun table that large has already earned.
// This is the one such count an operator grows by direct action.
const row_count = std.math.cast(u32, rows.len) orelse return error.OutOfMemory;
try self.digests.ensureTotalCapacity(self.gpa, row_count);
for (rows) |row| {
// Dupe BEFORE inserting. `row.hash` belongs to the store and dies
// with the handle, so the key must be owned — and inserting the
// borrowed slice first and replacing it after leaves a BORROWED key
// in an owned-key map on the one error this function can return.
// Every unwind path then runs `clear`, which frees the store's own
// live digest buffer: a double free at `log.deinit`, and a
// use-after-free of `shun_entries[i].hash` while the store is open.
const owned = try self.gpa.dupe(u8, row.hash);
// Capacity is already reserved for every row, so this cannot fail.
const gop = self.digests.getOrPutAssumeCapacity(owned);
// Two rows naming the same digest collapse to one entry, and the
// second dupe is surplus — say so rather than leaking it.
if (gop.found_existing) self.gpa.free(owned) else gop.key_ptr.* = owned;
}
}
fn clear(self: *RepoShunSet) void {
var it = self.digests.keyIterator();
while (it.next()) |k| self.gpa.free(k.*);
self.digests.clearAndFree(self.gpa);
}
fn deinit(self: *RepoShunSet) void {
self.clear();
}
fn shunSet(self: *RepoShunSet) zit.sync.ShunSet {
return .{ .context = self, .vtable = &vtable };
}
const vtable: zit.sync.ShunSet.VTable = .{ .contains = containsWireName };
/// Exact-key lookup on the digest a COMPLETE wire name decodes to. A name
/// whose WIDTH is not one of artifact-format §3.2 L347-350's two forms is
/// rejected before the decode, which is the order `isShunned`'s caller
/// contract fixes; a correct-width token that is not hex fails the decode.
/// Neither can be a key any shun row carries (`putShun` rejects a
/// non-complete key with `error.BadKey`, and the open-time scan fails
/// `error.Corrupt` on one), so `false` is the exact answer. Never a prefix
/// match, never a truncate, never a pad: a prefix must not be able to
/// masquerade as a shun-table hit.
///
/// An UPPERCASE-hex spelling of a shunned name decodes to the same digest
/// and therefore answers TRUE. artifact-format §3.3 L370-377 makes such a
/// token invalid but explicitly lets an implementation accept it at name
/// validation and fail it at verification, and this direction is the safe
/// one: it over-suppresses a name that can never verify against content
/// anyway. No spelling of a shunned name can BYPASS the filter, which is the
/// only direction §3.3 L553-556 cares about.
fn containsWireName(context: *anyopaque, name: []const u8) bool {
const self: *RepoShunSet = @ptrCast(@alignCast(context));
if (self.digests.count() == 0) return false;
if (name.len != 2 * logstore.sha1_digest_len and name.len != 2 * logstore.sha3_256_digest_len) return false;
var digest_buf: [digest_len]u8 = undefined;
const digest = decodeWireName(name, &digest_buf) orelse return false;
std.debug.assert(logstore.isCompleteDigest(digest.len));
return self.digests.contains(digest);
}
};
/// Opens `dir/.zit` read-only and snapshots its stored artifacts into `ms` — the
/// held/phantom split the pull client announces via `igot` (sync §3.2/§3.3) —
/// and its shun table into `shun` (sync §3.3 L553-556), both off the one open.
/// Built by `loadContentStore`, which is the CLIENT's builder and not `serve`'s:
/// `serve` builds a `LogIndexStore` instead, whose §8.2 slot-preserving mask,
/// masked write doors and `read_fault` channel this snapshot has no use for. The
/// two share the `digestToWireName` encoding and the held/phantom split, and
/// nothing else — a filter added to one does NOT reach the other, which is why
/// the §3.3 masked-basis guard is stated in both. The
/// read-only handle is closed before `persistStore` reopens the log read-write,
/// which is why `shun` owns its keys rather than borrowing the store's.
/// sync §7.7 L1463-1467 — this repository's EFFECTIVE hash-policy value, as the
/// §7.7 target-admission predicate needs it: read from the repository's own
/// stored config, never from peer state or a card field.
///
/// The stored byte is opaque to storage (`logstore.Config.hash_policy`), so two
/// cases have no defined policy: a repository carrying no config frame at all,
/// and a byte naming no artifact-format §7.1 Table 7 ordinal. Neither is
/// shun-sha1 — the ONLY distinction §7.7 draws — so both answer `.auto` and the
/// receive path admits, rather than failing a whole sync over a field this
/// clause reads for one boolean. That is a read-side default, not a repair: the
/// stored byte is left exactly as found.
///
/// A READ FAULT IS NOT ONE OF THOSE TWO CASES. `readConfig` also returns
/// `Storage` (the frame would not read) and `Corrupt` (the frame is not a config
/// frame, or does not parse), and both mean the policy is UNKNOWN rather than
/// undefined. Answering `.auto` there would silently disable the shun-sha1 half
/// of the §7.7 predicate and admit SHA1-named content this repository is
/// configured to refuse, with nothing failing — the same fail-open `serveRound`
/// refuses for the shun-table half ("must fail the round rather than silently
/// serve with no shun table"). So they propagate, exactly as `RepoShunSet.load`
/// does three lines above the serve-side call site.
fn effectiveHashPolicy(gpa: std.mem.Allocator, log: *logstore.LogStore) error{ OutOfMemory, Storage }!artifact.HashPolicy {
var oc = (log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
// A config frame that will not read or will not parse is a storage-level
// failure of this repository, not a policy answer.
error.Storage, error.Corrupt => return error.Storage,
}) orelse return .auto;
defer oc.deinit(gpa);
return std.enums.fromInt(artifact.HashPolicy, oc.config.hash_policy) orelse .auto;
}
/// sync §3.3 L553-556 / §7.7 L1469-1472 — the two repository-local inputs to the
/// receive-side filters, as ONE value. They are halves of one question (§7.7's
/// combined target predicate reads both; §3.3's phantom scan reads the table),
/// they are always read off the same open, and a client wired with one but not
/// the other silently applies half a filter with nothing failing. Travelling as
/// a pair is what makes that unrepresentable: `syncClient` takes this struct, so
/// there is exactly one place either field is handed to a `Client`.
///
/// `shun` borrows the caller's `RepoShunSet`, which must outlive the run.
const RepoAdmission = struct {
shun: zit.sync.ShunSet,
hash_policy: artifact.HashPolicy,
/// RFC section 5.2 rule 3 -- `.none` for every dense store.
elided: zit.sync.ShunSet = .none,
/// RFC section 6.1 -- the store cone's raw members, borrowed from the
/// run-lifetime cone; empty for a dense store. The client encodes them
/// as `pragma cone MEMBER` cards so a cone-aware peer filters replies.
cone_members: []const []const u8 = &.{},
/// The store cone's exclusion members (`pragma cone-not MEMBER`).
cone_not_members: []const []const u8 = &.{},
};
/// The ONE construction of a sync `Client` from repository state. `pullRun`,
/// `pushRun` and `syncRun` differ only in the operation tag and whether a
/// credential is named, so everything else — the servercode, the version, and
/// both halves of `RepoAdmission` — is stated here once. Three copies of this
/// literal is how a repository filter comes to be wired into two verbs and not
/// the third.
fn syncClient(
op: zit.sync.Operation,
identity: *const RepoSyncIdentity,
credential: ?zit.sync.Credential,
admission: RepoAdmission,
) zit.sync.Client {
return .{
.state = zit.sync.initialState(op),
.operation = op,
.self_servercode = &identity.server_code,
.project_code = identity.project_code,
.version = sync_client_version_info,
// sync §6 — a named credential makes `buildRequest` emit the §6.1 login
// card; null leaves the session anonymous.
.credential = credential,
// sync §3.3 L553-556 — the per-round phantom scan's shun filter.
.shun = admission.shun,
.elided = admission.elided,
.cone_members = admission.cone_members,
.cone_not_members = admission.cone_not_members,
// sync §7.7 L1463-1467 — this repository's effective hash-policy, the
// SHA1-name half of the receive-side target-admission predicate.
.hash_policy = admission.hash_policy,
// submanifests §8.1/§7.2 — the durable indicator and the §8.1 emission
// setting together arm the §7.2 upload gate. A pull uploads nothing, so
// both are inert there; they are set for every operation so no caller has
// to remember which ones upload, which is the same reason the credential
// and the admission pair live here rather than at three call sites.
.seen_submanifest = identity.seen_submanifest,
.submanifests_enabled = identity.enable_submanifests,
};
}
/// The sync client's holdings view, log-backed and lazy. The clients
/// used to snapshot every stored artifact into a `MemStore` (`loadRepoHoldings`)
/// before the first wire byte, which resident-loaded the whole store's
/// plaintext per pull/push/sync — 3.7 GB for a 147,920-artifact store, an OOM
/// kill at tcl scale — just to announce igot names. This seats the client on
/// the serve side's `LogIndexStore` instead: names, held/phantom, the
/// shun mask, the hash-policy, and the config facts come off the frame index;
/// content and wire forms materialize per-rid on demand; received artifacts
/// land in the index's appendix `MemStore` and are the only new bytes
/// resident. `persistStore` then walks the same store — every pre-held rid
/// early-outs on the append log's own `lookup`, so persist stays flat too.
///
/// The read handle stays open across the run and the persist; readers take no
/// lock (`cli.openStoreRead`), so the append side's exclusive lock is not
/// contested by it.
const ClientHoldings = struct {
file: File,
index: *LogIndexStore,
fn open(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) error{ OutOfMemory, Storage }!ClientHoldings {
var file = cli.openStoreRead(io, dir) catch return error.Storage;
errdefer file.close(io);
const st = file.stat(io) catch return error.Storage;
const index = LogIndexStore.openFromFile(gpa, io, file, st.size) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
};
return .{ .file = file, .index = index };
}
fn store(self: *ClientHoldings) repostate.ArtifactStore {
return self.index.store();
}
/// The number of already-durable, log-backed artifacts. Received
/// artifacts are appended beyond this, so persist begins here and never
/// re-walks the holdings a pull did not change.
fn logBackedCount(self: *ClientHoldings) repostate.Rid {
return @intCast(self.index.rids.len);
}
/// The §3.3/§7.7 admission pair, borrowed from the index — which outlives
/// the run exactly as the caller-owned `RepoShunSet` used to.
fn admission(self: *ClientHoldings) RepoAdmission {
return .{ .shun = self.index.shun.shunSet(), .hash_policy = self.index.hash_policy };
}
fn deinit(self: *ClientHoldings, io: std.Io) void {
self.index.destroy();
self.file.close(io);
}
};
/// sync §9.1 — the repository's stored project code, copied into `buf`, or the
/// zero placeholder when the repo carries none (a repo predating the stored
/// code) or an out-of-spec-length one. `init` derives a project code and a
/// clone adopts the origin's (`storeClonedProjectCode`), so a real repo
/// announces its adopted code here; the pull/push client emits it verbatim as
/// the operation card's PROJECTCODE, and a real peer (fossil §9.1, ABNF
/// `pull-card = "pull" SP servercode SP projectcode`) refuses a mismatch — so a
/// zero placeholder would be rejected. Bounded by `max_projectcode_len`: a
/// longer stored field is out of spec and never truncated into the wire.
fn repoProjectCode(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
buf: *[zit.sync.max_projectcode_len]u8,
) error{ OutOfMemory, Storage }![]const u8 {
return (try repoSyncIdentity(gpa, io, dir, "test-fixture-repo", buf)).project_code;
}
/// The durable config facts every sync-family run needs before it can build
/// a peer: §9.1's project code and draft-fossil-submanifests-00 §8.1's
/// seen-submanifest indicator and emission setting. Read together because they
/// come out of the same config frame — a second open to fetch another field
/// would be one more way for them to disagree about which frame they came from.
const RepoSyncIdentity = struct {
/// §9.1 — the adopted project code, or the zero placeholder.
project_code: []const u8,
/// submanifests §8.1 — has this repository ever stored a §7.2-gated
/// artifact? False for every repository predating this draft, and the
/// second thing that arms the §7.2 send gate (see `Server.seen_submanifest`).
seen_submanifest: bool,
/// submanifests §8.1 — the emission setting. The client twin of
/// `serveContext`'s server-side refresh: it is the OTHER half of the §7.2
/// arming disjunction (`Client.mayHoldSubmanifests`). Today it is
/// belt-and-braces, because any repository actually holding a gated artifact
/// also has `seen_submanifest` set — the two are written in one `log.commit`
/// with the first tree. It is threaded anyway so the field is never read
/// stale: leaving it at its `false` default would silently disarm the gate
/// for any future surface that commits mid-session, where the setting is on
/// but the indicator has not been written yet.
enable_submanifests: bool,
/// §9.1 — this instance's own SERVERCODE, derived from the checkout's
/// absolute path (`deriveServerCode`); informational on the wire and
/// never adopted from a peer. By value so the client's borrow has no
/// lifetime to manage beyond the identity itself.
server_code: zit.sync.HashHex,
};
fn repoSyncIdentity(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
abs: []const u8,
buf: *[zit.sync.max_projectcode_len]u8,
) error{ OutOfMemory, Storage }!RepoSyncIdentity {
var file = cli.openStoreRead(io, dir) catch return error.Storage;
defer file.close(io);
const st = file.stat(io) catch return error.Storage;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
};
defer log.deinit(gpa);
var oc = (log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
}) orelse return .{
.project_code = sync_projectcode,
.seen_submanifest = false,
.enable_submanifests = zit.sync.submanifest_emission_default,
.server_code = zit.sync.deriveServerCode(&.{abs}),
};
defer oc.deinit(gpa);
const seen = oc.config.seen_submanifest;
const enabled = oc.config.enable_submanifests;
const server_code = zit.sync.deriveServerCode(&.{abs});
const code = oc.config.project_code;
if (code.len == 0 or code.len > buf.len) return .{
.project_code = sync_projectcode,
.seen_submanifest = seen,
.enable_submanifests = enabled,
.server_code = server_code,
};
@memcpy(buf[0..code.len], code);
return .{
.project_code = buf[0..code.len],
.seen_submanifest = seen,
.enable_submanifests = enabled,
.server_code = server_code,
};
}
/// The number of content-bearing (non-phantom) artifacts in `store` — the held
/// set (§3.3 r24). A pull measures its yield as the held count after convergence
/// minus the count before, so "already up to date" is an exact zero, never a guess.
fn contentCount(store: repostate.ArtifactStore) usize {
// `held`, never `content`: on the log-backed client store a
// `content()` probe MATERIALIZES the rid's plaintext — asking it of every
// rid re-inflates the whole repository to produce a number, which is the
// exact whole-store cost this store exists to remove. `held` answers from
// the index, and `persistStore` counts `written` by the same predicate,
// so the `written >= held_before` arithmetic stays aligned.
return store.heldCount();
}
// ---------------------------------------------------------------------------
// `zit push URL` — the mirror of pull: UPLOAD this repository's artifacts into an
// EXISTING served repo. It opens the cwd's `.zit`, announces what it holds so the
// peer requests only what it lacks (sync §3.2/§3.3 igot/gimme), and uploads the
// missing artifacts as `file` cards, driving to §3.7 termination. The peer's
// `zit serve` persists what it receives; nothing changes on this side (§3.3 r24).
// Anonymous only.
// ---------------------------------------------------------------------------
/// What one push run reports: the number of artifacts uploaded and the round
/// count it took. `sent` reads the client's own upload counter, so it is the
/// exact count of `file` cards emitted — never an estimate.
/// submanifests §7.2 — `refused` counts artifacts the §7.2 gate declined to
/// upload because the peer's announced version predates sub-manifest support.
/// Carried alongside `sent` so a run that withheld an artifact can never be
/// reported as a success: the draft rules out silent omission.
const PushOutcome = struct { rounds: usize, sent: usize, refused: usize = 0, uv_skipped: u64 = 0 };
// ---------------------------------------------------------------------------
// `zit uv` -- unversioned files, moved between a server and plain files.
//
// quarry keeps a repository's unversioned files in its own storage and answers
// the uv cards itself, so a zit repository holds no copy of them: this client
// lists, fetches, sends and deletes them on the server and writes nothing to
// `.zit`. The wire is fossil's (sync.wiki §3.11): `pragma uv-hash`, answered
// by `uv-push-ok` or `uv-pull-only` and a `uvigot` catalog, then `uvgimme` and
// `uvfile`. A uv exchange carries no `pull` or `push` card; that is what a
// stock fossil client sends for `fossil uv sync`, captured on the wire.
// ---------------------------------------------------------------------------
/// The catalog hash this client announces. A fossil server answers only when
/// the client's hash differs from its own, so a client announcing an empty
/// catalog would hear nothing from a server holding no files, which reads the
/// same as a server that does not share. No catalog hashes to this.
const uv_hash_never = "0000000000000000000000000000000000000000";
/// Rounds a pull may take: a server may answer a batch of `uvgimme` with only
/// some files when a reply would be too large, so the rest are asked again.
const uv_max_rounds: usize = 64;
/// Fossil's limit on an unversioned file's name.
const uv_max_name: usize = 500;
const UvAccess = enum { none, pull_only, push_ok };
const UvEntry = struct {
name: []const u8,
mtime: i64,
/// `-` for a tombstone.
hash: []const u8,
size: u32,
fn deleted(self: UvEntry) bool {
return std.mem.eql(u8, self.hash, "-");
}
};
const UvFile = struct { name: []const u8, content: []const u8 };
/// What one reply said, arena-owned.
const UvReply = struct {
access: UvAccess = .none,
catalog: std.ArrayList(UvEntry) = .empty,
files: std.ArrayList(UvFile) = .empty,
/// The server's own words from an `error` card, decoded.
refusal: ?[]const u8 = null,
/// sync §5.1/§10.1 — the body ended inside a uvfile payload; the cards
/// before it stand and the rest of the reply was not read.
truncated: bool = false,
fn find(self: *const UvReply, name: []const u8) ?UvEntry {
for (self.catalog.items) |e| if (std.mem.eql(u8, e.name, name)) return e;
return null;
}
};
/// Fossil's rules for an unversioned name: no whitespace or control
/// characters, at most 500 bytes, a relative path with no `.` or `..`
/// component. `releases/1.2.3/x.tar.gz` is just a name.
fn uvNameOk(name: []const u8) bool {
if (name.len == 0 or name.len > uv_max_name) return false;
if (name[0] == '/' or name[name.len - 1] == '/') return false;
for (name) |c| if (c <= ' ' or c == 0x7f or c == '\\') return false;
var it = std.mem.splitScalar(u8, name, '/');
while (it.next()) |part| {
if (part.len == 0 or std.mem.eql(u8, part, ".") or std.mem.eql(u8, part, "..")) return false;
}
return true;
}
/// One uv round: the client-version pragma and `cards`, signed with a login
/// card when there is a credential, sent, and the reply parsed. A reply card
/// this client has no use for (`pragma server-version`, a `#` comment) is
/// passed over; a malformed uv card or content that does not match its hash
/// is a protocol error.
fn uvRound(
gpa: std.mem.Allocator,
arena: std.mem.Allocator,
tr: *transport.Transport,
credential: ?zit.sync.Credential,
cards: []const u8,
) !UvReply {
var tail: std.ArrayList(u8) = .empty;
defer tail.deinit(gpa);
var line_buf: [zit.sync.max_line_len]u8 = undefined;
try tail.appendSlice(gpa, try zit.sync.writeClientVersion(sync_client_version_info, &line_buf));
try tail.append(gpa, '\n');
try tail.appendSlice(gpa, cards);
var request: std.ArrayList(u8) = .empty;
defer request.deinit(gpa);
if (credential) |cred| {
const nonce = zit.sync.computeNonce(tail.items);
const signature = zit.sync.computeSignature(&nonce, &cred.secret_hex);
try request.appendSlice(gpa, try zit.sync.writeLogin(.{ .user = cred.user, .nonce = &nonce, .signature = &signature }, &line_buf));
try request.append(gpa, '\n');
}
try request.appendSlice(gpa, tail.items);
const body = try tr.exchange(gpa, request.items);
defer gpa.free(body);
return parseUvReply(arena, try arena.dupe(u8, body));
}
/// Read one uv reply body. `owned` lives in `arena`, as do the names and file
/// contents the reply points into.
fn parseUvReply(arena: std.mem.Allocator, owned: []const u8) !UvReply {
var reply: UvReply = .{};
var pos: usize = 0;
while (pos < owned.len) {
const eol = std.mem.indexOfScalarPos(u8, owned, pos, '\n') orelse owned.len;
const line = owned[pos..eol];
pos = @min(eol + 1, owned.len);
if (line.len == 0 or line[0] == '#') continue;
const tokens = zit.sync.tokenizeCard(line);
if (tokens.len == 0) continue;
const op = tokens.items[0];
if (std.mem.eql(u8, op, "pragma") and tokens.len >= 2) {
if (std.mem.eql(u8, tokens.items[1], "uv-push-ok")) reply.access = .push_ok;
if (std.mem.eql(u8, tokens.items[1], "uv-pull-only") and reply.access != .push_ok) reply.access = .pull_only;
} else if (std.mem.eql(u8, op, "uvigot")) {
const card = zit.sync.parseUvigotCard(tokens) catch return error.ProtocolError;
try reply.catalog.append(arena, .{ .name = try uvDecodeName(arena, card.name), .mtime = card.mtime, .hash = card.hash, .size = card.size });
} else if (std.mem.eql(u8, op, "uvfile")) {
const card = zit.sync.parseUvfileCard(tokens) catch return error.ProtocolError;
const payload = zit.sync.consumeUvfilePayload(owned, pos, card) catch {
reply.truncated = true;
break;
};
pos = payload.next;
if (card.isDeletion() or card.contentOmitted()) continue;
if (!zit.sync.verifyUvContent(card.hash, payload.bytes)) return error.ProtocolError;
try reply.files.append(arena, .{ .name = try uvDecodeName(arena, card.name), .content = payload.bytes });
} else if (std.mem.eql(u8, op, "error") and tokens.len >= 2) {
reply.refusal = try uvDecodeName(arena, tokens.items[1]);
}
}
return reply;
}
fn uvDecodeName(arena: std.mem.Allocator, wire: []const u8) ![]const u8 {
const out = try arena.alloc(u8, wire.len);
return artifact.decodeEscapes(wire, out);
}
/// The catalog round every subcommand starts with.
fn uvCatalog(gpa: std.mem.Allocator, arena: std.mem.Allocator, tr: *transport.Transport, credential: ?zit.sync.Credential) !UvReply {
return uvRound(gpa, arena, tr, credential, "pragma uv-hash " ++ uv_hash_never ++ "\n");
}
pub fn runUv(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
const parsed = switch (cli.parseUvArgs(gpa, verb_args) catch return stateOom(streams, "uv")) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "uv", .unknown_option, tok),
.bad_subcommand => |tok| return cli.usageError(streams, "uv", .bad_subcommand, tok),
};
defer gpa.free(parsed.positionals);
if (parsed.help) {
streams.out.writeAll(main.uv_help) catch {};
return .ok;
}
const sub = parsed.sub orelse return cli.usageError(streams, "uv", .bad_subcommand, "");
const url = parsed.url orelse return cli.usageError(streams, "uv", .no_url, "");
switch (uvCheckArgs(streams, sub, parsed)) {
.ok => {},
else => |rc| return rc,
}
if (parsed.user) |u| if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) {
streams.err.writeAll("zit uv: --user must not contain whitespace\n") catch {};
return .failure;
};
// The login is the repository's, as for push: the shared secret is
// derived from the project code the server serves.
const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "uv")) orelse {
streams.err.writeAll(not_a_repo_msg) catch {};
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined;
const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "uv"),
error.Storage => return repoReadFail(streams, "uv"),
};
var pw_buf: [max_password_len]u8 = undefined;
defer std.crypto.secureZero(u8, &pw_buf);
var user_buf: [logstore.max_config_str]u8 = undefined;
const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf, &user_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "uv"),
error.Storage => return repoReadFail(streams, "uv"),
error.NoPassword => {
streams.err.writeAll("zit uv: no password supplied for --user\n") catch {};
return .failure;
},
};
var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "uv", url, e);
defer tr.close();
notePlaintext(streams, "uv", url);
if (resolved) |r| armBasicAuth(&tr, r.credential.user, r.plaintext) catch return stateOom(streams, "uv");
const credential: ?zit.sync.Credential = if (resolved) |r| r.credential else null;
const catalog = uvCatalog(gpa, arena, &tr, credential) catch |e| return uvFail(streams, url, e, &tr);
reportTruncatedPayloads(streams, "uv", @intFromBool(catalog.truncated));
if (catalog.refusal) |why| {
streams.err.print("zit uv: the server refused: {s}\n", .{why}) catch {};
return .failure;
}
if (catalog.access == .none) {
streams.err.print("zit uv: {s} does not share unversioned files with {s}\n", .{ url, if (credential) |c| c.user else "an anonymous user" }) catch {};
return .failure;
}
return switch (sub) {
.list => uvList(streams, &catalog, parsed.long),
.pull => uvPull(gpa, io, arena, streams, &tr, credential, &catalog, parsed, url),
.push => uvPush(gpa, io, arena, streams, &tr, credential, &catalog, parsed, amb, url),
.rm => uvRm(gpa, arena, streams, &tr, credential, &catalog, parsed, amb, url),
};
}
/// The flags each subcommand takes and the operands it needs, before any
/// network or store work.
fn uvCheckArgs(streams: cli.Streams, sub: cli.UvSub, p: cli.UvArgs) cli.Exit {
const misuse = struct {
fn say(s: cli.Streams, comptime msg: []const u8) cli.Exit {
s.err.writeAll("zit uv: " ++ msg ++ "\ntry 'zit uv --help'\n") catch {};
return .usage;
}
};
if (p.long and sub != .list) return misuse.say(streams, "-l belongs to 'uv list'");
if ((p.all or p.out_dir != null) and sub != .pull) return misuse.say(streams, "--all and -o belong to 'uv pull'");
if ((p.as != null) and sub != .push) return misuse.say(streams, "--as belongs to 'uv push'");
if ((p.mtime != null) and sub != .push and sub != .rm) return misuse.say(streams, "--mtime belongs to 'uv push' and 'uv rm'");
switch (sub) {
.list => if (p.items.len != 0) return misuse.say(streams, "'uv list' takes only the URL"),
.pull => {
if (p.all and p.items.len != 0) return misuse.say(streams, "'uv pull' takes NAMEs or --all, not both");
if (!p.all and p.items.len == 0) return misuse.say(streams, "'uv pull' needs a NAME or --all");
},
.push => {
if (p.items.len == 0) return misuse.say(streams, "'uv push' needs a FILE");
if (p.as != null and p.items.len != 1) return misuse.say(streams, "--as names one FILE");
},
.rm => if (p.items.len == 0) return misuse.say(streams, "'uv rm' needs a NAME"),
}
if (p.mtime) |m| _ = std.fmt.parseInt(i64, m, 10) catch return misuse.say(streams, "--mtime takes seconds since 1970 (date +%s)");
for (if (sub == .push) (if (p.as) |a| &[_][]const u8{a} else p.items) else p.items) |name| {
if (!uvNameOk(name)) {
streams.err.print("zit uv: '{s}' is not an unversioned file name: no spaces, no '.' or '..' parts, no leading '/', at most {d} bytes\n", .{ name, uv_max_name }) catch {};
return .usage;
}
}
return .ok;
}
fn uvList(streams: cli.Streams, catalog: *const UvReply, long: bool) cli.Exit {
const entries = catalog.catalog.items;
std.mem.sort(UvEntry, entries, {}, struct {
fn lt(_: void, a: UvEntry, b: UvEntry) bool {
return std.mem.order(u8, a.name, b.name) == .lt;
}
}.lt);
for (entries) |e| {
if (long) {
if (e.deleted())
streams.out.print("{s} {d} deleted 0\n", .{ e.name, e.mtime }) catch {}
else
streams.out.print("{s} {d} {s} {d}\n", .{ e.name, e.mtime, e.hash, e.size }) catch {};
} else if (!e.deleted()) {
streams.out.print("{s}\n", .{e.name}) catch {};
}
}
return .ok;
}
fn uvPull(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
streams: cli.Streams,
tr: *transport.Transport,
credential: ?zit.sync.Credential,
catalog: *const UvReply,
p: cli.UvArgs,
url: []const u8,
) cli.Exit {
var rc: cli.Exit = .ok;
var wanted: std.ArrayList([]const u8) = .empty;
if (p.all) {
for (catalog.catalog.items) |e| if (!e.deleted()) wanted.append(arena, e.name) catch return stateOom(streams, "uv");
} else for (p.items) |name| {
const e = catalog.find(name) orelse {
streams.err.print("zit uv: the server holds no '{s}'\n", .{name}) catch {};
rc = .failure;
continue;
};
if (e.deleted()) {
streams.err.print("zit uv: '{s}' is deleted on the server\n", .{name}) catch {};
rc = .failure;
continue;
}
wanted.append(arena, name) catch return stateOom(streams, "uv");
}
var out = std.Io.Dir.cwd().createDirPathOpen(io, p.out_dir orelse ".", .{}) catch {
streams.err.print("zit uv: cannot open '{s}'\n", .{p.out_dir orelse "."}) catch {};
return .failure;
};
defer out.close(io);
var round: usize = 0;
while (wanted.items.len != 0 and round < uv_max_rounds) : (round += 1) {
var cards: std.ArrayList(u8) = .empty;
for (wanted.items) |name| {
var name_buf: [2 * uv_max_name]u8 = undefined;
cards.print(arena, "uvgimme {s}\n", .{artifact.encodeEscapes(name, &name_buf)}) catch return stateOom(streams, "uv");
}
const reply = uvRound(gpa, arena, tr, credential, cards.items) catch |e| return uvFail(streams, url, e, tr);
reportTruncatedPayloads(streams, "uv", @intFromBool(reply.truncated));
if (reply.files.items.len == 0) break; // the server sent nothing more
for (reply.files.items) |f| {
const i = for (wanted.items, 0..) |w, j| {
if (std.mem.eql(u8, w, f.name)) break j;
} else continue; // not asked for
if (!uvNameOk(f.name)) continue;
zit.update.restoreFile(io, out, f.name, f.content, .regular) catch {
streams.err.print("zit uv: cannot write '{s}'\n", .{f.name}) catch {};
return .failure;
};
_ = wanted.swapRemove(i);
}
}
for (wanted.items) |name| {
streams.err.print("zit uv: the server did not send '{s}'\n", .{name}) catch {};
rc = .failure;
}
return rc;
}
/// The hash a pushed file is named by: the catalog's own algorithm, so a
/// server on SHA1 names compares equal names, else SHA3-256.
fn uvHashAlgo(catalog: *const UvReply) artifact.HashAlgo {
for (catalog.catalog.items) |e| if (!e.deleted()) return artifact.nameAlgo(e.hash.len) orelse .sha3_256;
return .sha3_256;
}
fn uvPush(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
streams: cli.Streams,
tr: *transport.Transport,
credential: ?zit.sync.Credential,
catalog: *const UvReply,
p: cli.UvArgs,
amb: cli.Ambient,
url: []const u8,
) cli.Exit {
if (catalog.access != .push_ok) return uvNoPush(streams, url, credential);
const mtime: i64 = if (p.mtime) |m| std.fmt.parseInt(i64, m, 10) catch unreachable else amb.now_epoch;
const algo = uvHashAlgo(catalog);
var rc: cli.Exit = .ok;
var cards: std.ArrayList(u8) = .empty;
const Sent = struct { name: []const u8, hash: []const u8 };
var sent: std.ArrayList(Sent) = .empty;
for (p.items) |path| {
const name = p.as orelse path;
const content = std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(zit.sync.reply_body_cap)) catch {
streams.err.print("zit uv: cannot read '{s}'\n", .{path}) catch {};
rc = .failure;
continue;
};
var hash_buf: [64]u8 = undefined;
const hash = arena.dupe(u8, artifact.artifactName(algo, content, &hash_buf)) catch return stateOom(streams, "uv");
if (catalog.find(name)) |held| {
if (std.mem.eql(u8, held.hash, hash)) continue; // already there
if (held.mtime >= mtime) {
streams.err.print("zit uv: the server's '{s}' is as new or newer; pass a later --mtime to replace it\n", .{name}) catch {};
rc = .failure;
continue;
}
}
var name_buf: [2 * uv_max_name]u8 = undefined;
cards.print(arena, "uvfile {s} {d} {s} {d} 0\n", .{ artifact.encodeEscapes(name, &name_buf), mtime, hash, content.len }) catch return stateOom(streams, "uv");
cards.appendSlice(arena, content) catch return stateOom(streams, "uv");
sent.append(arena, .{ .name = name, .hash = hash }) catch return stateOom(streams, "uv");
}
if (sent.items.len == 0) return rc;
return uvSendAndConfirm(gpa, arena, streams, tr, credential, cards.items, sent.items, url, rc);
}
fn uvRm(
gpa: std.mem.Allocator,
arena: std.mem.Allocator,
streams: cli.Streams,
tr: *transport.Transport,
credential: ?zit.sync.Credential,
catalog: *const UvReply,
p: cli.UvArgs,
amb: cli.Ambient,
url: []const u8,
) cli.Exit {
if (catalog.access != .push_ok) return uvNoPush(streams, url, credential);
const mtime: i64 = if (p.mtime) |m| std.fmt.parseInt(i64, m, 10) catch unreachable else amb.now_epoch;
var rc: cli.Exit = .ok;
var cards: std.ArrayList(u8) = .empty;
const Sent = struct { name: []const u8, hash: []const u8 };
var sent: std.ArrayList(Sent) = .empty;
for (p.items) |name| {
const held = catalog.find(name) orelse {
streams.err.print("zit uv: the server holds no '{s}'\n", .{name}) catch {};
rc = .failure;
continue;
};
if (held.deleted()) continue; // already deleted
if (held.mtime >= mtime) {
streams.err.print("zit uv: the server's '{s}' is as new or newer; pass a later --mtime to delete it\n", .{name}) catch {};
rc = .failure;
continue;
}
var name_buf: [2 * uv_max_name]u8 = undefined;
cards.print(arena, "uvfile {s} {d} - 0 0\n", .{ artifact.encodeEscapes(name, &name_buf), mtime }) catch return stateOom(streams, "uv");
sent.append(arena, .{ .name = name, .hash = "-" }) catch return stateOom(streams, "uv");
}
if (sent.items.len == 0) return rc;
return uvSendAndConfirm(gpa, arena, streams, tr, credential, cards.items, sent.items, url, rc);
}
/// Send the `uvfile` cards, then read the catalog again: a fossil server
/// keeps whichever copy is newer and says nothing when it keeps its own, so
/// only the catalog shows whether each file was taken.
fn uvSendAndConfirm(
gpa: std.mem.Allocator,
arena: std.mem.Allocator,
streams: cli.Streams,
tr: *transport.Transport,
credential: ?zit.sync.Credential,
cards: []const u8,
sent: anytype,
url: []const u8,
rc_in: cli.Exit,
) cli.Exit {
var rc = rc_in;
const reply = uvRound(gpa, arena, tr, credential, cards) catch |e| return uvFail(streams, url, e, tr);
reportTruncatedPayloads(streams, "uv", @intFromBool(reply.truncated));
if (reply.refusal) |why| {
streams.err.print("zit uv: the server refused: {s}\n", .{why}) catch {};
return .failure;
}
const after = uvCatalog(gpa, arena, tr, credential) catch |e| return uvFail(streams, url, e, tr);
reportTruncatedPayloads(streams, "uv", @intFromBool(after.truncated));
for (sent) |s| {
const now = after.find(s.name);
if (now == null or !std.mem.eql(u8, now.?.hash, s.hash)) {
streams.err.print("zit uv: the server did not take '{s}'\n", .{s.name}) catch {};
rc = .failure;
}
}
return rc;
}
fn uvNoPush(streams: cli.Streams, url: []const u8, credential: ?zit.sync.Credential) cli.Exit {
streams.err.print("zit uv: {s} lets {s} read unversioned files but not change them (that takes the 'y' capability)\n", .{ url, if (credential) |c| c.user else "an anonymous user" }) catch {};
return .failure;
}
fn uvFail(streams: cli.Streams, url: []const u8, err: anyerror, tr: *const transport.Transport) cli.Exit {
switch (err) {
error.OutOfMemory => return stateOom(streams, "uv"),
error.ProtocolError => {
if (tr.last_status) |st| if (st != .ok) {
streams.err.print("zit uv: {s} answered HTTP {d} ({s})\n", .{ url, @backingInt(st), st.phrase() orelse "unrecognized" }) catch {};
return .failure;
};
streams.err.print("zit uv: {s} sent a reply this client cannot read\n", .{url}) catch {};
},
else => streams.err.print("zit uv: cannot talk to {s}: {s}\n", .{ url, @errorName(err) }) catch {},
}
return .failure;
}
/// `zit push URL` — open the current repository, announce its holdings to the
/// peer at URL, and upload every artifact the peer lacks until §3.7 termination.
/// Thin wiring over the same deep modules pull uses: `transport` (HTTP/IO),
/// `sync.Client` + `runSync` (sans-IO §3.7 loop). With URL omitted the saved
/// remote (`zit remote`) is used; a repo with neither is a usage error.
pub fn runPush(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
const parsed = switch (cli.parsePushArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "push", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "push", .extra_argument, tok),
};
if (parsed.help) {
streams.out.writeAll(push_help) catch {};
return .ok;
}
// sync §6.3 — a login name is a whitespace-free token (the shared-secret
// derivation embeds it between "/" separators). Reject a malformed --user
// as a usage failure rather than reaching the derivation's assertion.
if (parsed.user) |u| {
if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) {
streams.err.writeAll("zit push: --user must not contain whitespace\n") catch {};
return .failure;
}
}
// Locate the current repository before any network I/O: a push outside a repo
// is a usage failure, not a wasted connection.
const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "push")) orelse {
streams.err.writeAll(not_a_repo_msg) catch {};
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
// sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved
// remote; a repo with neither is a usage error. `url_buf` outlives `open`.
var url_buf: [logstore.max_config_str]u8 = undefined;
const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "push"),
error.Storage => return repoReadFail(streams, "push"),
error.NoRemote => return noRemoteError(streams, "push"),
};
var tr = transport.Transport.open(io, gpa, url) catch |e|
return syncOpenFail(streams, "push", url, e);
defer tr.close();
notePlaintext(streams, "push", url);
// The client's holdings view — log-backed and lazy, the igot
// announcement plus the upload source: the peer requests only what it
// lacks, and only those artifacts' content materializes.
var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "push"),
error.Storage => return repoReadFail(streams, "push"),
};
defer holdings.deinit(io);
var admission = holdings.admission();
// Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once
// and suppress spontaneous requests for elided phantoms. Both the
// classification and the wrapper outlive the run below.
var sync_cone: ?zit.cone.Cone = null;
defer if (sync_cone) |*c| c.deinit(gpa);
var sync_classification: ?SparseClassification = null;
defer if (sync_classification) |*c| c.deinit(gpa);
var sync_elided: RepoElidedSet = undefined;
if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| {
sync_cone = c;
sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch
return stateOom(streams, "push");
sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() };
admission.elided = sync_elided.shunSet();
}
// §9.1 — announce this repository's adopted project code in the push card
// (same identity gate as pull; a real peer refuses a mismatch). The buffer
// outlives the run — the client borrows it.
// The cluster index: what the held clusters already announce
// (never announced again individually) and what they promised that this
// repository never collected (seeded as phantoms, so the run asks for it).
// The clusters' plaintext is memoised by the holdings adapter while it
// resolves; drop it — the index keeps only names and bits.
var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "push");
defer clusters.deinit();
holdings.index.releaseTransient();
var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined;
const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "push"),
error.Storage => return repoReadFail(streams, "push"),
};
// sync §6 — resolve the login credential (null for an anonymous push). The
// password plaintext is confined to `pw_buf`/argv/env; only the derived
// 40-hex secret (§6.3) is signed with and — on success — persisted.
var pw_buf: [max_password_len]u8 = undefined;
// Defense-in-depth: once the 40-hex secret is derived the plaintext in
// pw_buf is spent, so scrub it on every exit from this scope — success and
// error alike — rather than leave it resident until the stack is reused.
defer std.crypto.secureZero(u8, &pw_buf);
var user_buf: [logstore.max_config_str]u8 = undefined;
const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf, &user_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "push"),
error.Storage => return repoReadFail(streams, "push"),
error.NoPassword => {
streams.err.writeAll("zit push: no password supplied for --user\n") catch {};
return .failure;
},
};
var peer_error: PeerErrorCopy = .{};
const stderr_tty = std.Io.File.stderr().isTty(io) catch false;
var progress: CloneProgress = .{ .err = streams.err };
const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null;
defer progress.finishLine();
if (resolved) |r| armBasicAuth(&tr, r.credential.user, r.plaintext) catch return stateOom(streams, "push");
var notes: ServerNotes = .{};
defer notes.deinit(gpa);
var truncated: u64 = 0;
defer reportTruncatedPayloads(streams, "push", truncated);
const outcome = pushRun(gpa, &tr, holdings.store(), identity, if (resolved) |r| r.credential else null, admission, &peer_error, ¬es, sink, &clusters, &truncated) catch |e| {
// A peer that stated its refusal is quoted rather than flattened into
// the generic mapping — the whole point of a server-side refusal is that the operator learns
// WHICH file and WHICH rule without a second round trip.
if (e == error.FatalProtocolError) {
if (peer_error.text()) |text| {
streams.err.print("zit push: the peer refused: {s}\n", .{text}) catch {};
return .failure;
}
}
return syncRunFail(streams, "push", url, e, tr.last_status, basicStateOf(&tr));
};
// DECISION-3 — persist the derived secret (never the plaintext) at 0600 only
// after the peer accepted the authenticated push, and only when it was
// freshly derived: a reused stored secret is already durable.
if (resolved) |r| {
if (r.persist) {
storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "push"),
else => return repoReadFail(streams, "push"),
};
}
}
// submanifests §7.2 — "the sender MUST refuse to transmit that artifact,
// with an explicit error -- never by converting, downgrading, or silently
// omitting it." A client cannot put an `error` card in a REQUEST (it is
// outside the §3.9 strict-accept request vocabulary), so the explicit error
// is this: the operator is told exactly what was withheld and why, and the
// run fails. Reported before the success line, because there was none.
if (outcome.refused > 0) {
streams.err.print(
"zit push: {d} artifacts were withheld: {s}\n",
.{ outcome.refused, zit.sync.submanifest_peer_version_error_text },
) catch {};
return .failure;
}
notes.show(streams);
if (outcome.sent == 0) {
streams.out.print("push to {s}: already in sync\n", .{url}) catch {};
} else {
streams.out.print("push to {s}: sent {d} artifact{s} in {d} round{s}\n", .{ url, outcome.sent, plural(outcome.sent), outcome.rounds, plural(outcome.rounds) }) catch {};
reportUvSkipped(streams, "push", outcome.uv_skipped);
}
// The server kept a pushed check-in on an attempt branch. Until the move
// arrives here, this repository still shows it where it was committed, and
// the next commit would build on a picture the server no longer agrees
// with; so fetch it now, from the same server.
if (notes.diverted) return runPull(gpa, io, arena, &.{url}, streams, amb);
return .ok;
}
/// The upper bound on an interactively-typed or `$ZIT_PASSWORD` password. A
/// login password far shorter than this in practice; the fixed bound keeps the
/// no-echo read on the stack with no unbounded allocation (TigerStyle).
const max_password_len: usize = 1024;
/// sync §6.3 — a resolved push credential and whether its secret was freshly
/// derived (so it should be persisted after a successful push).
const ResolvedCredential = struct {
credential: zit.sync.Credential,
persist: bool,
/// The plaintext password when this run had one (--password, the
/// environment, or the prompt): what an HTTP Basic challenge is answered
/// with. A stored login has none, and a Basic challenge then fails with
/// a message naming the flag. Borrows argv, the environment, or pw_buf.
plaintext: ?[]const u8 = null,
};
/// The causes `resolvePushCredential` can surface: store read failure, OOM, or
/// a `--user` with no obtainable password (`NoPassword`). Closed set so the
/// caller switches exhaustively.
const ResolveError = error{ OutOfMemory, Storage, NoPassword };
/// sync §6.3/DECISION-3 — turn `push`'s `--user`/`--password` into a signing
/// credential. An empty `--user` is an anonymous push (null result). An
/// ABSENT `--user` falls back to the login a previous authenticated push
/// stored: that credential is reused as-is and this branch never
/// prompts — there is no operator at a keyboard by definition, since it is
/// how autosync pushes — so with nothing stored the push is anonymous. That is
/// fossil's shape: a remembered login authenticates every later sync.
/// Otherwise the 40-hex secret is resolved, in order, from: a stored credential
/// for this exact login (reused verbatim, already durable), `--password`,
/// `$ZIT_PASSWORD`, or a no-echo terminal prompt. A freshly derived secret is
/// flagged `persist`; the plaintext lives only in `pw_buf`/argv/env and is
/// never returned or stored. `user_buf` holds a stored login's name for the
/// returned credential to borrow; it must outlive the credential.
/// Give the transport the plaintext this run holds, so a peer that gates the
/// READ with an HTTP Basic challenge (a forge in front of a private
/// repository) can be answered; nothing is sent unless the peer asks.
fn armBasicAuth(tr: *transport.Transport, user: []const u8, plaintext: ?[]const u8) error{OutOfMemory}!void {
const pw = plaintext orelse return;
try tr.setBasicAuth(user, pw);
}
/// The user named in a remote URL's `user@host` component, or null. A
/// `user:password@host` form yields the user only; the password part of a URL
/// is never read from here (the saved remote strips it by construction).
pub fn urlUser(url: []const u8) ?[]const u8 {
const scheme_end = std.mem.indexOf(u8, url, "://") orelse return null;
const rest = url[scheme_end + 3 ..];
const authority_end = std.mem.indexOfScalar(u8, rest, '/') orelse rest.len;
const authority = rest[0..authority_end];
const at = std.mem.lastIndexOfScalar(u8, authority, '@') orelse return null;
const userinfo = authority[0..at];
const colon = std.mem.indexOfScalar(u8, userinfo, ':') orelse userinfo.len;
const user = userinfo[0..colon];
return if (user.len == 0) null else user;
}
test "urlUser: the user@ component, without any password, or null" {
try std.testing.expectEqualStrings("alice", urlUser("https://[email protected]/r/p").?);
try std.testing.expectEqualStrings("alice", urlUser("https://alice:[email protected]:8443/r/p").?);
try std.testing.expect(urlUser("https://h.example/r/p") == null);
try std.testing.expect(urlUser("https://h.example/r/p@ath") == null);
try std.testing.expect(urlUser("") == null);
}
fn resolvePushCredential(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
streams: cli.Streams,
project_code: []const u8,
parsed: cli.PushArgs,
amb: cli.Ambient,
pw_buf: *[max_password_len]u8,
user_buf: *[logstore.max_config_str]u8,
) ResolveError!?ResolvedCredential {
// --user, then the remote URL's user@, then the login a previous
// authenticated round stored; none of the three means anonymous.
const explicit = parsed.user != null;
const user = parsed.user orelse urlUser(parsed.url) orelse {
const stored = (try storedLogin(gpa, io, dir)) orelse return null;
@memcpy(user_buf[0..stored.user_len], stored.user());
return .{
.credential = .{ .user = user_buf[0..stored.user_len], .secret_hex = stored.secret_hex },
.persist = false,
};
};
if (user.len == 0) return null; // an empty --user is an anonymous push
std.debug.assert(!zit.sync.containsFossilWhitespace(user)); // pre-validated in runPush
// A stored secret for this exact login is reused as-is: no re-derive, no
// prompt, no re-persist. Only consulted when no explicit password is given.
if (parsed.password == null and amb.push_password == null) {
if (try storedSecretFor(gpa, io, dir, user)) |sec| {
return .{ .credential = .{ .user = user, .secret_hex = sec }, .persist = false };
}
}
// Otherwise derive from a plaintext password. Precedence: --password, then
// $ZIT_PASSWORD, then an interactive no-echo prompt. A user named only by
// the URL is a hint, not a demand: with no password to be had, the round
// goes anonymous and says so, so a private repository's 404 at least
// points at the credential it wanted. --user with no password is an error.
const password: []const u8 = parsed.password orelse amb.push_password orelse
(readPasswordNoEcho(streams, user, pw_buf) orelse {
if (explicit) return error.NoPassword;
streams.err.print("zit: no password for '{s}' from the remote URL (set ZIT_PASSWORD or pass --user/--password); continuing anonymously\n", .{user}) catch {};
return null;
});
const secret = zit.sync.deriveSharedSecret(project_code, user, password);
return .{ .credential = .{ .user = user, .secret_hex = secret }, .persist = true, .plaintext = password };
}
/// The login a successful authenticated push left in this repo's config
/// (DECISION-3): the name and its 40-hex secret, both copied out — the config
/// frame is freed here — so the value outlives the read.
const StoredLogin = struct {
user_buf: [logstore.max_config_str]u8,
user_len: usize,
secret_hex: [40]u8,
fn user(self: *const StoredLogin) []const u8 {
return self.user_buf[0..self.user_len];
}
};
/// The stored login, or null when none is stored. Read-only.
fn storedLogin(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
) error{ OutOfMemory, Storage }!?StoredLogin {
var file = cli.openStoreRead(io, dir) catch return error.Storage;
defer file.close(io);
const st = file.stat(io) catch return error.Storage;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
};
defer log.deinit(gpa);
var oc = (log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
}) orelse return null;
defer oc.deinit(gpa);
if (oc.config.login_secret_hex.len != 40) return null;
const name = oc.config.login_user;
if (name.len == 0 or name.len > logstore.max_config_str) return null;
var out: StoredLogin = .{ .user_buf = undefined, .user_len = name.len, .secret_hex = undefined };
@memcpy(out.user_buf[0..name.len], name);
@memcpy(&out.secret_hex, oc.config.login_secret_hex[0..40]);
return out;
}
/// DECISION-3 — the 40-hex secret stored for `user` in this repo's config, or
/// null when none is stored or it belongs to a different login.
fn storedSecretFor(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
user: []const u8,
) error{ OutOfMemory, Storage }!?[40]u8 {
const stored = (try storedLogin(gpa, io, dir)) orelse return null;
if (!std.mem.eql(u8, stored.user(), user)) return null;
return stored.secret_hex;
}
/// DECISION-3 — persist the derived login credential into the repo config,
/// preserving every other field, then tighten the repository file to 0600 so
/// the stored secret is owner-only at rest. The permission tightening is
/// best-effort: the secret is already durable, so a filesystem that refuses
/// chmod does not fail the push (perms + FDE are the boundary, not the sole
/// guarantee — per DECISION-3). NEVER writes the plaintext password.
fn storePushCredential(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
credential: zit.sync.Credential,
) PersistLockedError!void {
std.debug.assert(credential.secret_hex.len == 40);
// The field-preserving RMW writes the credential without disturbing the
// project code or a saved remote (a bare `putConfig` here would drop them).
try updateConfig(gpa, io, dir, .{ .login = credential });
// Owner-only now that the file carries a credential (best-effort; see above).
var store_append = cli.openStoreAppend(io, dir) catch return;
defer store_append.close(io);
const file = store_append.file;
file.setPermissions(io, File.Permissions.fromMode(0o600)) catch {};
}
/// CLI/IO seam — prompt on stderr and read one password line from the terminal
/// with echo disabled, returning the trimmed line in `buf` (never its newline),
/// or null on a read failure or an empty line. Echo is restored on every exit
/// path; when stdin is not a terminal (`tcgetattr` fails) the line is read with
/// no termios toggle. Not exercised by the automated gates (they pass
/// `--password`/`$ZIT_PASSWORD`) — this is the interactive path.
fn readPasswordNoEcho(
streams: cli.Streams,
user: []const u8,
buf: *[max_password_len]u8,
) ?[]const u8 {
// No terminal, no prompt: a script, an autosync, or a test would otherwise
// see a prompt it cannot answer and a read error underneath it.
_ = std.posix.tcgetattr(std.posix.STDIN_FILENO) catch return null;
streams.err.print("Password for {s}: ", .{user}) catch {};
streams.err.flush() catch {};
const fd = std.posix.STDIN_FILENO;
const restore: ?std.posix.termios = blk: {
var t = std.posix.tcgetattr(fd) catch break :blk null;
const saved = t;
t.lflag.ECHO = false;
std.posix.tcsetattr(fd, .FLUSH, t) catch break :blk null;
break :blk saved;
};
defer if (restore) |saved| {
std.posix.tcsetattr(fd, .FLUSH, saved) catch {};
streams.err.writeAll("\n") catch {};
streams.err.flush() catch {};
};
const n = std.posix.read(fd, buf) catch return null;
if (n == 0) return null;
var line = buf[0..n];
// Trim a single trailing CRLF/LF; interior bytes (including spaces) are kept.
if (line.len > 0 and line[line.len - 1] == '\n') line = line[0 .. line.len - 1];
if (line.len > 0 and line[line.len - 1] == '\r') line = line[0 .. line.len - 1];
if (line.len == 0) return null;
return line;
}
/// Drives a push client (§3.2/§3.3) to convergence over `tr`, uploading the
/// artifacts the peer lacks; returns the upload count and round count. The client
/// announces `ms`'s holdings via igot and uploads what the peer gimme's (§3.3
/// r24: push sends content but never obtains it). Sans-IO on this side of
/// `tr.exchange`. Its closed error set is the driver's own causes unioned with
/// the transport's, so `runPush` switches it exhaustively.
fn pushRun(
gpa: std.mem.Allocator,
tr: *transport.Transport,
holdings: repostate.ArtifactStore,
identity: RepoSyncIdentity,
credential: ?zit.sync.Credential,
admission: RepoAdmission,
/// A server that REFUSED states why in its error card. Copied out
/// here because the client owns the buffer and is deinit'd on the way out,
/// so the caller cannot reach it afterwards; the clone path does the same
/// for the sparse-origin refusal.
peer_error_out: *PeerErrorCopy,
notes: *ServerNotes,
sink: ?sync_driver.RoundSink,
clusters: ?*const clusterindex.Resolved,
truncated_out: *u64,
) sync_driver.RunError(*transport.Transport)!PushOutcome {
// A push client still RECEIVES — §3.3 answers the server's gimme-driven
// `file` cards in the same session — so `admission` is load-bearing here too.
var client = syncClient(.push, &identity, credential, admission);
defer client.deinit(gpa);
defer truncated_out.* = client.payloads_truncated;
defer notes.adopt(&client);
if (clusters) |c| {
client.clustered = &c.clustered;
try client.seedClusterPhantoms(gpa, holdings, c.missing);
}
errdefer peer_error_out.set(client.peerErrorText());
const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null);
recordSyncStats(stats);
// Read the upload counter before `deinit` runs (the return expression is
// evaluated first): the exact number of `file` cards this session emitted.
return .{ .rounds = stats.rounds, .sent = client.content_sent, .refused = client.submanifests_refused, .uv_skipped = client.uv_skipped };
}
// ---------------------------------------------------------------------------
// `zit sync [URL]` — BIDIRECTIONAL convergence with a peer in ONE session.
// Unlike pull-then-push, a single `operation=.sync` client announces igot,
// requests gimme, AND uploads on one §3.7 exchange (§3.2 round structure: a sync
// request carries both push and pull each round) — not a third copy of the driver
// loop over two transport sessions.
// Opens the cwd's `.zit`, resolves the URL (arg → saved remote), may
// authenticate (sync uploads), persists what it receives like pull, and reports
// both directions. A saved `zit remote` lets the URL be omitted.
// ---------------------------------------------------------------------------
/// What one sync run reports: uploads (`sent`) and the round count. Received is
/// measured by the caller as the persisted-count growth (as pull does), so it is
/// not carried here.
/// submanifests §7.2 — see `PushOutcome.refused`.
const SyncOutcome = struct { rounds: usize, sent: usize, refused: usize = 0, uv_skipped: u64 = 0 };
pub fn runSync(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
const parsed = switch (cli.parseSyncArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "sync", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "sync", .extra_argument, tok),
};
if (parsed.help) {
streams.out.writeAll(sync_help) catch {};
return .ok;
}
// sync §6.3 — a login name is whitespace-free (mirrors push). Reject a
// malformed --user as usage before the secret derivation's assertion.
if (parsed.user) |u| {
if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) {
streams.err.writeAll("zit sync: --user must not contain whitespace\n") catch {};
return .failure;
}
}
// Locate the current repository before any network I/O.
const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "sync")) orelse {
streams.err.writeAll(not_a_repo_msg) catch {};
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
// sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved
// remote; a repo with neither is a usage error. `url_buf` outlives `open`.
var url_buf: [logstore.max_config_str]u8 = undefined;
const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "sync"),
error.Storage => return repoReadFail(streams, "sync"),
error.NoRemote => return noRemoteError(streams, "sync"),
};
var tr = transport.Transport.open(io, gpa, url) catch |e|
return syncOpenFail(streams, "sync", url, e);
defer tr.close();
notePlaintext(streams, "sync", url);
// The client's holdings view — log-backed and lazy: the igot
// announcement, the upload source, and the baseline for the received
// count. The held set can only grow (§3.3 storage only adds content).
var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "sync"),
error.Storage => return repoReadFail(streams, "sync"),
};
defer holdings.deinit(io);
var admission = holdings.admission();
// Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once
// and suppress spontaneous requests for elided phantoms. Both the
// classification and the wrapper outlive the run below.
var sync_cone: ?zit.cone.Cone = null;
defer if (sync_cone) |*c| c.deinit(gpa);
var sync_classification: ?SparseClassification = null;
defer if (sync_classification) |*c| c.deinit(gpa);
var sync_elided: RepoElidedSet = undefined;
if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| {
sync_cone = c;
sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch
return stateOom(streams, "sync");
sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() };
admission.elided = sync_elided.shunSet();
admission.cone_members = sync_cone.?.members;
admission.cone_not_members = sync_cone.?.not_members;
}
const held_before = contentCount(holdings.store());
// §9.1 — announce this repository's adopted project code (a real peer refuses
// a mismatch). The buffer outlives the run — the client borrows it.
// The cluster index: what the held clusters already announce
// (never announced again individually) and what they promised that this
// repository never collected (seeded as phantoms, so the run asks for it).
// The clusters' plaintext is memoised by the holdings adapter while it
// resolves; drop it — the index keeps only names and bits.
var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "sync");
defer clusters.deinit();
holdings.index.releaseTransient();
var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined;
const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "sync"),
error.Storage => return repoReadFail(streams, "sync"),
};
// sync §6 — resolve the login credential (null for anonymous). A sync
// uploads, so it may authenticate exactly as push does; the plaintext lives
// only in `pw_buf`/argv/env and is scrubbed on every exit from this scope.
var pw_buf: [max_password_len]u8 = undefined;
defer std.crypto.secureZero(u8, &pw_buf);
var user_buf: [logstore.max_config_str]u8 = undefined;
const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf, &user_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "sync"),
error.Storage => return repoReadFail(streams, "sync"),
error.NoPassword => {
streams.err.writeAll("zit sync: no password supplied for --user\n") catch {};
return .failure;
},
};
const stderr_tty = std.Io.File.stderr().isTty(io) catch false;
var progress: CloneProgress = .{ .err = streams.err };
const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null;
defer progress.finishLine();
if (resolved) |r| armBasicAuth(&tr, r.credential.user, r.plaintext) catch return stateOom(streams, "sync");
var peer_error: PeerErrorCopy = .{};
var notes: ServerNotes = .{};
defer notes.deinit(gpa);
var truncated: u64 = 0;
defer reportTruncatedPayloads(streams, "sync", truncated);
const outcome = syncRun(gpa, &tr, holdings.store(), identity, if (resolved) |r| r.credential else null, admission, &peer_error, ¬es, sink, &clusters, &truncated) catch |e| {
// As for push: a refusal is quoted in the server's own words.
if (e == error.FatalProtocolError) {
if (peer_error.text()) |text| {
streams.err.print("zit sync: the peer refused: {s}\n", .{text}) catch {};
return .failure;
}
}
return syncRunFail(streams, "sync", url, e, tr.last_status, basicStateOf(&tr));
};
notes.show(streams);
// Sparse-cone RFC section 6.3 applies to EVERY receive: a dense peer
// volunteers content the client never announced, so the filter runs at
// persist. Reclassified over the post-run store, so a brand-new commit's
// blobs classify by their own F cards -- new in-cone content persists,
// new out-of-cone content elides with the rest.
var post_classification: ?SparseClassification = null;
defer if (post_classification) |*c| c.deinit(gpa);
if (sync_cone) |*c| {
post_classification = classifyKeepRids(gpa, holdings.store(), c) catch
return stateOom(streams, "sync");
}
const persisted = persistStore(gpa, io, root_dir, holdings.store(), if (post_classification) |*c| &c.keep else null, 0) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "sync"),
error.Storage => return repoReadFail(streams, "sync"),
error.StoreBusy => return .failure, // the lock notice already said it
error.Corrupt => return renderStorage(streams, root.abs, "the peer sent a malformed artifact name"),
};
const written = persisted.written;
std.debug.assert(written >= held_before);
const received = written - held_before;
// DECISION-3 — persist a freshly derived secret only after the peer accepted
// the authenticated session, and only when it was freshly derived (mirrors push).
if (resolved) |r| {
if (r.persist) {
storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "sync"),
else => return repoReadFail(streams, "sync"),
};
}
}
// submanifests §7.2 — "the sender MUST refuse to transmit that artifact,
// with an explicit error -- never by converting, downgrading, or silently
// omitting it." A client cannot put an `error` card in a REQUEST (it is
// outside the §3.9 strict-accept request vocabulary), so the explicit error
// is this: the operator is told exactly what was withheld and why, and the
// run fails. Reported before the success line, because there was none.
if (outcome.refused > 0) {
streams.err.print(
"zit sync: {d} artifacts were withheld: {s}\n",
.{ outcome.refused, zit.sync.submanifest_peer_version_error_text },
) catch {};
return .failure;
}
// submanifests §7.1/§7.3 — see `runPull`: convergence against the peer is
// not the same fact as a complete tree closure.
if (persisted.submanifest_gaps > 0) {
streams.err.print(
"zit sync: {d} sub-manifest references are missing; some check-ins cannot be enumerated\n",
.{persisted.submanifest_gaps},
) catch {};
return .failure;
}
if (outcome.sent == 0 and received == 0) {
streams.out.print("sync with {s}: already in sync\n", .{url}) catch {};
} else {
streams.out.print("sync with {s}: sent {d}, received {d} in {d} rounds\n", .{ url, outcome.sent, received, outcome.rounds }) catch {};
reportUvSkipped(streams, "sync", outcome.uv_skipped);
}
return .ok;
}
/// Drives a bidirectional sync client (§3.2 round structure — both directions on
/// ONE §3.7 session) to convergence over `tr`; returns the upload count and round
/// count.
/// The received count is the caller's persisted-count growth. Sans-IO on this
/// side of `tr.exchange`; its closed error set is switched exhaustively by
/// `runSync`. The sibling of `pullRun`/`pushRun`, differing only in the
/// operation tag (`.sync`, which drives igot + gimme + upload on one session).
fn syncRun(
gpa: std.mem.Allocator,
tr: *transport.Transport,
holdings: repostate.ArtifactStore,
identity: RepoSyncIdentity,
credential: ?zit.sync.Credential,
admission: RepoAdmission,
peer_error_out: *PeerErrorCopy,
notes: *ServerNotes,
sink: ?sync_driver.RoundSink,
clusters: ?*const clusterindex.Resolved,
truncated_out: *u64,
) sync_driver.RunError(*transport.Transport)!SyncOutcome {
var client = syncClient(.sync, &identity, credential, admission);
defer client.deinit(gpa);
defer truncated_out.* = client.payloads_truncated;
defer notes.adopt(&client);
if (clusters) |c| {
client.clustered = &c.clustered;
try client.seedClusterPhantoms(gpa, holdings, c.missing);
}
errdefer peer_error_out.set(client.peerErrorText());
const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null);
recordSyncStats(stats);
return .{ .rounds = stats.rounds, .sent = client.content_sent, .refused = client.submanifests_refused, .uv_skipped = client.uv_skipped };
}
// ---------------------------------------------------------------------------
// `zit remote [URL | --unset]` — read, set, or clear the single default peer URL
// (sync §4/§6). Bare prints the saved remote; a URL saves it; --unset clears it.
// A local, non-synced config value (`config.remote_url`) — anonymous only, no
// credential in the URL — that lets pull/push/sync omit their URL argument. Not
// a git-style named-remote registry: one value, matching the single-value config.
// ---------------------------------------------------------------------------
/// `zit settings [NAME [VALUE]]` — the local-settings surface, holding
/// exactly one setting today: `autosync`. Bare lists; NAME prints; NAME VALUE
/// sets. The value lives in the config frame like `remote_url` — local, never
/// synced — inverted in storage (the flag records OFF) so absence means the
/// fossil default, on.
pub fn runSettings(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
const parsed = switch (cli.parseSettingsArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "settings", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "settings", .extra_argument, tok),
};
if (parsed.help) {
streams.out.writeAll(settings_help) catch {};
return .ok;
}
const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "settings")) orelse {
main.reportNoRoot(streams, io, "settings", amb.repo_override);
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
const zset = zit.settings;
// The tier the operator named. `--global` selects WHERE a write lands and
// which stored value a bare NAME reports; it never changes the fact that a
// listing reports the value in EFFECT.
const tier: Tier = if (parsed.global) .global else .repository;
if (parsed.name) |name| {
const setting = zset.Setting.byName(name) orelse {
var names_buf: [256]u8 = undefined;
streams.err.print(
"zit settings: unknown setting '{s}' (known: {s})\n",
.{ name, zset.knownNames(&names_buf) },
) catch {};
return .failure;
};
if (parsed.value) |value| {
return writeSetting(gpa, io, arena, root_dir, streams, amb, setting, value, tier);
}
return showSetting(gpa, io, arena, root_dir, streams, amb, setting, parsed.changed, parsed.value_only);
}
inline for (comptime std.enums.values(zset.Setting)) |setting| {
const exit = showSetting(gpa, io, arena, root_dir, streams, amb, setting, parsed.changed, parsed.value_only);
if (exit != .ok) return exit;
}
return .ok;
}
/// Write one setting into the tier the operator named.
///
/// Validation happens BEFORE either tier is touched, so a rejected value never
/// leaves a half-written global file or a config frame carrying a value the
/// reader will refuse.
fn writeSetting(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
root_dir: std.Io.Dir,
streams: cli.Streams,
amb: cli.Ambient,
setting: zit.settings.Setting,
value: []const u8,
tier: Tier,
) cli.Exit {
const zset = zit.settings;
switch (setting.kind()) {
.boolean => if (zset.parseBool(value) == null and !(setting == .autosync and std.mem.eql(u8, value, "pullonly"))) {
streams.err.print(
"zit settings: {s} takes 'on' or 'off'{s}\ntry 'zit settings --help'\n",
.{ setting.name(), if (setting == .autosync) " (or 'pullonly')" else "" },
) catch {};
return .usage;
},
.string => if (tier == .repository and value.len > logstore.max_config_str) {
// Refusing beats truncating: a truncated glob list silently stops
// protecting files, and a truncated command silently runs a
// different program.
if (setting == .ignore_glob) {
streams.err.print(
"zit settings: ignore-glob is limited to {d} bytes; use the versioned {s} file for longer lists\n",
.{ logstore.max_config_str, zit.ignore.versioned_path },
) catch {};
} else {
streams.err.print(
"zit settings: {s} is limited to {d} bytes\n",
.{ setting.name(), logstore.max_config_str },
) catch {};
}
return .failure;
},
}
if (setting.tier() == .repository) {
if (tier == .global) {
streams.err.print("zit settings: {s} belongs to one repository and has no --global value\n", .{setting.name()}) catch {};
return .usage;
}
if (!branchNameOk(value)) {
streams.err.print("zit settings: '{s}' is not a branch name: it may not contain spaces or control characters\n", .{value}) catch {};
return .usage;
}
if (value.len != 0) noteNoSuchBranch(gpa, io, root_dir, streams, value);
}
if (tier == .global) {
writeGlobalSetting(gpa, io, arena, amb, setting, value) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "settings"),
error.NoGlobalHome => {
streams.err.writeAll(
"zit settings: no global settings location (set $ZIT_CONFIG, $XDG_CONFIG_HOME or $HOME)\n",
) catch {};
return .failure;
},
else => {
streams.err.writeAll("zit settings: cannot write the global settings file\n") catch {};
return .failure;
},
};
streams.out.print("{s} {s}\n", .{ setting.name(), value }) catch {};
// A global write that is shadowed locally changes nothing an operator
// can see, which is precisely the confusion the provenance work exists
// to end -- so say so at the moment it happens.
const eff = resolveSetting(gpa, io, arena, root_dir, amb, setting) catch return .ok;
if (eff.source != .global) {
streams.err.print(
"note: the {s} tier still supplies {s} here, so this global value is not in effect\n",
.{ eff.source.label(), setting.name() },
) catch {};
}
_ = noteVersionedShadow(io, root_dir, streams, setting, eff);
if (setting == .ignore_glob) noteGitignorePresent(io, root_dir, streams);
return .ok;
}
const patch = repositoryPatch(setting, value) orelse {
streams.err.print(
"zit settings: {s} cannot be stored in this repository yet\n",
.{setting.name()},
) catch {};
return .failure;
};
updateConfig(gpa, io, root_dir, patch) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "settings"),
else => return storeAppendFail(streams, "settings", e),
};
streams.out.print("{s} {s}\n", .{ setting.name(), value }) catch {};
// The value just written IS the repository tier, so the shadow question is
// answerable without re-opening the store.
const r = resolveSettingWith(io, arena, root_dir, amb, setting, value) catch
return .ok;
_ = noteVersionedShadow(io, root_dir, streams, setting, r);
if (setting == .ignore_glob) noteGitignorePresent(io, root_dir, streams);
return .ok;
}
/// A name a branch tag can carry: one token, so no whitespace or control
/// characters. Empty is allowed and resets the setting to its default.
fn branchNameOk(name: []const u8) bool {
for (name) |c| if (c <= ' ' or c == 0x7f) return false;
return true;
}
/// Setting the main branch to a name no open branch has is allowed, as in
/// Fossil: the branch may be about to arrive. Said on stderr, because a typo
/// here points `update`, `merge` and the forge at nothing.
fn noteNoSuchBranch(gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams, name: []const u8) void {
var repo: Repository = undefined;
repo.open(gpa, io, root_dir, .{ .mode = .full }) catch return;
defer repo.deinit(gpa);
const leaves = repo.state().openLeavesOf(gpa, name) catch return;
defer gpa.free(leaves);
if (leaves.len == 0) streams.err.print("note: no open branch is named '{s}' yet\n", .{name}) catch {};
}
/// The config-frame patch for a repository-tier write, or null for a setting
/// the frame does not yet carry.
fn repositoryPatch(setting: zit.settings.Setting, value: []const u8) ?ConfigPatch {
const on = zit.settings.parseBool(value);
return switch (setting) {
// Both booleans record the explicit-set bit alongside the value, which
// is what makes "explicitly the default" distinguishable from untouched.
.autosync => if (std.mem.eql(u8, value, "pullonly"))
.{ .autosync_off = false, .autosync_set = true, .autosync_mode = "pullonly" }
else
.{ .autosync_off = !(on orelse return null), .autosync_set = true, .autosync_mode = "" },
.editor => .{ .editor = value },
.clearsign => .{ .clearsign = on orelse return null, .clearsign_set = true },
.ignore_glob => .{ .ignore_glob = value },
.pgp_command => .{ .pgp_command = value },
.keep_glob => .{ .keep_glob = value },
.manifest => .{ .manifest = value },
.before_commit => .{ .before_commit = value },
.binary_glob => .{ .binary_glob = value },
.crlf_glob => .{ .crlf_glob = value },
.crnl_glob => .{ .crnl_glob = value },
.clean_glob => .{ .clean_glob = value },
.empty_dirs => .{ .empty_dirs = value },
.encoding_glob => .{ .encoding_glob = value },
// Boolean-valued, stored as the operator's own text: `settings
// dotfiles yes` round-trips as `yes`, and `settings.isTruth` is the
// one thing that reads it.
.dotfiles => .{ .dotfiles = value },
.mimetypes => .{ .mimetypes = value },
.attachment_size_limit => .{ .attachment_size_limit = value },
.main_branch => .{ .default_branch = value },
};
}
/// Read-modify-write the operator-scope settings file, creating its directory
/// when absent. Read-modify-write rather than append so a second edit of the
/// same setting replaces the first instead of leaving a dead line above it.
fn writeGlobalSetting(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
amb: cli.Ambient,
setting: zit.settings.Setting,
value: []const u8,
) !void {
const zset = zit.settings;
var path_buf: [1024]u8 = undefined;
const g = zset.globalPath(amb.environ, &path_buf) orelse return error.NoGlobalHome;
const cwd = std.Io.Dir.cwd();
const existing = cwd.readFileAlloc(io, g.path, arena, .limited(zset.max_versioned_bytes)) catch "";
const next = try zset.globalSet(gpa, existing, setting, value);
defer gpa.free(next);
if (g.dir.len != 0) cwd.createDirPath(io, g.dir) catch {};
var f = cwd.createFile(io, g.path, .{ .truncate = true }) catch return error.Storage;
defer f.close(io);
f.writePositionalAll(io, next, 0) catch return error.Storage;
}
/// Which stored tier a `zit settings` invocation addresses.
const Tier = enum { repository, global };
/// Print one setting as `NAME VALUE` plus the tier that supplied it.
///
/// The provenance column is the point. zit has four places a value can
/// live, and a display that showed only the value would let an operator edit
/// the shadowed tier -- twice -- while the effective answer never moved. The
/// note is on stderr so `zit settings NAME` remains a clean value on stdout for
/// a script.
fn showSetting(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
root_dir: std.Io.Dir,
streams: cli.Streams,
amb: cli.Ambient,
setting: zit.settings.Setting,
only_changed: bool,
value_only: bool,
) cli.Exit {
const r = resolveSetting(gpa, io, arena, root_dir, amb, setting) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "settings"),
else => return repoReadFail(streams, "settings"),
};
if (only_changed and r.source == .default) return .ok; // --changed
if (value_only) {
streams.out.print("{s}\n", .{r.value}) catch {}; // --value
} else {
streams.out.print("{s} {s}\n", .{ setting.name(), r.value }) catch {};
}
// Only say where it came from when that is not the obvious answer: naming
// the tier for every default would bury the two lines that matter. One
// fact, one line: when a versioned file is shadowing a stored value the
// note SAYS so and replaces the provenance line, and when `.no-warn`
// silences that, the provenance line still names the file that is live.
if (!setting.versionable()) noteVersionedIgnored(io, root_dir, streams, setting);
if (noteVersionedShadow(io, root_dir, streams, setting, r)) return .ok;
if (r.source == .versioned) {
const dir = r.versioned_dir orelse .zit;
streams.err.print(
"note: {s} is set in {s}/{s}\n",
.{ setting.name(), dir.dirName(), setting.name() },
) catch {};
} else if (r.source != .default and r.source != .repository) {
streams.err.print(
"note: {s} is set in the {s} tier\n",
.{ setting.name(), r.source.label() },
) catch {};
}
return .ok;
}
/// Gather every tier's answer and apply `settings.resolve`. The reads live here
/// -- the model module stays sans-IO so precedence is a pure, tested function.
/// Public so the COMMIT path resolves signing settings through the same
/// precedence a listing reports. They must never disagree: a `zit settings`
/// that says `clearsign on` while the commit quietly does not sign is worse
/// than having no global tier at all.
pub fn resolveSetting(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
root_dir: std.Io.Dir,
amb: cli.Ambient,
setting: zit.settings.Setting,
) !zit.settings.Resolved {
const repository = try repositorySetting(gpa, io, arena, root_dir, setting);
return resolveSettingWith(io, arena, root_dir, amb, setting, repository);
}
/// The effective `crlf-glob`, with fossil's older `crnl-glob` as the
/// fallback SPELLING of the same setting. fossil reads
/// `db_get("crlf-glob", db_get("crnl-glob",""))` (checkin.c), so `crnl-glob`
/// is consulted only when no tier supplies `crlf-glob` at all -- a stored
/// empty `crlf-glob` is a value and wins, exactly as it does for every other
/// glob list.
///
/// The COMMIT gate only. The wire twin (`sync.PushPolicy`, built from the
/// serve-side snapshot) keeps reading `crlf-glob` alone: fossil's server never
/// gates content, so there is no parity to mirror there.
/// `config` is the caller's already-read frame, as for `localSetting`: the
/// commit gate holds the store's write lock and reads its config once, so both
/// spellings answer from that one frame rather than re-opening the store twice.
pub fn resolveCrlfGlob(
io: std.Io,
arena: std.mem.Allocator,
root_dir: std.Io.Dir,
amb: cli.Ambient,
config: ?*const logstore.Config,
) error{OutOfMemory}!zit.settings.Resolved {
const crlf = try localSetting(io, arena, root_dir, amb, config, .crlf_glob);
if (crlf.source != .default) return crlf;
return localSetting(io, arena, root_dir, amb, config, .crnl_glob);
}
/// `resolveSetting` with the REPOSITORY tier supplied by the caller instead of
/// read here. `repositorySetting` opens the store and pays a full frame-header
/// scan to reach one config frame -- fine for a one-shot verb, ruinous in the
/// serve loop, where it turned every round of a large-store clone into seconds
/// of dead re-scanning. The server passes the values its per-round
/// snapshot already read; every other tier is a cheap local read and stays
/// here, so the resolution order cannot fork between the two paths.
pub fn resolveSettingWith(
io: std.Io,
arena: std.mem.Allocator,
root_dir: std.Io.Dir,
amb: cli.Ambient,
setting: zit.settings.Setting,
repository: ?[]const u8,
) !zit.settings.Resolved {
const zset = zit.settings;
var c: zset.Candidates = .{};
// The versioned file is read ONLY for a versionable setting. `resolve`
// enforces the same rule, so this is belt and braces on the boundary that
// keeps a cloned repository from choosing what this machine executes.
if (setting.versionable()) {
// fossil's spelling first, zit's as the fallback -- a mirrored
// Fossil repository's own policy is honoured without adopting zit's
// directory name, and when both exist the fossil file wins.
// Two buffers: the candidates are materialized before the loop, and a
// shared buffer would leave the first slice pointing at the second's
// bytes.
var fossil_buf: [128]u8 = undefined;
var zit_buf: [128]u8 = undefined;
const candidates = .{ setting.fossilVersionedPath(&fossil_buf), setting.versionedPath(&zit_buf) };
const dirs = [_]zset.VersionedDir{ .fossil, .zit };
inline for (candidates, 0..) |maybe_path, i| {
if (c.versioned == null) {
if (maybe_path) |path| {
if (root_dir.readFileAlloc(io, path, arena, .limited(zset.max_versioned_bytes))) |text| {
// fossil's `db_get_versioned` strips `#` lines
// and trims before anything reads the value, so every
// versionable setting -- in either directory -- gets
// the same treatment from the one reader.
c.versioned = zset.stripCommentLines(text);
c.versioned_dir = dirs[i];
} else |_| {}
}
}
}
}
return resolveSettingTiers(io, arena, amb, setting, &c, repository);
}
/// `resolveSettingWith` with the versioned tier read from
/// the check-in `rid` instead of the working tree: what `open` needs, since
/// its tree does not exist yet. Same two file names in the same order, the
/// same comment stripping, the same tiers below; fossil's `db_get_versioned`
/// reads the file out of the check-in through `historical_blob` too.
fn checkinSetting(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
repo: *Repository,
rid: Repository.Rid,
amb: cli.Ambient,
config: ?*const logstore.Config,
setting: zit.settings.Setting,
) !zit.settings.Resolved {
const zset = zit.settings;
var c: zset.Candidates = .{};
if (setting.versionable()) {
var fossil_buf: [128]u8 = undefined;
var zit_buf: [128]u8 = undefined;
const candidates = .{ setting.fossilVersionedPath(&fossil_buf), setting.versionedPath(&zit_buf) };
const dirs = [_]zset.VersionedDir{ .fossil, .zit };
inline for (candidates, 0..) |maybe_path, i| {
if (c.versioned == null) {
if (maybe_path) |path| {
if (checkinFileText(gpa, arena, repo, rid, path)) |text| {
c.versioned = zset.stripCommentLines(text);
c.versioned_dir = dirs[i];
}
}
}
}
}
const repository = if (config) |cf| try configValue(arena, cf, setting) else null;
return resolveSettingTiers(io, arena, amb, setting, &c, repository);
}
/// The content of `path` in check-in `rid`, copied into `arena` and bounded
/// like a versioned file on disk; null when the check-in has no such file or
/// it cannot be read (the tiers below then answer, as for an unreadable file).
fn checkinFileText(gpa: std.mem.Allocator, arena: std.mem.Allocator, repo: *Repository, rid: Repository.Rid, path: []const u8) ?[]u8 {
const card = (repo.fileAt(gpa, rid, path) catch return null) orelse return null;
defer Repository.freeFileCard(gpa, card);
const hash = card.hash orelse return null;
const bytes = (repo.blob(gpa, hash) catch return null) orelse return null;
defer gpa.free(bytes);
if (bytes.len > zit.settings.max_versioned_bytes) return null;
return arena.dupe(u8, bytes) catch null;
}
/// The environment, repository and global tiers on top of an already-read
/// versioned candidate -- the tail both readers above share, so the order the
/// tiers are consulted in cannot fork between them.
fn resolveSettingTiers(
io: std.Io,
arena: std.mem.Allocator,
amb: cli.Ambient,
setting: zit.settings.Setting,
c: *zit.settings.Candidates,
repository: ?[]const u8,
) !zit.settings.Resolved {
if (setting.envVar()) |name| c.environment = amb.environ.getPosix(name);
c.repository = repository;
// A repository-tier setting has no global value to fall back to.
c.global = if (setting.tier() == .repository) null else try globalSetting(io, arena, amb, setting);
return zit.settings.resolve(setting, c.*);
}
/// Serialize and atomically store the content log's scan index (the same
/// best-effort refresh `Repository.logStore` performs): a failure costs the
/// next open one scan, nothing else.
fn refreshSettingLogIndex(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *const logstore.LogStore) void {
const idx = log.serializeIndex(gpa) catch return;
defer gpa.free(idx);
zit.repocache.storeLogIndex(io, dir, idx);
}
/// Open a read-only log over `sf`, adopting the persisted scan index when the
/// checkout's store cache holds a fresh one — the CLI pattern, shared so
/// every read verb stops paying the full frame-header scan (diff and
/// status carried an O(store) floor of ~0.2s per invocation on a
/// 148k-artifact store from exactly this). A refused index (absent, stale,
/// corrupt) falls back to the scan and refreshes the sidecar, so the next
/// open is cheap again. The sidecar lives with the STORE, never a worktree,
/// so the cache directory resolves through the checkout's store pointer.
pub fn openLogAdoptingIndex(
gpa: std.mem.Allocator,
io: std.Io,
root_dir: std.Io.Dir,
sf: storage.StorageFile,
/// Whether a scan (or a tail-extended adoption) may persist the refreshed
/// sidecar. `.read_only` exists for verbs PINNED as tree-untouching —
/// extras' B-extras-16 requires that a run create nothing, `.zit-cache`
/// included — which still adopt a sidecar someone else already wrote.
sidecar: enum { refresh_on_scan, read_only },
) logstore.OpenError!logstore.LogStore {
var resolved = checkout.storeDirOf(io, root_dir) catch null;
defer if (resolved) |*r| r.close(io);
const cache_dir = if (resolved) |r| r else root_dir;
indexed: {
const idx = zit.repocache.tryLoadLogIndex(gpa, io, cache_dir) orelse break :indexed;
// An adopted store TAKES `idx` and frees it in `deinit` -- its
// entry hashes are slices of it. Every refusal path leaves it ours.
var taken = false;
defer if (!taken) gpa.free(idx);
var extended = false;
const adopted = logstore.LogStore.openReadOnlyIndexed(gpa, sf, idx, &extended) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => break :indexed,
};
var log = adopted orelse break :indexed;
taken = true;
if (extended and sidecar == .refresh_on_scan) refreshSettingLogIndex(gpa, io, cache_dir, &log);
return log;
}
var log = try logstore.LogStore.openReadOnly(gpa, sf);
// The scan just paid for the product the index persists.
if (sidecar == .refresh_on_scan) refreshSettingLogIndex(gpa, io, cache_dir, &log);
return log;
}
/// Open this checkout's store read-only for a one-shot config read, adopting
/// the persisted scan index when the cache holds a fresh one (the CLI
/// half). The plain `openReadOnly` walks every frame header, which `open` and
/// `update` were paying once per setting on every materialization. A
/// refused index (absent, stale, corrupt) falls back to the scan and refreshes
/// the sidecar, as `Repository.logStore` does. The sidecar lives with the
/// STORE, never a worktree, so the cache directory is resolved through the
/// checkout's store pointer first.
const OpenedStoreLog = struct {
file: std.Io.File,
fs: storage.FileStorage,
log: logstore.LogStore,
fn close(self: *OpenedStoreLog, gpa: std.mem.Allocator, io: std.Io) void {
self.log.deinit(gpa);
self.file.close(io);
gpa.destroy(self);
}
};
/// Heap-allocated: the log holds a pointer into `fs`, so the record must
/// never move after `storageFile()` is taken.
fn openStoreLogForRead(gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir) !*OpenedStoreLog {
var file = cli.openStoreRead(io, root_dir) catch return error.Storage;
errdefer file.close(io);
const st = file.stat(io) catch return error.Storage;
const out = gpa.create(OpenedStoreLog) catch return error.OutOfMemory;
errdefer gpa.destroy(out);
out.* = .{ .file = file, .fs = storage.FileStorage.init(io, file, st.size), .log = undefined };
out.log = openLogAdoptingIndex(gpa, io, root_dir, out.fs.storageFile(), .refresh_on_scan) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
};
return out;
}
/// This repository's stored answer, or null when it has no opinion. The
/// booleans consult their explicit-set bit: without it a repository would
/// always have an opinion and the global tier could never be reached.
fn repositorySetting(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
root_dir: std.Io.Dir,
setting: zit.settings.Setting,
) !?[]const u8 {
const opened = try openStoreLogForRead(gpa, io, root_dir);
defer opened.close(gpa, io);
var oc = (opened.log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
}) orelse return null;
defer oc.deinit(gpa);
return configValue(arena, &oc.config, setting);
}
/// One config frame's answer for one setting, or null when the frame has no
/// opinion. PURE: the caller supplies the frame, so a verb that has already
/// read `Config` once resolves every name it needs without a second store
/// open (now the rule rather than the exception). The arena
/// dupe is the only way this can fail.
pub fn configValue(
arena: std.mem.Allocator,
config: *const logstore.Config,
setting: zit.settings.Setting,
) error{OutOfMemory}!?[]const u8 {
return switch (setting) {
.autosync => if (config.autosync_set)
(if (config.autosync_off) "off" else if (std.mem.eql(u8, config.autosync_mode, "pullonly")) "pullonly" else "on")
else
null,
.editor => if (config.editor.len == 0) null else try arena.dupe(u8, config.editor),
.clearsign => if (config.clearsign_set)
(if (config.clearsign) "on" else "off")
else
null,
// A stored empty string is "no opinion" for these: the config frame has
// no separate absent marker for its string fields, and an operator who
// wants the empty value spells it by clearing, which is the same thing.
.ignore_glob => if (config.ignore_glob.len == 0) null else try arena.dupe(u8, config.ignore_glob),
.pgp_command => if (config.pgp_command.len == 0) null else try arena.dupe(u8, config.pgp_command),
.keep_glob => if (config.keep_glob.len == 0) null else try arena.dupe(u8, config.keep_glob),
.manifest => if (config.manifest.len == 0) null else try arena.dupe(u8, config.manifest),
.before_commit => if (config.before_commit.len == 0) null else try arena.dupe(u8, config.before_commit),
.binary_glob => if (config.binary_glob.len == 0) null else try arena.dupe(u8, config.binary_glob),
.crlf_glob => if (config.crlf_glob.len == 0) null else try arena.dupe(u8, config.crlf_glob),
.crnl_glob => if (config.crnl_glob.len == 0) null else try arena.dupe(u8, config.crnl_glob),
.clean_glob => if (config.clean_glob.len == 0) null else try arena.dupe(u8, config.clean_glob),
.empty_dirs => if (config.empty_dirs.len == 0) null else try arena.dupe(u8, config.empty_dirs),
.encoding_glob => if (config.encoding_glob.len == 0) null else try arena.dupe(u8, config.encoding_glob),
.dotfiles => if (config.dotfiles.len == 0) null else try arena.dupe(u8, config.dotfiles),
.mimetypes => if (config.mimetypes.len == 0) null else try arena.dupe(u8, config.mimetypes),
.attachment_size_limit => if (config.attachment_size_limit.len == 0) null else try arena.dupe(u8, config.attachment_size_limit),
// `init` stores the default explicitly, so a stored `trunk` says no
// more than the default does; reporting it as the default keeps
// `--changed` meaning "this repository chose something else".
.main_branch => if (config.default_branch.len == 0 or std.mem.eql(u8, config.default_branch, zit.settings.Setting.main_branch.default())) null else try arena.dupe(u8, config.default_branch),
};
}
/// Open this checkout's store, read its config ONCE, and hand back the owned
/// result; null on any failure, exactly as every reader of an optional
/// repository value degrades today. `.read_only`, because the listing verbs
/// are pinned tree-untouching (B-extras-16) and must never mint a sidecar.
/// The `OwnedConfig` owns its backing bytes, so the log and its file are
/// closed here and the config outlives them.
pub fn readRepositoryConfig(
gpa: std.mem.Allocator,
io: std.Io,
root_dir: std.Io.Dir,
) ?logstore.OwnedConfig {
var file = cli.openStoreRead(io, root_dir) catch return null;
defer file.close(io);
const st = file.stat(io) catch return null;
var fs = zit.storage.FileStorage.init(io, file, st.size);
// Indexed open: this config read rides every extras/clean/
// addremove listing, each of which was paying its own full header scan.
var log = openLogAdoptingIndex(gpa, io, root_dir, fs.storageFile(), .read_only) catch return null;
defer log.deinit(gpa);
return log.readConfig(gpa) catch null;
}
/// Resolve one setting from a config the caller already holds. The whole job
/// is to make "resolve a name from the frame in my hand" one call, so a verb
/// that reads `Config` once can answer for every name it needs.
///
/// The error set is written out rather than inferred: `resolveSettingWith`'s
/// own failures are all `catch return null`, so `configValue`'s arena dupe is
/// the only thing left that can fail. The day a read there stops degrading,
/// the compiler names it at this boundary.
pub fn localSetting(
io: std.Io,
arena: std.mem.Allocator,
root_dir: std.Io.Dir,
amb: cli.Ambient,
config: ?*const logstore.Config,
setting: zit.settings.Setting,
) error{OutOfMemory}!zit.settings.Resolved {
const repository = if (config) |c| try configValue(arena, c, setting) else null;
return resolveSettingWith(io, arena, root_dir, amb, setting, repository);
}
/// The operator-scope answer, or null when there is no global file or it says
/// nothing about this setting. A missing file is never an error: no global
/// tier is the state of every installation that never set one.
fn globalSetting(
io: std.Io,
arena: std.mem.Allocator,
amb: cli.Ambient,
setting: zit.settings.Setting,
) !?[]const u8 {
const zset = zit.settings;
var path_buf: [1024]u8 = undefined;
const g = zset.globalPath(amb.environ, &path_buf) orelse return null;
const text = std.Io.Dir.cwd().readFileAlloc(io, g.path, arena, .limited(zset.max_versioned_bytes)) catch return null;
const found = zset.globalLookup(text, setting) orelse return null;
return found;
}
/// The local settings `zit settings` edits, as one table rather than an
/// if-chain per name. Every one of them is repo-LOCAL and never crosses the
/// wire: `autosync` and `ignore-glob` because they describe this working
/// copy's habits, `clearsign` and `pgp-command` because fossil classifies the
/// latter SENSITIVE and puts the former in no config group. The settings
/// that DO propagate are a separate mechanism — versioned `.zit-settings/`
/// files and the sync §9.3 config groups — not this verb.
const LocalSetting = enum {
autosync,
ignore_glob,
clearsign,
pgp_command,
const known_names = "autosync, ignore-glob, clearsign, pgp-command";
const Kind = enum { boolean, string };
fn wireName(self: LocalSetting) []const u8 {
return switch (self) {
.autosync => "autosync",
.ignore_glob => "ignore-glob",
.clearsign => "clearsign",
.pgp_command => "pgp-command",
};
}
fn byName(name: []const u8) ?LocalSetting {
inline for (comptime std.enums.values(LocalSetting)) |s| {
if (std.mem.eql(u8, name, s.wireName())) return s;
}
return null;
}
fn kind(self: LocalSetting) Kind {
return switch (self) {
.autosync, .clearsign => .boolean,
.ignore_glob, .pgp_command => .string,
};
}
/// This setting's stored value as display text, borrowed from `current`.
fn read(self: LocalSetting, current: Settings) []const u8 {
return switch (self) {
// autosync is stored as the DEVIATION, so the flag reads inverted.
.autosync => if (current.autosync_off) "off" else "on",
.clearsign => if (current.clearsign) "on" else "off",
.ignore_glob => current.ignore_glob,
.pgp_command => current.pgp_command,
};
}
/// The patch that sets this setting to `value`, or null when `value` is
/// not one this setting accepts.
fn patch(self: LocalSetting, value: []const u8) ?ConfigPatch {
switch (self) {
.autosync, .clearsign => {
const on = if (std.mem.eql(u8, value, "on"))
true
else if (std.mem.eql(u8, value, "off"))
false
else
return null;
// Inverted in storage for autosync (absent = fossil's default,
// on); plain for clearsign (absent = fossil's default, off).
return switch (self) {
.autosync => .{ .autosync_off = !on },
.clearsign => .{ .clearsign = on },
else => unreachable,
};
},
.ignore_glob, .pgp_command => {
// The config frame caps every string field; refusing beats
// truncating, which would silently change what runs or what
// is ignored.
if (value.len > logstore.max_config_str) return null;
return switch (self) {
.ignore_glob => .{ .ignore_glob = value },
.pgp_command => .{ .pgp_command = value },
else => unreachable,
};
},
}
}
fn reportBadValue(self: LocalSetting, streams: cli.Streams, value: []const u8) void {
switch (self.kind()) {
.boolean => streams.err.print(
"zit settings: {s} takes 'on' or 'off'\ntry 'zit settings --help'\n",
.{self.wireName()},
) catch {},
.string => {
_ = value;
if (self == .ignore_glob) {
streams.err.print(
"zit settings: ignore-glob is limited to {d} bytes; use the versioned {s} file for longer lists\n",
.{ logstore.max_config_str, zit.ignore.versioned_path },
) catch {};
} else {
streams.err.print(
"zit settings: {s} is limited to {d} bytes\n",
.{ self.wireName(), logstore.max_config_str },
) catch {};
}
},
}
}
fn print(self: LocalSetting, streams: cli.Streams, value: []const u8) void {
streams.out.print("{s} {s}\n", .{ self.wireName(), value }) catch {};
}
};
/// Every local setting's value, read in ONE store open. Four separate readers
/// would open, scan and close the log four times for a bare `zit settings`.
const Settings = struct {
autosync_off: bool = false,
clearsign: bool = false,
ignore_glob: []const u8 = "",
pgp_command: []const u8 = "",
};
/// Backing store for the two string settings a `Settings` borrows: the config
/// frame's own buffer dies with the read, so the values are copied out.
const SettingsBuf = [2 * logstore.max_config_str]u8;
fn readSettings(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
buf: *SettingsBuf,
) !Settings {
var file = cli.openStoreRead(io, dir) catch return error.Storage;
defer file.close(io);
const st = file.stat(io) catch return error.Storage;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
};
defer log.deinit(gpa);
var oc = (log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
}) orelse return .{};
defer oc.deinit(gpa);
const glob = oc.config.ignore_glob;
const pgp = oc.config.pgp_command;
std.debug.assert(glob.len + pgp.len <= buf.len);
@memcpy(buf[0..glob.len], glob);
@memcpy(buf[glob.len..][0..pgp.len], pgp);
return .{
.autosync_off = oc.config.autosync_off,
.clearsign = oc.config.clearsign,
.ignore_glob = buf[0..glob.len],
.pgp_command = buf[glob.len..][0..pgp.len],
};
}
/// fossil's posture: when the working tree carries the versioned
/// ignore file, the sweeps use IT, and a display of the local setting must
/// say so — a value that reads as live while being shadowed is how an
/// operator edits the wrong one twice.
fn noteVersionedShadow(
io: std.Io,
root_dir: std.Io.Dir,
streams: cli.Streams,
setting: zit.settings.Setting,
r: zit.settings.Resolved,
) bool {
const shadowed = r.shadowed orelse return false;
const dir = r.versioned_dir orelse return false;
if (noWarnPresent(io, root_dir, setting)) return false;
streams.err.print(
"note: {s}/{s} overrides the {s} setting in the {s} tier; create {s}/{s}.no-warn to silence\n",
.{ dir.dirName(), setting.name(), setting.name(), shadowed.label(), dir.dirName(), setting.name() },
) catch {};
return true;
}
/// Whether either settings directory holds `<name>.no-warn`.
///
/// EITHER silences the note, whichever directory holds the winning file:
/// fossil has one directory and cannot pin this, and zit picks the permissive
/// rule because the two directories are one setting with two spellings -- a
/// silence the operator asked for should not evaporate because a later commit
/// chose the other spelling.
fn noWarnPresent(io: std.Io, root_dir: std.Io.Dir, setting: zit.settings.Setting) bool {
inline for (comptime std.enums.values(zit.settings.VersionedDir)) |d| {
var buf: [160]u8 = undefined;
if (std.fmt.bufPrint(&buf, "{s}/{s}.no-warn", .{ d.dirName(), setting.name() })) |path| {
if (root_dir.statFile(io, path, .{ .follow_symlinks = false })) |_| return true else |_| {}
} else |_| {}
}
return false;
}
/// One line for a versioned file that cannot supply its
/// setting. `autosync` describes this machine's relationship to the network,
/// so no tracked file may set it; fossil ignores such a file silently, and zit
/// says so once rather than leaving the operator to wonder why their committed
/// value does nothing.
fn noteVersionedIgnored(
io: std.Io,
root_dir: std.Io.Dir,
streams: cli.Streams,
setting: zit.settings.Setting,
) void {
std.debug.assert(!setting.versionable());
inline for (comptime std.enums.values(zit.settings.VersionedDir)) |d| {
var buf: [160]u8 = undefined;
if (std.fmt.bufPrint(&buf, "{s}/{s}", .{ d.dirName(), setting.name() })) |path| {
if (root_dir.statFile(io, path, .{ .follow_symlinks = false })) |_| {
// Two reasons a file cannot supply a setting, and the note says
// which: network posture belongs to the machine, and an
// executable-tier value a clone could set would be remote code
// execution.
const why: []const u8 = switch (setting.tier()) {
.local => "a local setting",
.executable => "never supplied by a tracked file",
.repository => "kept in the repository's own config",
.data, .consented => unreachable, // versionable; not this path
};
streams.err.print(
"note: {s} is versioned and ignored; {s} is {s}\n",
.{ path, setting.name(), why },
) catch {};
} else |_| {}
} else |_| {}
}
}
/// `.zitignore` is not a settings file -- it is a second DIALECT that applies
/// alongside whatever `ignore-glob` resolves to, so its presence is
/// worth one line but is not a shadowing.
fn noteGitignorePresent(io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams) void {
if (root_dir.statFile(io, zit.ignore.gitignore_path, .{ .follow_symlinks = false })) |_| {
streams.err.print(
"note: {s} is present; its patterns apply in addition to the ignore-glob setting\n",
.{zit.ignore.gitignore_path},
) catch {};
} else |_| {}
}
/// The autosync hook: a best-effort pull (before commit/update/merge)
/// or push (after a successful commit) against the saved remote. Silent no-op
/// when autosync is off or no remote is saved (a nag would punish the
/// single-repository workflow); any failure is ONE warning line and the
/// caller proceeds — autosync must never turn a working offline operation
/// into an error (fossil's posture). The exchange itself is the real verb
/// runner, so behavior, diagnostics, and persistence never fork from a
/// hand-run `zit pull`/`zit push`.
pub const AutosyncOp = enum { pull, push };
pub fn autosyncPull(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, verb: []const u8) void {
autosync(gpa, io, arena, streams, verb, .pull, null);
}
pub fn autosyncPush(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, verb: []const u8, amb: cli.Ambient) void {
autosync(gpa, io, arena, streams, verb, .push, amb);
}
fn autosync(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
streams: cli.Streams,
verb: []const u8,
op: AutosyncOp,
amb: ?cli.Ambient,
) void {
{
const root = (cli.discoverRoot(arena, io) catch return) orelse return;
var root_dir = root.dir;
defer root_dir.close(io);
// Once per invocation: the pull hook runs before the push one on
// every verb that has both, so this is the first hook to fire.
if (op == .pull) noteVersionedIgnored(io, root_dir, streams, .autosync);
const off = readAutosyncOff(gpa, io, root_dir) catch return;
if (off) return;
// `autosync pullonly`: pull as usual, never push.
if (op == .push) {
if (amb) |a| {
const r = resolveSetting(gpa, io, arena, root_dir, a, .autosync) catch return;
if (std.mem.eql(u8, r.value, "pullonly")) return;
}
}
var url_buf: [logstore.max_config_str]u8 = undefined;
_ = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch return;
}
// The exchange reports on STDERR only: the hosting verb owns stdout (a
// commit's contract is ONE check-in line there), and autosync is ambient
// narration, not the verb's answer.
const side_streams: cli.Streams = .{ .out = streams.err, .err = streams.err };
const rc = switch (op) {
.pull => runPull(gpa, io, arena, &.{}, side_streams, amb orelse .{ .now_epoch = 0, .user_env = null }),
.push => runPush(gpa, io, arena, &.{}, side_streams, amb.?),
};
if (rc != .ok) {
streams.err.print(
"zit {s}: autosync {s} failed; continuing locally\n",
.{ verb, @tagName(op) },
) catch {};
}
}
/// The stored autosync deviation, read off the config frame. Storage faults
/// propagate; a repository with no config frame reads as the default (on).
fn readAutosyncOff(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !bool {
const opened = try openStoreLogForRead(gpa, io, dir);
defer opened.close(gpa, io);
var oc = (opened.log.readConfig(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
}) orelse return false;
defer oc.deinit(gpa);
return oc.config.autosync_off;
}
pub fn runRemote(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
repo_override: ?[]const u8,
) cli.Exit {
const parsed = switch (cli.parseRemoteArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "remote", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "remote", .extra_argument, tok),
.not_a_url => |tok| {
streams.err.print("zit remote: '{s}' is not an http:// or https:// URL\nzit remote: no subcommands: 'zit remote' shows the saved remote, 'zit remote URL' sets it, 'zit remote --unset' clears it\n", .{tok}) catch {};
return .usage;
},
};
if (parsed.help) {
streams.out.writeAll(remote_help) catch {};
return .ok;
}
const root = (cli.resolveRoot(arena, io, repo_override) catch return stateOom(streams, "remote")) orelse {
streams.err.writeAll(not_a_repo_msg) catch {};
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
if (parsed.unset) {
// Overlay remote_url with empty — clears the saved value, every other
// config field preserved by the field-preserving RMW.
updateConfig(gpa, io, root_dir, .{ .remote_url = "" }) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "remote"),
else => return storeAppendFail(streams, "remote", e),
};
streams.out.writeAll("remote cleared\n") catch {};
return .ok;
}
if (parsed.url.len > 0) {
// A URL over the config-field cap cannot be stored (the putConfig assert);
// reject it as usage rather than trip the assertion.
if (parsed.url.len > logstore.max_config_str) {
streams.err.print("zit remote: URL too long (max {d} bytes)\n", .{logstore.max_config_str}) catch {};
return .failure;
}
updateConfig(gpa, io, root_dir, .{ .remote_url = parsed.url }) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "remote"),
else => return storeAppendFail(streams, "remote", e),
};
streams.out.print("remote set to {s}\n", .{parsed.url}) catch {};
return .ok;
}
// Bare `zit remote` — print the saved remote (reusing resolveRemoteUrl's read
// with an empty arg; NoRemote means none is saved).
var url_buf: [logstore.max_config_str]u8 = undefined;
const url = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "remote"),
error.Storage => return repoReadFail(streams, "remote"),
error.NoRemote => {
streams.out.writeAll("no remote set\n") catch {};
return .ok;
},
};
streams.out.print("{s}\n", .{url}) catch {};
return .ok;
}
test "serve §3.3: a pushed file card is persisted durably into the served repository's log" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// A fresh, empty on-disk repository (header only) — the "server" a peer pushes
// new content into. Built through the real LogStore, not seeded from fossil.
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
{
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer seed.deinit(gpa);
try seed.commit();
}
// A pushed full-content file card for a 64-hex (SHA3-256 width) artifact the
// server lacks (sync §7.1: `file HASH SIZE\n` + payload; §4.1 zlib container).
// The name is the artifact's real SHA3-256 (computed by our own hasher, not
// fossil) so the server's §7.7 commit-hook recompute accepts it.
const push_body = "pushed artifact bytes\n"; // 22 bytes
var name_buf: [64]u8 = undefined;
const push_hash = zit.artifact.artifactName(.sha3_256, push_body, &name_buf);
const request_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\npush sc pc\nfile {s} {d}\n{s}", .{ push_hash, push_body.len, push_body });
defer gpa.free(request_plain);
const request = try zit.container.encode(gpa, request_plain, .default);
defer gpa.free(request);
var digest_buf: [digest_len]u8 = undefined;
const push_digest = decodeWireName(push_hash, &digest_buf).?;
const server = zit.sync.Server{ .version = serve_mod.server_version };
// Answer the push twice through the real `serve.Responder` seam over a SINGLE
// open handle (§3.8): the second answer re-reads the current on-disk log from
// the same handle and sees the first push's frame, so it is an idempotent
// re-push (content is addressed by hash — no dup frame). A stale startup
// snapshot would instead re-append the artifact; this handle is never reopened.
{
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
for (0..2) |_| {
const reply = try serveAnswer(&ctx, gpa, request, null, .{}, true);
gpa.free(reply.body);
}
}
// THE PROOF: re-open the log from disk in a fresh handle; the pushed artifact
// is present with its exact content — durably stored, not discarded — and
// appears exactly once (the idempotent re-push added no second frame).
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
var fs = storage.FileStorage.init(io, file, st.size);
var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile());
defer log.deinit(gpa);
const rid = log.lookup(push_digest) orelse return error.PushedArtifactMissing;
const stored = (try log.content(gpa, rid)).?;
defer gpa.free(stored);
try std.testing.expectEqualStrings(push_body, stored);
const rids = try log.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
var artifacts: usize = 0;
for (rids) |r| {
if (log.kindOf(r) != .config) artifacts += 1;
}
try std.testing.expectEqual(@as(usize, 1), artifacts);
}
// --- D4: client verbatim retention + the §7.7 persist hook ------------------
/// Create a fresh, empty (header-only, committed) `.zit` in `dir` so a
/// `persistStore` test can persist into it, exactly like a just-cloned repo.
fn d4SeedEmptyRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void {
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer seed.deinit(gpa);
try seed.commit();
}
/// Reopen the persisted `.zit` in `dir` for reading. `fs_out` is a caller-owned
/// `FileStorage` that MUST outlive the returned `LogStore` (its `storageFile()`
/// vtable borrows `fs_out`); the caller closes `fs_out.file` and deinits `log`.
fn d4OpenLog(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, fs_out: *storage.FileStorage) !logstore.LogStore {
const file = try dir.openFile(io, repo_filename, .{ .mode = .read_write });
const st = try file.stat(io);
fs_out.* = storage.FileStorage.init(io, file, st.size);
return try logstore.LogStore.open(gpa, fs_out.storageFile(), 0);
}
fn d4SeedConfiguredRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void {
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer seed.deinit(gpa);
try seed.putConfig(.{
.hash_policy = 9,
.default_branch = "stable",
.project_id = "project",
.project_code = "init-code",
.login_user = "operator",
.login_secret_hex = repeat("0", 40),
.remote_url = "https://example.test/remote",
.seen_submanifest = true,
.enable_submanifests = false,
.submanifest_k = 3,
.submanifest_auto_retune = false,
.submanifest_settled_cost_shape = @as([32]u8, @splat(0x42)),
});
try seed.commit();
}
test "with no --user the resolver reuses the stored login, never prompts, and an empty --user still means anonymous" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedConfiguredRepo(gpa, io, tmp.dir); // stores login "operator" / "0"*40
var sink: [256]u8 = undefined;
var w = Io.Writer.fixed(&sink);
const s: cli.Streams = .{ .out = &w, .err = &w };
const amb: cli.Ambient = .{ .now_epoch = 0, .user_env = null };
var pw_buf: [max_password_len]u8 = undefined;
var user_buf: [logstore.max_config_str]u8 = undefined;
// Absent --user: the stored login stands in, verbatim, and nothing is
// written to either stream — this is the branch autosync takes, and a
// prompt here would block a commit.
const r = (try resolvePushCredential(gpa, io, tmp.dir, s, "init-code", .{ .url = "" }, amb, &pw_buf, &user_buf)) orelse
return error.AnonymousDespiteStoredLogin;
try std.testing.expectEqualStrings("operator", r.credential.user);
try std.testing.expectEqualStrings(repeat("0", 40), &r.credential.secret_hex);
try std.testing.expect(!r.persist);
try std.testing.expectEqual(@as(usize, 0), w.buffered().len);
// An explicit empty --user is the way to push anonymously regardless.
try std.testing.expect((try resolvePushCredential(gpa, io, tmp.dir, s, "init-code", .{ .url = "", .user = "" }, amb, &pw_buf, &user_buf)) == null);
// Nothing stored — no secret, and separately a secret with no name, which
// is not a login either — absent --user is anonymous, and still no prompt.
for ([_][2][]const u8{ .{ "", "" }, .{ "", repeat("0", 40) } }) |login| {
var bare = std.testing.tmpDir(.{});
defer bare.cleanup();
{
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = try bare.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer seed.deinit(gpa);
try seed.putConfig(.{
.hash_policy = 9,
.default_branch = "trunk",
.project_id = "project",
.project_code = "init-code",
.login_user = login[0],
.login_secret_hex = login[1],
.remote_url = "",
.seen_submanifest = false,
.enable_submanifests = false,
.submanifest_k = 0,
.submanifest_auto_retune = false,
.submanifest_settled_cost_shape = @as([32]u8, @splat(0)),
});
try seed.commit();
}
try std.testing.expect((try resolvePushCredential(gpa, io, bare.dir, s, "init-code", .{ .url = "" }, amb, &pw_buf, &user_buf)) == null);
try std.testing.expectEqual(@as(usize, 0), w.buffered().len);
}
}
test "sync clone persistence: open-log artifacts and one combined config overlay preserve the wrapper result" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var wrapped_dir = std.testing.tmpDir(.{});
defer wrapped_dir.cleanup();
var open_dir = std.testing.tmpDir(.{});
defer open_dir.cleanup();
try d4SeedConfiguredRepo(gpa, io, wrapped_dir.dir);
try d4SeedConfiguredRepo(gpa, io, open_dir.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const content = "one content-bearing clone artifact";
var name_buf: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, content, &name_buf);
const container = try zit.container.encode(gpa, content, .default);
defer gpa.free(container);
const rid = try ms.store().put(name, content);
try ms.store().retain(rid, .{ .full = container });
const patch: ConfigPatch = .{
.project_code = "origin-code",
.enable_submanifests = true,
.submanifest_k = 5,
.submanifest_auto_retune = true,
};
const wrapped = try persistStore(gpa, io, wrapped_dir.dir, ms.store(), null, 0);
try updateConfig(gpa, io, wrapped_dir.dir, patch);
var open_fs: storage.FileStorage = undefined;
var open_log = try d4OpenLog(gpa, io, open_dir.dir, &open_fs);
defer open_fs.file.close(io);
defer open_log.deinit(gpa);
const opened = try persistStoreOpen(gpa, &open_log, ms.store(), null, 0, null);
try updateConfigOpen(gpa, &open_log, patch);
try open_log.commit();
try std.testing.expectEqual(wrapped.written, opened.written);
try std.testing.expectEqual(wrapped.submanifest_gaps, opened.submanifest_gaps);
try std.testing.expectEqual(@as(u64, 3), (try open_log.logKey()).frame_count);
var wrapped_fs: storage.FileStorage = undefined;
var wrapped_log = try d4OpenLog(gpa, io, wrapped_dir.dir, &wrapped_fs);
defer wrapped_fs.file.close(io);
defer wrapped_log.deinit(gpa);
var wrapped_cfg = (try wrapped_log.readConfig(gpa)).?;
defer wrapped_cfg.deinit(gpa);
var opened_cfg = (try open_log.readConfig(gpa)).?;
defer opened_cfg.deinit(gpa);
try std.testing.expectEqualDeep(wrapped_cfg.config, opened_cfg.config);
try std.testing.expectEqualStrings("origin-code", opened_cfg.config.project_code);
try std.testing.expectEqualStrings("operator", opened_cfg.config.login_user);
try std.testing.expectEqualStrings("https://example.test/remote", opened_cfg.config.remote_url);
try std.testing.expect(opened_cfg.config.seen_submanifest);
try std.testing.expect(opened_cfg.config.enable_submanifests);
try std.testing.expectEqual(@as(u8, 5), opened_cfg.config.submanifest_k);
try std.testing.expect(opened_cfg.config.submanifest_auto_retune);
try std.testing.expectEqualDeep(@as([32]u8, @splat(0x42)), opened_cfg.config.submanifest_settled_cost_shape.?);
var digest_buf: [digest_len]u8 = undefined;
const stored = (try open_log.content(gpa, open_log.lookup(decodeWireName(name, &digest_buf).?).?)).?;
defer gpa.free(stored);
try std.testing.expectEqualStrings(content, stored);
}
test "sync clone config: absent origin settings append no frame and preserve init values" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedConfiguredRepo(gpa, io, tmp.dir);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
const patch: ConfigPatch = .{};
try std.testing.expect(!patch.any());
if (patch.any()) try updateConfigOpen(gpa, &log, patch);
try log.commit();
try std.testing.expectEqual(@as(u64, 1), (try log.logKey()).frame_count);
var cfg = (try log.readConfig(gpa)).?;
defer cfg.deinit(gpa);
try std.testing.expectEqual(@as(u8, 9), cfg.config.hash_policy);
try std.testing.expectEqualStrings("stable", cfg.config.default_branch);
try std.testing.expectEqualStrings("project", cfg.config.project_id);
try std.testing.expectEqualStrings("init-code", cfg.config.project_code);
try std.testing.expectEqualStrings("operator", cfg.config.login_user);
try std.testing.expectEqualStrings("https://example.test/remote", cfg.config.remote_url);
try std.testing.expect(cfg.config.seen_submanifest);
try std.testing.expect(!cfg.config.enable_submanifests);
try std.testing.expectEqual(@as(u8, 3), cfg.config.submanifest_k);
try std.testing.expect(!cfg.config.submanifest_auto_retune);
try std.testing.expectEqualDeep(@as([32]u8, @splat(0x42)), cfg.config.submanifest_settled_cost_shape.?);
}
test "streaming persist writes the byte-identical log and releases every persisted entry" {
const gpa = std.testing.allocator;
const io = std.testing.io;
// Two stores fed the same reverse-delta chain (older-as-delta-of-newer,
// fossil's wire shape), one persisted batched and one streamed in
// increments with release. The logs must match byte for byte — streaming
// reorders nothing — and the streamed store must end empty of payload:
// every entry held, none resident.
const links = 6;
var bodies: [links][]u8 = undefined;
var made: usize = 0;
defer for (bodies[0..made]) |b| gpa.free(b);
var names: [links][64]u8 = undefined;
var nlen: [links]usize = undefined;
for (0..links) |i| {
bodies[i] = try std.fmt.allocPrint(gpa, "streamed chain body: one long shared prefix kept across every revision -- revision {d}", .{i});
made += 1;
nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len;
}
const seed = struct {
fn one(a: std.mem.Allocator, s: repostate.ArtifactStore, i: usize, bs: *const [links][]u8, ns: *const [links][64]u8, nl: *const [links]usize) !void {
const rid = try s.put(ns[i][0..nl[i]], bs[i]);
if (i == links - 1) {
const c = try zit.container.encode(a, bs[i], .default);
defer a.free(c);
try s.retain(rid, .{ .full = c });
} else {
const dr = try zit.delta.create(a, bs[i + 1], bs[i]);
defer a.free(dr);
const c = try zit.container.encode(a, dr, .default);
defer a.free(c);
try s.retain(rid, .{ .delta = .{ .src = ns[i + 1][0..nl[i + 1]], .container = c, .target_size = bs[i].len } });
}
}
}.one;
// Batched reference.
var tmp_a = std.testing.tmpDir(.{});
defer tmp_a.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp_a.dir);
var ms_a = repostate.MemStore.init(gpa);
defer ms_a.deinit();
for (0..links) |i| try seed(gpa, ms_a.store(), i, &bodies, &names, &nlen);
{
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
const sum = try persistStoreOpen(gpa, &log, ms_a.store(), null, 0, null);
try std.testing.expectEqual(@as(usize, links), sum.written);
try log.commit();
}
// Streamed: seed in two "rounds", draining after each, releasing as it goes.
var tmp_b = std.testing.tmpDir(.{});
defer tmp_b.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp_b.dir);
var ms_b = repostate.MemStore.init(gpa);
defer ms_b.deinit();
{
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp_b.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var pass = try PersistPass.init(gpa, &log, ms_b.store(), null, &ms_b, 0, null);
defer pass.deinit();
for (0..3) |i| try seed(gpa, ms_b.store(), i, &bodies, &names, &nlen);
try pass.drainNew(); // rids 1..3 are deltas of unseen bases: all defer
for (3..links) |i| try seed(gpa, ms_b.store(), i, &bodies, &names, &nlen);
try pass.drainNew(); // the full form lands and the whole chain drains
const sum = try pass.finish();
try std.testing.expectEqual(@as(usize, links), sum.written);
try std.testing.expectEqual(@as(usize, 0), sum.submanifest_gaps);
try log.commit();
}
// Every streamed entry is now frame-backed and released: held, no bytes.
for (1..links + 1) |rid_us| {
const rid: repostate.Rid = @intCast(rid_us);
try std.testing.expect(ms_b.store().held(rid));
try std.testing.expect(ms_b.store().wire(rid) == null);
try std.testing.expect(ms_b.store().content(rid) == null);
}
// The two logs are byte-identical: streaming changed when frames were
// written, never what or in what order.
const a_bytes = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 20));
defer gpa.free(a_bytes);
const b_bytes = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 20));
defer gpa.free(b_bytes);
try std.testing.expectEqualSlices(u8, a_bytes, b_bytes);
}
test "a basis interned as a phantom before the cursor passed it is revisited and persisted" {
const gpa = std.testing.allocator;
const io = std.testing.io;
// The fossil-peer streaming shape: a delta cfile arrives naming an unseen
// DELTASRC, which interns the basis as a PHANTOM at a low rid; the basis
// content lands at that same rid in a later round, behind the drain's
// cursor. The pass must come back for it — a drain that only moves forward
// leaves the basis unwritten and its waiters unresolvable.
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s_store = ms.store();
const base_body = "phantom-first basis body: the long shared prefix every revision keeps\n";
const delta_body = "phantom-first basis body: the long shared prefix every revision keeps, edited\n";
var base_name: [64]u8 = undefined;
const bn = zit.artifact.artifactName(.sha3_256, base_body, &base_name).len;
var delta_name: [64]u8 = undefined;
const dn = zit.artifact.artifactName(.sha3_256, delta_body, &delta_name).len;
// Round 1: the basis is interned as a phantom (rid 1), then the delta
// arrives as content (rid 2).
const base_rid = try s_store.intern(base_name[0..bn]);
const delta_rid = try s_store.put(delta_name[0..dn], delta_body);
{
const dr = try zit.delta.create(gpa, base_body, delta_body);
defer gpa.free(dr);
const c = try zit.container.encode(gpa, dr, .default);
defer gpa.free(c);
try s_store.retain(delta_rid, .{ .delta = .{ .src = base_name[0..bn], .container = c, .target_size = delta_body.len } });
}
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var pass = try PersistPass.init(gpa, &log, s_store, null, &ms, 0, null);
defer pass.deinit();
try pass.drainNew(); // cursor passes the phantom; the delta defers on it
// Round 2: the basis content lands at the SAME rid, behind the cursor.
{
const got = try s_store.put(base_name[0..bn], base_body);
try std.testing.expectEqual(base_rid, got);
const c = try zit.container.encode(gpa, base_body, .default);
defer gpa.free(c);
try s_store.retain(base_rid, .{ .full = c });
}
try pass.drainNew(); // the revisit sweep drives it, unblocking the delta
const sum = try pass.finish();
try std.testing.expectEqual(@as(usize, 2), sum.written);
try log.commit();
// Both artifacts are durable frames, and both entries were released.
var d: [digest_len]u8 = undefined;
try std.testing.expect(log.lookup(decodeWireName(base_name[0..bn], &d).?) != null);
try std.testing.expect(log.lookup(decodeWireName(delta_name[0..dn], &d).?) != null);
try std.testing.expect(s_store.wire(base_rid) == null and s_store.held(base_rid));
try std.testing.expect(s_store.wire(delta_rid) == null and s_store.held(delta_rid));
}
test "sync persistStoreOpen: malformed artifact retains the wrapper's corrupt error and writes no frame" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const content = "content that the wire name does not actually name\n";
const lying_name = repeat("0", 64);
const rid = try ms.store().put(lying_name, content);
const container = try zit.container.encode(gpa, content, .default);
defer gpa.free(container);
try ms.store().retain(rid, .{ .full = container });
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
try std.testing.expectError(error.Corrupt, persistStoreOpen(gpa, &log, ms.store(), null, 0, null));
try log.commit();
var digest_buf: [digest_len]u8 = undefined;
try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null);
}
test "sync §7.7: persistStore recomputes the artifact hash and aborts on a lying container, writing no frame" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// A received full cfile whose wire NAME does not name its content (§7.7 —
// a clone skips the transfer-layer pre-storage check, so a lying container
// reaches the persist commit hook).
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const content = "content that the wire name does not actually name\n";
const lying_name = repeat("0", 64); // not SHA3-256(content)
const rid = try s.put(lying_name, content);
const container = try zit.container.encode(gpa, content, .default);
defer gpa.free(container);
try s.retain(rid, .{ .full = container });
// §7.7 L1496-1498 — recompute and abort before ANY frame is written.
try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, ms.store(), null, 0));
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var digest_buf: [digest_len]u8 = undefined;
const digest = decodeWireName(lying_name, &digest_buf).?;
try std.testing.expect(log.lookup(digest) == null); // no frame landed
}
test "sync §7.2/§4.1: persistStore stores received full and delta wire forms verbatim; a re-serve preserves the delta container byte-for-byte" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Model a clone reply in §8.2 RID order: a full base cfile then a delta
// cfile against it. Names are the real SHA3-256 so the §7.7 hook accepts.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const base = "the base artifact with a long shared run so the delta stays a compact copy payload distinct in size from the reconstructed target";
const target = base ++ " plus a small trailing mutation";
var bn: [64]u8 = undefined;
var tn: [64]u8 = undefined;
const base_name = zit.artifact.artifactName(.sha3_256, base, &bn);
const target_name = zit.artifact.artifactName(.sha3_256, target, &tn);
const base_container = try zit.container.encode(gpa, base, .default);
defer gpa.free(base_container);
const base_rid = try s.put(base_name, base);
try s.retain(base_rid, .{ .full = base_container });
const raw = try zit.delta.create(gpa, base, target);
defer gpa.free(raw);
const delta_container = try zit.container.encode(gpa, raw, .default);
defer gpa.free(delta_container);
const ts = try zit.delta.targetSize(raw);
const delta_rid = try s.put(target_name, target);
try s.retain(delta_rid, .{ .delta = .{ .src = base_name, .container = delta_container, .target_size = ts } });
const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 2), written.written);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// The target persisted as a `.delta` frame that reconstructs to the original.
var tdig: [digest_len]u8 = undefined;
const target_digest = decodeWireName(target_name, &tdig).?;
const trid = log.lookup(target_digest).?;
try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(trid));
const got = (try log.content(gpa, trid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(target, got);
// Clone-of-clone: a re-serve snapshot emits the SAME delta container bytes
// (§7.2 retransmitted verbatim) — the compactness survives the second hop.
var serve_ms = repostate.MemStore.init(gpa);
defer serve_ms.deinit();
try loadContentStore(gpa, &log, &serve_ms);
const serve_rid = serve_ms.store().lookup(target_name).?;
const wf = serve_ms.store().wire(serve_rid).?;
try std.testing.expect(wf.delta != null);
try std.testing.expectEqualSlices(u8, delta_container, wf.container);
try std.testing.expectEqual(@as(u64, target.len), wf.delta.?.target_size);
}
/// A tree manifest naming `subs` (submanifests §3.6), plus the §6.2 Z card, so
/// the fixture is a well-formed card sequence rather than a bare S run.
fn treeManifestBytes(gpa: std.mem.Allocator, subs: []const []const u8) ![]u8 {
var body: std.ArrayListUnmanaged(u8) = .empty;
defer body.deinit(gpa);
try body.appendSlice(gpa, "C tree\nD 2026-07-27T12:00:00.000\n");
// §3.4 — the S run ascends by filename; `a`, `b`, ... keeps it legal.
for (subs, 0..) |h, i| {
try body.print(gpa, "S {c}.txt {s}\n", .{ @as(u8, 'a') + @as(u8, @intCast(i)), h });
}
try body.appendSlice(gpa, "U u\n");
var digest: [16]u8 = undefined;
std.crypto.hash.Md5.hash(body.items, &digest, .{});
return std.fmt.allocPrint(gpa, "{s}Z {x}\n", .{ body.items, &digest });
}
test "submanifests §7.1: the persist commit hook phantomizes every absent S reference" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Two leaves that were never received — the state a clone from a peer whose
// OWN closure is incomplete lands in. Nothing on the receive path can see
// this: a clone stores wire-backed and inflates nothing.
const leaf_a = "F a.txt " ++ (repeat("1", 64)) ++ "\n";
const leaf_b = "F b.txt " ++ (repeat("2", 64)) ++ "\n";
var an: [64]u8 = undefined;
var bn: [64]u8 = undefined;
const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an);
const name_b = zit.artifact.artifactName(.sha3_256, leaf_b, &bn);
const tree = try treeManifestBytes(gpa, &.{ name_a, name_b });
defer gpa.free(tree);
var tn: [64]u8 = undefined;
const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const rid = try s.put(tree_name, tree);
const container = try zit.container.encode(gpa, tree, .default);
defer gpa.free(container);
try s.retain(rid, .{ .full = container });
// §7.1: "Storing a structural artifact that carries S cards whose referenced
// sub-manifests are absent MUST create a phantom for each missing reference."
const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), summary.written);
try std.testing.expectEqual(@as(usize, 2), summary.submanifest_gaps);
}
test "persist scoped past the holdings writes nothing and re-walks only received" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
// Five full-form blobs (the already-durable holdings), named by true hash.
inline for (.{ "alpha", "bravo", "charlie", "delta", "echo" }) |body| {
var nm: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, body, &nm);
const rid = try s.put(name, body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
}
const first = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 5), first.written);
// A pull that received nothing: scope past all five holdings. The walk
// never begins, so nothing is persisted and nothing is re-visited (this
// was O(store) — every holding read once from disk to be discarded).
const noop = try persistStore(gpa, io, tmp.dir, ms.store(), null, 5);
try std.testing.expectEqual(@as(usize, 0), noop.written);
// Three artifacts arrive (rids 6..8); scope past the original five. Only the
// new three are persisted — a mirror's incremental pull no longer re-walks
// the millions it already holds.
inline for (.{ "foxtrot", "golf", "hotel" }) |body| {
var nm: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, body, &nm);
const rid = try s.put(name, body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
}
const incr = try persistStore(gpa, io, tmp.dir, ms.store(), null, 5);
try std.testing.expectEqual(@as(usize, 3), incr.written);
// A full re-persist (start 0) early-returns every artifact and reaches the
// same eight-artifact total, so scoping removed work, never content.
const full = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 8), full.written);
}
test "submanifests §7.1/§8.1: the S-reference gap survives a RESTART, rebuilt from the durable indicator" {
// §7.1's phantom is an interned entry with no frame, "exactly as an absent
// delta source creates a source phantom" — and `LogStore.intern` states the
// consequence outright: "Phantoms never reach disk." So a restart loses
// them, and the only thing that can make them correct across one is a
// REBUILD from the durable content-bearing frames.
//
// This test is that invariant: run 1 mints the gap, the process ends, run 2
// re-derives the identical gap over a store loaded from `.zit` alone. Remove
// the rebuild (`rescan_persisted`) and run 2 reports ZERO gaps — a
// repository that quietly declares an unenumerable check-in converged.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
const leaf_a = "F a.txt " ++ (repeat("1", 64)) ++ "\n";
const leaf_b = "F b.txt " ++ (repeat("2", 64)) ++ "\n";
var an: [64]u8 = undefined;
var bn: [64]u8 = undefined;
const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an);
const name_b = zit.artifact.artifactName(.sha3_256, leaf_b, &bn);
const tree = try treeManifestBytes(gpa, &.{ name_a, name_b });
defer gpa.free(tree);
var tn: [64]u8 = undefined;
const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn);
// RUN 1 — the tree lands; neither sub-manifest is held.
{
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(tree_name, tree);
const first = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), first.written);
try std.testing.expectEqual(@as(usize, 2), first.submanifest_gaps);
}
// RESTART — nothing survives but the bytes in `.zit`.
var ms2 = repostate.MemStore.init(gpa);
defer ms2.deinit();
{
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// §8.1 — the indicator IS durable: "record a local seen-submanifest
// indicator". It is the one byte the rebuild rests on.
var oc = (try log.readConfig(gpa)) orelse return error.NoConfig;
defer oc.deinit(gpa);
try std.testing.expect(oc.config.seen_submanifest);
// …and the phantoms are NOT durable, asserted rather than assumed —
// this is the premise the whole rebuild exists to answer.
var adig: [digest_len]u8 = undefined;
try std.testing.expect(log.lookup(decodeWireName(name_a, &adig).?) == null);
// The snapshot a fresh process starts every verb from.
try loadContentStore(gpa, &log, &ms2);
}
// Only the tree came back: the store a second run works from holds no trace
// of the gap.
try std.testing.expectEqual(@as(usize, 1), ms2.store().count());
// RUN 2 — the gap is re-derived, identically, from the durable frames.
const second = try persistStore(gpa, io, tmp.dir, ms2.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), second.written);
try std.testing.expectEqual(@as(usize, 2), second.submanifest_gaps);
}
test "submanifests §8.1: a repository predating this draft records no indicator and rescans nothing" {
// The compatibility half of the rebuild: the indicator is false, so the
// rescan is unreachable and a second persist over the same store is exactly
// the sweep it always was — no config frame appended, no gap invented.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
const flat = "C msg\nD 2026-07-27T12:00:00.000\nF a.txt " ++ (repeat("1", 64)) ++ "\nU u\n";
var fn_buf: [64]u8 = undefined;
const flat_name = zit.artifact.artifactName(.sha3_256, flat, &fn_buf);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(flat_name, flat);
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
const again = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 0), again.submanifest_gaps);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// No indicator was written at all: `d4SeedEmptyRepo` stores no config, and
// nothing above gave it a reason to start one.
try std.testing.expect((try log.readConfig(gpa)) == null);
}
test "submanifests §7.1: a persisted tree whose sub-manifests are held reports no gap" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
const leaf_a = "F a.txt " ++ (repeat("1", 64)) ++ "\n";
var an: [64]u8 = undefined;
const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an);
const tree = try treeManifestBytes(gpa, &.{name_a});
defer gpa.free(tree);
var tn: [64]u8 = undefined;
const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
// The sub-manifest is stored at a LOWER rid than the tree in one direction
// and a higher one in the other; the gap verdict must not depend on which,
// which is why it is taken once, after the whole sweep.
_ = try s.put(tree_name, tree);
_ = try s.put(name_a, leaf_a);
const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 2), summary.written);
try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps);
// The §7.1 phantom was minted for the leaf BEFORE the leaf's own frame was
// written (the tree persists first), so this asserts the phantom-upgrade
// path storage actually takes: the frame must read back byte-for-byte, not
// merely exist. A §7.1 duty that corrupted a frame would be worse than an
// unmet one.
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var ldig: [digest_len]u8 = undefined;
const leaf_digest = decodeWireName(name_a, &ldig).?;
const lrid = log.lookup(leaf_digest).?;
try std.testing.expect(!log.isPhantom(lrid));
const got = (try log.content(gpa, lrid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(leaf_a, got);
}
test "submanifests §7.1: an artifact predating this draft costs the persist hook no gap and no phantom" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// The backwards-compatibility guarantee as a test: a flat manifest carries
// no S card, so §7.1's duty over it is empty and the persisted bytes and
// frame count are exactly what they were before this rule existed.
const flat = "C msg\nD 2026-07-27T12:00:00.000\nF a.txt " ++ (repeat("1", 64)) ++ "\nU u\n";
var fn_buf: [64]u8 = undefined;
const flat_name = zit.artifact.artifactName(.sha3_256, flat, &fn_buf);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(flat_name, flat);
const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), summary.written);
try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
const rids = try log.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
var artifacts: usize = 0;
for (rids) |r| {
if (log.kindOf(r) != .config) artifacts += 1;
}
try std.testing.expectEqual(@as(usize, 1), artifacts);
}
test "submanifests §7.1: a malformed S run mints no phantom at the persist hook" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// §3.2: a third token on S is rejected. Validate-then-apply means the FIRST
// (well-formed) S card mints nothing either — a half-read artifact never
// leaves a half-minted phantom set behind.
const bad = "C tree\nS a.txt " ++ (repeat("1", 64)) ++ "\nS b.txt " ++ (repeat("2", 64)) ++ " extra\nU u\n";
var bn: [64]u8 = undefined;
const bad_name = zit.artifact.artifactName(.sha3_256, bad, &bn);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(bad_name, bad);
const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), summary.written);
try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps);
}
test "sync §3.3: persistStore falls back to full form when a retained delta's basis is absent, never a phantom-based delta" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Only the delta is present; its basis was never received/stored (a legal
// §3.3 outcome that must not persist a delta over a phantom base).
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const base = "an absent basis whose long shared run would make a compact delta if it were here at all";
const target = base ++ " mutated once";
var bn: [64]u8 = undefined;
var tn: [64]u8 = undefined;
const base_name = zit.artifact.artifactName(.sha3_256, base, &bn);
const target_name = zit.artifact.artifactName(.sha3_256, target, &tn);
const raw = try zit.delta.create(gpa, base, target);
defer gpa.free(raw);
const dc = try zit.container.encode(gpa, raw, .default);
defer gpa.free(dc);
const ts = try zit.delta.targetSize(raw);
const rid = try s.put(target_name, target);
try s.retain(rid, .{ .delta = .{ .src = base_name, .container = dc, .target_size = ts } });
const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), written.written);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// The target persisted as a FULL (.zlib) frame — the fallback, not a delta.
var tdig: [digest_len]u8 = undefined;
const target_digest = decodeWireName(target_name, &tdig).?;
const trid = log.lookup(target_digest).?;
try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(trid));
const got = (try log.content(gpa, trid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(target, got);
// No phantom basis frame was minted for the absent DELTASRC.
var bdig: [digest_len]u8 = undefined;
const base_digest = decodeWireName(base_name, &bdig).?;
try std.testing.expect(log.lookup(base_digest) == null);
}
test "sync §7.7/§3.3: persistStore stores a retained delta chain up to the read bound and falls back to full form beyond it" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const cap: usize = logstore.max_delta_chain; // the READ bound (32)
const n: usize = cap + 2; // links 0..cap+1: link cap+1's basis sits at depth cap
// A chain grown one byte per link so each delta is tiny. Link i's basis is
// link i-1 (§8.2 RID order); the persist path stores a `.delta` iff the base
// depth is under `max_delta_chain`, so link cap+1 falls back to full form.
var content: std.ArrayList(u8) = .empty;
defer content.deinit(gpa);
try content.appendSlice(gpa, "chain-root-artifact-with-a-long-shared-run-for-compact-forward-deltas");
var prev_name: [64]u8 = undefined;
var prev_name_len: usize = 0;
var deep_name: [64]u8 = undefined; // link cap (deepest stored delta)
var deep_len: usize = 0;
var last_name: [64]u8 = undefined; // link cap+1 (the fallback)
var last_len: usize = 0;
var i: usize = 0;
while (i < n) : (i += 1) {
try content.append(gpa, 'a' + @as(u8, @intCast(i % 26)));
const cur = content.items;
const prev = cur[0 .. cur.len - 1];
var nbuf: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, cur, &nbuf);
const rid = try s.put(name, cur);
if (i == 0) {
const c = try zit.container.encode(gpa, cur, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
} else {
const raw = try zit.delta.create(gpa, prev, cur);
defer gpa.free(raw);
const c = try zit.container.encode(gpa, raw, .default);
defer gpa.free(c);
const tsz = try zit.delta.targetSize(raw);
try s.retain(rid, .{ .delta = .{ .src = prev_name[0..prev_name_len], .container = c, .target_size = tsz } });
}
@memcpy(prev_name[0..name.len], name);
prev_name_len = name.len;
if (i == cap) {
@memcpy(deep_name[0..name.len], name);
deep_len = name.len;
}
if (i == n - 1) {
@memcpy(last_name[0..name.len], name);
last_len = name.len;
}
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// Link cap is the deepest `.delta` the write bound allows; link cap+1 fell
// back to a full `.zlib` frame yet still reconstructs (never a Corrupt read).
var ddig: [digest_len]u8 = undefined;
const deep_rid = log.lookup(decodeWireName(deep_name[0..deep_len], &ddig).?).?;
try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(deep_rid));
var ldig: [digest_len]u8 = undefined;
const last_rid = log.lookup(decodeWireName(last_name[0..last_len], &ldig).?).?;
try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(last_rid));
const got = (try log.content(gpa, last_rid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(content.items, got);
}
test "sync §7.2/§7.4/§3.3: a dangling reverse-delta cfile persists as a verbatim .delta frame that re-serves byte-identically" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Fossil's clone stream: a reverse delta (the OLDER revision stored as a
// delta of the NEWER one) arrives BEFORE its basis, so it is dangling. The
// client retains the §4.1 container through the pending queue and, when the
// basis lands, persists it as a compact `.delta` — not a full re-compression.
const newer = "artifact NEWER: a long shared body kept verbatim so the reverse delta reconstructing the older revision stays a compact copy op";
const older = "artifact NEWER: a long shared body kept verbatim so the reverse delta reconstructing the older revision stays a compact copy op minus tail";
var nn: [64]u8 = undefined;
var on: [64]u8 = undefined;
const newer_name = zit.artifact.artifactName(.sha3_256, newer, &nn);
const older_name = zit.artifact.artifactName(.sha3_256, older, &on);
const newer_container = try zit.container.encode(gpa, newer, .default);
defer gpa.free(newer_container);
const raw = try zit.delta.create(gpa, newer, older); // OLDER = delta ∘ NEWER
defer gpa.free(raw);
const delta_container = try zit.container.encode(gpa, raw, .default);
defer gpa.free(delta_container);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
var client = zit.sync.Client{ .operation = .clone, .state = .clone_round0 };
defer client.deinit(gpa);
var body: std.ArrayList(u8) = .empty;
defer body.deinit(gpa);
try body.appendSlice(gpa, "clone_seqno 0\npush 1111111111111111 aa87ea41b0bb05ba528f7a3bdba700d7e23d6318\n");
// Reverse (dangling) order: the delta cfile precedes its basis cfile.
const dh = try std.fmt.allocPrint(gpa, "cfile {s} {s} {d} {d}\n", .{ older_name, newer_name, older.len, delta_container.len });
defer gpa.free(dh);
try body.appendSlice(gpa, dh);
try body.appendSlice(gpa, delta_container);
try body.append(gpa, '\n');
const bh = try std.fmt.allocPrint(gpa, "cfile {s} {d} {d}\n", .{ newer_name, newer.len, newer_container.len });
defer gpa.free(bh);
try body.appendSlice(gpa, bh);
try body.appendSlice(gpa, newer_container);
try body.append(gpa, '\n');
_ = try client.consumeReply(gpa, ms.store(), body.items);
const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 2), written.written);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// §3.3 — the older revision persisted as a `.delta`, not a full re-compression,
// and reconstructs the original bytes (delta ∘ basis).
var od: [digest_len]u8 = undefined;
const older_rid = log.lookup(decodeWireName(older_name, &od).?).?;
try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(older_rid));
const got = (try log.content(gpa, older_rid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(older, got);
// §7.2 — a clone-of-clone re-serve re-emits the SAME delta container byte-for-byte.
var serve_ms = repostate.MemStore.init(gpa);
defer serve_ms.deinit();
try loadContentStore(gpa, &log, &serve_ms);
const srid = serve_ms.store().lookup(older_name).?;
const wf = serve_ms.store().wire(srid).?;
try std.testing.expect(wf.delta != null);
try std.testing.expectEqualSlices(u8, delta_container, wf.container);
try std.testing.expectEqualStrings(newer_name, wf.delta.?.src_hash);
}
test "sync §7.2/§3.3: persistStore stores a whole reverse-delta chain, ordering every basis frame before the delta that points at it" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// A reverse-delta chain in fossil's storage shape: revision i is stored as a
// delta of the NEWER revision i+1, the newest stored full. The store rids run
// oldest..newest (put order), so EVERY delta's basis sits at a HIGHER rid than
// the delta — the worst case for physical ordering. persist's worklist must
// still write each basis frame before its delta so `delta_of` points backward;
// naive rid-order would degrade all but the oldest link to full form.
const links = 6;
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
var bodies: [links][]u8 = undefined;
var made: usize = 0;
defer for (bodies[0..made]) |b| gpa.free(b);
var names: [links][64]u8 = undefined;
var nlen: [links]usize = undefined;
for (0..links) |i| {
bodies[i] = try std.fmt.allocPrint(gpa, "chain body: one long shared prefix kept across every revision for compact reverse deltas -- revision {d}", .{i});
made += 1;
nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len;
}
// put oldest..newest (store rids 1..links); older i is a reverse delta of newer i+1.
for (0..links) |i| {
const rid = try s.put(names[i][0..nlen[i]], bodies[i]);
if (i == links - 1) {
const c = try zit.container.encode(gpa, bodies[i], .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
} else {
const dr = try zit.delta.create(gpa, bodies[i + 1], bodies[i]); // older from newer
defer gpa.free(dr);
const c = try zit.container.encode(gpa, dr, .default);
defer gpa.free(c);
try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } });
}
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// The newest link is full; every older link is a `.delta` whose basis frame is
// physically earlier (a LOWER log rid = a lower offset), and all reconstruct.
for (0..links) |i| {
var d: [digest_len]u8 = undefined;
const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?;
const rgot = (try log.content(gpa, rid)).?;
defer gpa.free(rgot);
try std.testing.expectEqualStrings(bodies[i], rgot);
if (i == links - 1) {
try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(rid));
} else {
try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(rid));
var bd: [digest_len]u8 = undefined;
const brid = log.lookup(decodeWireName(names[i + 1][0..nlen[i + 1]], &bd).?).?;
try std.testing.expect(brid < rid); // basis frame precedes the reverse-delta
}
}
}
test "sync §7.7: persistStore verifies a retained delta's reconstruction and refuses a lying delta container, writing no frame" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// A retained delta whose stored plaintext does NOT hash to its wire name (a
// clone skips the transfer-layer pre-storage check, §7.7 r6, so a lying
// delta reaches the persist commit hook). The §7.7 recompute runs BEFORE the
// wire-form branch, so a delta-form entry is rejected exactly like a full one.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const content = "content that the wire name does not actually name, delivered as a delta\n";
const lying_name = repeat("0", 64); // not SHA3-256(content)
const some_basis = repeat("1", 64); // an absent basis; verify fails before it is even consulted
const raw = try zit.delta.create(gpa, "a plausible basis blob", content);
defer gpa.free(raw);
const dc = try zit.container.encode(gpa, raw, .default);
defer gpa.free(dc);
const rid = try s.put(lying_name, content);
try s.retain(rid, .{ .delta = .{ .src = some_basis, .container = dc, .target_size = content.len } });
try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, ms.store(), null, 0));
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var digest_buf: [digest_len]u8 = undefined;
try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null); // no frame landed
}
fn persistReverseChainNoLeak(allocator: std.mem.Allocator) !void {
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(allocator, io, tmp.dir);
// A small reverse-delta chain built on the (failing) allocator so injected
// OOM is exercised in both the store build and the persist worklist.
var ms = repostate.MemStore.init(allocator);
defer ms.deinit();
const s = ms.store();
const links = 4;
var bodies: [links][]u8 = undefined;
var made: usize = 0;
defer for (bodies[0..made]) |b| allocator.free(b);
var names: [links][64]u8 = undefined;
var nlen: [links]usize = undefined;
for (0..links) |i| {
bodies[i] = try std.fmt.allocPrint(allocator, "reverse chain oom: one shared body across revisions -- revision {d}", .{i});
made += 1;
nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len;
}
for (0..links) |i| {
const rid = try s.put(names[i][0..nlen[i]], bodies[i]);
if (i == links - 1) {
const c = try zit.container.encode(allocator, bodies[i], .default);
defer allocator.free(c);
try s.retain(rid, .{ .full = c });
} else {
const dr = try zit.delta.create(allocator, bodies[i + 1], bodies[i]);
defer allocator.free(dr);
const c = try zit.container.encode(allocator, dr, .default);
defer allocator.free(c);
try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } });
}
}
_ = try persistStore(allocator, io, tmp.dir, ms.store(), null, 0);
}
test "sync §7.2/§3.3: persistStore drains the reverse-delta worklist without leaking under allocation failure" {
try std.testing.checkAllAllocationFailures(std.testing.allocator, persistReverseChainNoLeak, .{});
}
/// Drives `persistArtifact` over a FORWARD delta chain (each link a delta of the
/// immediately older, already-persisted lower-rid link) in rid order, so a test
/// owns the shared basis memo and can read its hit counter. Every basis precedes
/// its delta, so `allow_defer=false` never defers and no worklist is needed —
/// mirrors `persistStore`'s open/commit around a plain rid-order drive.
fn persistForwardChain(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
ms: *repostate.MemStore,
cache: *logstore.PlaintextCache,
) !void {
var store_append = try cli.openStoreAppend(io, dir);
defer store_append.close(io);
const file = store_append.file;
const st = try file.stat(io);
var fs = storage.FileStorage.init(io, file, st.size);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0);
defer log.deinit(gpa);
const store = ms.store();
const n = store.count();
var srefs: SubManifestRefs = .{};
defer srefs.deinit(gpa);
var rid: repostate.Rid = 1;
while (rid <= n) : (rid += 1) {
switch (try persistArtifact(gpa, ms.store(), &log, rid, false, false, cache, &srefs, false, null)) {
.persisted => {},
.deferred => unreachable, // forward chain: no basis is ever absent
}
}
try log.commit();
}
test "sync §7.2/§7.4: the basis memo reuses each basis (hits) and reconstructs a byte-identical store to the uncached path" {
const gpa = std.testing.allocator;
const io = std.testing.io;
const links = 8;
// A forward delta chain: link 0 is full, every later link is a delta that
// reconstructs its body FROM the older body (its basis a lower, already-
// persisted rid) — the case where the memo turns each basis reconstruction
// into a lookup for the delta built on it. Store owns its copies (retain
// dupes), but the bodies/names back the reconstructed-content assertions.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
var bodies: [links][]u8 = undefined;
var made: usize = 0;
defer for (bodies[0..made]) |b| gpa.free(b);
var names: [links][64]u8 = undefined;
var nlen: [links]usize = undefined;
for (0..links) |i| {
bodies[i] = try std.fmt.allocPrint(gpa, "forward chain shared prefix across every revision -- revision {d}", .{i});
made += 1;
nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len;
}
for (0..links) |i| {
const rid = try s.put(names[i][0..nlen[i]], bodies[i]);
if (i == 0) {
const c = try zit.container.encode(gpa, bodies[i], .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
} else {
const dr = try zit.delta.create(gpa, bodies[i - 1], bodies[i]); // newer body FROM its older basis
defer gpa.free(dr);
const c = try zit.container.encode(gpa, dr, .default);
defer gpa.free(c);
try s.retain(rid, .{ .delta = .{ .src = names[i - 1][0..nlen[i - 1]], .container = c, .target_size = bodies[i].len } });
}
}
// (A) a normal memo; (B) a zero-budget memo that can hold nothing — the exact
// uncached reconstruction. Same store, two freshly-seeded repos.
var tmp_a = std.testing.tmpDir(.{});
defer tmp_a.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp_a.dir);
var cache_a = logstore.PlaintextCache.init();
defer cache_a.deinit(gpa);
try persistForwardChain(gpa, io, tmp_a.dir, &ms, &cache_a);
var tmp_b = std.testing.tmpDir(.{});
defer tmp_b.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp_b.dir);
var cache_b = logstore.PlaintextCache.init();
cache_b.max_bytes = 0; // never memoises: forces a fresh reconstruction each time
defer cache_b.deinit(gpa);
try persistForwardChain(gpa, io, tmp_b.dir, &ms, &cache_b);
// The memo reused every basis exactly once (one hit per delta link); the
// zero-budget run reused nothing — the two ways the same bytes get built.
try std.testing.expectEqual(@as(u64, links - 1), cache_a.hits);
try std.testing.expectEqual(@as(u64, 0), cache_b.hits);
// Byte-identical store: the memo changes only the work, never the bytes.
const bytes_a = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(bytes_a);
const bytes_b = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(bytes_b);
try std.testing.expectEqualSlices(u8, bytes_b, bytes_a);
// Every link reconstructs to its exact body (§7.7 chain correctness).
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
for (0..links) |i| {
var d: [digest_len]u8 = undefined;
const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?;
const got = (try log.content(gpa, rid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(bodies[i], got);
}
}
test "sync §9.1: repoProjectCode returns the stored code, else the zero placeholder" {
const gpa = std.testing.allocator;
const io = std.testing.io;
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
const seed = struct {
fn withCode(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, code: []const u8) !void {
var f = try dir.createFile(i, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(i);
var fs = storage.FileStorage.init(i, f, 0);
var log = try logstore.LogStore.open(g, fs.storageFile(), hk);
defer log.deinit(g);
try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = code });
try log.commit();
}
};
// (a) a repo whose config stores a real 40-hex project code: it is returned
// verbatim — the code the pull/push card announces (§9.1), never the zeros
// fossil would refuse.
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const code = repeat("a1b2c3d4", 5); // 40 hex chars
try seed.withCode(gpa, io, tmp.dir, hash_kind, code);
var buf: [zit.sync.max_projectcode_len]u8 = undefined;
const got = try repoProjectCode(gpa, io, tmp.dir, &buf);
try std.testing.expectEqualStrings(code, got);
try std.testing.expect(!std.mem.eql(u8, got, sync_projectcode));
// (b) a repo predating the stored code (empty project_code): the placeholder
// is the honest fallback (no adopted identity to announce yet).
var tmp2 = std.testing.tmpDir(.{});
defer tmp2.cleanup();
try seed.withCode(gpa, io, tmp2.dir, hash_kind, "");
var buf2: [zit.sync.max_projectcode_len]u8 = undefined;
const got2 = try repoProjectCode(gpa, io, tmp2.dir, &buf2);
try std.testing.expectEqualStrings(sync_projectcode, got2);
}
test "sync §9.1: the pull request card carries the adopted project code, not the placeholder" {
const gpa = std.testing.allocator;
// The pull client, constructed exactly as `pullRun` does, with a stored code.
const code = repeat("a1b2c3d4", 5); // a real (non-zero) 40-hex project code
var client = zit.sync.Client{
.state = zit.sync.initialState(.pull),
.operation = .pull,
.self_servercode = sync_servercode,
.project_code = code,
.version = .{ .version = sync_client_version },
};
defer client.deinit(gpa);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
var buf: std.ArrayListUnmanaged(u8) = .empty;
defer buf.deinit(gpa);
const request = try client.buildRequest(gpa, ms.store(), &buf);
// §9.1 ABNF `pull-card = "pull" SP servercode SP projectcode`: the adopted
// code MUST be the card's PROJECTCODE (a real peer refuses a mismatch).
const expected = "pull " ++ sync_servercode ++ " " ++ (repeat("a1b2c3d4", 5)) ++ "\n";
try std.testing.expect(std.mem.indexOf(u8, request, expected) != null);
// And the all-zero placeholder is NOT the project code the card announces.
const rejected = "pull " ++ sync_servercode ++ " " ++ sync_projectcode ++ "\n";
try std.testing.expect(std.mem.indexOf(u8, request, rejected) == null);
}
test "serve §3.8: a per-request store-load on one serve handle sees a later append (no restart)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const append = struct {
fn artifactFrame(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, digest: []const u8, bytes: []const u8) !void {
var f = dir.openFile(i, repo_filename, .{ .mode = .read_write }) catch
try dir.createFile(i, repo_filename, .{ .read = true });
defer f.close(i);
const st = try f.stat(i);
var fs = storage.FileStorage.init(i, f, st.size);
var log = try logstore.LogStore.open(g, fs.storageFile(), hk);
defer log.deinit(g);
_ = try log.put(digest, bytes);
try log.commit();
}
};
// Seed the repo holding exactly one content artifact (a 32-byte raw digest).
const digest_a = repeat("a", digest_len);
try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_a, "content a\n");
// Open ONE read handle — the handle `serve` holds for its whole lifetime.
var handle = try tmp.dir.openFile(io, repo_filename, .{});
defer handle.close(io);
// First per-request read: one content artifact.
var index1 = try LogIndexStore.openFromFile(gpa, io, handle, (try handle.stat(io)).size);
defer index1.destroy();
const n1 = contentCount(index1.store());
try std.testing.expectEqual(@as(usize, 1), n1);
// A SEPARATE writer appends a second artifact while the handle stays open —
// exactly a `zit commit` running alongside a live `zit serve`.
const digest_b = repeat("b", digest_len);
try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_b, "content b\n");
// Second per-request read on the SAME handle sees the new artifact: proof the
// server re-reads the current on-disk log per request (§3.8). A startup
// snapshot would still report n1 — no restart is needed here.
var index2 = try LogIndexStore.openFromFile(gpa, io, handle, (try handle.stat(io)).size);
defer index2.destroy();
try std.testing.expectEqual(n1 + 1, contentCount(index2.store()));
}
/// A test-only `storage.StorageFile` that wraps another and tallies the bytes
/// every `pread` actually returned — the instrument for "how many times does a
/// prime read the log?". Every other vtable method delegates verbatim, so a log
/// opened over it behaves identically while its read traffic is measured.
const CountingStorage = struct {
inner: storage.StorageFile,
read_bytes: usize = 0,
fn storageFile(self: *CountingStorage) storage.StorageFile {
return .{ .context = self, .vtable = &vtable };
}
const vtable = storage.StorageFile.VTable{
.append = appendImpl,
.pread = preadImpl,
.sync = syncImpl,
.size = sizeImpl,
.truncate = truncateImpl,
.refresh_end = refreshEndImpl,
};
/// One writer by construction, so there is no other writer to exclude.
fn refreshEndImpl(_: *anyopaque) storage.StorageError!void {}
fn appendImpl(ctx: *anyopaque, bytes: []const u8) storage.StorageError!void {
const self: *CountingStorage = @ptrCast(@alignCast(ctx));
return self.inner.append(bytes);
}
fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize {
const self: *CountingStorage = @ptrCast(@alignCast(ctx));
const n = try self.inner.pread(offset, buf);
self.read_bytes += n;
return n;
}
fn syncImpl(ctx: *anyopaque) storage.StorageError!void {
const self: *CountingStorage = @ptrCast(@alignCast(ctx));
return self.inner.sync();
}
fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 {
const self: *CountingStorage = @ptrCast(@alignCast(ctx));
return self.inner.size();
}
fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void {
const self: *CountingStorage = @ptrCast(@alignCast(ctx));
return self.inner.truncate(len);
}
};
/// Seed a `.zit` under `dir` holding a full+delta+manifest+config mix (via
/// `packSeedDeltaStore`) and return its committed byte size — the fixture the
/// serve-prime read-traffic and index-equivalence tests both build over.
fn seedServeRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !u64 {
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = try dir.createFile(io, repo_filename, .{ .read = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer log.deinit(gpa);
var tip: [64]u8 = undefined;
try packSeedDeltaStore(gpa, &log, &tip);
return log.size();
}
test "serve prime reads the served log at most once (no second full-container copy)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const file_size = try seedServeRepo(gpa, io, tmp.dir);
var handle = try tmp.dir.openFile(io, repo_filename, .{});
defer handle.close(io);
var fs = storage.FileStorage.init(io, handle, file_size);
var counting = CountingStorage{ .inner = fs.storageFile() };
// Build the serve base snapshot over the counting storage and measure how
// many bytes priming read. `openReadOnly`'s scan already whole-frame-CRC'd
// every byte once (the warm-start CRC concern); an index prime adds NOTHING on top, so
// total prime reads stay at roughly one pass over the log. The copy prime
// (`loadContentStore`) read every container a SECOND time — this bound fails
// for it, which is exactly the cost the log-offset index removes.
var index = try LogIndexStore.open(gpa, counting.storageFile());
defer index.destroy();
try std.testing.expect(counting.read_bytes <= file_size + file_size / 2);
}
test "log-backed index answers count/hashOf/held/content/wire identically to the MemStore snapshot" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const file_size = try seedServeRepo(gpa, io, tmp.dir);
// Reference: the whole-container MemStore snapshot `loadContentStore` builds.
var ref_handle = try tmp.dir.openFile(io, repo_filename, .{});
defer ref_handle.close(io);
var ref_fs = storage.FileStorage.init(io, ref_handle, file_size);
var ref_log = try logstore.LogStore.openReadOnly(gpa, ref_fs.storageFile());
defer ref_log.deinit(gpa);
var ref = repostate.MemStore.init(gpa);
defer ref.deinit();
try loadContentStore(gpa, &ref_log, &ref);
const rs = ref.store();
// Subject: the log-backed index over the same file.
var idx_handle = try tmp.dir.openFile(io, repo_filename, .{});
defer idx_handle.close(io);
var index = try LogIndexStore.openFromFile(gpa, io, idx_handle, file_size);
defer index.destroy();
const is = index.store();
try std.testing.expectEqual(rs.count(), is.count());
try std.testing.expect(rs.count() > 0);
var rid: repostate.Rid = 1;
while (rid <= rs.count()) : (rid += 1) {
// Identity, held-ness, and lookup round-trip.
try std.testing.expectEqualStrings(rs.hashOf(rid), is.hashOf(rid));
try std.testing.expectEqual(rs.held(rid), is.held(rid));
try std.testing.expectEqual(@as(?repostate.Rid, rid), is.lookup(rs.hashOf(rid)));
// Plaintext byte-for-byte.
const rc = rs.content(rid);
const ic = is.content(rid);
try std.testing.expectEqual(rc == null, ic == null);
if (rc) |rb| try std.testing.expectEqualSlices(u8, rb, ic.?);
// Wire form: container bytes, and the delta arm's DELTASRC + target size.
const rw = rs.wire(rid);
const iw = is.wire(rid);
try std.testing.expectEqual(rw == null, iw == null);
if (rw) |rwf| {
try std.testing.expectEqualSlices(u8, rwf.container, iw.?.container);
try std.testing.expectEqual(rwf.delta == null, iw.?.delta == null);
if (rwf.delta) |rd| {
try std.testing.expectEqualStrings(rd.src_hash, iw.?.delta.?.src_hash);
try std.testing.expectEqual(rd.target_size, iw.?.delta.?.target_size);
}
}
}
}
test "appendix honours intern/put/retain for hashes the log does not carry" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const file_size = try seedServeRepo(gpa, io, tmp.dir);
var handle = try tmp.dir.openFile(io, repo_filename, .{});
defer handle.close(io);
var index = try LogIndexStore.openFromFile(gpa, io, handle, file_size);
defer index.destroy();
const s = index.store();
// The serve base is contractually read-only — every write lands in the
// per-request OverlayStore (ticket constraint 3), so these appendix arms never
// fire in production. They exist so the store honours the whole ArtifactStore
// interface instead of trapping on a write; exercise them directly here so that
// completeness is not dead-untested surface.
const persisted = s.count();
try std.testing.expect(persisted > 0);
// A hash the log cannot hold (all-0xAB digest, hex-named as the wire does).
var d1: [digest_len]u8 = undefined;
@memset(&d1, 0xAB);
var h1_buf: [2 * digest_len]u8 = undefined;
const h1 = digestToWireName(&d1, &h1_buf);
try std.testing.expectEqual(@as(?repostate.Rid, null), s.lookup(h1));
// intern mints an appendix phantom ABOVE the persisted rids: not held (no
// content yet), count grows by one, identity round-trips, and it is idempotent.
const r1 = try s.intern(h1);
try std.testing.expect(r1 > persisted);
try std.testing.expectEqual(r1, try s.intern(h1));
try std.testing.expectEqual(@as(?repostate.Rid, r1), s.lookup(h1));
try std.testing.expectEqualStrings(h1, s.hashOf(r1));
try std.testing.expect(!s.held(r1));
try std.testing.expectEqual(@as(?[]const u8, null), s.content(r1));
try std.testing.expectEqual(persisted + 1, s.count());
// put upgrades that phantom in place: held flips true, content reads back, and
// the persisted count is unchanged (an off-log hash never touches the index).
const body = "off-log body bytes";
try std.testing.expectEqual(r1, try s.put(h1, body));
try std.testing.expect(s.held(r1));
try std.testing.expectEqualSlices(u8, body, s.content(r1).?);
try std.testing.expectEqual(persisted + 1, s.count());
// retain on a PERSISTED rid is the advisory no-op (sync §7.2 first-form-wins;
// the log IS the retained form): it returns without error and ignores ingest.
try s.retain(1, .{ .full = "ignored — a persisted rid keeps its log form" });
// None of the appendix traffic left persisted-round memos resident.
try std.testing.expectEqual(@as(usize, 0), index.residentBytes());
}
test "index materialises only the container a round reads and releases it between rounds" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const file_size = try seedServeRepo(gpa, io, tmp.dir);
var handle = try tmp.dir.openFile(io, repo_filename, .{});
defer handle.close(io);
var index = try LogIndexStore.openFromFile(gpa, io, handle, file_size);
defer index.destroy();
const s = index.store();
// Right after the prime the index holds ZERO container/plaintext bytes — the
// whole point: it did not copy the store into RAM.
try std.testing.expectEqual(@as(usize, 0), index.residentBytes());
// Find one wire-bearing rid (a zlib/delta frame).
var target: ?repostate.Rid = null;
var rid: repostate.Rid = 1;
while (rid <= s.count()) : (rid += 1) {
if (s.wire(rid) != null) {
target = rid;
break;
}
}
try std.testing.expect(target != null);
// Reading it materialised only its container; drop the round's memos.
try std.testing.expect(index.residentBytes() > 0);
index.releaseTransient();
try std.testing.expectEqual(@as(usize, 0), index.residentBytes());
// A single round's residency is EXACTLY the one container it emitted — not
// the whole store's containers (which the old snapshot held resident).
const one = s.wire(target.?).?;
try std.testing.expectEqual(one.container.len, index.residentBytes());
}
/// A test-only `storage.StorageFile` that reads faithfully until `fail_reads` is
/// flipped, then fails every `pread` — the instrument for a fault that strikes
/// AFTER `openReadOnly` has scanned and whole-frame-CRC'd the log, exactly the
/// post-scan bitrot / transient EIO the on-demand serve reads must not swallow
/// silently. Every other method delegates verbatim.
const FaultAfterScanStorage = struct {
inner: storage.StorageFile,
fail_reads: bool = false,
fn storageFile(self: *FaultAfterScanStorage) storage.StorageFile {
return .{ .context = self, .vtable = &vtable };
}
const vtable = storage.StorageFile.VTable{
.append = appendImpl,
.pread = preadImpl,
.sync = syncImpl,
.size = sizeImpl,
.truncate = truncateImpl,
.refresh_end = refreshEndImpl,
};
/// One writer by construction, so there is no other writer to exclude.
fn refreshEndImpl(_: *anyopaque) storage.StorageError!void {}
fn appendImpl(ctx: *anyopaque, bytes: []const u8) storage.StorageError!void {
const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx));
return self.inner.append(bytes);
}
fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize {
const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx));
if (self.fail_reads) return error.Io;
return self.inner.pread(offset, buf);
}
fn syncImpl(ctx: *anyopaque) storage.StorageError!void {
const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx));
return self.inner.sync();
}
fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 {
const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx));
return self.inner.size();
}
fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void {
const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx));
return self.inner.truncate(len);
}
};
test "a post-scan read fault on a held rid is recorded, not silently swallowed" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const file_size = try seedServeRepo(gpa, io, tmp.dir);
var handle = try tmp.dir.openFile(io, repo_filename, .{});
defer handle.close(io);
var fs = storage.FileStorage.init(io, handle, file_size);
var faulting = FaultAfterScanStorage{ .inner = fs.storageFile() };
// Open succeeds: the scan CRCs the whole log BEFORE any fault is armed.
var index = try LogIndexStore.open(gpa, faulting.storageFile());
defer index.destroy();
const s = index.store();
// From here every on-demand container/plaintext read faults (post-scan EIO).
faulting.fail_reads = true;
// Walk the store as the §8.2 clone pager does: a held rid whose wire() AND
// content() both return null is one the pager silently skips while still
// reporting the sequence complete — a clone finishing MISSING an artifact.
var silently_omitted = false;
var rid: repostate.Rid = 1;
while (rid <= s.count()) : (rid += 1) {
if (!s.held(rid)) continue;
if (s.wire(rid) == null and s.content(rid) == null) silently_omitted = true;
}
// The fault genuinely makes a held rid unemittable (the bug's precondition)...
try std.testing.expect(silently_omitted);
// ...and the store RECORDED it, so the serve layer can convert the round to an
// error instead of completing a broken clone. A copy-based prime could not
// reach this state: a read fault surfaced at prime, not silently at serve.
try std.testing.expect(index.read_fault);
}
test "serveAnswer converts a recorded read fault into a storage error (no silent short clone)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var hex_buf: [2 * digest_len]u8 = undefined;
try serveSeedRepo(gpa, io, tmp.dir, 4, &hex_buf);
const server = zit.sync.Server{
.version = serve_mod.server_version,
.self_servercode = serve_mod.server_servercode,
.project_code = serve_mod.server_projectcode,
};
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const clone_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 1\n", .{});
defer gpa.free(clone_plain);
const clone_req = try zit.container.encode(gpa, clone_plain, .default);
defer gpa.free(clone_req);
// A healthy round builds the cached snapshot and completes normally.
const ok = try serveAnswer(&ctx, gpa, clone_req, null, .{}, true);
gpa.free(ok.body);
// Simulate a post-scan read fault having struck THIS round — the test above
// proves the store records exactly this. serveAnswer MUST refuse to hand back
// a reply built while an artifact read faulted: otherwise the §8.2 pager's
// silent skip ships a clone missing an artifact and reports it complete.
ctx.snapshot.?.read_fault = true;
try std.testing.expectError(error.ServerStorage, serveAnswer(&ctx, gpa, clone_req, null, .{}, true));
// The flag is cleared with the round's memos, so an immediately retried round
// over the now-healthy snapshot succeeds again (a transient fault self-heals).
const recovered = try serveAnswer(&ctx, gpa, clone_req, null, .{}, true);
gpa.free(recovered.body);
}
/// Build a full-form 3-check-in chain over an evolving file into `log`, plus a
/// live config frame; returns the tip check-in's 64-hex name. Every blob and
/// manifest lands as a FULL frame — pack's re-delta target.
fn packSeedFullStore(gpa: std.mem.Allocator, log: *logstore.LogStore, tip_out: *[64]u8) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
var root_name: [64]u8 = undefined;
var mid_name: [64]u8 = undefined;
const v1 = try packEvolvingBlob(gpa, 1);
defer gpa.free(v1);
const b1 = try packSeedBlob(log, v1);
try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "f.txt", &b1, &root_name);
const v2 = try packEvolvingBlob(gpa, 2);
defer gpa.free(v2);
const b2 = try packSeedBlob(log, v2);
try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "f.txt", &b2, &mid_name);
const v3 = try packEvolvingBlob(gpa, 3);
defer gpa.free(v3);
const b3 = try packSeedBlob(log, v3);
try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &mid_name, "f.txt", &b3, tip_out);
try log.commit();
}
/// Store `content` under its sha3-256 name, delta-encoded against `base` when
/// one is given (the evolving blobs delta to a fraction, so the frame lands as a
/// `.delta`), and return its rid. Models a fossil-clone store that RETAINED
/// compact wire deltas rather than full frames.
fn packSeedDeltaBlob(log: *logstore.LogStore, content: []const u8, base: ?logstore.Rid) !logstore.Rid {
var digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(content, &digest, .{});
return log.putArtifact(&digest, content, base);
}
/// Build a clone-shaped store whose successive blobs are RETAINED as `.delta`
/// frames (the compact wire form), under DISTINCT filenames so pack's
/// same-path-blob lineage hint finds NO base for them — exactly the case where
/// the old pack reconstructed every delta to a full frame and inflated the
/// store. `.zit` therefore holds real delta frames whose bases pack cannot
/// re-derive, so a correct pack must preserve them verbatim. Returns the tip
/// check-in's 64-hex name.
fn packSeedDeltaStore(gpa: std.mem.Allocator, log: *logstore.LogStore, tip_out: *[64]u8) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
var root_name: [64]u8 = undefined;
var mid_name: [64]u8 = undefined;
const v1 = try packEvolvingBlob(gpa, 1);
defer gpa.free(v1);
const r1 = try packSeedDeltaBlob(log, v1, null); // first version: full frame (base of the chain)
var b1: [64]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, v1, &b1);
try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "f1.txt", &b1, &root_name);
const v2 = try packEvolvingBlob(gpa, 2);
defer gpa.free(v2);
const r2 = try packSeedDeltaBlob(log, v2, r1); // delta of v1
var b2: [64]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, v2, &b2);
try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "f2.txt", &b2, &mid_name);
const v3 = try packEvolvingBlob(gpa, 3);
defer gpa.free(v3);
_ = try packSeedDeltaBlob(log, v3, r2); // delta of v2
var b3: [64]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, v3, &b3);
try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &mid_name, "f3.txt", &b3, tip_out);
try log.commit();
}
/// Seed a store holding a wire delta that pack CANNOT beat by re-deltification,
/// so ONLY the §6.1 monotonic-shrink guard keeps the store from regressing. `B`
/// is a one-line edit of `A`, stored as a COMPACT zit delta against `A` — the
/// smallest form pack could ever reproduce. But `max_base_candidates` unrelated
/// filler blobs sit between `A` and `B`, so pack's bounded backward candidate
/// window (§5.2) saturates on the fillers and never reaches `A`; with no lineage
/// hint either (bare blobs, no F-card path), pack's best re-encoding of `B` is a
/// full zlib frame STRICTLY LARGER than the stored A-based delta. The pre-guard
/// pack replaced the delta with that larger full frame (the +33% clone
/// regression in miniature); the guard keeps the existing smaller frame. Fills
/// A's and B's 64-hex names.
fn packSeedUnbeatableWireDelta(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
a_name: *[64]u8,
b_name: *[64]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
// A: B's ideal base, stored full (zlib).
const a = try packEvolvingBlob(gpa, 1);
defer gpa.free(a);
var a_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(a, &a_digest, .{});
const a_rid = try log.putZlib(&a_digest, a);
_ = zit.artifact.artifactName(.sha3_256, a, a_name);
// `max_base_candidates` unrelated fillers between A and B: they saturate B's
// backward candidate window so A is out of reach when pack re-encodes B.
var f: usize = 0;
while (f < logstore.max_base_candidates) : (f += 1) {
var fbuf: std.ArrayList(u8) = .empty;
defer fbuf.deinit(gpa);
var i: usize = 0;
while (i < 200) : (i += 1) try fbuf.print(gpa, "filler {d} unrelated row {d}\n", .{ f, i });
var fdig: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(fbuf.items, &fdig, .{});
_ = try log.putZlib(&fdig, fbuf.items);
}
// B: a one-line edit of A, stored as a COMPACT zit delta against A (the
// smallest form pack could reproduce — and only against A, which pack's
// window cannot reach here).
const b = try packEvolvingBlob(gpa, 2);
defer gpa.free(b);
var b_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{});
_ = zit.artifact.artifactName(.sha3_256, b, b_name);
const raw = try zit.delta.create(gpa, a, b);
defer gpa.free(raw);
const container = try zit.container.encode(gpa, raw, .default);
defer gpa.free(container);
_ = try log.putDeltaContainer(&b_digest, a_rid, container);
try log.commit();
}
/// Assert the §6.1 per-frame monotonicity: NO artifact's packed frame is larger
/// on disk than its source frame. Pack keeps the smaller of {existing frame, new
/// encoding} for every artifact, so a repack is non-worsening frame by frame.
/// This holds universally over THESE fixtures because their chains stay below the
/// read cap; the one path that can enlarge a frame — `copySrcFrame`'s §3.3
/// deep-chain fallback re-storing a kept delta as full zlib — needs a base at
/// `max_delta_chain` depth, which the seeds never build.
fn packExpectNoFrameLarger(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void {
const src_rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(src_rids);
for (src_rids) |sr| {
const dr = dst.lookup(src.hashOf(sr)) orelse return error.PackedArtifactMissing;
try std.testing.expect(dst.frameSpanOf(dr) <= src.frameSpanOf(sr));
}
}
/// Count the `.delta` frames among a store's persisted artifacts.
fn packDeltaFrameCount(gpa: std.mem.Allocator, log: *logstore.LogStore) !usize {
const rids = try log.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
var n: usize = 0;
for (rids) |r| {
if (log.kindOf(r) == .delta) n += 1;
}
return n;
}
/// Assert that every artifact in `src` reconstructs byte-identically in `dst`
/// and the two artifact SETS are equal (no drop, no addition).
fn packExpectContentEqual(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void {
const src_rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(src_rids);
const dst_rids = try dst.ridsInPersistedOrder(gpa);
defer gpa.free(dst_rids);
try std.testing.expectEqual(src_rids.len, dst_rids.len);
for (src_rids) |sr| {
const digest = src.hashOf(sr);
const dr = dst.lookup(digest) orelse return error.PackedArtifactMissing;
const a = (try src.content(gpa, sr)).?;
defer gpa.free(a);
const b = (try dst.content(gpa, dr)).?;
defer gpa.free(b);
try std.testing.expectEqualSlices(u8, a, b);
}
}
test "pack: the packed store's artifact set + every reconstructed content byte-equal the original (config preserved)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// No drop: the packed content-frame count equals the source's, and every
// artifact reconstructs byte-for-byte.
try std.testing.expectEqual(src.entries.items.len, stats.artifacts);
try packExpectContentEqual(gpa, &src, &dst);
// The live config survives as exactly one frame with identical fields.
var soc = (try src.readConfig(gpa)).?;
defer soc.deinit(gpa);
var doc = (try dst.readConfig(gpa)).?;
defer doc.deinit(gpa);
try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy);
try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch);
try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id);
try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code);
}
test "pack re-deltas full-form frames (delta count rises, .zit shrinks) with byte-identical content" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// The source is all full-form: not one delta frame.
{
const rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
for (rids) |r| try std.testing.expect(src.kindOf(r) != .delta);
}
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// The two child blobs (v2 against v1, v3 against v2) re-delta against the
// same path's parent blob, so the delta count rises and the store shrinks —
// with content preserved exactly.
try std.testing.expect(stats.deltas >= 2);
try std.testing.expect(stats.bytes_after < stats.bytes_before);
try packExpectContentEqual(gpa, &src, &dst);
}
test "pack preserves wire deltas — a store with delta frames packs to <= source size (regression for the 8.65x inflation bug)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedDeltaStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// The source genuinely holds retained wire deltas (the compact form pack
// must not discard) whose bases pack's lineage hint cannot re-derive.
const src_deltas = try packDeltaFrameCount(gpa, &src);
try std.testing.expect(src_deltas >= 2);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// MONOTONIC: the packed store is never larger than the source. The old pack
// reconstructed each delta to a full zlib frame and inflated the store; a
// correct pack preserves the compact deltas verbatim.
try std.testing.expect(stats.bytes_after <= stats.bytes_before);
// Every source delta survives AS a delta frame — none reconstructed to full
// (the optional re-delta may add further deltas, so the total only grows).
{
const rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
for (rids) |sr| {
if (src.kindOf(sr) != .delta) continue;
const dr = dst.lookup(src.hashOf(sr)).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr));
}
}
try std.testing.expect(try packDeltaFrameCount(gpa, &dst) >= src_deltas);
// No drop, every reconstruction byte-exact.
try packExpectContentEqual(gpa, &src, &dst);
}
test "pack copies a delta frame verbatim re-linked to its base's new rid, content byte-identical" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedDeltaStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
_ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// For every delta frame in the source, the packed store holds it as a delta
// whose stored container is byte-identical (verbatim re-serve) and whose base
// names the SAME artifact (re-linked to the base's new rid). Content matches.
const rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
var checked: usize = 0;
for (rids) |sr| {
if (src.kindOf(sr) != .delta) continue;
const digest = src.hashOf(sr);
const dr = dst.lookup(digest).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr));
var sw = (try src.wire(gpa, sr)).?;
defer sw.deinit(gpa);
var dw = (try dst.wire(gpa, dr)).?;
defer dw.deinit(gpa);
// Verbatim container bytes and the same base hash (re-link preserves the
// logical base while its rid/offset differs in the packed generation).
try std.testing.expectEqualSlices(u8, sw.delta.container, dw.delta.container);
try std.testing.expectEqualSlices(u8, sw.delta.baseHash(), dw.delta.baseHash());
const a = (try src.content(gpa, sr)).?;
defer gpa.free(a);
const b = (try dst.content(gpa, dr)).?;
defer gpa.free(b);
try std.testing.expectEqualSlices(u8, a, b);
checked += 1;
}
try std.testing.expect(checked >= 2);
}
/// The rid naming the 64-hex `name` in `log`, or null when absent (test-only
/// name→digest→rid convenience over `lookup`).
fn packRidByName(log: *logstore.LogStore, name: *const [64]u8) ?logstore.Rid {
var d_buf: [digest_len]u8 = undefined;
const d = hexToDigest(name, &d_buf) orelse return null;
return log.lookup(d);
}
/// Seed a store modeling a peer-supplied (verbatim wire) delta whose sender
/// chose a POOR base. `A` (a large file) and `C` (an unrelated large blob) are
/// stored full; `B` (a one-line edit of `A`) is stored as a hand-crafted §4.1
/// delta container against `C` — a large, suboptimal delta the sender picked.
/// `A` precedes `B` chronologically and is B's ideal base, so pack's best-of-N
/// re-deltification (§6.1) must re-base `B` onto `A` rather than trust the
/// sender's `C`. Fills the 64-hex names of A/B/C.
fn packSeedSuboptimalWireDelta(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
a_name: *[64]u8,
b_name: *[64]u8,
c_name: *[64]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
// A: the ideal base for B (near-identical predecessor), stored full.
const a = try packEvolvingBlob(gpa, 1);
defer gpa.free(a);
a_name.* = try packSeedBlob(log, a);
// C: an unrelated large blob (a poor base for B), stored full.
var cbuf: std.ArrayList(u8) = .empty;
defer cbuf.deinit(gpa);
var i: usize = 0;
while (i < 200) : (i += 1) try cbuf.print(gpa, "unrelated content row {d}\n", .{i});
const c = cbuf.items;
c_name.* = try packSeedBlob(log, c);
const c_rid = packRidByName(log, c_name).?;
// B: a one-line edit of A. delta(A,B) is tiny; delta(C,B) is large.
const b = try packEvolvingBlob(gpa, 2);
defer gpa.free(b);
_ = zit.artifact.artifactName(.sha3_256, b, b_name);
var b_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{});
// Hand-craft the verbatim wire delta against the POOR base C — models a peer
// that chose C as base, retained at rest by `putDeltaContainer`.
const raw = try zit.delta.create(gpa, c, b);
defer gpa.free(raw);
const container = try zit.container.encode(gpa, raw, .default);
defer gpa.free(container);
_ = try log.putDeltaContainer(&b_digest, c_rid, container);
try log.commit();
}
test "pack §6.1: a verbatim wire delta is re-deltified against the size-minimizing base, not the sender's" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var a_name: [64]u8 = undefined;
var b_name: [64]u8 = undefined;
var c_name: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedSuboptimalWireDelta(gpa, &wlog, &a_name, &b_name, &c_name);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// Precondition: the source really holds B as a delta whose base is C (the
// sender's suboptimal choice) — the exact frame the old pack copied verbatim.
{
const sb = packRidByName(&src, &b_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb));
var sw = (try src.wire(gpa, sb)).?;
defer sw.deinit(gpa);
var cbin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&cbin, &c_name);
try std.testing.expectEqualSlices(u8, &cbin, sw.delta.baseHash());
}
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// After pack, B is still a delta, but its base is now A (the size-minimizing
// candidate) — the sender's base C is no longer trusted at rest (§6.1/§3).
const db = packRidByName(&dst, &b_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db));
var dw = (try dst.wire(gpa, db)).?;
defer dw.deinit(gpa);
var abin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&abin, &a_name);
try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash());
// Every artifact still reconstructs byte-for-byte and the store shrank (a
// tiny A-based delta replaced the large C-based one).
try packExpectContentEqual(gpa, &src, &dst);
try std.testing.expect(stats.bytes_after < stats.bytes_before);
try std.testing.expectEqual(dst.size(), stats.bytes_after);
try std.testing.expectEqual(src.size(), stats.bytes_before);
}
/// Seed the §6.2 same-path scenario: two versions of ONE tracked file whose ideal
/// base (the earlier version) sits OUTSIDE pack's bounded chronological window.
/// `v1` of "tracked.txt" is stored full, then `max_base_candidates` unrelated
/// filler blobs saturate the backward candidate window, then `v2` (a one-line
/// edit of `v1`) is stored full. With ONLY the chronological window (step §6.1)
/// pack cannot reach `v1` from `v2` — the fillers fill every slot — so `v2` stays
/// a full frame and the store does not shrink. The mlink-analog same-path walk
/// (§6.2) reads both check-in manifests, learns tracked.txt: [v1, v2], and offers
/// `v1` as `v2`'s priority candidate, so pack re-deltifies `v2` onto `v1`. Both
/// check-ins are FULL frames (pack's re-delta target); fills v1/v2's 64-hex names.
fn packSeedSamePathOutOfWindow(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
v1_name: *[64]u8,
v2_name: *[64]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
// v1 of tracked.txt (full), and the root check-in that names it.
const v1 = try packEvolvingBlob(gpa, 1);
defer gpa.free(v1);
v1_name.* = try packSeedBlob(log, v1);
var root_name: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &root_name);
// `max_base_candidates` unrelated fillers between v1 and v2: they saturate
// v2's bounded backward window so v1 is out of chronological reach.
var f: usize = 0;
while (f < logstore.max_base_candidates) : (f += 1) {
var fbuf: std.ArrayList(u8) = .empty;
defer fbuf.deinit(gpa);
var i: usize = 0;
while (i < 200) : (i += 1) try fbuf.print(gpa, "samepath filler {d} unrelated row {d}\n", .{ f, i });
_ = try packSeedBlob(log, fbuf.items);
}
// v2 of tracked.txt (a one-line edit of v1), stored FULL, and its check-in.
const v2 = try packEvolvingBlob(gpa, 2);
defer gpa.free(v2);
v2_name.* = try packSeedBlob(log, v2);
var tip_name: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "tracked.txt", v2_name, &tip_name);
try log.commit();
}
test "pack §6.2: a same-path predecessor beyond the chronological window becomes v2's base (store shrinks)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var v1_name: [64]u8 = undefined;
var v2_name: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedSamePathOutOfWindow(gpa, &wlog, &v1_name, &v2_name);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// Precondition: the source holds v2 as a NON-delta (full) frame — the case
// step §6.1's chronological window alone cannot improve (v1 is out of reach).
{
const sv2 = packRidByName(&src, &v2_name).?;
try std.testing.expect(src.kindOf(sv2) != .delta);
}
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// After pack, v2 is a delta whose base is the same-path predecessor v1 — the
// §6.2 mlink analog reached a base the chronological window could not.
const dv2 = packRidByName(&dst, &v2_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dv2));
var dw = (try dst.wire(gpa, dv2)).?;
defer dw.deinit(gpa);
var v1bin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&v1bin, &v1_name);
try std.testing.expectEqualSlices(u8, &v1bin, dw.delta.baseHash());
// The store shrank (a tiny v1-based delta replaced the full v2 frame), every
// artifact still reconstructs byte-for-byte, and no frame grew.
try std.testing.expect(stats.bytes_after < stats.bytes_before);
try packExpectContentEqual(gpa, &src, &dst);
try packExpectNoFrameLarger(gpa, &src, &dst);
}
/// Seed the §6.2 same-path REVERT scenario: one tracked file whose content goes
/// A→B→A across three check-ins — the third restores a byte-identical earlier
/// blob. Because the store is content-addressed, the reverted version resolves to
/// the SAME blob rid as v1, so the pack manifest walk records that rid twice,
/// non-consecutively, for one path. Exercises `deriveCandidateHints`'s
/// distinct-rid compaction: a blob must never be offered as its own same-path
/// predecessor. Fills v1/v2's 64-hex names.
fn packSeedSamePathRevert(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
v1_name: *[64]u8,
v2_name: *[64]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
const v1 = try packEvolvingBlob(gpa, 1);
defer gpa.free(v1);
const v2 = try packEvolvingBlob(gpa, 2);
defer gpa.free(v2);
// v1 of tracked.txt (full) + its root check-in.
v1_name.* = try packSeedBlob(log, v1);
var c1: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &c1);
// v2 (a one-line edit) + its check-in.
v2_name.* = try packSeedBlob(log, v2);
var c2: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &c1, "tracked.txt", v2_name, &c2);
// Revert: tracked.txt returns to v1's byte-identical content. The blob is
// already stored (content-addressed), so this check-in re-references v1's rid
// — the non-consecutive duplicate that recordPathBlob's consecutive-only dedup
// lets through.
var c3: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &c2, "tracked.txt", v1_name, &c3);
try log.commit();
}
test "pack §6.2: a same-path revert (A→B→A) re-references one blob rid twice and packs cleanly (no self-predecessor panic)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var v1_name: [64]u8 = undefined;
var v2_name: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedSamePathRevert(gpa, &wlog, &v1_name, &v2_name);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
// Before the distinct-rid compaction in deriveCandidateHints this panicked on
// a self-predecessor (v1 sorted adjacent to its own duplicate ⇒ equal offsets
// fail the strict offset-below assert). Now the pack completes and every
// artifact reconstructs byte-for-byte with no frame enlarged.
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
try packExpectContentEqual(gpa, &src, &dst);
try packExpectNoFrameLarger(gpa, &src, &dst);
try std.testing.expectEqual(dst.size(), stats.bytes_after);
try std.testing.expectEqual(src.size(), stats.bytes_before);
}
test "pack §6.1 monotonic guard: a wire delta pack cannot beat is KEPT, not replaced by a larger frame (no regression)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var a_name: [64]u8 = undefined;
var b_name: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedUnbeatableWireDelta(gpa, &wlog, &a_name, &b_name);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// Precondition: the source holds B as a compact delta whose base is A.
{
const sb = packRidByName(&src, &b_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb));
var sw = (try src.wire(gpa, sb)).?;
defer sw.deinit(gpa);
var abin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&abin, &a_name);
try std.testing.expectEqualSlices(u8, &abin, sw.delta.baseHash());
}
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// (a) decode-all byte-identical pre/post.
try packExpectContentEqual(gpa, &src, &dst);
// (b) NO frame is larger post-pack than its pre-pack encoding.
try packExpectNoFrameLarger(gpa, &src, &dst);
// (c) the whole store did not regress on this wire-delta-heavy fixture.
try std.testing.expect(stats.bytes_after <= stats.bytes_before);
// Accounting exact: reported sizes are the real store sizes.
try std.testing.expectEqual(src.size(), stats.bytes_before);
try std.testing.expectEqual(dst.size(), stats.bytes_after);
// B stayed a delta (the compact A-based frame was kept, re-linked to A's new
// rid) — the guard refused to replace it with the larger full zlib pack's
// window-only re-encoding would have produced.
const db = packRidByName(&dst, &b_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db));
var dw = (try dst.wire(gpa, db)).?;
defer dw.deinit(gpa);
var abin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&abin, &a_name);
try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash());
// The kept frame is no larger than the source's (equal — copied verbatim).
try std.testing.expect(dst.frameSpanOf(db) <= src.frameSpanOf(packRidByName(&src, &b_name).?));
}
/// Seed a store exercising EVERY frame class pack must handle in one place: a
/// raw frame, a zlib frame, a locally-created delta, a verbatim wire delta, an
/// interned phantom (no frame on disk), and the config singleton. Fills the
/// content-bearing artifacts' 64-hex names plus the phantom's binary digest.
fn packSeedMixedStore(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
raw_name: *[64]u8,
zlib_name: *[64]u8,
delta_name: *[64]u8,
wire_name: *[64]u8,
phantom_digest: *[digest_len]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
// raw: a unique blob large enough to be a real (if unrelated) base.
var rbuf: std.ArrayList(u8) = .empty;
defer rbuf.deinit(gpa);
var i: usize = 0;
while (i < 120) : (i += 1) try rbuf.print(gpa, "raw-only content row {d}\n", .{i});
const rbytes = rbuf.items;
raw_name.* = try packSeedBlob(log, rbytes); // `put` => .raw
// zlib: a large evolving blob stored as a full zlib container.
const z = try packEvolvingBlob(gpa, 1);
defer gpa.free(z);
var z_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(z, &z_digest, .{});
const z_rid = try log.putZlib(&z_digest, z);
_ = zit.artifact.artifactName(.sha3_256, z, zlib_name);
// local delta: an edit of the zlib blob, delta-encoded against it.
const d = try packEvolvingBlob(gpa, 2);
defer gpa.free(d);
var d_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(d, &d_digest, .{});
_ = try log.putArtifact(&d_digest, d, z_rid);
_ = zit.artifact.artifactName(.sha3_256, d, delta_name);
// wire delta: a hand-crafted §4.1 delta container against the raw blob (a
// poor base) — the verbatim-retained peer form.
const w = try packEvolvingBlob(gpa, 3);
defer gpa.free(w);
var w_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(w, &w_digest, .{});
_ = zit.artifact.artifactName(.sha3_256, w, wire_name);
const r_rid = packRidByName(log, raw_name).?;
const raw_delta = try zit.delta.create(gpa, rbytes, w);
defer gpa.free(raw_delta);
const container = try zit.container.encode(gpa, raw_delta, .default);
defer gpa.free(container);
_ = try log.putDeltaContainer(&w_digest, r_rid, container);
// phantom: an interned forward reference that never reaches disk.
std.crypto.hash.sha3.Sha3_256.hash("absent forward-referenced artifact", phantom_digest, .{});
_ = try log.intern(phantom_digest);
try log.commit();
}
test "pack §6.1/§3: a mixed raw/zlib/delta/wire-delta/phantom/config store round-trips, config+phantom preserved, accounting exact" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var raw_name: [64]u8 = undefined;
var zlib_name: [64]u8 = undefined;
var delta_name: [64]u8 = undefined;
var wire_name: [64]u8 = undefined;
var phantom_digest: [digest_len]u8 = undefined;
// A phantom never reaches disk, so we pack the in-memory writable store
// directly (a reopen would drop it) — the case pack must tolerate without
// choking or emitting a garbage frame.
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedMixedStore(gpa, &wlog, &raw_name, &zlib_name, &delta_name, &wire_name, &phantom_digest);
// The seeded store really holds one frame of each class plus the phantom.
try std.testing.expectEqual(logstore.FrameKind.raw, wlog.kindOf(packRidByName(&wlog, &raw_name).?));
try std.testing.expectEqual(logstore.FrameKind.zlib, wlog.kindOf(packRidByName(&wlog, &zlib_name).?));
try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &delta_name).?));
try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &wire_name).?));
try std.testing.expect(wlog.isPhantom(wlog.lookup(&phantom_digest).?));
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), wlog.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &wlog, &dst, Filter.keep_all, &.{});
// Decode-all byte-identical pre/post, and the artifact SETS match (the
// phantom is excluded from both, so counts still agree).
try packExpectContentEqual(gpa, &wlog, &dst);
// §6.1 per-frame monotonicity: no frame grew in the repack.
try packExpectNoFrameLarger(gpa, &wlog, &dst);
// Config preserved verbatim in fields.
var soc = (try wlog.readConfig(gpa)).?;
defer soc.deinit(gpa);
var doc = (try dst.readConfig(gpa)).?;
defer doc.deinit(gpa);
try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy);
try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch);
try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id);
try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code);
// Phantom preserved as a contentless forward reference: it carries no bytes,
// so pack writes no frame for it (a phantom never reaches disk) and it is not
// resurrected as a garbage artifact in the packed store.
try std.testing.expect(dst.lookup(&phantom_digest) == null);
// Accounting exact: the reported sizes are the actual store sizes and the
// artifact tally is the source's real (non-phantom) frame count.
try std.testing.expectEqual(wlog.size(), stats.bytes_before);
try std.testing.expectEqual(dst.size(), stats.bytes_after);
const src_rids = try wlog.ridsInPersistedOrder(gpa);
defer gpa.free(src_rids);
try std.testing.expectEqual(src_rids.len, stats.artifacts);
}
test "pack §7.7: packLog aborts on a source frame whose content does not name it (no packed frame written)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// A frame keyed by a name that is NOT sha3-256 of its bytes: `put` stores
// bytes verbatim under any key, so this models a corrupt/lying source frame
// that survives `openReadOnly` (its CRC + embedded-hash label are intact).
const lying_name = @as([digest_len]u8, @splat(0xab));
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
_ = try wlog.put(&lying_name, "content these bytes do not name");
try wlog.commit();
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
try std.testing.expectError(error.Corrupt, packLog(gpa, &src, &dst, Filter.keep_all, &.{}));
try std.testing.expect(dst.lookup(&lying_name) == null); // no frame landed on the abort
}
test "pack is idempotent: pack∘pack is byte-identical (0 additional re-deltas)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
// First pack: source (full) → L1.
{
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var f1 = try tmp.dir.createFile(io, "pack1", .{ .read = true, .truncate = true });
defer f1.close(io);
var f1fs = storage.FileStorage.init(io, f1, 0);
var dst = try logstore.LogStore.open(gpa, f1fs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
_ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
}
// Second pack: L1 → L2, byte-for-byte equal, and no further re-deltaing.
var l1fs: storage.FileStorage = undefined;
var l1 = try packOpenReadOnly2(gpa, io, tmp.dir, "pack1", &l1fs);
defer l1fs.file.close(io);
defer l1.deinit(gpa);
var f2 = try tmp.dir.createFile(io, "pack2", .{ .read = true, .truncate = true });
defer f2.close(io);
var f2fs = storage.FileStorage.init(io, f2, 0);
var l2 = try logstore.LogStore.open(gpa, f2fs.storageFile(), l1.hash_kind);
defer l2.deinit(gpa);
const stats2 = try packLog(gpa, &l1, &l2, Filter.keep_all, &.{});
try std.testing.expectEqual(l1.size(), l2.size());
const a = try tmp.dir.readFileAlloc(io, "pack1", gpa, .limited(1 << 24));
defer gpa.free(a);
const b = try tmp.dir.readFileAlloc(io, "pack2", gpa, .limited(1 << 24));
defer gpa.free(b);
try std.testing.expectEqualSlices(u8, a, b); // byte-identical: pack∘pack == pack
// An already-packed store still reports its deltas (they were re-created
// identically), and packing it does not shrink further.
try std.testing.expectEqual(l1.size(), stats2.bytes_after);
}
/// Reopen a named log file in `dir` read-only (a sibling to `packOpenReadOnly`
/// that takes an arbitrary name, for the multi-generation idempotence test).
fn packOpenReadOnly2(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, name: []const u8, fs_out: *storage.FileStorage) !logstore.LogStore {
const f = try dir.openFile(io, name, .{});
const st = try f.stat(io);
fs_out.* = storage.FileStorage.init(io, f, st.size);
return logstore.LogStore.openReadOnly(gpa, fs_out.storageFile());
}
test "pack: packLog never writes the source (a crash before the rename leaves the original byte-identical)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
const before = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(before);
// Pack into a separate temp (the new generation). packLog only ever reads
// the source; the atomic swap in `runPack` is the sole mutation of `.zit`.
{
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var f = try tmp.dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = true });
defer f.close(io);
var ffs = storage.FileStorage.init(io, f, 0);
var dst = try logstore.LogStore.open(gpa, ffs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
_ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
}
// No rename happened (the "crash"): the original `.zit` is byte-for-byte
// unchanged and still opens.
const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, before, after);
}
/// The allocation-failure probe: pack a prebuilt source `.zit` into a fresh temp
/// destination, using the failing allocator throughout. Every OOM point must
/// unwind cleanly (no leak) and a fully-funded run must succeed.
fn packOomProbe(alloc: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void {
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(alloc, io, dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(alloc);
var f = try dir.createFile(io, "oomdst", .{ .read = true, .truncate = true });
defer f.close(io);
var ffs = storage.FileStorage.init(io, f, 0);
var dst = try logstore.LogStore.open(alloc, ffs.storageFile(), src.hash_kind);
defer dst.deinit(alloc);
_ = try packLog(alloc, &src, &dst, Filter.keep_all, &.{});
}
test "pack: checkAllAllocationFailures on packLog (every OOM path unwinds cleanly)" {
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(std.testing.allocator, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(std.testing.allocator);
try packSeedFullStore(std.testing.allocator, &wlog, &tip);
}
try std.testing.checkAllAllocationFailures(std.testing.allocator, packOomProbe, .{ io, tmp.dir });
}
test "pack∘derive == from-scratch derive: the packed store derives the identical repo view" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var src_view: zit.repoview.RepoView = undefined;
try src_view.load(gpa, &src);
defer src_view.deinit(gpa);
var pfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer pfile.close(io);
var pfs = storage.FileStorage.init(io, pfile, 0);
var dst = try logstore.LogStore.open(gpa, pfs.storageFile(), src.hash_kind);
_ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
dst.deinit(gpa);
var rfs: storage.FileStorage = undefined;
var packed_log = try packOpenReadOnly2(gpa, io, tmp.dir, "packed", &rfs);
defer rfs.file.close(io);
defer packed_log.deinit(gpa);
var packed_view: zit.repoview.RepoView = undefined;
try packed_view.load(gpa, &packed_log);
defer packed_view.deinit(gpa);
// Same derived view: main branch, artifact count, and the leaf set + order
// (§7.5) are identical — pack changed only the storage layout.
try std.testing.expectEqualStrings(src_view.main_branch, packed_view.main_branch);
try std.testing.expectEqual(src_view.store.store().count(), packed_view.store.store().count());
var la = src_view.state.leaves();
var lb = packed_view.state.leaves();
while (la.next()) |ra| {
const rb = lb.next() orelse return error.LeafCountMismatch;
try std.testing.expectEqualStrings(src_view.hashOf(ra), packed_view.hashOf(rb));
}
try std.testing.expect(lb.next() == null);
}
test "renderPackStats: 'packed' when the store shrank, 'already packed' otherwise" {
var buf: [128]u8 = undefined;
{
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 1695, .deltas = 1373, .dropped = 0, .shuns_recorded = 0, .bytes_before = 9_680_000, .bytes_after = 920_000 });
try std.testing.expectEqualStrings(
"packed: 1695 artifacts walked, 1373 re-deltaed, 9680000 -> 920000 bytes\n",
w.buffered(),
);
}
{
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 3, .deltas = 2, .dropped = 0, .shuns_recorded = 0, .bytes_before = 4096, .bytes_after = 4096 });
try std.testing.expectEqualStrings(
"already packed: 3 artifacts walked, 2 re-deltaed, 4096 -> 4096 bytes\n",
w.buffered(),
);
}
}
test "renderPackStats: a reclaiming run names the dropped count; a run that drops nothing stays silent" {
// Reclamation is the one pack outcome an operator must not have to infer
// from a shrinking byte count: bytes shrink on an ordinary re-deltify too.
// So the clause appears exactly when `dropped > 0`, and nowhere else.
var buf: [128]u8 = undefined;
{
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 3, .shuns_recorded = 0, .bytes_before = 8192, .bytes_after = 4096 });
try std.testing.expectEqualStrings(
"packed: 12 artifacts walked, 4 re-deltaed, 3 shunned reclaimed, 8192 -> 4096 bytes\n",
w.buffered(),
);
}
{
// `artifacts` counts the source frames WALKED, so a run that dropped
// every one of them still reports 12 walked and 12 reclaimed.
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 12, .deltas = 0, .dropped = 12, .shuns_recorded = 0, .bytes_before = 8192, .bytes_after = 64 });
try std.testing.expectEqualStrings(
"packed: 12 artifacts walked, 0 re-deltaed, 12 shunned reclaimed, 8192 -> 64 bytes\n",
w.buffered(),
);
}
{
// A reclaim that GREW the log is still "packed". Reclaiming an artifact
// other artifacts were deltified against moves its content into one of
// them, so the log can legitimately end up larger — and "already packed"
// on a run that just irreversibly deleted content would be the most
// misleading line this verb could print.
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 7, .deltas = 5, .dropped = 1, .shuns_recorded = 0, .bytes_before = 5760, .bytes_after = 6248 });
try std.testing.expectEqualStrings(
"packed: 7 artifacts walked, 5 re-deltaed, 1 shunned reclaimed, 5760 -> 6248 bytes\n",
w.buffered(),
);
}
}
test "rewriteAndSwap: a run the concurrency guard aborts leaves .zit untouched and records NO tombstone" {
// THE "ONE DURABLE TRANSITION" INVARIANT, made observable. `zit shun --force`
// must never leave a tombstone behind without also having removed the content
// it names; the mechanism is that the rows are written into the TEMP and only
// the rename makes anything durable. This drives the abort seam that is a live
// guard rather than an injected fault — the no-repo-lock concurrency guard: a
// concurrent writer extends `.zit` between our scan and the rename, so `validateTail`
// fails after `packLog` has already written every row into the temp. If the
// rows had been appended to the live log first, they would still be here.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const doomed_body = repeat("content the operator named in this invocation ", 4);
var doomed: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(doomed_body, &doomed, .{});
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
_ = try packSeedBlob(&wlog, doomed_body);
_ = try packSeedBlob(&wlog, repeat("content with nothing against it ", 4));
try wlog.commit();
}
// The scan the rewrite will be built from, and the tail key that pins it.
var rfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &rfs);
defer src.deinit(gpa);
const src_key = try src.logKey();
// The concurrent writer. One appended frame is exactly what the guard exists
// to detect; the snapshot in `src` is now stale.
{
var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write });
defer f.close(io);
const st = try f.stat(io);
var wfs = storage.FileStorage.init(io, f, st.size);
var wlog = try logstore.LogStore.open(gpa, wfs.storageFile(), 0);
defer wlog.deinit(gpa);
_ = try packSeedBlob(&wlog, repeat("a frame a concurrent writer appended ", 3));
try wlog.commit();
}
const extended = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(extended);
var set: NameSet = .{};
defer set.deinit(gpa);
try set.add(gpa, &doomed);
const rows = [_]logstore.ShunRecord{
.{ .hash = &doomed, .epoch = 1753000077, .reason = "takedown notice 4417" },
};
try std.testing.expectError(error.TailChanged, rewriteAndSwap(
gpa,
io,
tmp.dir,
&src,
src_key,
set.filter(),
&rows,
));
// THE PINS, tombstone FIRST because it is the one this test exists for: the
// aborted run left no row, `.zit` is the concurrent writer's version byte for
// byte (our stale rewrite never landed), the content the run intended to
// remove is intact, and the temp is gone.
var vfs: storage.FileStorage = undefined;
var after_log = try packOpenReadOnly(gpa, io, tmp.dir, &vfs);
defer after_log.deinit(gpa);
try std.testing.expectEqual(@as(usize, 0), after_log.shunList().len);
try std.testing.expect(!after_log.isShunned(&doomed));
try std.testing.expect(after_log.lookup(&doomed) != null);
{
const after = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, extended, after);
}
if (tmp.dir.openFile(io, pack_tmp_filename, .{})) |f| {
var mf = f;
mf.close(io);
return error.TempSurvivedTheAbort;
} else |_| {}
}
test "rewriteAndSwap: a run the concurrency guard aborts leaves .zit untouched and records NO tombstone (competing RENAME)" {
// THE SIBLING SHAPE, and the one the caller's own file handle cannot see. The
// test above simulates a concurrent writer that APPENDS to `.zit`; here a
// competing rewrite (`zit pack`, or another `zit shun --force`) renames a whole
// new generation over the name while our snapshot's handle stays open. That
// handle then names an ORPHAN inode whose size and tail can never move again,
// so a guard that re-stats it agrees with its own key forever and this run
// renames a pre-competitor snapshot over the winner — silently restoring
// content the winner shunned, with both runs exiting 0. Reopening `.zit` by
// NAME is what turns it into `error.TailChanged`.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const doomed_body = repeat("content the operator named in this invocation ", 4);
var doomed: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(doomed_body, &doomed, .{});
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
_ = try packSeedBlob(&wlog, doomed_body);
_ = try packSeedBlob(&wlog, repeat("content with nothing against it ", 4));
try wlog.commit();
}
// The scan the rewrite will be built from, and the tail key that pins it. This
// handle is the one the competing rename is about to orphan.
var rfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &rfs);
defer src.deinit(gpa);
const src_key = try src.logKey();
// The competing rewrite: a NEW generation built beside `.zit` and renamed over
// it. Nothing touches the inode `src` is reading, which is the whole point.
{
var f = try tmp.dir.createFile(io, "gen2", .{ .read = true, .truncate = true });
defer f.close(io);
var gfs = storage.FileStorage.init(io, f, 0);
var glog = try logstore.LogStore.open(gpa, gfs.storageFile(), src.hash_kind);
defer glog.deinit(gpa);
_ = try packSeedBlob(&glog, repeat("the generation the competing rewrite produced ", 3));
try glog.commit();
}
try tmp.dir.rename("gen2", tmp.dir, repo_filename, io);
const winner = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(winner);
var set: NameSet = .{};
defer set.deinit(gpa);
try set.add(gpa, &doomed);
const rows = [_]logstore.ShunRecord{
.{ .hash = &doomed, .epoch = 1753000077, .reason = "takedown notice 4417" },
};
try std.testing.expectError(error.TailChanged, rewriteAndSwap(
gpa,
io,
tmp.dir,
&src,
src_key,
set.filter(),
&rows,
));
// The winner's generation survives byte for byte, and the aborted run recorded
// nothing — the loser did not resurrect its own snapshot's contents.
{
const after = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, winner, after);
}
var vfs: storage.FileStorage = undefined;
var after_log = try packOpenReadOnly(gpa, io, tmp.dir, &vfs);
defer after_log.deinit(gpa);
try std.testing.expectEqual(@as(usize, 0), after_log.shunList().len);
try std.testing.expect(after_log.lookup(&doomed) == null);
if (tmp.dir.openFile(io, pack_tmp_filename, .{})) |f| {
var mf = f;
mf.close(io);
return error.TempSurvivedTheAbort;
} else |_| {}
}
/// Read `dir/.zit` whole (caller frees) — the byte-identical oracle for a run
/// that must have changed nothing.
fn storeSnapshot(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) ![]u8 {
var file = try dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
const buf = try gpa.alloc(u8, @intCast(st.size));
errdefer gpa.free(buf);
const n = try file.readPositionalAll(io, buf, 0);
try std.testing.expectEqual(buf.len, n);
return buf;
}
// --- `zit shun`'s pure renderers -------------------------------------------
//
// Every operator-visible byte this verb emits is produced by a function of data
// alone, for the same reason `renderPackStats` is: the exact wording of a
// refusal that precedes irreversible deletion is the part most worth pinning,
// and pinning it through a subprocess would make the e2e tests the only place
// it is stated.
const shun_name_a = "1c04b7" ++ repeat("0", 58);
const shun_name_b = "7bd1e0" ++ repeat("1", 58);
const shun_name_c = "9f2a1c" ++ repeat("2", 58);
test "renderShunRefusal: the counts and the word refusing share the first line, then one line per hash" {
// The register is `clean`'s (`cleanRefuse`): an operator must not have to read
// six lines of inventory to discover that nothing happened.
var buf: [1024]u8 = undefined;
var w: std.Io.Writer = .fixed(&buf);
renderShunRefusal(&w, &.{
.{ .name = shun_name_a, .kind = .manifest, .disposition = .remove },
.{ .name = shun_name_b, .kind = null, .disposition = .remove },
.{ .name = shun_name_c, .disposition = .absent },
}, 0, false);
try std.testing.expectEqualStrings(
"zit shun: 3 hashes would be shunned in this repository; 2 artifacts would be permanently removed; refusing without --force:\n" ++
"\n" ++
" " ++ shun_name_a ++ " checkin a check-in; its descendants become unresolvable\n" ++
" " ++ shun_name_b ++ " file file content; every check-in listing it loses that file\n" ++
" " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++
"\n" ++
"This is irreversible: zit undo does not recover a shun, and the content\n" ++
"cannot be re-fetched from a peer.\n" ++
"run 'zit shun --force <HASH>...' to proceed\n",
w.buffered(),
);
}
test "renderShunRefusal: singular agreement, an already-shunned line, and the nothing-held wording" {
var buf: [1024]u8 = undefined;
{
// One target, held here: "1 hash", "1 artifact".
var w: std.Io.Writer = .fixed(&buf);
renderShunRefusal(&w, &.{
.{ .name = shun_name_a, .kind = .wiki, .disposition = .remove },
.{ .name = shun_name_b, .disposition = .already },
}, 0, false);
try std.testing.expectEqualStrings(
"zit shun: 1 hash would be shunned in this repository; 1 artifact would be permanently removed; refusing without --force:\n" ++
"\n" ++
" " ++ shun_name_a ++ " wiki structural content; what it renders stops rendering\n" ++
" " ++ shun_name_b ++ " already shunned; content already gone\n" ++
"\n" ++
"This is irreversible: zit undo does not recover a shun, and the content\n" ++
"cannot be re-fetched from a peer.\n" ++
"run 'zit shun --force <HASH>...' to proceed\n",
w.buffered(),
);
}
{
// A purely pre-emptive run: the tombstones are still irreversible, so the
// refusal stands, but nothing would be removed and the line says so
// instead of claiming a removal that is not going to happen.
var w: std.Io.Writer = .fixed(&buf);
renderShunRefusal(&w, &.{.{ .name = shun_name_c, .disposition = .absent }}, 0, false);
try std.testing.expectEqualStrings(
"zit shun: 1 hash would be shunned in this repository; nothing you named holds content to remove; refusing without --force:\n" ++
"\n" ++
" " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++
"\n" ++
"This is irreversible: zit undo does not recover a shun, and the content\n" ++
"cannot be re-fetched from a peer.\n" ++
"run 'zit shun --force <HASH>...' to proceed\n",
w.buffered(),
);
}
}
test "renderShunRefusal: a reclaim-only run, and the collateral an earlier --defer left behind" {
// THE CONSENT GATE MUST NAME EVERYTHING THE REWRITE DESTROYS, including what the
// operator did not name. `renderShunStats` is honest after the fact that
// `dropped` can exceed `recorded`; a refusal that only counted this
// invocation's own targets would have the operator consent to one deletion and
// get several.
var buf: [1024]u8 = undefined;
var w: std.Io.Writer = .fixed(&buf);
renderShunRefusal(&w, &.{
.{ .name = shun_name_a, .kind = null, .disposition = .reclaim },
}, 2, false);
try std.testing.expectEqualStrings(
"zit shun: no new hash would be recorded; 1 artifact would be permanently removed; " ++
"the same rewrite also removes 2 artifacts shunned earlier and not yet reclaimed; " ++
"refusing without --force:\n" ++
"\n" ++
" " ++ shun_name_a ++ " file already shunned; content still here and would be removed\n" ++
"\n" ++
"This is irreversible: zit undo does not recover a shun, and the content\n" ++
"cannot be re-fetched from a peer.\n" ++
"run 'zit shun --force <HASH>...' to proceed\n",
w.buffered(),
);
}
test "renderShunRefusal: a --defer run is described as a --defer run, and suggests --defer --force" {
// THE CONSENT GATE MUST DESCRIBE THE OPERATION THE OPERATOR ASKED FOR. A
// `--defer` run removes nothing now, so it must not say "would be permanently
// removed"; it must not attribute rewrite collateral to a run that performs no
// rewrite; and — the reason this is pinned byte for byte — the command it
// suggests must be the deferred one. An operator who copies `zit shun --force`
// out of a `--defer` refusal escalates a tombstone into a full-store reclaim of
// the whole union set, including collateral this run never named.
var buf: [1024]u8 = undefined;
{
// `deferred_held` is 2 here and the clause must NOT appear: nothing on this
// path reclaims those rows.
var w: std.Io.Writer = .fixed(&buf);
renderShunRefusal(&w, &.{
.{ .name = shun_name_a, .kind = .manifest, .disposition = .remove },
.{ .name = shun_name_b, .kind = null, .disposition = .remove },
.{ .name = shun_name_c, .disposition = .absent },
}, 2, true);
try std.testing.expectEqualStrings(
"zit shun: 3 hashes would be shunned in this repository; " ++
"the bytes of 2 artifacts stay on disk and stay reconstructible until 'zit pack'; " ++
"refusing without --force:\n" ++
"\n" ++
" " ++ shun_name_a ++ " checkin a check-in; its descendants become unresolvable\n" ++
" " ++ shun_name_b ++ " file file content; every check-in listing it loses that file\n" ++
" " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++
"\n" ++
"This is irreversible: zit undo does not recover a shun, and the content\n" ++
"cannot be re-fetched from a peer.\n" ++
"run 'zit shun --defer --force <HASH>...' to proceed\n",
w.buffered(),
);
}
{
// Singular agreement on the deferred clause, and the `.reclaim` line: under
// `--defer` the content stays rather than being removed.
var w: std.Io.Writer = .fixed(&buf);
renderShunRefusal(&w, &.{
.{ .name = shun_name_a, .kind = .wiki, .disposition = .reclaim },
}, 0, true);
try std.testing.expectEqualStrings(
"zit shun: no new hash would be recorded; " ++
"the bytes of 1 artifact stay on disk and stay reconstructible until 'zit pack'; " ++
"refusing without --force:\n" ++
"\n" ++
" " ++ shun_name_a ++ " wiki already shunned; content still here and stays\n" ++
"\n" ++
"This is irreversible: zit undo does not recover a shun, and the content\n" ++
"cannot be re-fetched from a peer.\n" ++
"run 'zit shun --defer --force <HASH>...' to proceed\n",
w.buffered(),
);
// The escalating suggestion must be ABSENT, not merely reworded: this is the
// one substring an operator is most likely to paste.
try std.testing.expect(std.mem.indexOf(u8, w.buffered(), "run 'zit shun --force") == null);
}
{
// Nothing named holds content: the wording is the same on both paths, and
// only the suggested command differs.
var w: std.Io.Writer = .fixed(&buf);
renderShunRefusal(&w, &.{.{ .name = shun_name_c, .disposition = .absent }}, 0, true);
try std.testing.expectEqualStrings(
"zit shun: 1 hash would be shunned in this repository; " ++
"nothing you named holds content to remove; refusing without --force:\n" ++
"\n" ++
" " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++
"\n" ++
"This is irreversible: zit undo does not recover a shun, and the content\n" ++
"cannot be re-fetched from a peer.\n" ++
"run 'zit shun --defer --force <HASH>...' to proceed\n",
w.buffered(),
);
}
}
test "shunConsequence/shunTypeLabel: one arm per consequence, total over ArtifactType" {
// The per-type diagnostic, pinned type by type. The switch is total, so this
// test and the compiler together guarantee a type added later cannot ship
// with no warning attached to its removal.
try std.testing.expectEqualStrings("file", shunTypeLabel(null));
try std.testing.expectEqualStrings(
"file content; every check-in listing it loses that file",
shunConsequence(null),
);
try std.testing.expectEqualStrings("checkin", shunTypeLabel(.manifest));
try std.testing.expectEqualStrings(
"a check-in; its descendants become unresolvable",
shunConsequence(.manifest),
);
try std.testing.expectEqualStrings("cluster", shunTypeLabel(.cluster));
try std.testing.expectEqualStrings("sync housekeeping only; regenerable", shunConsequence(.cluster));
// The six that all stop rendering share one arm; their LABELS still differ,
// so the operator can tell a ticket from a forum post.
const rendering = "structural content; what it renders stops rendering";
inline for (.{ .control, .wiki, .ticket, .attachment, .event, .forum }) |k| {
try std.testing.expectEqualStrings(rendering, shunConsequence(k));
}
try std.testing.expectEqualStrings("tag", shunTypeLabel(.control));
try std.testing.expectEqualStrings("technote", shunTypeLabel(.event));
try std.testing.expectEqualStrings("forumpost", shunTypeLabel(.forum));
}
test "renderShunBlockers: the unconditional stops, one line each, with no --force offered" {
// The current-checkout stop is the only refusal in this verb with no
// override, so its text must NOT mention --force — an operator who reads a
// flag out of it will try the flag and be refused twice.
var buf: [1024]u8 = undefined;
var w: std.Io.Writer = .fixed(&buf);
renderShunBlockers(&w, &.{
.{ .name = shun_name_a, .kind = .manifest, .disposition = .current_checkout },
.{ .name = "nothex", .disposition = .not_a_name },
.{ .name = "1c04", .disposition = .unresolved },
.{ .name = "7bd1", .disposition = .ambiguous },
// A permissive target contributes nothing to this block.
.{ .name = shun_name_c, .disposition = .remove },
});
try std.testing.expectEqualStrings(
"zit shun: " ++ shun_name_a ++ " is the check-in this checkout is on; refusing\n" ++
"run 'zit update' to a different check-in first\n" ++
"zit shun: 'nothex' is not an artifact name\n" ++
"zit shun: no artifact matches '1c04'; give the complete name to shun a hash this repository does not hold\n" ++
"zit shun: hash '7bd1' is ambiguous\n",
w.buffered(),
);
try std.testing.expect(std.mem.indexOf(u8, w.buffered(), "--force") == null);
}
test "renderShunStats/renderShunDeferred: the two success lines" {
var buf: [256]u8 = undefined;
{
var w: std.Io.Writer = .fixed(&buf);
renderShunStats(&w, .{
.artifacts = 12,
.deltas = 4,
.dropped = 3,
.shuns_recorded = 3,
.bytes_before = 481232,
.bytes_after = 402118,
});
try std.testing.expectEqualStrings(
"shunned: 3 hashes recorded, 3 artifacts removed, 12 -> 9 remaining, 481232 -> 402118 bytes\n",
w.buffered(),
);
}
{
// `dropped` legitimately EXCEEDS `recorded`: the same rewrite reclaims
// shuns deferred by earlier runs, and an operator must be told that
// rather than left to infer it from the byte count.
var w: std.Io.Writer = .fixed(&buf);
renderShunStats(&w, .{
.artifacts = 12,
.deltas = 4,
.dropped = 4,
.shuns_recorded = 1,
.bytes_before = 8192,
.bytes_after = 4096,
});
try std.testing.expectEqualStrings(
"shunned: 1 hash recorded, 4 artifacts removed, 12 -> 8 remaining, 8192 -> 4096 bytes\n",
w.buffered(),
);
}
{
// THE RECORDED COUNT IS THE PACK'S, NOT THE CALLER'S. `shuns_recorded` is
// measured off the new generation's own shun table, so this line cannot
// over-report a takedown even if a caller's intent and the pack's result ever
// diverged. Two tombstones, and this rewrite reclaimed nothing.
var w: std.Io.Writer = .fixed(&buf);
renderShunStats(&w, .{
.artifacts = 12,
.deltas = 4,
.dropped = 0,
.shuns_recorded = 2,
.bytes_before = 8192,
.bytes_after = 8320,
});
try std.testing.expectEqualStrings(
"shunned: 2 hashes recorded, 0 artifacts removed, 12 -> 12 remaining, 8192 -> 8320 bytes\n",
w.buffered(),
);
}
{
// --defer promises no byte count: nothing on that path measured one.
var w: std.Io.Writer = .fixed(&buf);
renderShunDeferred(&w, 2, 2);
try std.testing.expectEqualStrings(
"shunned: 2 hashes recorded; run 'zit pack' to reclaim the space\n",
w.buffered(),
);
}
{
var w: std.Io.Writer = .fixed(&buf);
renderShunDeferred(&w, 1, 1);
try std.testing.expectEqualStrings(
"shunned: 1 hash recorded; run 'zit pack' to reclaim the space\n",
w.buffered(),
);
}
{
// Re-deferring a hash already deferred: no row to add, and yet content still
// to lose. "0 hashes recorded" would read as a no-op over live bytes.
var w: std.Io.Writer = .fixed(&buf);
renderShunDeferred(&w, 0, 1);
try std.testing.expectEqualStrings(
"shunned: no new hash to record; 1 artifact still holds content; run 'zit pack' to reclaim the space\n",
w.buffered(),
);
}
{
// The purely pre-emptive success line: no rewrite ran, so it claims no byte
// count AND does not claim a removal.
var w: std.Io.Writer = .fixed(&buf);
renderShunRecordedOnly(&w, 1);
try std.testing.expectEqualStrings(
"shunned: 1 hash recorded; none is held here, so nothing was removed\n",
w.buffered(),
);
}
}
test "renderShunList: persisted order, the recorded date, and the reason only when given" {
var buf: [512]u8 = undefined;
{
var w: std.Io.Writer = .fixed(&buf);
renderShunList(&w, &.{});
try std.testing.expectEqualStrings("the shun table is empty\n", w.buffered());
}
{
// 1753660800 = 2025-07-28T00:00:00Z; 1752192000 = 2025-07-11T00:00:00Z.
// A row with no reason prints no trailing separator at all.
var w: std.Io.Writer = .fixed(&buf);
renderShunList(&w, &.{
.{ .hash = "\x9f\x2a\x1c" ++ repeat("\x00", 29), .epoch = 1753660800, .reason = "legal takedown ref #4417" },
.{ .hash = "\x1c\x04\xb7" ++ repeat("\x00", 17), .epoch = 1753660800, .reason = "" },
.{ .hash = "\x7b\xd1\xe0" ++ repeat("\x00", 29), .epoch = 1752192000, .reason = "committed credential" },
});
try std.testing.expectEqualStrings(
"9f2a1c" ++ repeat("00", 29) ++ " 2025-07-28 legal takedown ref #4417\n" ++
"1c04b7" ++ repeat("00", 17) ++ " 2025-07-28\n" ++
"7bd1e0" ++ repeat("00", 29) ++ " 2025-07-11 committed credential\n",
w.buffered(),
);
}
}
/// How many rows the store in `dir` holds in its shun table.
fn shunRowsIn(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !usize {
var fs: storage.FileStorage = undefined;
var log = try packOpenReadOnly(gpa, io, dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
return log.shunList().len;
}
test "shunAppendRows: the append aborts on a changed tail and on a held temp lock, recording nothing" {
// THE APPEND PATH'S TWO CONCURRENCY GUARDS, driven directly because neither is
// reachable from the CLI in one process: both windows open between the moment
// `runShun` classified its targets and the moment this function commits.
//
// The TAIL guard is what makes the append path's success lines honest. They are
// claims about STORE STATE — "none is held here, so nothing was removed" — and a
// `zit commit` landing a named hash in that window falsifies them: the tombstone
// would be recorded, exit 0, and an operator under a takedown order told nothing
// was held while the bytes were live and unreclaimed.
//
// The LOCK guard is what stops the row being written to an ORPHANED INODE. A
// `zit pack` renaming its new generation over `.zit` after our `openFile` takes
// the append with it, and the run would still report success — a deferred shun
// that silently does not exist, having just told the operator to run `zit pack`
// to finish it.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
const digest_a = @as([digest_len]u8, @splat(0xab));
const digest_b = @as([digest_len]u8, @splat(0xcd));
const rows_a = [_]logstore.ShunRecord{.{ .hash = &digest_a, .epoch = 1_753_660_800, .reason = "" }};
const rows_b = [_]logstore.ShunRecord{.{ .hash = &digest_b, .epoch = 1_753_660_800, .reason = "" }};
// The key the runner would have captured off its read-only snapshot.
var key_1: logstore.LogStore.LogKey = undefined;
{
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
key_1 = try src.logKey();
}
// Control: with a current key the append lands, so the guards below are failing
// for the reason under test rather than because this path never works.
try shunAppendRows(gpa, io, tmp.dir, key_1, &rows_a);
try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir));
// No temp left behind: the lock file is removed on the success path too.
try std.testing.expectError(error.FileNotFound, tmp.dir.openFile(io, pack_tmp_filename, .{}));
// That append is itself the "concurrent writer": `key_1` is now stale.
const before = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(before);
try std.testing.expectError(error.TailChanged, shunAppendRows(gpa, io, tmp.dir, key_1, &rows_b));
// NOTHING WRITTEN: the check precedes the first `putShun`.
try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir));
{
const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, before, after);
}
var key_2: logstore.LogStore.LogKey = undefined;
{
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
key_2 = try src.logKey();
}
{
// Stand in for the concurrent rewrite with a rewrite lock naming a live
// process that is not this one (this test's parent), which makes this
// deterministic rather than a race.
var host_buf: [std.posix.HOST_NAME_MAX]u8 = undefined;
const host = try std.posix.gethostname(&host_buf);
const parent: i64 = if (@import("builtin").os.tag == .linux) std.os.linux.getppid() else std.c.getppid();
var rec_buf: [256]u8 = undefined;
const rec = try std.fmt.bufPrint(&rec_buf, "pid={d} host={s} time=1\n", .{ parent, host });
try tmp.dir.writeFile(io, .{ .sub_path = rewrite_lock_name, .data = rec });
defer tmp.dir.deleteFile(io, rewrite_lock_name) catch {};
try std.testing.expectError(error.RewriteBusy, shunAppendRows(gpa, io, tmp.dir, key_2, &rows_b));
try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir));
const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, before, after);
}
// Released and current: the same call now lands.
try shunAppendRows(gpa, io, tmp.dir, key_2, &rows_b);
try std.testing.expectEqual(@as(usize, 2), try shunRowsIn(gpa, io, tmp.dir));
}
/// Render one `ApplyError` through `openApplyFail` into `buf`.
fn openFailText(buf: []u8, err: update.ApplyError) []const u8 {
var w = Io.Writer.fixed(buf);
var sink: [1]u8 = undefined;
var wnull = Io.Writer.fixed(&sink);
const s: cli.Streams = .{ .out = &wnull, .err = &w };
_ = openApplyFail(s, err);
return w.buffered();
}
// Same guard as the `zit update`, `zit merge` and `zit bisect` mappers.
// `Corrupt` alleges the stored bytes do not match the name they are stored
// under; `StateRecordInvalid` says only that an artifact was unusable as a
// state record. A clone of an intact fossil repository reaches the second
// without anything being damaged, so it must never be answered with a
// corruption claim.
test "open: the two read-path failures do not share a diagnostic" {
var a: [256]u8 = undefined;
var b: [256]u8 = undefined;
const corrupt = openFailText(&a, error.Corrupt);
const state = openFailText(&b, error.StateRecordInvalid);
try std.testing.expect(!std.mem.eql(u8, corrupt, state));
try std.testing.expect(std.mem.indexOf(u8, corrupt, "corrupt") != null);
try std.testing.expect(std.mem.indexOf(u8, state, "corrupt") == null);
}
// ---------------------------------------------------------------------------
// `zit serve` must answer one sync round in time bounded by the
// REQUEST, not by the served store. Two per-request costs made a clone
// O(rounds x store): counting held artifacts by forcing a full inflate of every
// artifact, and rebuilding the entire in-memory snapshot from the log on every
// request. The guards below pin both as SCALING properties measured with work
// counters (plaintext bytes materialised, snapshot builds performed) — never
// wall-clock thresholds, which flake and prove nothing about the mechanism.
// ---------------------------------------------------------------------------
/// Seed `ms` with `n` distinct wire-backed artifacts exactly as
/// `loadContentStore` does for a served repository: one §4.1 full container per
/// artifact, with NO plaintext materialised. `ms.plaintextBytesHeld()` is 0 on
/// return, so any later non-zero reading is inflation this store did not ask for.
fn serveSeedWireStore(gpa: std.mem.Allocator, ms: *repostate.MemStore, n: usize) !void {
var i: usize = 0;
while (i < n) : (i += 1) {
var body_buf: [64]u8 = undefined;
const body = try std.fmt.bufPrint(&body_buf, "issue nine artifact {d}\n", .{i});
var name_buf: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, body, &name_buf);
const container = try zit.container.encode(gpa, body, .default);
defer gpa.free(container);
_ = try ms.putWire(name, container);
}
std.debug.assert(ms.plaintextBytesHeld() == 0);
}
test "serve: the held-artifact count serve uses never materialises plaintext (constant in store size)" {
const gpa = std.testing.allocator;
// Two store sizes: the work this count performs must not grow with n. The
// count VALUE alone would pass before and after the fix — it is the
// materialised-plaintext reading that observes the defect, so that is what
// is asserted, at both sizes, as the scaling statement.
for ([_]usize{ 8, 64 }) |n| {
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
try serveSeedWireStore(gpa, &ms, n);
try std.testing.expectEqual(n, ms.store().heldCount());
try std.testing.expectEqual(@as(usize, 0), ms.plaintextBytesHeld());
}
}
test "serve: a held artifact whose container does not inflate is still counted as servable" {
const gpa = std.testing.allocator;
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
// §4.1 shape (4-byte length prefix) with a payload that is not a valid zlib
// stream: `held()==true`, `content()==null` — the case `held()`'s contract at
// repostate.zig names. `respond` retransmits it verbatim and `persistStore`
// counts it, so serve's own count must agree with them.
const bogus = "\x00\x00\x00\x0anot-zlib-bytes";
_ = try ms.putWire(repeat("f", 64), bogus);
try std.testing.expectEqual(@as(usize, 1), ms.store().heldCount());
try std.testing.expect(ms.store().held(1));
try std.testing.expect(ms.store().content(1) == null);
}
/// Append one content frame (raw `digest` naming `bytes`) to the `.zit` in
/// `dir`, creating the repository on first call — a separate writer's committed
/// append, exactly what a `zit commit` alongside a live `zit serve` performs.
fn serveAppendFrame(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, digest: []const u8, bytes: []const u8) !void {
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch
try dir.createFile(io, repo_filename, .{ .read = true });
defer f.close(io);
const st = try f.stat(io);
var fs = storage.FileStorage.init(io, f, st.size);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer log.deinit(gpa);
_ = try log.put(digest, bytes);
try log.commit();
}
/// A §4.1-framed read-only request naming `hash` in a `gimme` card.
fn serveGimmeRequest(gpa: std.mem.Allocator, card: []const u8, hash: []const u8) ![]u8 {
const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\n{s} {s}\n", .{ card, hash });
defer gpa.free(plain);
return zit.container.encode(gpa, plain, .default);
}
test "serve: repeated rounds over an unchanged log build the snapshot once (constant in round count)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const digest_a = repeat("a", digest_len);
try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n");
var hex_buf: [2 * digest_len]u8 = undefined;
const hex_a = digestToWireName(digest_a, &hex_buf);
const request = try serveGimmeRequest(gpa, "gimme", hex_a);
defer gpa.free(request);
const server = zit.sync.Server{ .version = serve_mod.server_version };
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
// Eight rounds, log unchanged: the snapshot is a function of the log, not of
// the request, so the number of builds must not grow with the round count.
for (0..8) |_| {
const reply = try serveAnswer(&ctx, gpa, request, null, .{}, true);
gpa.free(reply.body);
}
try std.testing.expectEqual(@as(u64, 1), ctx.loads);
}
test "serve: a responder's phantom interning cannot leak into the next round" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const digest_a = repeat("a", digest_len);
try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n");
var hex_buf: [2 * digest_len]u8 = undefined;
const hex_a = digestToWireName(digest_a, &hex_buf);
// §3.3 — an `igot` for a hash the server lacks makes `respond` intern a
// phantom into whatever store it answers over. That mutation must not
// survive into the reused snapshot.
const unknown_hex = repeat("c", (2 * digest_len));
const igot_req = try serveGimmeRequest(gpa, "igot", unknown_hex);
defer gpa.free(igot_req);
const gimme_req = try serveGimmeRequest(gpa, "gimme", hex_a);
defer gpa.free(gimme_req);
const server = zit.sync.Server{ .version = serve_mod.server_version };
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
// NOTE ON THIS GUARD'S ROLE: it is green against the pre-fix code (which
// rebuilds per request and so cannot leak) — the round-count guard above is
// the red one. This guard exists because the obvious WRONG way to satisfy
// that guard — answer directly over one shared mutable snapshot — passes it
// and truncates real clones, since the phantoms and memoisations `respond`
// writes change what later rounds serve. Without this assertion that
// implementation ships.
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const r1 = try serveAnswer(&ctx, gpa, igot_req, null, .{}, true);
gpa.free(r1.body);
const snap = ctx.snapshot.?.store();
try std.testing.expectEqual(@as(usize, 1), snap.count()); // the one on-disk artifact, nothing more
try std.testing.expect(snap.lookup(unknown_hex) == null); // the phantom did not land here
const r2 = try serveAnswer(&ctx, gpa, gimme_req, null, .{}, true);
defer gpa.free(r2.body);
try std.testing.expectEqual(@as(u64, 1), ctx.loads); // still one build: the log never changed
// The reply is byte-identical to the one a server with no prior round produces.
var fresh_file = try tmp.dir.openFile(io, repo_filename, .{});
defer fresh_file.close(io);
var fresh_ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &fresh_file, .server = &server };
defer fresh_ctx.dropSnapshot();
const expected = try serveAnswer(&fresh_ctx, gpa, gimme_req, null, .{}, true);
defer gpa.free(expected.body);
try std.testing.expectEqualSlices(u8, expected.body, r2.body);
}
test "serve §3.8: a reused snapshot still sees a concurrent commit's append" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const digest_a = repeat("a", digest_len);
const digest_b = repeat("b", digest_len);
try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n");
var hex_a_buf: [2 * digest_len]u8 = undefined;
var hex_b_buf: [2 * digest_len]u8 = undefined;
const hex_a = digestToWireName(digest_a, &hex_a_buf);
const hex_b = digestToWireName(digest_b, &hex_b_buf);
const req_a = try serveGimmeRequest(gpa, "gimme", hex_a);
defer gpa.free(req_a);
const req_b = try serveGimmeRequest(gpa, "gimme", hex_b);
defer gpa.free(req_b);
const server = zit.sync.Server{ .version = serve_mod.server_version };
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const r1 = try serveAnswer(&ctx, gpa, req_a, null, .{}, true);
gpa.free(r1.body);
try std.testing.expectEqual(@as(usize, 1), ctx.snapshot.?.store().count());
// A SEPARATE writer appends while the serve handle stays open.
try serveAppendFrame(gpa, io, tmp.dir, digest_b, "content b\n");
const r2 = try serveAnswer(&ctx, gpa, req_b, null, .{}, true);
gpa.free(r2.body);
// The log grew, so the snapshot was rebuilt and the new artifact is servable:
// a cache that never invalidates would still report one artifact and one load.
try std.testing.expectEqual(@as(u64, 2), ctx.loads);
try std.testing.expectEqual(@as(usize, 2), ctx.snapshot.?.store().count());
try std.testing.expect(ctx.snapshot.?.store().lookup(hex_b) != null);
}
/// Seed `dir`'s `.zit` with `n` distinct compressible artifacts, returning the
/// wire (hex) name of the first. Compressible so the log retains a §4.1
/// container per frame — the shape a served repository actually has, and the
/// only shape in which "answering a round does not inflate the store" has
/// anything to say.
fn serveSeedRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, n: usize, first_hex: *[2 * digest_len]u8) !void {
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch
try dir.createFile(io, repo_filename, .{ .read = true });
defer f.close(io);
const st = try f.stat(io);
var fs = storage.FileStorage.init(io, f, st.size);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer log.deinit(gpa);
var i: usize = 0;
while (i < n) : (i += 1) {
var body: [512]u8 = undefined;
@memset(&body, 'a' + @as(u8, @intCast(i % 26)));
var digest: [digest_len]u8 = undefined;
@memset(&digest, 0);
std.mem.writeInt(u64, digest[0..8], i + 1, .big);
_ = try log.putZlib(&digest, &body);
if (i == 0) _ = digestToWireName(&digest, first_hex);
}
try log.commit();
}
test "serve: answering a round inflates only what the round emits (constant in store size)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
// The same read round against two store sizes. The reply is identical, so
// the plaintext the server inflates to produce it must be identical too: any
// dependence on n is the server paying for artifacts the round never names.
// This is the assertion the whole change exists to make true — the pre-fix
// held-count walked `content()` over every rid, and so did inflate the store.
var inflated: [2]usize = undefined;
for ([_]usize{ 4, 40 }, 0..) |n, k| {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var hex_buf: [2 * digest_len]u8 = undefined;
try serveSeedRepo(gpa, io, tmp.dir, n, &hex_buf);
const request = try serveGimmeRequest(gpa, "gimme", &hex_buf);
defer gpa.free(request);
const server = zit.sync.Server{
.version = serve_mod.server_version,
.self_servercode = serve_mod.server_servercode,
.project_code = serve_mod.server_projectcode,
};
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
for (0..3) |_| {
const reply = try serveAnswer(&ctx, gpa, request, null, .{}, true);
gpa.free(reply.body);
// Nothing the round materialised is still resident: the cached index
// reads containers on demand and frees them between rounds, never
// holding the repository's uncompressed history (or its containers).
try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.residentBytes());
}
const emitted = ctx.snapshot.?.materialized_bytes;
// A CLONE round is the worst case for residency: it emits every artifact
// and proves each one's declared size against the plaintext, so it
// legitimately inflates the whole store. What must not survive it is the
// plaintext — a snapshot that outlives the round would otherwise end up
// holding the entire repository uncompressed, for the process's life.
const clone_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 1\n", .{});
defer gpa.free(clone_plain);
const clone_req = try zit.container.encode(gpa, clone_plain, .default);
defer gpa.free(clone_req);
const clone_reply = try serveAnswer(&ctx, gpa, clone_req, null, .{}, true);
gpa.free(clone_reply.body);
try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.residentBytes());
try std.testing.expect(ctx.snapshot.?.materialized_bytes >= emitted);
try std.testing.expectEqual(@as(usize, n), ctx.snapshot.?.store().count());
inflated[k] = emitted;
}
try std.testing.expectEqual(inflated[0], inflated[1]);
// ... and the constant is the round's own emission, not the store's: a
// 40-artifact store holds 20480 bytes of plaintext, so an equality that
// happened to hold at "everything" would still be caught here.
try std.testing.expect(inflated[1] < 4 * 512);
}
test "serve: the pushing round does not rebuild inside its deadline; maintain rebuilds between connections" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const hash_kind: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
{
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind);
defer seed.deinit(gpa);
try seed.commit();
}
const push_body = "pushed artifact bytes\n";
var name_buf: [64]u8 = undefined;
const push_hash = zit.artifact.artifactName(.sha3_256, push_body, &name_buf);
const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\npush sc pc\nfile {s} {d}\n{s}", .{ push_hash, push_body.len, push_body });
defer gpa.free(plain);
const push_req = try zit.container.encode(gpa, plain, .default);
defer gpa.free(push_req);
const read_req = try serveGimmeRequest(gpa, "gimme", push_hash);
defer gpa.free(read_req);
const server = zit.sync.Server{ .version = serve_mod.server_version };
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const r0 = try serveAnswer(&ctx, gpa, read_req, null, .{}, true);
gpa.free(r0.body);
try std.testing.expectEqual(@as(u64, 1), ctx.loads);
// The push makes the snapshot stale, but answering it must NOT pay the
// O(store) rebuild: the reply is written inside a connection deadline, and
// The failure is exactly the whole-repository rebuild landing inside that
// budget. The rebuild is deferred to `maintain`, which the accept loop runs
// after the connection closes — outside every deadline. So the pushing round
// leaves `loads` untouched.
const r1 = try serveAnswer(&ctx, gpa, push_req, null, .{}, true);
gpa.free(r1.body);
try std.testing.expectEqual(@as(u64, 1), ctx.loads); // no rebuild inside the round
// The snapshot is momentarily stale — it predates the push — yet the push is
// already durable on disk: `persistStore` ran inside the round.
try std.testing.expect(ctx.snapshot != null);
try std.testing.expect(ctx.snapshot.?.store().lookup(push_hash) == null);
// The accept loop's post-connection maintenance hook does the O(store)
// rebuild, between connections, with no connection open to charge it to.
ServeCtx.maintain(&ctx, gpa);
try std.testing.expectEqual(@as(u64, 2), ctx.loads);
try std.testing.expect(ctx.snapshot.?.store().lookup(push_hash) != null);
// The rebuilt snapshot describes the grown log exactly (the append-only stat
// gate that `baseSnapshot` already carries).
const st = try file.stat(io);
try std.testing.expectEqual(st.size, ctx.snapshot_log_size);
// A later read round is stat-gated to no rebuild, and still serves the push.
const r2 = try serveAnswer(&ctx, gpa, read_req, null, .{}, true);
defer gpa.free(r2.body);
try std.testing.expectEqual(@as(u64, 2), ctx.loads);
}
// ---------------------------------------------------------------------------
// sync §3.3 L553-556 / §8.8 L1839-1841 — the durable side of the phantom-scan
// shun filter: the repository shun table, snapshotted off the log and answered
// as a wire-name question. `RepoShunSet` lives above with `LogIndexStore`; these
// tests pin the key domain and the end-to-end suppression.
// ---------------------------------------------------------------------------
/// The shunned artifact's digest and its complete 64-hex wire name. 's' is 0x73,
/// so the name is the byte pattern spelled out — no runtime encode in the vector.
const shun_test_digest = repeat("s", digest_len);
const shun_test_name = repeat("73", digest_len);
/// A second artifact the repo actually holds ('h' is 0x68), to prove the filter
/// suppresses only the shunned name.
const shun_test_held_digest = repeat("h", digest_len);
const shun_test_held_name = repeat("68", digest_len);
/// A third shunned digest whose hex spelling contains LETTER nibbles, so the
/// uppercase-spelling question below is a real one.
const shun_test_letter_digest = repeat("\xab", digest_len);
const shun_test_letter_name = repeat("ab", digest_len);
comptime {
std.debug.assert(shun_test_name.len == 2 * digest_len);
std.debug.assert(logstore.isCompleteDigest(shun_test_digest.len));
}
/// Seed `dir`'s repository with one held artifact and one shun row naming an
/// artifact it does NOT hold — exactly the state a `zit shun` on a hash the peer
/// still offers leaves behind.
fn seedShunnedRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void {
const hk: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = try dir.createFile(io, repo_filename, .{ .read = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer log.deinit(gpa);
try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" });
_ = try log.put(shun_test_held_digest, "held content\n");
try log.putShun(shun_test_digest, "operator request", 1);
try log.putShun(shun_test_letter_digest, "operator request", 1);
try log.commit();
}
test "sync §3.3 L553-556: a repository shun row is a durable wire-name hit, keyed exactly" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedShunnedRepo(gpa, io, tmp.dir);
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const shun = &holdings.index.shun;
const set = holdings.admission().shun;
// The complete name is the key: a hit.
try std.testing.expect(set.contains(shun_test_name));
// The held artifact is not shunned.
try std.testing.expect(!set.contains(shun_test_held_name));
// A 12-hex PREFIX of the shunned name is a DIFFERENT key, not a hit. The
// lookup never truncates, pads, or prefix-matches — a prefix must not be
// able to masquerade as a shun-table hit. Nor may an incomplete name reach
// `LogStore.isShunned`, whose caller contract (src/logstore.zig L1746-1757)
// binds this boundary: the width check is a CHECKED rejection, not an
// assert, because an assert is nothing in ReleaseFast and these names come
// off the wire.
//
// These OUT-OF-DOMAIN widths are probed against `containsWireName` directly,
// not through `zit.sync.ShunSet.contains`: the release-checked rule lives
// here, in the implementation that owns it, while the wrapper asserts the
// §3.2 width its own key domain promises. Going through the wrapper would
// pin the assert instead of the rejection these lines mean to pin.
try std.testing.expect(!RepoShunSet.containsWireName(shun, shun_test_name[0..12]));
try std.testing.expect(!RepoShunSet.containsWireName(shun, shun_test_name[0 .. 2 * digest_len - 1])); // odd length
try std.testing.expect(!RepoShunSet.containsWireName(shun, "")); // empty
try std.testing.expect(!RepoShunSet.containsWireName(shun, repeat("ab", 48))); // 96 hex — not one of the two forms
// Correct WIDTH, not hex: in the wrapper's key domain, so probed through it.
try std.testing.expect(!set.contains(repeat("z", (2 * digest_len)))); // not hex
// An UPPERCASE spelling of the shunned name decodes to the same digest and
// so answers TRUE. artifact-format §3.3 L370-377 makes the token invalid but
// lets an implementation accept it at name validation and fail it at
// verification; over-suppressing a name that can never verify is the safe
// direction. Pinned so the fail-closed direction is a decision, not an
// accident of `hexToBytes` accepting case.
try std.testing.expect(set.contains(shun_test_letter_name));
var upper: [2 * digest_len]u8 = undefined;
for (shun_test_letter_name, 0..) |c, i| upper[i] = std.ascii.toUpper(c);
try std.testing.expect(!std.mem.eql(u8, &upper, shun_test_letter_name)); // the case really differs
try std.testing.expect(set.contains(&upper));
}
test "sync §3.3 L553-556: an OOM inside the shun-set load leaves no borrowed key in the owned-key map" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedShunnedRepo(gpa, io, tmp.dir);
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
var fs = storage.FileStorage.init(io, file, st.size);
var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile());
defer log.deinit(gpa);
try std.testing.expect(log.shunList().len >= 2);
// `load` dupes each row's digest because `shunList` borrows the STORE's own
// bytes, and the dupe must PRECEDE the insert: insert-then-replace leaves the
// borrowed slice live as a key on the one error path, after which `deinit`
// frees the store's digest and `log.deinit` frees it again. A manual sweep
// (not `checkAllAllocationFailures`) because the obligation is about which
// POINTER survives the unwind, not just about leaking: the store is
// allocated from the UNWRAPPED allocator, so a key that is not `load`'s own
// dupe is freed through the wrap and the double free lands on
// `testing.allocator`, which catches it. `fail_index` indexes `load`'s
// allocations alone for the same reason.
var i: usize = 0;
while (i < 64) : (i += 1) {
var fa = std.testing.FailingAllocator.init(gpa, .{ .fail_index = i });
var shun = RepoShunSet.init(fa.allocator());
defer shun.deinit();
if (shun.load(&log)) |_| {
// The fail point lay at or past the last allocation: no failure was
// forced, so every allocation site is now covered and the set is
// whole.
try std.testing.expect(shun.digests.count() >= 2);
break;
} else |e| {
try std.testing.expectEqual(error.OutOfMemory, e);
}
} else return error.SweepNeverCompleted;
}
test "sync §3.3 L553-556: a pull client carrying the repository shun set creates the phantom but never gimme's it" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedShunnedRepo(gpa, io, tmp.dir);
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
// The pull client, constructed exactly as `pullRun` does.
var client = zit.sync.Client{
.state = zit.sync.initialState(.pull),
.operation = .pull,
.self_servercode = sync_servercode,
.project_code = sync_projectcode,
.version = .{ .version = sync_client_version },
.shun = holdings.admission().shun,
};
defer client.deinit(gpa);
// The peer announces BOTH artifacts. §3.3 r24(1) is unconditional on the
// shun table, so the shunned one still becomes a phantom.
const reply = "pragma server-version 1\nigot " ++ shun_test_name ++ "\n";
const obs = try client.consumeReply(gpa, holdings.store(), reply);
try std.testing.expect(obs.phantom_created);
try std.testing.expectEqual(@as(usize, 1), client.phantoms.count());
// …and the request never asks for it, in any round.
var buf: std.ArrayListUnmanaged(u8) = .empty;
defer buf.deinit(gpa);
var round: usize = 0;
while (round < 3) : (round += 1) {
const request = try client.buildRequest(gpa, holdings.store(), &buf);
try std.testing.expect(std.mem.indexOf(u8, request, "gimme " ++ shun_test_name) == null);
// The held artifact is still announced — nothing else changed.
try std.testing.expect(std.mem.indexOf(u8, request, "igot " ++ shun_test_held_name) != null);
}
}
/// Does the decoded reply body carry `gimme NAME` as a whole card line?
fn replyHasGimme(gpa: std.mem.Allocator, reply_container: []const u8, name: []const u8) !bool {
const body = try zit.container.decode(gpa, reply_container, @intCast(logstore.max_payload));
defer gpa.free(body);
var it = std.mem.splitScalar(u8, body, '\n');
while (it.next()) |line| {
if (line.len == 6 + name.len and
std.mem.startsWith(u8, line, "gimme ") and
std.mem.eql(u8, line[6..], name)) return true;
}
return false;
}
test "sync §3.3 L553-556 / §8.8 L1839-1841: a served reply never gimme's a shunned phantom, and a shun row written between rounds is honoured by the next" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// A repo holding one artifact and NO shun row yet.
const hk: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
{
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer seed.deinit(gpa);
_ = try seed.put(shun_test_held_digest, "held content\n");
try seed.commit();
}
// A pushing client's `igot` for a hash this repo lacks: §3.3 r24(1)
// L511-514 makes the server a receiver, so `respond` interns the phantom
// and would beg for it.
const igot_req = try serveGimmeRequest(gpa, "push sc pc\nigot", shun_test_name);
defer gpa.free(igot_req);
const server = zit.sync.Server{ .version = serve_mod.server_version };
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
// Round 1, no shun row: the reply asks for it. This is the behaviour the
// filter must change, so the test is red on an unfiltered server.
{
const r = try serveAnswer(&ctx, gpa, igot_req, null, .{}, true);
defer gpa.free(r.body);
try std.testing.expect(try replyHasGimme(gpa, r.body, shun_test_name));
}
// The operator shuns it BETWEEN rounds — an append to the same log.
{
var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write });
defer f.close(io);
const st = try f.stat(io);
var fs = storage.FileStorage.init(io, f, st.size);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer log.deinit(gpa);
try log.putShun(shun_test_digest, "operator request", 1);
try log.commit();
}
// Round 2 honours it with no restart: the append grew the log, so
// `baseSnapshot`'s stat gate rebuilds the index — and the rebuild re-loads
// the shun table (`LogIndexStore.shun`). That is the freshness mechanism this
// set rides; nothing here is memoised past a build.
{
const r = try serveAnswer(&ctx, gpa, igot_req, null, .{}, true);
defer gpa.free(r.body);
try std.testing.expect(!try replyHasGimme(gpa, r.body, shun_test_name));
// The unshunned holding is still advertised: only the shunned name went quiet.
const body = try zit.container.decode(gpa, r.body, @intCast(logstore.max_payload));
defer gpa.free(body);
try std.testing.expect(std.mem.indexOf(u8, body, "igot " ++ shun_test_held_name) != null);
}
try std.testing.expectEqual(@as(u64, 2), ctx.loads); // one build per log state, not per round
}
// ---------------------------------------------------------------------------
// sync §9.5 L2042-2046 — the SEND side of the shun filter.
//
// "A subsequent read-authorized transfer path requests igot for every
// eligible holding after the Section 3.3 shun, phantom, private, and
// peer-has filters"
//
// `LogIndexStore`'s enumeration is that "every eligible holding", so the shun
// filter belongs in the enumeration itself: a shunned holding is masked, which
// makes the server neither `igot`-announce it (the §3.2 walk's `held()` gate
// answers false) nor serve it in a clone batch (no wire form, no content), and
// makes an inbound `gimme` for it land on §3.3 L569-572's *"If absent, silently
// skip"* with no code on that path — the name is simply not in `by_name`.
//
// Which of those three the MASK is load-bearing for, stated honestly: the igot
// announcement and the `gimme` miss are new here — nothing else suppressed them.
// The clone batch was already covered by `LogStore.wire`/`content`'s own rid-keyed
// read mask, which the pager gates on; the mask's contribution there is that the
// pager's cursor stays contiguous over the survivors (see the clone-pager test).
//
// The mask is a MASK, not a removal, and §8.2 L1551-1553 is why:
//
// "SEQNO is a server blob-table RID. Iterate from supplied RID through
// current maximum, silently skipping gaps; clients MUST NOT infer count
// or size from range."
//
// Dropping the shunned frame from the enumeration would renumber every survivor
// above it. The server is stateless (§8.9 L1855-1856) — the client's SEQNO *is* the
// clone cursor — and a shun row appended between two rounds of one clone grows
// the log, which is exactly what trips `ServeCtx.baseSnapshot`'s append-only stat
// gate and forces a rebuild. Under renumbering the resuming client's cursor would
// then step over a survivor and still be told the sequence was complete: a
// silently short clone, the failure class `read_fault` exists to prevent. Keeping
// the slot and answering it as a §8.2 GAP makes adapter-rid identity a pure
// function of the log, independent of the shun table.
// ---------------------------------------------------------------------------
/// Five artifact digests in persisted order. Distinct byte fills, so each wire
/// name is the pattern spelled out and no vector needs a runtime encode.
const send_shun_digests = [5][]const u8{
repeat("\x11", digest_len),
repeat("\x22", digest_len),
repeat("\x33", digest_len),
repeat("\x44", digest_len),
repeat("\x55", digest_len),
};
const send_shun_names = [5][]const u8{
repeat("11", digest_len),
repeat("22", digest_len),
repeat("33", digest_len),
repeat("44", digest_len),
repeat("55", digest_len),
};
/// The FIRST, a MIDDLE and the LAST of the five are shunned, so an off-by-one or
/// a misindexed mask cannot pass: the survivors are interleaved among them.
fn sendShunIsShunned(i: usize) bool {
return i % 2 == 0;
}
/// Seed `dir`'s repository holding all five `send_shun_digests` in order, with a
/// live config frame BEFORE the first and another BETWEEN the second and third,
/// and shun rows naming the first, middle and last.
///
/// The interleaving is what an off-by-one cannot survive: shunning the FIRST, a
/// MIDDLE and the LAST slot means a mask built by any walk other than the
/// enumeration's own loop names one artifact and suppresses another, and the
/// alignment test below reads that straight off the log. The config frames are
/// here for a WEAKER reason than they look: they never enter
/// `ridsInPersistedOrder` at all (a config frame is not interned — `LogStore.scan`
/// only records its offset), so they exercise the `.config` guards' *presence*, not
/// a live skip; they do not by themselves make a second walk misalign.
fn seedSendShunRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void {
const hk: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer log.deinit(gpa);
try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" });
_ = try log.put(send_shun_digests[0], "content zero\n");
_ = try log.put(send_shun_digests[1], "content one\n");
try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "cccc" });
_ = try log.put(send_shun_digests[2], "content two\n");
_ = try log.put(send_shun_digests[3], "content three\n");
_ = try log.put(send_shun_digests[4], "content four\n");
for (send_shun_digests, 0..) |d, i| {
if (sendShunIsShunned(i)) try log.putShun(d, "operator request", 1);
}
try log.commit();
}
test "sync §9.5 L2042-2046: a shunned holding is masked out of the adapter enumeration and its name is absent from by_name" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedSendShunRepo(gpa, io, tmp.dir);
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
var index = try LogIndexStore.openFromFile(gpa, io, file, st.size);
defer index.destroy();
const s = index.store();
// §8.2 L1551-1553 — the rid space is the LOG's, not the survivors': all five
// artifact frames still hold their slot (config frames were never in it).
try std.testing.expectEqual(@as(usize, 5), index.rids.len);
try std.testing.expectEqual(@as(usize, 5), s.count());
for (send_shun_names, 0..) |name, i| {
const rid: repostate.Rid = @intCast(i + 1);
if (sendShunIsShunned(i)) {
// Not resolvable by name: the §3.2 gimme arm keys on `lookup`, so
// this IS §3.3 L569-572's "If absent, silently skip".
try std.testing.expect(index.by_name.get(name) == null);
try std.testing.expect(s.lookup(name) == null);
// Not announced (§3.2's igot walk gates on `held`), and nothing to
// send in either form.
try std.testing.expect(!s.held(rid));
try std.testing.expect(s.content(rid) == null);
try std.testing.expect(s.wire(rid) == null);
} else {
try std.testing.expectEqual(@as(?repostate.Rid, rid), s.lookup(name));
try std.testing.expect(s.held(rid));
try std.testing.expect(s.content(rid) != null);
}
}
// A masked frame is a policy gap, never a read fault: the round must not be
// converted to `error.ServerStorage` by `serveAnswer`.
try std.testing.expect(!index.read_fault);
}
/// Does the decoded reply body carry `VERB NAME` as a whole card line?
fn replyHasCardFor(body: []const u8, verb: []const u8, name: []const u8) bool {
var it = std.mem.splitScalar(u8, body, '\n');
while (it.next()) |line| {
if (line.len != verb.len + 1 + name.len) continue;
if (!std.mem.startsWith(u8, line, verb)) continue;
if (line[verb.len] != ' ') continue;
if (std.mem.eql(u8, line[verb.len + 1 ..], name)) return true;
}
return false;
}
test "sync §9.5 L2042-2046: a served round igot-announces the survivors and never a masked holding" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedSendShunRepo(gpa, io, tmp.dir);
const server = zit.sync.Server{ .version = serve_mod.server_version };
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
// A round that makes the server run its §3.2 inventory walk over every rid.
// The announced hash is one this repo does not hold, so nothing else about
// the request steers which holdings the walk reaches.
const req = try serveGimmeRequest(gpa, "igot", shun_test_name);
defer gpa.free(req);
const reply = try serveAnswer(&ctx, gpa, req, null, .{}, true);
defer gpa.free(reply.body);
const body = try zit.container.decode(gpa, reply.body, @intCast(logstore.max_payload));
defer gpa.free(body);
for (send_shun_names, 0..) |name, i| {
if (sendShunIsShunned(i)) {
// §9.5 L2042-2046 — not among "every eligible holding": the walk's
// `held()` gate answers false, so no igot names it.
try std.testing.expect(!replyHasCardFor(body, "igot", name));
// Nor does it fall out of the walk as a PHANTOM to beg for. The
// enumeration now hands the walk a non-held rid, and it is `Server.shun`
// — the §3.3 L553-556 arm this repository's own table feeds through
// `serveRound` — that keeps the reply quiet. That check is not dead
// after this slice; the mask is what makes it load-bearing.
try std.testing.expect(!replyHasCardFor(body, "gimme", name));
} else {
try std.testing.expect(replyHasCardFor(body, "igot", name));
}
}
// A masked holding is not an error condition on any card.
try std.testing.expect(std.mem.indexOf(u8, body, "error") == null);
}
test "sync §9.5 L2042-2046: the masked enumeration stays aligned — adapter rid r names rids[r-1] with shunned first, middle and last" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedSendShunRepo(gpa, io, tmp.dir);
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
var index = try LogIndexStore.openFromFile(gpa, io, file, st.size);
defer index.destroy();
const s = index.store();
// `rids`, `names`, `masked`, `content_memo` and `wire_memo` are PARALLEL
// arrays and `by_name` maps a wire name to that same index. Filtering or
// masking one out of step with the others makes adapter rid r name a
// DIFFERENT artifact's bytes — silent corruption on the wire, not an error.
// Pin the whole alignment against the log itself, entry by entry.
var shunned_seen: usize = 0;
for (index.rids, 0..) |lr, a| {
var buf: [2 * digest_len]u8 = undefined;
const expect_name = digestToWireName(index.log.hashOf(lr), &buf);
const rid: repostate.Rid = @intCast(a + 1);
try std.testing.expectEqualStrings(expect_name, index.names[a]);
try std.testing.expectEqualStrings(expect_name, s.hashOf(rid));
try std.testing.expectEqualStrings(send_shun_names[a], expect_name);
// The mask indexes the SAME slot as `rids[a]` — the invariant a mask
// built over any second walk over the log breaks (verified: a mask
// written one slot out of step turns this expectation red).
try std.testing.expectEqual(index.shun.digests.contains(index.log.hashOf(lr)), index.masked[a]);
if (index.masked[a]) {
shunned_seen += 1;
try std.testing.expect(index.by_name.get(expect_name) == null);
} else {
try std.testing.expectEqual(@as(?repostate.Rid, rid), index.by_name.get(expect_name));
}
}
// The fixture really does interleave: three shunned, two survivors.
try std.testing.expectEqual(@as(usize, 3), shunned_seen);
try std.testing.expectEqual(@as(usize, 2), index.by_name.count());
}
test "sync §9.5 L2042-2046 / §7.7 L1472-1475: a write against a masked wire name does not unmask it — no second identity, still not held, still not servable" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedSendShunRepo(gpa, io, tmp.dir);
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
var index = try LogIndexStore.openFromFile(gpa, io, file, st.size);
defer index.destroy();
const s = index.store();
// The masked name is absent from `by_name`, so every write door below misses the
// dedup key the survivors hit and would otherwise fall through to the appendix —
// minting a SECOND, HELD entry for a name §9.5 says this server does not send,
// and putting it back on the §3.2 walk. The mask outranks the write instead.
const masked_name = send_shun_names[0];
try std.testing.expect(index.masked[0]);
const before = s.count();
try std.testing.expectEqual(@as(repostate.Rid, 1), try s.intern(masked_name));
try std.testing.expectEqual(@as(repostate.Rid, 1), try s.put(masked_name, "content zero\n"));
// Even bytes that DISAGREE with the frame: the mask is not a content check, and
// this path must not read (nor assert on) a masked frame's plaintext.
try std.testing.expectEqual(@as(repostate.Rid, 1), try s.put(masked_name, "different bytes\n"));
try s.retain(1, .{ .full = "not a real container" });
// Nothing was created and nothing was unmasked.
try std.testing.expectEqual(before, s.count());
try std.testing.expectEqual(@as(usize, 2), index.by_name.count());
try std.testing.expect(!s.held(1));
try std.testing.expect(s.content(1) == null);
try std.testing.expect(s.wire(1) == null);
// §3.3 L569-572 still applies to an inbound `gimme` for it: absent, silently
// skipped, no second rid for the walk to trip over.
try std.testing.expect(s.lookup(masked_name) == null);
try std.testing.expect(!index.read_fault);
// NEGATIVE SPACE: the appendix arm is intact for a name the log genuinely does
// not carry — the guard above narrows the masked case only.
const absent_name = repeat("99", digest_len);
const appendix_rid = try s.intern(absent_name);
try std.testing.expectEqual(@as(repostate.Rid, 6), appendix_rid);
try std.testing.expectEqual(before + 1, s.count());
try std.testing.expect(!s.held(appendix_rid));
}
test "sync §7.7 L1479-1485 / §9.5 L2042-2046: the adapter owns its content mask — the frame's plaintext is still on disk and still readable through the maintenance door" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedSendShunRepo(gpa, io, tmp.dir);
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
var index = try LogIndexStore.openFromFile(gpa, io, file, st.size);
defer index.destroy();
const s = index.store();
// §9.5's filter is a POLICY answer about what this server SENDS, not a deletion:
// the frame, its bytes and its checksums are untouched on disk. §7.7 L1479-1485
// depends on exactly that — "For an admitted target whose DELTASRC name is in
// the repository shun table, the receiver does not apply admission independently
// to that source: content still held locally MAY serve as the delta basis" — so
// "still held locally" is a property this slice must not have quietly destroyed.
// `LogStore.contentUnmasked` is the maintenance door that reads under no mask
// (`pack.zig`'s reader), and it is the only way to distinguish "the bytes are
// gone" from "the send side refuses to hand them out".
try std.testing.expect(index.masked[0]);
const on_disk = try index.log.contentUnmasked(gpa, index.rids[0], null);
defer if (on_disk) |b| gpa.free(b);
try std.testing.expectEqualStrings("content zero\n", on_disk.?);
// ...and the store still refuses, in both forms, without calling it a fault.
try std.testing.expect(s.content(1) == null);
try std.testing.expect(s.wire(1) == null);
try std.testing.expect(!index.read_fault);
// Nothing was materialised on the way: a refusal is answered from the mask, so
// a masked frame is never inflated into the round's memo (which is also what
// keeps a shunned artifact's plaintext out of a server's memory).
try std.testing.expectEqual(@as(usize, 0), index.materialized_bytes);
try std.testing.expectEqual(@as(usize, 0), index.residentBytes());
// NEGATIVE SPACE: the maintenance door is not a blanket unmask of the store —
// a SURVIVOR reads identically through both, so the asymmetry above is the
// mask's doing and not an artefact of which reader was called.
const survivor = try index.log.contentUnmasked(gpa, index.rids[1], null);
defer if (survivor) |b| gpa.free(b);
try std.testing.expectEqualStrings(survivor.?, s.content(2).?);
}
test "sync §9.5 L2042-2046 / §7.7 L1472-1475: a served round's writes land in the round-local overlay — the base enumeration's write doors are never reached" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// `seedSendShunRepo`'s digests are synthetic, so no peer could legitimately push
// one back — the responder hashes the payload and would reject the name long
// before reaching any store door. This fixture names its two artifacts by their
// REAL content address, so the push below is one a real peer could send.
const shunned_body = "shunned artifact bytes\n";
const survivor_body = "surviving artifact bytes\n";
var shunned_buf: [64]u8 = undefined;
var survivor_buf: [64]u8 = undefined;
const shunned_name = zit.artifact.artifactName(.sha3_256, shunned_body, &shunned_buf);
const survivor_name = zit.artifact.artifactName(.sha3_256, survivor_body, &survivor_buf);
var shunned_dbuf: [digest_len]u8 = undefined;
var survivor_dbuf: [digest_len]u8 = undefined;
const shunned_digest = hexToDigest(shunned_name, &shunned_dbuf) orelse return error.BadTestName;
const survivor_digest = hexToDigest(survivor_name, &survivor_dbuf) orelse return error.BadTestName;
const hk: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
{
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer seed.deinit(gpa);
try seed.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" });
_ = try seed.put(shunned_digest, shunned_body);
_ = try seed.put(survivor_digest, survivor_body);
try seed.putShun(shunned_digest, "operator request", 1);
try seed.commit();
}
const server = zit.sync.Server{
.version = serve_mod.server_version,
.self_servercode = serve_mod.server_servercode,
.project_code = serve_mod.server_projectcode,
};
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
// A peer pushes the very name this repository has shunned.
const push_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\npush sc pc\nfile {s} {d}\n{s}", .{ shunned_name, shunned_body.len, shunned_body });
defer gpa.free(push_plain);
const push_req = try zit.container.encode(gpa, push_plain, .default);
defer gpa.free(push_req);
{
const reply = try serveAnswer(&ctx, gpa, push_req, null, .{}, true);
defer gpa.free(reply.body);
// THE PROPERTY. `putImpl`/`internImpl` deliberately NARROW the store contract
// for a masked name — they answer a rid whose `content` is null and whose
// `held` is false — and the vtable's `error{OutOfMemory}` cannot signal that.
// What keeps the narrowing safe is that a served round never reaches those
// doors at all: `respond` writes into a per-request `repostate.OverlayStore`
// that is discarded at round end. `appendix` is where any write that DID
// reach this base would land, so its emptiness is that claim, measured.
const base = ctx.snapshot.?;
try std.testing.expectEqual(@as(usize, 0), base.appendix.store().count());
try std.testing.expectEqual(@as(usize, 2), base.store().count());
// ...and the push did not unmask the name it carried.
try std.testing.expect(base.masked[0]);
try std.testing.expect(base.store().lookup(shunned_name) == null);
try std.testing.expect(!base.store().held(1));
try std.testing.expect(base.store().content(1) == null);
try std.testing.expect(!base.read_fault);
// The survivor is untouched by any of it.
try std.testing.expectEqual(@as(?repostate.Rid, 2), base.store().lookup(survivor_name));
}
// NEGATIVE SPACE: it is not the MASK that keeps the base's doors shut, it is the
// overlay. A push of a name this repository has never seen — the case that has a
// real appendix arm waiting for it — must land in the round-local store just the
// same. (This round's `baseSnapshot` rebuilds over the grown log first, so the
// snapshot read below is the one this round actually served from.)
const fresh_body = "a name this repository has never held\n";
var fresh_buf: [64]u8 = undefined;
const fresh_name = zit.artifact.artifactName(.sha3_256, fresh_body, &fresh_buf);
const fresh_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\npush sc pc\nfile {s} {d}\n{s}", .{ fresh_name, fresh_body.len, fresh_body });
defer gpa.free(fresh_plain);
const fresh_req = try zit.container.encode(gpa, fresh_plain, .default);
defer gpa.free(fresh_req);
const reply2 = try serveAnswer(&ctx, gpa, fresh_req, null, .{}, true);
defer gpa.free(reply2.body);
try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.appendix.store().count());
}
/// The `clone_seqno NEXT` cursor a clone reply carries (§8.2 L1573-1576: "An
/// accepting clone reply emits zero or more clone cfile cards in RID order … then
/// clone_seqno NEXT, push SERVERCODE PROJECTCODE, and final comment. … NEXT zero
/// signals exhaustion."), read off the decoded body.
fn cloneSeqnoOf(body: []const u8) !u64 {
var it = std.mem.splitScalar(u8, body, '\n');
while (it.next()) |line| {
if (!std.mem.startsWith(u8, line, "clone_seqno ")) continue;
return std.fmt.parseInt(u64, line["clone_seqno ".len..], 10) catch return error.BadCloneSeqno;
}
return error.NoCloneSeqno;
}
/// Does the decoded reply body carry a `cfile` card for `name`? Keyed on the card
/// prefix plus the separating space, so a name that is a prefix of another cannot
/// answer for it.
fn replyHasCfile(gpa: std.mem.Allocator, body: []const u8, name: []const u8) !bool {
const needle = try std.fmt.allocPrint(gpa, "cfile {s} ", .{name});
defer gpa.free(needle);
return std.mem.indexOf(u8, body, needle) != null;
}
// WHAT THIS TEST PINS, and what it does NOT. The masked frames' absence from the
// clone stream is NOT this slice's doing: the pager gates on `store.wire()` then
// `store.content()`, and `LogStore.wire`/`content` already answered null for a
// shunned entry through their own rid-keyed read mask before the enumeration was
// masked at all — so this test passes with `LogIndexStore`'s exclusion removed
// (confirmed by mutation). What it does pin, and what nothing else pins, is that
// the mask keeps the pager's persisted-order cursor CONTIGUOUS over the survivors:
// each survivor arrives exactly once across paged batches and the sequence
// terminates at NEXT 0. A mask written out of step with `rids` turns it red.
test "sync §8.2 L1551-1553 / §9.5 L2042-2046: the clone pager walks the survivors contiguously across paged batches and terminates at NEXT 0" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedSendShunRepo(gpa, io, tmp.dir);
// A one-artifact batch cap forces the pager to page, so the persisted-order
// cursor is exercised at every masked frame rather than swallowed by a single
// full-store batch.
const server = zit.sync.Server{
.version = serve_mod.server_version,
.self_servercode = serve_mod.server_servercode,
.project_code = serve_mod.server_projectcode,
.clone_batch_max = 1,
};
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
var delivered = @as([5]usize, @splat(0));
var seqno: u64 = 1;
var rounds: usize = 0;
while (rounds < 16) : (rounds += 1) {
const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 {d}\n", .{seqno});
defer gpa.free(plain);
const req = try zit.container.encode(gpa, plain, .default);
defer gpa.free(req);
const reply = try serveAnswer(&ctx, gpa, req, null, .{}, true);
defer gpa.free(reply.body);
// §8.1 L1529-1531 / §4.1 L885-887 — a clone reply is the RAW card/cfile stream under
// `application/x-fossil-uncompressed`, never a zlib container.
try std.testing.expectEqualStrings("application/x-fossil-uncompressed", reply.content_type);
const body = reply.body;
for (send_shun_names, 0..) |name, i| {
if (try replyHasCfile(gpa, body, name)) delivered[i] += 1;
}
const next = try cloneSeqnoOf(body);
if (next == 0) break;
// A LOOP GUARD, not a protocol requirement: §8.2 L1577-1579 says the
// opposite — "The client records a digit-only NEXT and does not validate
// that a positive value advances from the supplied SEQNO". A pager that
// returned a non-advancing cursor would spin this test to its round cap
// instead of failing on the thing it is here to measure, so the guard
// states the pager's own contract, with no normative claim behind it.
try std.testing.expect(next > seqno);
seqno = next;
} else return error.CloneNeverExhausted;
// THE PROOF: every survivor arrived EXACTLY ONCE across the paged batches, and
// the sequence reached NEXT 0. The zero counts for the masked slots are carried
// by the log's own read mask (see the note above the test), not by this slice;
// the "exactly once" and the termination are the mask's contribution — a slot
// masked out of step with `rids` drops or duplicates a survivor here.
for (delivered, 0..) |n, i| {
const want: usize = if (sendShunIsShunned(i)) 0 else 1;
try std.testing.expectEqual(want, n);
}
}
test "sync §9.5 L2042-2046: a shun row written between clone rounds is honoured by the next round WITHOUT renumbering the survivors" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// A repo holding all five artifacts and NO shun row yet.
const hk: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
{
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer seed.deinit(gpa);
try seed.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" });
for (send_shun_digests, 0..) |d, i| {
var body_buf: [32]u8 = undefined;
_ = try seed.put(d, try std.fmt.bufPrint(&body_buf, "distinct content {d}\n", .{i}));
}
try seed.commit();
}
const server = zit.sync.Server{
.version = serve_mod.server_version,
.self_servercode = serve_mod.server_servercode,
.project_code = serve_mod.server_projectcode,
};
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const clone_plain = "pragma client-version 20000\nclone 3 1\n";
const clone_req = try zit.container.encode(gpa, clone_plain, .default);
defer gpa.free(clone_req);
// Round 1, no shun row: every artifact is served, and artifact 3 sits at
// adapter rid 3. This is the behaviour the mask must change, so the test is
// red on an unfiltered server.
{
const reply = try serveAnswer(&ctx, gpa, clone_req, null, .{}, true);
defer gpa.free(reply.body);
const body = reply.body; // §8.1 L1529-1531 / §4.1 L885-887 — uncompressed clone stream
for (send_shun_names) |name| try std.testing.expect(try replyHasCfile(gpa, body, name));
try std.testing.expectEqual(@as(?repostate.Rid, 3), ctx.snapshot.?.store().lookup(send_shun_names[2]));
try std.testing.expectEqual(@as(?repostate.Rid, 5), ctx.snapshot.?.store().lookup(send_shun_names[4]));
}
// The operator shuns the MIDDLE artifact between rounds — an append, which is
// exactly what trips `baseSnapshot`'s append-only stat gate and rebuilds.
{
var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write });
defer f.close(io);
const st = try f.stat(io);
var fs = storage.FileStorage.init(io, f, st.size);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer log.deinit(gpa);
try log.putShun(send_shun_digests[1], "operator request", 1);
try log.commit();
}
// Round 2 honours it with no restart — AND the survivors above it keep their
// adapter rids. A dense filter would shift artifact 3 to rid 2 and artifact 5
// to rid 4, so a client resuming this clone at cursor 3 would step over
// artifact 3 entirely and still be told NEXT 0 (§8.2 L1551-1553 licenses the
// gap; it does not license the shift).
{
const reply = try serveAnswer(&ctx, gpa, clone_req, null, .{}, true);
defer gpa.free(reply.body);
const body = reply.body; // §8.1 L1529-1531 / §4.1 L885-887 — uncompressed clone stream
try std.testing.expect(!try replyHasCfile(gpa, body, send_shun_names[1]));
for (send_shun_names, 0..) |name, i| {
if (i == 1) continue;
try std.testing.expect(try replyHasCfile(gpa, body, name));
}
try std.testing.expectEqual(@as(usize, 5), ctx.snapshot.?.store().count());
try std.testing.expectEqual(@as(?repostate.Rid, 3), ctx.snapshot.?.store().lookup(send_shun_names[2]));
try std.testing.expectEqual(@as(?repostate.Rid, 5), ctx.snapshot.?.store().lookup(send_shun_names[4]));
// THE MASK ITSELF, at THIS layer. The absent `cfile` above does not prove it:
// `LogStore.wire`/`content` answer null for a shunned entry through their own
// read mask, so the clone stream would omit the frame even with the
// enumeration unmasked. The rebuilt index is where a shun table read once at
// startup instead of per generation shows up, so assert on it directly.
try std.testing.expect(ctx.snapshot.?.masked[1]);
try std.testing.expect(ctx.snapshot.?.store().lookup(send_shun_names[1]) == null);
try std.testing.expect(!ctx.snapshot.?.store().held(2));
}
try std.testing.expectEqual(@as(u64, 2), ctx.loads); // one build per log state
}
test "serve §3.8 / sync §9.5 L2042-2046: the snapshot cache key is not the size alone — a same-length write to the log invalidates it" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try seedSendShunRepo(gpa, io, tmp.dir);
const server = zit.sync.Server{ .version = serve_mod.server_version };
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const req = try serveGimmeRequest(gpa, "igot", shun_test_name);
defer gpa.free(req);
// Two read rounds over an untouched log: built once, then stat-gated. This is
// the property the mtime term must NOT cost — a read round still does not
// rebuild, so the cache is still a cache.
for (0..2) |_| {
const reply = try serveAnswer(&ctx, gpa, req, null, .{}, true);
gpa.free(reply.body);
}
try std.testing.expectEqual(@as(u64, 1), ctx.loads);
const before = try file.stat(io);
// A write that leaves the log BYTE-IDENTICAL and exactly as long: the last byte
// is lifted off and put straight back. Nothing a `zit` writer does today looks
// like this — they all append — but "every writer only appends" is an assumption
// about all present and future writers, not a checked property, and since this
// snapshot began carrying the §3.3 shun table a missed rebuild no longer costs
// stale CONTENT (self-correcting) but a stale SUPPRESSION TABLE: an artifact the
// operator shunned served on, with nothing failing and nothing logged.
{
var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, before.size);
var last: [1]u8 = undefined;
try std.testing.expectEqual(@as(usize, 1), try fs.storageFile().pread(before.size - 1, &last));
try fs.storageFile().truncate(before.size - 1);
var reopened = storage.FileStorage.init(io, f, before.size - 1);
try reopened.storageFile().append(&last);
try reopened.storageFile().sync();
}
// THE PREMISE, asserted rather than assumed: the log really is the same length
// and the filesystem really did move the modification time. A filesystem that
// could not express this scenario fails here, loudly, instead of letting the
// conclusion below pass vacuously.
const after = try file.stat(io);
try std.testing.expectEqual(before.size, after.size);
try std.testing.expect(after.mtime.nanoseconds != before.mtime.nanoseconds);
// THE CONCLUSION: the size-only gate would have returned the stale snapshot and
// its stale shun table. The size+mtime key rebuilds.
{
const reply = try serveAnswer(&ctx, gpa, req, null, .{}, true);
gpa.free(reply.body);
}
try std.testing.expectEqual(@as(u64, 2), ctx.loads);
try std.testing.expectEqual(after.mtime.nanoseconds, ctx.snapshot_log_mtime_ns);
// ...and having rebuilt, it settles again: the key is stable, not a rebuild on
// every round.
{
const reply = try serveAnswer(&ctx, gpa, req, null, .{}, true);
gpa.free(reply.body);
}
try std.testing.expectEqual(@as(u64, 2), ctx.loads);
}
test "sync §7.2 L1265-1266 / §9.5 L2042-2046: a surviving delta whose basis is masked is served in FULL form, never as a dangling DELTASRC" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// `zit shun --defer` records the row without rewriting the generation, so a
// SURVIVING frame in the current generation can still be a delta rooted on
// the shunned one. Handing that delta's wire form out would emit a `cfile`
// whose DELTASRC is never sent: the peer stores a permanently dangling delta
// and gimme's a basis it will be silently skipped on — CORRUPT, not absent.
const base_plain = "the quick brown fox jumps over the lazy dog, and then again once more";
const child_plain = "the quick brown fox jumps over the lazy dog, and then twice more, at last";
const hk: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
{
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer log.deinit(gpa);
try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" });
const base_rid = try log.put(send_shun_digests[0], base_plain);
const d = try zit.delta.create(gpa, base_plain, child_plain);
defer gpa.free(d);
_ = try log.putDelta(send_shun_digests[1], base_rid, d);
try log.putShun(send_shun_digests[0], "operator request", 1);
try log.commit();
}
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
{
var index = try LogIndexStore.openFromFile(gpa, io, file, st.size);
defer index.destroy();
const s = index.store();
// The premise: the child really is a delta frame rooted on the masked one.
try std.testing.expectEqual(logstore.FrameKind.delta, index.log.kindOf(index.rids[1]));
// The masked basis is a §8.2 gap...
try std.testing.expect(!s.held(1));
try std.testing.expect(s.wire(1) == null);
// ...so the child's DELTA wire form is withheld — the responder falls back
// to compressing `content()` on the fly (§7.2), which reconstructs against
// the raw log and is unaffected by the name-level mask.
try std.testing.expect(s.held(2));
try std.testing.expect(s.wire(2) == null);
try std.testing.expectEqualStrings(child_plain, s.content(2).?);
// Withholding a wire form for policy is not a read fault.
try std.testing.expect(!index.read_fault);
}
// End to end through a clone round: the child arrives, the basis does not, and
// the shunned NAME appears nowhere in the reply — no DELTASRC to dangle on.
const server = zit.sync.Server{
.version = serve_mod.server_version,
.self_servercode = serve_mod.server_servercode,
.project_code = serve_mod.server_projectcode,
};
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const clone_req = try zit.container.encode(gpa, "pragma client-version 20000\nclone 3 1\n", .default);
defer gpa.free(clone_req);
const reply = try serveAnswer(&ctx, gpa, clone_req, null, .{}, true);
defer gpa.free(reply.body);
const body = reply.body; // §8.1 L1529-1531 / §4.1 L885-887 — uncompressed clone stream
try std.testing.expect(try replyHasCfile(gpa, body, send_shun_names[1]));
try std.testing.expect(std.mem.indexOf(u8, body, send_shun_names[0]) == null);
}
test "sync §3.3 L553-556: the CLIENT's holdings snapshot carries the same masked-basis guard — a surviving delta on a masked base is held as full content" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// The same `zit shun --defer` shape as the serve-side test above, asked of
// the CLIENT's builder: `ClientHoldings`, which every `zit pull`/`push`/
// `sync` client announces from. Serving the delta wire form would give the
// snapshot an entry that `held()` calls true but whose DELTASRC names a
// masked basis — so §3.2's igot walk would announce a name the gimme send
// gate can never answer, and a push would spin the round with nothing to
// send while reporting success. The artifact itself is NOT shunned; only
// its basis is.
const base_plain = "the quick brown fox jumps over the lazy dog, and then again once more";
const child_plain = "the quick brown fox jumps over the lazy dog, and then twice more, at last";
const hk: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
{
var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer log.deinit(gpa);
try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" });
const base_rid = try log.put(send_shun_digests[0], base_plain);
const d = try zit.delta.create(gpa, base_plain, child_plain);
defer gpa.free(d);
_ = try log.putDelta(send_shun_digests[1], base_rid, d);
try log.putShun(send_shun_digests[0], "operator request", 1);
try log.commit();
}
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const s = holdings.store();
// The masked basis is ABSENT from the name map — `lookup` answers null,
// which IS §3.3 L569-572's "If absent, silently skip" on the gimme path —
// and a peer-announced intern lands on the masked slot rather than minting
// a second identity: held by nothing, readable as nothing.
try std.testing.expect(s.lookup(send_shun_names[0]) == null);
const base_rid = try s.intern(send_shun_names[0]);
try std.testing.expect(!s.held(base_rid));
try std.testing.expect(s.content(base_rid) == null);
// The survivor is held AND readable — held-ness and readability agree, which
// is the property a recorded dangling delta would have broken.
const child_rid = s.lookup(send_shun_names[1]).?;
try std.testing.expect(s.held(child_rid));
try std.testing.expectEqualStrings(child_plain, s.content(child_rid).?);
try std.testing.expect(s.wire(child_rid) == null); // full content, no DELTASRC to dangle on
}
// ---------------------------------------------------------------------------
// sync §7.7 L1463-1467 — the hash-policy half of the receive-side admission
// predicate, from the REPOSITORY's stored config to the receiver that asks it.
//
// "shun-sha1 applies exactly when the receiving repository's effective
// hash-policy value at the start of processing that card is the shun-sha1
// value defined by [FossilArtifact], Section 7."
//
// The tests below exist because every §7.7 unit test in `sync_test.zig` sets
// `Client.hash_policy` / `Server.hash_policy` on a struct literal. That pins the
// RULE and proves nothing about the WIRING, so a regression that dropped
// `effectiveHashPolicy` — or read the wrong config field — would ship green.
// These two close that: one over the stored byte, one over the served round.
// ---------------------------------------------------------------------------
/// Seed `dir` with a repository whose config frame names `policy` and which
/// holds nothing. The empty holdings matter: the push below must be the only
/// thing that could ever land.
fn s77SeedPolicyRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, policy: artifact.HashPolicy) !void {
const hk: u8 = @backingInt(zit.artifact.defaultPolicy(false, false));
var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true });
defer f.close(io);
var fs = storage.FileStorage.init(io, f, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk);
defer log.deinit(gpa);
try log.putConfig(.{
.hash_policy = @backingInt(policy),
.default_branch = "trunk",
.project_id = "",
.project_code = "",
});
try log.commit();
}
/// True when `dir`'s repository resolves `name` — read back off disk, after the
/// served round has persisted, so this answers "did it LAND", not "was it
/// staged".
fn s77RepoHolds(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, name: []const u8) !bool {
var file = try dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
var index = try LogIndexStore.openFromFile(gpa, io, file, st.size);
defer index.destroy();
return index.store().lookup(name) != null;
}
test "sync §7.7 L1463-1467: the SERVED round's admission predicate is read from the repository's stored hash-policy, not defaulted" {
const gpa = std.testing.allocator;
const io = std.testing.io;
const content = "a SHA1-named artifact a peer pushes at us\n";
var nbuf: [64]u8 = undefined;
const sha1_name = zit.artifact.artifactName(.sha1, content, &nbuf);
try std.testing.expectEqual(@as(usize, 40), sha1_name.len);
const plain = try std.fmt.allocPrint(
gpa,
"pragma client-version 20000\npush {s} {s}\nfile {s} {d}\n{s}",
.{ serve_mod.server_servercode, serve_mod.server_projectcode, sha1_name, content.len, content },
);
defer gpa.free(plain);
const request = try zit.container.encode(gpa, plain, .default);
defer gpa.free(request);
const server = zit.sync.Server{
.version = serve_mod.server_version,
.self_servercode = serve_mod.server_servercode,
.project_code = serve_mod.server_projectcode,
};
// CONTROL: the same repository under `sha3` stores the pushed SHA1 name. If
// this half ever stops holding, the shun-sha1 half below proves nothing.
{
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try s77SeedPolicyRepo(gpa, io, tmp.dir, .sha3);
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const reply = try serveAnswer(&ctx, gpa, request, null, .{}, true);
gpa.free(reply.body);
try std.testing.expect(try s77RepoHolds(gpa, io, tmp.dir, sha1_name));
}
// THE PROOF: the identical round against a repository whose stored config
// names shun-sha1 discards it — with no error card (L1437-1440) — and the
// ONLY difference between the two runs is that stored byte. A `serveRound`
// that defaulted the policy instead of reading the snapshot would store here.
{
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1);
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const reply = try serveAnswer(&ctx, gpa, request, null, .{}, true);
defer gpa.free(reply.body);
const plain_reply = try zit.container.decode(gpa, reply.body, 1 << 20);
defer gpa.free(plain_reply);
try std.testing.expect(std.mem.indexOf(u8, plain_reply, "error ") == null);
try std.testing.expect(!try s77RepoHolds(gpa, io, tmp.dir, sha1_name));
}
}
test "sync §7.7 L1463-1467: ClientHoldings carries the stored hash-policy into the one place a Client is built from repository state" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1);
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const admission = holdings.admission();
try std.testing.expectEqual(artifact.HashPolicy.shun_sha1, admission.hash_policy);
// `syncClient` is the sole construction `pullRun`/`pushRun`/`syncRun` share,
// so this one assertion covers all three verbs: a client that dropped either
// half of `RepoAdmission` would apply half a filter with nothing failing.
// A placeholder identity: this test is about `RepoAdmission`, and the
// sub-manifest half of the same client is covered by the §7.2 tests.
const identity: RepoSyncIdentity = .{ .project_code = "", .seen_submanifest = false, .enable_submanifests = false, .server_code = zit.sync.deriveServerCode(&.{"test-fixture-repo"}) };
for ([_]zit.sync.Operation{ .pull, .push, .sync }) |op| {
const client = syncClient(op, &identity, null, admission);
try std.testing.expectEqual(op, client.operation);
try std.testing.expectEqual(artifact.HashPolicy.shun_sha1, client.hash_policy);
// A 40-hex name is rejected by the very predicate this policy feeds.
try std.testing.expectEqual(
zit.sync.Admission.reject,
zit.sync.admitInbound(client.hash_policy, client.shun, repeat("a", 40)),
);
}
// And the `.auto` repository next door admits the same name, so the
// assertion above is reading the stored byte and not a constant.
var tmp2 = std.testing.tmpDir(.{});
defer tmp2.cleanup();
try s77SeedPolicyRepo(gpa, io, tmp2.dir, .auto);
var holdings2 = try ClientHoldings.open(gpa, io, tmp2.dir);
defer holdings2.deinit(io);
const admission2 = holdings2.admission();
try std.testing.expectEqual(artifact.HashPolicy.auto, admission2.hash_policy);
const c2 = syncClient(.pull, &identity, null, admission2);
try std.testing.expectEqual(
zit.sync.Admission.admit,
zit.sync.admitInbound(c2.hash_policy, c2.shun, repeat("a", 40)),
);
}
test "sync §7.7 L1463-1467: a DAMAGED repository never yields a hash-policy — the admission predicate fails the load rather than defaulting" {
// The property: no damaged repository may hand the receiver a policy value.
// `readConfig` answers `Corrupt`/`Storage` for a config frame that will not
// read or will not parse, and that is UNKNOWN, not "no defined policy":
// answering `.auto` there would silently disable shun-sha1 and admit
// SHA1-named content this repository is configured to refuse, with nothing
// failing. §3.3's shun half already refuses to fail open (`RepoShunSet.load`
// propagates, and `serveRound` returns `error.ServerStorage` rather than
// serving tableless); the two halves of one predicate must not differ.
//
// HONESTY ABOUT WHAT THIS PINS: with the injury below, the fault is caught
// UPSTREAM, at `LogStore.openReadOnly`, so this test does not by itself kill
// a mutant that restores `effectiveHashPolicy`'s old blanket
// `catch return .auto` — verified by applying that mutant, which leaves the
// suite green. It is here as the end-to-end guard: the day the log open grows
// tolerant of a damaged frame, the read below must still not answer `.auto`.
// Pinning `effectiveHashPolicy`'s own catch would need a storage layer that
// faults a single frame read, which this tree does not have.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1);
// Damage the config frame in place: its default_branch bytes are on disk
// verbatim, so overwriting them is a one-frame injury that leaves every other
// frame — and the log's own scan — intact.
{
var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write });
defer f.close(io);
const st = try f.stat(io);
const buf = try gpa.alloc(u8, st.size);
defer gpa.free(buf);
var r = f.reader(io, buf);
try r.interface.readSliceAll(buf);
const at = std.mem.indexOf(u8, buf, "trunk") orelse return error.TestUnexpectedResult;
var w = f.writer(io, &.{});
try w.seekTo(at);
try w.interface.writeAll("\xff\xff\xff\xff\xff");
try w.interface.flush();
}
try std.testing.expectError(error.Storage, ClientHoldings.open(gpa, io, tmp.dir));
}
test "submanifests §8.1/§7.2: a repository that HOLDS a leaf arms its serve-side gate against a pre-S peer" {
// The PRODUCTION wiring of the §7.2 gate, end to end through the durable
// store: `persistStore` records §8.1's seen-submanifest indicator for a
// stored SUBMANIFEST (which carries no S card at all, §4.4), the serve
// snapshot re-reads it, and `serveRound` arms the gate with it. Every link
// is load-bearing:
//
// * drop the `transferGatedArtifact` line in `phantomizeSubManifestRefs`
// and the indicator stays false — a leaf is not "S-bearing" literally;
// * drop `round_server.seen_submanifest = …` in `serveRound` and the gate
// is unarmed, because §8.1's emission setting is false on every zit;
// * drop `or self.seen_submanifest` from `Server.submanifestGateArmed`
// and it is unarmed for the same reason.
//
// Each of the three leaves the leaf in the reply to a fossil-2.28 peer,
// which §8 says can only hold it as "unreadable ... opaque content".
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var hex_buf: [2 * digest_len]u8 = undefined;
try serveSeedRepo(gpa, io, tmp.dir, 1, &hex_buf);
// §4.3: an F run with no D card is type SUBMANIFEST. The flat check-in
// manifest beside it is everything that predates this draft.
const leaf = "F a.txt " ++ (repeat("3", 64)) ++ "\n";
const flat = "C m\nD 2026-07-27T12:00:00.000\nF a.txt " ++ (repeat("2", 64)) ++ "\nU u\n";
var leaf_name: [2 * digest_len]u8 = undefined;
var flat_name: [2 * digest_len]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, leaf, &leaf_name);
_ = zit.artifact.artifactName(.sha3_256, flat, &flat_name);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(&leaf_name, leaf);
_ = try ms.store().put(&flat_name, flat);
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
// §8.1 — the indicator is DURABLE and covers §7.2's type shape, not only
// its S-card shape. Read back through the same function the CLI uses.
var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined;
const identity = try repoSyncIdentity(gpa, io, tmp.dir, "test-fixture-repo", &pc_buf);
try std.testing.expect(identity.seen_submanifest);
// …and the emission setting is the OTHER, independent half of the arming
// disjunction: this repository holds a gated artifact it received, and has
// never been told to emit one. A single flag could not tell the two apart.
try std.testing.expect(!identity.enable_submanifests);
// …and the served reply refuses the leaf to a peer announcing fossil 2.28,
// while still handing it the artifact it can read.
const server = zit.sync.Server{
.version = serve_mod.server_version,
.self_servercode = serve_mod.server_servercode,
.project_code = serve_mod.server_projectcode,
};
try std.testing.expect(!server.submanifests_enabled); // §8.1's default, stated
var file = try tmp.dir.openFile(io, repo_filename, .{});
defer file.close(io);
var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server };
defer ctx.dropSnapshot();
const plain = try std.fmt.allocPrint(
gpa,
"pragma client-version 22800\ngimme {s}\ngimme {s}\n",
.{ &leaf_name, &flat_name },
);
defer gpa.free(plain);
const request = try zit.container.encode(gpa, plain, .default);
defer gpa.free(request);
const reply = try serveAnswer(&ctx, gpa, request, null, .{}, true);
defer gpa.free(reply.body);
const body = try zit.container.decode(gpa, reply.body, zit.sync.reply_body_cap);
defer gpa.free(body);
// Built through the same writer the server uses (§11.1's escape encoding
// applies to the text), so the expectation cannot drift from the emission.
var card_buf: [2 * zit.sync.submanifest_peer_version_error_text.len + 16]u8 = undefined;
const refusal = zit.sync.writeSignalCard(.err, zit.sync.submanifest_peer_version_error_text, &card_buf);
try std.testing.expect(std.mem.indexOf(u8, body, refusal) != null);
try std.testing.expect(std.mem.indexOf(u8, body, &leaf_name) == null);
try std.testing.expect(std.mem.indexOf(u8, body, &flat_name) != null);
}
test "submanifests §7.2/§10.1: the CLI announces a DATED client-version so the peer's version can reach the gate" {
// §10.1 r11/r12 — the server replies `pragma server-version` exactly when
// the received client-version carried DATE and TIME, and §7.2's gate reads
// "the remote peer's announced version". An undated pragma therefore leaves
// `Client.peer.version` at 0 for the whole session and an armed client
// refuses its own trees to an opted-in server: pullable, never pushable.
// (`sync_driver_test` drives that consequence end to end; this pins the one
// CLI constant the whole chain hangs off.)
const gpa = std.testing.allocator;
try std.testing.expect(sync_client_version_info.datetime != null);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
var client = zit.sync.Client{
.state = zit.sync.initialState(.pull),
.operation = .pull,
.self_servercode = sync_servercode,
.project_code = sync_projectcode,
.version = sync_client_version_info,
};
defer client.deinit(gpa);
var buf: std.ArrayListUnmanaged(u8) = .empty;
defer buf.deinit(gpa);
const request = try client.buildRequest(gpa, ms.store(), &buf);
// The emitted pragma carries at least §10.1 r11's five tokens.
const eol = std.mem.indexOfScalar(u8, request, '\n').?;
var tokens: usize = 0;
var it = std.mem.tokenizeScalar(u8, request[0..eol], ' ');
while (it.next()) |_| tokens += 1;
try std.testing.expect(std.mem.startsWith(u8, request, "pragma client-version "));
try std.testing.expect(tokens >= zit.sync.min_datetime_tokens);
try std.testing.expect(zit.sync.serverEmitsServerVersion(.{
.version = sync_client_version,
.datetime = sync_client_version_info.datetime,
}));
}
test "configValue reads every name from one frame, and empty means no opinion" {
var arena_state: std.heap.ArenaAllocator = .init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
// An untouched frame has no opinion about anything, so the versioned and
// global tiers stay reachable.
const bare: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p" };
try std.testing.expect((try configValue(arena, &bare, .ignore_glob)) == null);
try std.testing.expect((try configValue(arena, &bare, .autosync)) == null);
try std.testing.expect((try configValue(arena, &bare, .clearsign)) == null);
try std.testing.expect((try configValue(arena, &bare, .dotfiles)) == null);
try std.testing.expect((try configValue(arena, &bare, .crnl_glob)) == null);
const set: logstore.Config = .{
.hash_policy = 0,
.default_branch = "trunk",
.project_id = "p",
.ignore_glob = "*.o",
.crnl_glob = "*.bat",
.clean_glob = "*.tmp",
.empty_dirs = "logs",
.encoding_glob = "*.utf16",
.dotfiles = "yes",
.mimetypes = "md text/markdown",
.clearsign_set = true,
.clearsign = false,
.autosync_set = true,
.autosync_off = true,
};
try std.testing.expectEqualStrings("*.o", (try configValue(arena, &set, .ignore_glob)).?);
try std.testing.expectEqualStrings("*.bat", (try configValue(arena, &set, .crnl_glob)).?);
try std.testing.expectEqualStrings("*.tmp", (try configValue(arena, &set, .clean_glob)).?);
try std.testing.expectEqualStrings("logs", (try configValue(arena, &set, .empty_dirs)).?);
try std.testing.expectEqualStrings("*.utf16", (try configValue(arena, &set, .encoding_glob)).?);
// Verbatim: the operator wrote `yes`, so the listing says `yes`.
try std.testing.expectEqualStrings("yes", (try configValue(arena, &set, .dotfiles)).?);
try std.testing.expectEqualStrings("md text/markdown", (try configValue(arena, &set, .mimetypes)).?);
// The booleans answer through their explicit-set bit, as they always have.
try std.testing.expectEqualStrings("off", (try configValue(arena, &set, .clearsign)).?);
try std.testing.expectEqualStrings("off", (try configValue(arena, &set, .autosync)).?);
}
test "resolveCrlfGlob falls back to crnl-glob from one frame" {
// The commit gate holds the store's write lock and reads its config once,
// so both spellings are answered from that frame -- no second store open,
// and no way for the two reads to see different bytes.
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
const amb: cli.Ambient = .{ .now_epoch = 0, .user_env = null };
// The older spelling answers while the newer one is absent.
const older: logstore.Config = .{
.hash_policy = 0,
.default_branch = "trunk",
.project_id = "p",
.crnl_glob = "*.txt",
};
const from_crnl = try resolveCrlfGlob(io, arena, tmp.dir, amb, &older);
try std.testing.expectEqualStrings("*.txt", from_crnl.value);
try std.testing.expectEqual(zit.settings.Source.repository, from_crnl.source);
// `crlf-glob` present REPLACES it, exactly as fossil's
// `db_get("crlf-glob", db_get("crnl-glob",""))` does.
const newer: logstore.Config = .{
.hash_policy = 0,
.default_branch = "trunk",
.project_id = "p",
.crlf_glob = "*.c",
.crnl_glob = "*.txt",
};
const from_crlf = try resolveCrlfGlob(io, arena, tmp.dir, amb, &newer);
try std.testing.expectEqualStrings("*.c", from_crlf.value);
// No frame and no versioned file: nothing is declared, so the class is
// not checked at all.
const none = try resolveCrlfGlob(io, arena, tmp.dir, amb, null);
try std.testing.expectEqual(zit.settings.Source.default, none.source);
try std.testing.expectEqualStrings("", none.value);
}
test "clone observer forwards phase edges without owning measurement state" {
const Probe = struct {
count: usize = 0,
phase: ClonePhase = .transport,
event: ClonePhaseEvent = .begin,
fn callback(context: *anyopaque, phase: ClonePhase, event: ClonePhaseEvent) void {
const self: *@This() = @ptrCast(@alignCast(context));
self.count += 1;
self.phase = phase;
self.event = event;
}
};
var probe = Probe{};
const observer = CloneObserver{ .context = &probe, .callback = Probe.callback };
observeClonePhase(&observer, .persist, .end);
try std.testing.expectEqual(@as(usize, 1), probe.count);
try std.testing.expectEqual(ClonePhase.persist, probe.phase);
try std.testing.expectEqual(ClonePhaseEvent.end, probe.event);
}
// ---------------------------------------------------------------------------
// `zit config pull AREA`: the explicit configuration request side
// ---------------------------------------------------------------------------
/// `zit config pull shun [URL]` — fossil's `configuration pull AREA ?URL?`.
///
/// Configuration carriage is otherwise CLONE-TIME ONLY, and deliberately so: a
/// shun table that refreshed on every sync would let any peer suppress content
/// in this repository on its own say-so. This verb is how an operator asks for
/// it anyway — explicitly, naming the area, against a peer they chose.
///
/// `shun` is the one area zit models. The rest of fossil's areas describe a web
/// interface and a user table zit has no equivalent for; naming one of those is
/// a clean refusal rather than a silent no-op, so an operator learns which
/// areas exist rather than watching nothing happen.
/// Apply the peer's user rows to this repository's table. Shares
/// `runConfigFromFile`'s two modes and its one-marker rule: `overwrite`
/// replaces, and the marker lands before the first row and only when there
/// is a row to land, so a pull that returned nothing never wipes the table it
/// could not replace.
///
/// An EMPTY result is reported rather than treated as success-with-nothing:
/// against fossil, "no rows" is what an under-privileged client sees, so an
/// operator who forgot `--user` must not read it as "the peer has no users".
fn applyPeerUsers(
gpa: std.mem.Allocator,
io: std.Io,
root_dir: std.Io.Dir,
rows: []const zit.sync.OwnedUserRow,
overwrite: bool,
streams: cli.Streams,
) cli.Exit {
if (rows.len == 0) {
streams.err.writeAll(
"zit config: the peer offered no user rows\n" ++
"zit config: fossil serves the user area to an administrator only -- " ++
"pass --user=NAME (and --password=, or $ZIT_PASSWORD)\n",
) catch {};
return .failure;
}
var store_append = cli.openStoreAppend(io, root_dir) catch |e| return storeAppendFail(streams, "config", e);
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch return repoReadFail(streams, "config");
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch
return repoReadFail(streams, "config");
defer log.deinit(gpa);
if (overwrite) log.putUserReset(0) catch return repoReadFail(streams, "config");
var skipped: usize = 0;
for (rows) |row| {
log.putUser(.{
.login = row.login,
.pw = row.pw,
.cap = row.cap,
.info = row.info,
.mtime = row.mtime,
}) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "config"),
error.Storage => return repoReadFail(streams, "config"),
error.LoginTooLong, error.FieldTooLong => {
skipped += 1;
continue;
},
};
}
log.commit() catch return repoReadFail(streams, "config");
const table = log.userTable(gpa) catch return stateOom(streams, "config");
defer gpa.free(table);
streams.out.print("user: {d} row{s} {s} from the peer; {d} in the table", .{
rows.len,
plural(rows.len),
if (overwrite) "adopted" else "merged",
table.len,
}) catch {};
if (skipped != 0) streams.out.print("; {d} unusable and skipped", .{skipped}) catch {};
streams.out.writeAll("\n") catch {};
if (skipped != 0) return .failure;
return .ok;
}
/// `zit config`'s three shapes, spelled as fossil spells them
/// (`configuration pull|import|merge`).
const ConfigSub = enum { pull, import, merge, @"export" };
const config_usage_line: []const u8 =
"zit config: usage: zit config pull shun|user [URL] | zit config import|merge FILE | zit config export shun|user [FILE]\n";
/// The largest configuration export this verb will read whole. A user table
/// is a handful of rows per operator; a file past this bound is not a
/// configuration export and is refused rather than allocated for.
const max_config_file_bytes: usize = 8 * 1024 * 1024;
/// The wire GROUPNAME an area name selects, or null. Fossil matches by PREFIX
/// (`configure_name_to_mask`) and orders `shun` before `user` in
/// `aGroupName`, so an ambiguous prefix resolves the way fossil resolves it.
fn configGroupOf(want: []const u8) ?[]const u8 {
if (want.len == 0) return null;
if (std.mem.startsWith(u8, "shun", want)) return zit.sync.config_shun_group;
if (std.mem.startsWith(u8, "user", want)) return zit.sync.config_user_group;
return null;
}
/// `zit config export AREA [FILE]` — fossil's `configuration export AREA
/// FILE`, the inverse of `import`/`merge`: a `#` banner, then one `config
/// /AREA SIZE` card per row, byte-identical to what the wire carries for the
/// area. So a table can be backed up before `import` replaces it. FILE
/// absent or `-` is stdout. Read-only: the store is opened without a lock,
/// as every reader opens it.
fn runConfigExport(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
group: []const u8,
file_arg: []const u8,
repo_override: ?[]const u8,
streams: cli.Streams,
) cli.Exit {
const root = (cli.resolveRoot(arena, io, repo_override) catch return stateOom(streams, "config")) orelse {
main.reportNoRoot(streams, io, "config", repo_override);
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
var file = cli.openStoreRead(io, root_dir) catch return repoReadFail(streams, "config");
defer file.close(io);
const st = file.stat(io) catch return repoReadFail(streams, "config");
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch
return repoReadFail(streams, "config");
defer log.deinit(gpa);
var out: std.ArrayList(u8) = .empty;
defer out.deinit(gpa);
const area_name = group[1..];
// fossil's banner shape. No timestamp: two exports of an unchanged table
// are the same bytes, so a backup can be compared with `cmp`.
out.print(gpa, "# The \"{s}\" configuration exported from\n# repository \"{s}\"\n", .{ area_name, std.fs.path.basename(root.abs) }) catch
return stateOom(streams, "config");
var line_buf: [64]u8 = undefined;
var rows: usize = 0;
if (std.mem.eql(u8, group, zit.sync.config_user_group)) {
const table = log.userTable(gpa) catch return stateOom(streams, "config");
defer gpa.free(table);
var payload_buf: [zit.sync.user_config_payload_max]u8 = undefined;
for (table) |u| {
const payload = zit.sync.writeUserConfigPayload(.{
.login = u.login,
.pw = u.pw,
.cap = u.cap,
.info = u.info,
.mtime = u.mtime,
}, &payload_buf) catch continue;
appendExportCard(gpa, &out, group, payload, &line_buf) catch return stateOom(streams, "config");
rows += 1;
}
} else {
var payload_buf: [zit.sync.shun_config_payload_max]u8 = undefined;
var hex_buf: [64]u8 = undefined;
for (log.shunList()) |r| {
if (r.hash.len > 32) continue;
const hex = std.fmt.bufPrint(&hex_buf, "{x}", .{r.hash}) catch unreachable;
const payload = zit.sync.writeShunConfigPayload(.{
.hash = hex,
.epoch = r.epoch,
.reason = r.reason,
}, &payload_buf) catch continue;
appendExportCard(gpa, &out, group, payload, &line_buf) catch return stateOom(streams, "config");
rows += 1;
}
}
if (file_arg.len == 0 or std.mem.eql(u8, file_arg, "-")) {
streams.out.writeAll(out.items) catch return .failure;
return .ok;
}
std.Io.Dir.cwd().writeFile(io, .{ .sub_path = file_arg, .data = out.items }) catch {
streams.err.print("zit config: cannot write '{s}'\n", .{file_arg}) catch {};
return .failure;
};
streams.out.print("{s}: {d} row{s} exported to {s}\n", .{ area_name, rows, plural(rows), file_arg }) catch {};
return .ok;
}
fn appendExportCard(
gpa: std.mem.Allocator,
out: *std.ArrayList(u8),
group: []const u8,
payload: []const u8,
line_buf: []u8,
) error{OutOfMemory}!void {
const card = zit.sync.writeConfigCard(group, @intCast(payload.len), line_buf) catch unreachable;
try out.appendSlice(gpa, card);
try out.append(gpa, '\n');
try out.appendSlice(gpa, payload);
try out.append(gpa, '\n');
}
/// Apply a fossil configuration export file: the output of
/// `fossil configuration export AREA FILE`, which is byte-identical to what
/// the sync protocol carries for the same area (`config /AREA SIZE\n` then
/// SIZE payload bytes then a newline), plus `#` comment lines at the top.
///
/// That identity is the whole reason this path exists: a repository's users
/// reach zit with no SQLite reader, no network, and no fossil server — one
/// `fossil configuration export` on the migration host, and the SAME parser
/// the wire path will use.
///
/// `replace` is fossil's `CONFIGSET_OVERWRITE`. Applied ONCE, before the
/// first row lands, exactly as `configure_receive`'s `configHasBeenReset`
/// guard does — a per-row reset would leave only the last row of the file.
fn runConfigFromFile(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
path: []const u8,
replace: bool,
repo_override: ?[]const u8,
streams: cli.Streams,
) cli.Exit {
// The user/shun areas are store tables; `-R` redirects which
// store they land in. The FILE is still read from the cwd (fossil's
// `configuration import FILENAME`), so only the store is redirected.
const root = (cli.resolveRoot(arena, io, repo_override) catch return stateOom(streams, "config")) orelse {
main.reportNoRoot(streams, io, "config", repo_override);
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
const text = std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(max_config_file_bytes)) catch {
streams.err.print("zit config: cannot read '{s}'\n", .{path}) catch {};
return .failure;
};
defer gpa.free(text);
var users: std.ArrayList(zit.sync.UserRow) = .empty;
defer users.deinit(gpa);
var value_arena = std.heap.ArenaAllocator.init(gpa);
defer value_arena.deinit();
const va = value_arena.allocator();
// Shun rows are held as hex off the file text's own unquote scratch, so
// they are copied onto the value arena before the next card reuses it.
var shuns: std.ArrayList(zit.sync.ShunRow) = .empty;
defer shuns.deinit(gpa);
var skipped: usize = 0;
var other_areas: usize = 0;
var pos: usize = 0;
while (pos < text.len) {
const line_end = std.mem.indexOfScalarPos(u8, text, pos, '\n') orelse text.len;
const line = text[pos..line_end];
// `configuration export` writes a `#` banner naming the repository and
// the export time; blank lines separate nothing but are harmless.
if (line.len == 0 or line[0] == '#') {
pos = line_end + 1;
continue;
}
const card = parseConfigCardLine(line) orelse {
streams.err.print("zit config: '{s}' is not a configuration export\n", .{path}) catch {};
return .failure;
};
const payload = zit.sync.consumeConfigPayload(text, line_end + 1, card.size) catch {
streams.err.print("zit config: '{s}' is truncated\n", .{path}) catch {};
return .failure;
};
pos = payload.next;
if (std.mem.eql(u8, card.area, zit.sync.config_shun_group)) {
var bufs: zit.sync.ShunPayloadBufs = undefined;
const row = zit.sync.parseShunConfigPayload(payload.bytes, &bufs) orelse {
skipped += 1;
continue;
};
const hash = va.dupe(u8, row.hash) catch return stateOom(streams, "config");
const reason = va.dupe(u8, row.reason) catch return stateOom(streams, "config");
shuns.append(gpa, .{ .hash = hash, .epoch = row.epoch, .reason = reason }) catch
return stateOom(streams, "config");
continue;
}
if (!std.mem.eql(u8, card.area, zit.sync.config_user_group)) {
other_areas += 1;
continue;
}
const row = zit.sync.parseUserConfigPayload(payload.bytes) catch {
skipped += 1;
continue;
};
const parsed = unquoteUserRow(va, row) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "config"),
error.Malformed => {
skipped += 1;
continue;
},
};
users.append(gpa, parsed) catch return stateOom(streams, "config");
}
if (other_areas != 0 and users.items.len == 0 and shuns.items.len == 0) {
streams.err.print(
"zit config: '{s}' carries no /user or /shun cards ({d} card{s} in areas zit does not model)\n",
.{ path, other_areas, plural(other_areas) },
) catch {};
return .failure;
}
var store_append = cli.openStoreAppend(io, root_dir) catch |e| return storeAppendFail(streams, "config", e);
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch return repoReadFail(streams, "config");
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch
return repoReadFail(streams, "config");
defer log.deinit(gpa);
// The marker lands before the first row and only when there is a row to
// land: an import whose file held nothing applicable must not wipe the
// table it could not replace.
if (replace and users.items.len != 0) {
log.putUserReset(0) catch return repoReadFail(streams, "config");
}
for (users.items) |row| {
log.putUser(.{
.login = row.login,
.pw = row.pw,
.cap = row.cap,
.info = row.info,
.mtime = row.mtime,
}) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "config"),
error.Storage => return repoReadFail(streams, "config"),
// A row this store cannot key or hold is reported, never silently
// dropped: an operator migrating an access-control table needs to
// know a login did not come across.
error.LoginTooLong, error.FieldTooLong => {
skipped += 1;
continue;
},
};
}
// Shun rows are ADDED in both modes, never replaced. Replacing the table
// would un-shun whatever the file happens not to name, and a shun is
// never undone here — not by `zit shun`, and so not as a side effect of
// restoring a backup. The same line `config pull` holds.
var shun_added: usize = 0;
for (shuns.items) |row| {
var digest_buf: [32]u8 = undefined;
const digest = digest_buf[0 .. row.hash.len / 2];
_ = std.fmt.hexToBytes(digest, row.hash) catch {
skipped += 1;
continue;
};
log.putShun(digest, row.reason, row.epoch) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "config"),
else => {
skipped += 1;
continue;
},
};
shun_added += 1;
}
log.commit() catch return repoReadFail(streams, "config");
if (users.items.len != 0 or shuns.items.len == 0) {
const table = log.userTable(gpa) catch return stateOom(streams, "config");
defer gpa.free(table);
streams.out.print("user: {d} row{s} {s}; {d} in the table\n", .{
users.items.len,
plural(users.items.len),
if (replace) "imported" else "merged",
table.len,
}) catch {};
}
if (shuns.items.len != 0) {
streams.out.print("shun: {d} row{s} added; {d} in the table\n", .{
shun_added,
plural(shun_added),
log.shunList().len,
}) catch {};
}
if (skipped != 0) streams.out.print("{d} unreadable and skipped\n", .{skipped}) catch {};
if (other_areas != 0) streams.out.print("{d} card{s} in other areas ignored\n", .{ other_areas, plural(other_areas) }) catch {};
if (skipped != 0) return .failure;
return .ok;
}
/// One `config /AREA SIZE` card header line.
const ConfigCardLine = struct { area: []const u8, size: u32 };
fn parseConfigCardLine(line: []const u8) ?ConfigCardLine {
const prefix = "config ";
if (!std.mem.startsWith(u8, line, prefix)) return null;
const rest = line[prefix.len..];
const sp = std.mem.indexOfScalar(u8, rest, ' ') orelse return null;
const area = rest[0..sp];
if (area.len == 0 or area[0] != '/') return null;
const size = std.fmt.parseInt(u32, rest[sp + 1 ..], 10) catch return null;
return .{ .area = area, .size = size };
}
/// Unquote a wire user row's fields onto `arena`. A field fossil omitted is
/// the empty string here — zit models "no password" as empty rather than as
/// a distinct NULL, which is also what it emits back on the wire.
fn unquoteUserRow(arena: std.mem.Allocator, row: zit.sync.QuotedUserRow) error{ OutOfMemory, Malformed }!zit.sync.UserRow {
const mtime = std.fmt.parseInt(i64, row.mtime_text, 10) catch return error.Malformed;
return .{
.login = try unquoteField(arena, row.quoted_login),
.pw = try unquoteField(arena, row.quoted_pw),
.cap = try unquoteField(arena, row.quoted_cap),
.info = try unquoteField(arena, row.quoted_info),
.mtime = mtime,
};
}
fn unquoteField(arena: std.mem.Allocator, quoted: ?[]const u8) error{ OutOfMemory, Malformed }![]const u8 {
const q = quoted orelse return "";
const buf = try arena.alloc(u8, q.len);
const out = zit.sync.fossilUnquote(q, buf) catch return error.Malformed;
return out;
}
pub fn runConfig(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
var area: ?[]const u8 = null;
var url_arg: []const u8 = "";
var sub: ?ConfigSub = null;
var overwrite = false;
var login_user: ?[]const u8 = null;
var login_password: ?[]const u8 = null;
for (verb_args) |tok| {
if (std.mem.eql(u8, tok, "-h") or std.mem.eql(u8, tok, "--help")) {
streams.out.writeAll(main.config_help) catch {};
return .ok;
} else if (std.mem.eql(u8, tok, "--overwrite")) {
overwrite = true;
} else if (std.mem.startsWith(u8, tok, "--user=")) {
login_user = tok["--user=".len..];
} else if (std.mem.startsWith(u8, tok, "--password=")) {
login_password = tok["--password=".len..];
} else if (tok.len > 1 and tok[0] == '-') {
return cli.usageError(streams, "config", .unknown_option, tok);
} else if (sub == null) {
sub = std.meta.stringToEnum(ConfigSub, tok) orelse
return cli.usageError(streams, "config", .unknown_option, tok);
} else if (area == null) {
area = tok;
} else if (url_arg.len == 0) {
url_arg = tok;
} else {
return cli.usageError(streams, "config", .extra_argument, tok);
}
}
if (sub == null or area == null) {
streams.err.writeAll(config_usage_line) catch {};
return .usage;
}
// `import`/`merge` take a FILE, not an area and a URL: fossil's own
// `configuration import|merge FILENAME`. The two differ in exactly one
// thing, which is why they share a body: `import` REPLACES the local table
// (fossil's `CONFIGSET_OVERWRITE` — `DELETE FROM user` then `REPLACE
// INTO`), `merge` folds rows in newest-mtime-wins.
switch (sub.?) {
.import, .merge => {
if (url_arg.len != 0) return cli.usageError(streams, "config", .extra_argument, url_arg);
// The file forms take their mode from the SUBCOMMAND, as fossil's
// do; accepting a contradicting --overwrite would give one
// invocation two ways to say the same thing and one way to lie.
if (overwrite) return cli.usageError(streams, "config", .unknown_option, "--overwrite");
if (login_user != null or login_password != null) {
streams.err.writeAll("zit config: --user/--password apply to 'config pull', not to a file\n") catch {};
return .usage;
}
return runConfigFromFile(gpa, io, arena, area.?, sub.? == .import, amb.repo_override, streams);
},
.@"export" => {
if (overwrite) return cli.usageError(streams, "config", .unknown_option, "--overwrite");
if (login_user != null or login_password != null) {
streams.err.writeAll("zit config: --user/--password apply to 'config pull', not to an export\n") catch {};
return .usage;
}
const group = configGroupOf(area.?) orelse {
streams.err.print("zit config: unknown configuration area '{s}' (known: shun, user)\n", .{area.?}) catch {};
return .failure;
};
return runConfigExport(gpa, io, arena, group, url_arg, amb.repo_override, streams);
},
.pull => {},
}
// Fossil matches area names by PREFIX (`configure_name_to_mask`), and so
// does the server side of this exchange, so the CLI accepts the same.
const want = area.?;
const group = configGroupOf(want) orelse {
streams.err.print("zit config: unknown configuration area '{s}' (known: shun, user)\n", .{want}) catch {};
return .failure;
};
const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "config")) orelse {
main.reportNoRoot(streams, io, "config", amb.repo_override);
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
var url_buf: [logstore.max_config_str]u8 = undefined;
const url = resolveRemoteUrl(gpa, io, root_dir, url_arg, &url_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "config"),
error.Storage => return repoReadFail(streams, "config"),
error.NoRemote => return noRemoteError(streams, "config"),
};
var tr = transport.Transport.open(io, gpa, url) catch |e|
return syncOpenFail(streams, "config", url, e);
defer tr.close();
notePlaintext(streams, "config", url);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
// §9.1 — the pull card carries this repository's adopted project code, and a
// real peer refuses a mismatch, so the identity is loaded even though this
// exchange moves no artifacts. The buffer outlives the run (the client
// borrows it).
var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined;
const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "config"),
error.Storage => return repoReadFail(streams, "config"),
};
// A pull-shaped session that asks for one group. The in-memory store stays
// EMPTY on purpose: this verb moves CONFIGURATION, never artifacts, so an
// operator asking for a shun table cannot be handed a repository's worth of
// content as a side effect.
// fossil serves the user area to an ADMINISTRATOR only
// (`configure.c`: `m &= ~(CONFIGSET_USER|CONFIGSET_SCRIBER)` for a client
// without Admin), so a `/user` pull is credentialed or it comes back
// empty. The §6.3 derivation is `resolvePushCredential`'s, reused rather
// than restated: one place turns a password into a signing secret.
var pw_buf: [max_password_len]u8 = undefined;
defer std.crypto.secureZero(u8, &pw_buf);
var user_buf: [logstore.max_config_str]u8 = undefined;
const resolved = resolvePushCredential(
gpa,
io,
root_dir,
streams,
identity.project_code,
.{ .url = url, .user = login_user, .password = login_password },
amb,
&pw_buf,
&user_buf,
) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "config"),
error.Storage => return repoReadFail(streams, "config"),
error.NoPassword => {
streams.err.writeAll("zit config: no password supplied for --user\n") catch {};
return .failure;
},
};
var client = zit.sync.Client{
.state = zit.sync.initialState(.pull),
.operation = .pull,
.self_servercode = &identity.server_code,
.project_code = identity.project_code,
.version = sync_client_version_info,
.request_config = true,
.config_group = group,
.credential = if (resolved) |r| r.credential else null,
};
defer client.deinit(gpa);
defer reportTruncatedPayloads(streams, "config", client.payloads_truncated);
_ = sync_driver.runSync(gpa, &client, ms.store(), &tr, max_stalled_sync_rounds) catch |e|
return syncRunFail(streams, "config", url, e, tr.last_status, basicStateOf(&tr));
if (std.mem.eql(u8, group, zit.sync.config_user_group)) {
return applyPeerUsers(gpa, io, root_dir, client.peerUserRows(), overwrite, streams);
}
const rows = client.peerShunRows();
if (rows.len == 0) {
streams.out.writeAll("no shun rows offered by the peer\n") catch {};
return .ok;
}
var store_append = cli.openStoreAppend(io, root_dir) catch |e| return storeAppendFail(streams, "config", e);
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch return repoReadFail(streams, "config");
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch
return repoReadFail(streams, "config");
defer log.deinit(gpa);
// Count by the TABLE's own size, not by successful calls: `putShun` is
// idempotent on a name already present, so counting calls would report rows
// the operator already had as newly adopted.
const before = log.shunList().len;
var skipped: usize = 0;
for (rows) |row| {
var digest_buf: [32]u8 = undefined;
if (row.hash.len != 40 and row.hash.len != 64) {
skipped += 1;
continue;
}
const digest = digest_buf[0 .. row.hash.len / 2];
_ = std.fmt.hexToBytes(digest, row.hash) catch {
skipped += 1;
continue;
};
// `putShun` is idempotent on a name already present, so re-running this
// verb converges rather than accumulating.
log.putShun(digest, row.reason, row.epoch) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "config"),
else => {
skipped += 1;
continue;
},
};
}
const added = log.shunList().len - before;
log.commit() catch return repoReadFail(streams, "config");
streams.out.print("shun: {d} new row{s} adopted", .{ added, plural(added) }) catch {};
if (skipped != 0) streams.out.print("; {d} unreadable and skipped", .{skipped}) catch {};
streams.out.writeAll("\n") catch {};
return .ok;
}
test "the cluster sidecar is created by persist, adopted without a walk, rebuilt by one walk when lost, extended by a later persist" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Two blobs and a cluster naming one of them, persisted into the fresh log.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
var names: [2][64]u8 = undefined;
inline for (.{ "clustered blob", "loose blob" }, 0..) |body, i| {
_ = zit.artifact.artifactName(.sha3_256, body, &names[i]);
const rid = try ms.store().put(&names[i], body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try ms.store().retain(rid, .{ .full = c });
}
const cluster = try std.fmt.allocPrint(gpa, "M {s}\nZ 00000000000000000000000000000000\n", .{&names[0]});
defer gpa.free(cluster);
var cn: [64]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, cluster, &cn);
{
const rid = try ms.store().put(&cn, cluster);
const c = try zit.container.encode(gpa, cluster, .default);
defer gpa.free(c);
try ms.store().retain(rid, .{ .full = c });
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
// 1. Adopted: persist created the index, so a load walks nothing.
{
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const before = clusterindex.walk_rids;
var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store());
defer r.deinit();
try std.testing.expectEqual(before, clusterindex.walk_rids);
try std.testing.expectEqual(@as(usize, 1), r.cluster_names.len);
try std.testing.expectEqualStrings(&cn, r.cluster_names[0]);
try std.testing.expect(r.clustered.isSet(holdings.store().lookup(&names[0]).?));
try std.testing.expect(!r.clustered.isSet(holdings.store().lookup(&names[1]).?));
try std.testing.expect(!r.clustered.isSet(holdings.store().lookup(&cn).?));
}
// 2. Lost: one full walk rebuilds it; the load after that adopts again.
try tmp.dir.deleteFile(io, zit.repocache.clusters_filename);
{
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const before = clusterindex.walk_rids;
var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store());
defer r.deinit();
try std.testing.expectEqual(before + 3, clusterindex.walk_rids);
try std.testing.expectEqual(@as(usize, 1), r.cluster_names.len);
const again = clusterindex.walk_rids;
var r2 = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store());
defer r2.deinit();
try std.testing.expectEqual(again, clusterindex.walk_rids);
}
// 3. Extended: a later persist records the cluster it writes; no walk.
const cluster2 = try std.fmt.allocPrint(gpa, "M {s}\nZ 00000000000000000000000000000000\n", .{&names[1]});
defer gpa.free(cluster2);
var cn2: [64]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, cluster2, &cn2);
{
const rid = try ms.store().put(&cn2, cluster2);
const c = try zit.container.encode(gpa, cluster2, .default);
defer gpa.free(c);
try ms.store().retain(rid, .{ .full = c });
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 3);
{
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const before = clusterindex.walk_rids;
var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store());
defer r.deinit();
try std.testing.expectEqual(before, clusterindex.walk_rids);
try std.testing.expectEqual(@as(usize, 2), r.cluster_names.len);
try std.testing.expect(r.clustered.isSet(holdings.store().lookup(&names[1]).?));
}
}
test "sync §5.1/§10.1: a uv reply cut short inside a uvfile payload keeps the files before it" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const abc_hash = "a9993e364706816aba3e25717850c26c9cd0d89d";
const body = "uvfile a 1 " ++ abc_hash ++ " 3 0\nabc" ++
"uvfile b 1 " ++ abc_hash ++ " 3 0\nab";
const reply = try parseUvReply(arena_state.allocator(), body);
try std.testing.expect(reply.truncated);
try std.testing.expectEqual(@as(usize, 1), reply.files.items.len);
try std.testing.expectEqualStrings("a", reply.files.items[0].name);
try std.testing.expectEqualStrings("abc", reply.files.items[0].content);
}