Artifact 9d3c90a622e3a9cbec399a62732bdf3a551405cad78af5b4d23b617ed4ecc86c:
- File
src/cmd/sync.zig
— part of check-in
[9033c13f21]
at
2026-09-17 17:04:25
on branch trunk
— Remove issue-tracker numbers from source and documents
The tracker is disposable and does not live with the source, so a number is a dangling pointer. Every zit#NN and #NN reference in code comments, test names, help text, smoke labels, identifiers, release notes and errata is replaced by enough context to stand alone, or by a short plain statement of the problem. No behaviour changed. (user: dmestas, size: 683418) [annotate] [blame] [check-ins using]
//! zit sync / network / store-transfer verbs (serve, clone, open, pack, pull, //! push, sync, remote) and the helpers and tests private to them. //! //! Extracted verbatim from src/main.zig (release step S2). Every declaration //! below is byte-identical to its main.zig source; only this import prologue is //! new. Helpers still shared with other verbs stay in main.zig and are aliased //! back here so the moved bodies call them unqualified. main.zig's verb-dispatch //! switch calls into this file (`cmd_sync.runServe` ...); its test block imports //! it so these inline tests keep running under `zig build test`. const std = @import("std"); const zit = @import("zit"); const SparseClassification = zit.sparse.SparseClassification; const classifyKeepRids = zit.sparse.classifyKeepRids; const cli = @import("zit").cli; const extras = @import("../extras.zig"); const logstore = @import("zit").logstore; const storage = @import("zit").storage; const clusterindex = @import("zit").clusterindex; const update = @import("zit").update; const checkout = @import("zit").checkout; const Repository = @import("zit").Repository; const serve_mod = @import("../serve.zig"); const transport = @import("../transport.zig"); const sync_driver = @import("../sync_driver.zig"); const Io = std.Io; const File = Io.File; const repostate = zit.repostate; const managed = zit.managed; const artifact = zit.artifact; // Shared CLI helpers and test fixtures that stay in main.zig (other verbs use // them too); aliased back so the moved code and tests reference them by their // original unqualified names. const main = @import("../main.zig"); const runInit = main.runInit; const repo_filename = main.repo_filename; const renderStorage = main.renderStorage; const reasonOf = main.reasonOf; const stateOom = main.stateOom; pub const not_a_repo_msg = main.not_a_repo_msg; const repoReadFail = main.repoReadFail; const repoOpenFail = main.repoOpenFail; const openRepo = main.openRepo; const updateApplyFail = main.updateApplyFail; const digest_len = main.digest_len; const hexToDigest = main.hexToDigest; const digestToWireName = main.digestToWireName; const decodeWireName = main.decodeWireName; const open_help = main.open_help; const pack_help = main.pack_help; const shun_help = main.shun_help; const serve_help = main.serve_help; const clone_help = main.clone_help; const pull_help = main.pull_help; const push_help = main.push_help; const sync_help = main.sync_help; const remote_help = main.remote_help; const settings_help = main.settings_help; const packSeedBlob = main.packSeedBlob; const packSeedCheckin = main.packSeedCheckin; const packEvolvingBlob = main.packEvolvingBlob; const packOpenWritable = main.packOpenWritable; const packOpenReadOnly = main.packOpenReadOnly; /// Clone phase edges let benchmark code attribute the real command without /// coupling clocks or counters into production behavior. pub const ClonePhase = enum { transport, initialize, exchange, persist, config, warm_cache }; pub const ClonePhaseEvent = enum { begin, end }; pub const CloneObserver = struct { context: *anyopaque, callback: *const fn (context: *anyopaque, phase: ClonePhase, event: ClonePhaseEvent) void, }; fn observeClonePhase(observer: ?*const CloneObserver, phase: ClonePhase, event: ClonePhaseEvent) void { if (observer) |value| value.callback(value.context, phase, event); } /// `zit serve [--port N]` — put an HTTP face on the stateless sync responder. /// Opens `.zit` read-only and keeps it open for the server's lifetime; every /// request reads a fresh content view of it and is answered independently /// (sync §3.8). `std.http`/socket I/O lives entirely in `serve.zig`; this only /// wires the repository and the listen address. pub fn runServe( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, /// Carried solely so the served rounds can resolve this server's own /// content policy through the shared settings resolver. amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseServeArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "serve", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "serve", .extra_argument, tok), .bad_port => |tok| { streams.err.print("zit serve: invalid --port value '{s}'\n", .{tok}) catch {}; return .usage; }, }; if (parsed.help) { streams.out.writeAll(serve_help) catch {}; return .ok; } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "serve")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; // serve answers from the STORE: its open handle, its swap-detect // stat and its config reads all follow the resolved store root. if (cli.storeDirOf(io, root_dir) catch { root_dir.close(io); return repoReadFail(streams, "serve"); }) |resolved| { root_dir.close(io); root_dir = resolved; } var file = root_dir.openFile(io, repo_filename, .{}) catch { root_dir.close(io); return repoReadFail(streams, "serve"); }; defer file.close(io); const log_st = file.stat(io) catch { root_dir.close(io); return repoReadFail(streams, "serve"); }; // Bring-up order is load-bearing (rationale in `serve_mod.listen`'s // doc): open the listener FIRST — before ANY O(store) work, which here means // both the config read below and the prime, each a full-log scan. A client // that connects during either queues in the accept backlog instead of being // refused, so the readiness window starts at the bind rather than after the // scans. Announce "listening" only once the bind has actually succeeded (so // it never lies about a listener that failed to come up); the accept loop // then arms each connection's deadline on accept. The backlog property this // rests on is pinned by serve.zig's "listen opens the accept backlog before // serve" test. const address = std.Io.net.IpAddress.parse("127.0.0.1", parsed.port) catch { streams.err.print("zit serve: invalid --port value '{d}'\n", .{parsed.port}) catch {}; return .usage; }; var listener = serve_mod.listen(io, address) catch { streams.err.print("zit serve: cannot bind 127.0.0.1:{d}\n", .{parsed.port}) catch {}; return .failure; }; defer listener.deinit(io); streams.err.print("zit serve: listening on http://127.0.0.1:{d}/xfer (Ctrl-C to stop)\n", .{parsed.port}) catch {}; streams.err.flush() catch {}; var fs = storage.FileStorage.init(io, file, log_st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| { root_dir.close(io); switch (e) { error.OutOfMemory => streams.err.writeAll("zit serve: out of memory\n") catch {}, else => streams.err.print("zit serve: cannot read repository: {s}\n", .{reasonOf(e)}) catch {}, } return .failure; }; defer log.deinit(gpa); defer root_dir.close(io); // §9.1/§8.2 — emit the repository's stored project code in the clone // reply's `push` card, so a clone learns the shared project identity. A // repo predating the stored code, or an unreadable config, falls back to // the fixed placeholder (informational; servercode never gates a clone). var maybe_cfg: ?logstore.OwnedConfig = log.readConfig(gpa) catch null; defer if (maybe_cfg) |*oc| oc.deinit(gpa); const project_code: []const u8 = if (maybe_cfg) |oc| (if (oc.config.project_code.len > 0) oc.config.project_code else serve_mod.server_projectcode) else serve_mod.server_projectcode; // §9.1 — a real instance identity in the clone reply's `push` card, // derived from this checkout's absolute path exactly as init derives the // project code (stable, distinct per instance, no ambient RNG). The // placeholder constant remains only as the test fixtures' server identity. const serve_code = zit.sync.deriveServerCode(&.{root.abs}); const server = zit.sync.Server{ .version = serve_mod.server_version, .credential = null, .self_servercode = &serve_code, .project_code = project_code, }; var ctx = ServeCtx{ .io = io, .root_dir = root_dir, .file = &file, .diag = streams.err, .server = &server, .amb = amb, .fronted = parsed.fronted }; defer ctx.dropSnapshot(); const responder = serve_mod.Responder{ .ctx = &ctx, .answerFn = serveAnswer, .maintainFn = ServeCtx.maintain, .fronted = parsed.fronted }; if (ctx.baseSnapshot(gpa)) |_| {} else |_| return repoReadFail(streams, "serve"); serve_mod.serve(io, gpa, &listener, responder, .{}) catch |e| switch (e) { error.Accept => { streams.err.writeAll("zit serve: connection accept failed\n") catch {}; return .failure; }, }; return .ok; } /// The per-request context threaded into `serve.Responder`: the open log to read /// a fresh store from, the server identity to answer with, and the injected IO + /// repository directory used to make pushed content durable (a push request /// ingests new artifacts into the per-request store, which must be persisted into /// the served repository's on-disk log — sync §3.3). const ServeCtx = struct { io: std.Io, root_dir: std.Io.Dir, /// The ambient environment, carried only so `resolveSetting` (the one /// settings resolver) is reachable per round. Reading the config frame /// directly here instead would be a second reader, and the two drifting is /// the bug `signingSettings` documents having shipped once already. amb: ?cli.Ambient = null, /// Behind a trusted front. Turns on the per-round membership /// rule and keeps the repository image warm between connections so that /// rule reads O(image), not O(store), after a push. fronted: bool = false, /// The open read handle on the served `.zit`. `respond` re-reads the CURRENT /// on-disk log from it (§3.8): a fresh stat fixes the committed end-of-log, /// so a check-in appended by a separate `zit commit` while `serve` runs is /// advertised on the next request — no restart. /// /// The handle pins an INODE, not a path. `zit pack` and `zit shun --force` /// replace `.zit` by rename, which would leave a held-open handle reading /// the replaced inode forever (its fstat never moves again — no cache key /// over the handle can detect the swap). `baseSnapshot` therefore stats /// the PATH beside the handle each round and REOPENS on an inode change: /// the swap is noticed at the next round boundary, so a shun that /// rode a generation rewrite stops being served with no restart, while an /// in-flight round keeps its consistent pre-swap view. A pointer into /// `runServe`'s frame so the reopen and the shutdown close agree on one /// current handle. file: *File, /// Where `serveAnswer` names a failed request, or null to stay silent. The /// tests and `driveHttp` leave it unset; `runServe` points it at stderr. diag: ?*std.Io.Writer = null, server: *const zit.sync.Server, /// The most recent snapshot of the on-disk log, reused across requests. A /// log-backed INDEX (not a full-container copy): it reads the containers it /// serves on demand and holds only the index between rounds. Built from /// the long-lived allocator `serve` threads into every answer, so it outlives /// the request that built it, and owned here — heap-allocated because it owns /// the `FileStorage` its log borrows; `dropSnapshot` frees it. Never handed to /// `respond` directly — see `serveAnswer`. snapshot: ?*LogIndexStore = null, /// The snapshot's cluster index, rebuilt with it. clusters: ?clusterindex.Resolved = null, /// The log size `snapshot` was built from — one half of the cache key that /// `baseSnapshot` stats per request (see there for why size alone is not it). snapshot_log_size: u64 = 0, /// The log's last-modification time (ns) when `snapshot` was built. Together /// with the size it is the cache key: a write that does not change the length /// still moves `mtime`. A `std.Io.Timestamp`, unwrapped to its `nanoseconds` /// here so the key is two scalar comparisons. snapshot_log_mtime_ns: i96 = 0, /// How many snapshots this server has built. Not used by the server; it is /// the observable that makes "snapshot builds do not grow with the round /// count" a testable property rather than a wall-clock claim. loads: u64 = 0, /// Free the cached snapshot, leaving the context usable: the next /// `baseSnapshot` rebuilds. Both a caller finishing with the context and the /// rebuild path itself want exactly this. fn dropSnapshot(self: *ServeCtx) void { if (self.clusters) |*c| c.deinit(); self.clusters = null; if (self.snapshot) |s| s.destroy(); self.snapshot = null; self.snapshot_log_size = 0; self.snapshot_log_mtime_ns = 0; } /// Drop plaintext the cached snapshot memoised while answering, keeping the /// retained containers it serves from. Called once per round, after every /// borrow of the snapshot has ended. /// /// Inflating is how `content()` answers a wire-backed entry, and the /// responder's own §7.2 size proofs call `content()` on every artifact they /// emit. Against a throwaway per-request store that plaintext died with the /// request; against a store that outlives the request it would accumulate /// until the server held the whole repository's uncompressed history. The /// containers stay, so the next round answers identically. /// /// The memo is given up rather than capped because in an optimized build /// there is almost nothing to give up: the size proofs are `runtime_safety` /// only, so a clone round inflates nothing and the memo would stay empty /// regardless. Measured over four clones of a 3001-artifact repository, /// ReleaseFast: 12 MB resident and 0.09 s of server CPU with this release, /// against 26 MB and 0.15 s without it. A safety-checked build does re-prove /// every emitted artifact each round and so pays to re-inflate; that cost /// belongs to the proofs, not to the cache. fn releaseSnapshotPlaintext(self: *ServeCtx) void { if (self.snapshot) |s| s.releaseTransient(); } /// §3.8 — the store describing the CURRENT on-disk log, rebuilt only when the /// log has actually grown. The §3.8 guarantee is that a check-in appended by /// a separate `zit commit` is advertised on the NEXT request with no restart; /// that needs a fresh `stat` per request, not a fresh rebuild per request. /// Rebuilding regardless made every round cost the whole repository, which is /// what turned a clone of a large store into O(rounds x store) server work. /// /// A torn tail from an append caught mid-write is rejected by the read-only /// scanner exactly as before; the previous snapshot is dropped only once a /// replacement is fully built, so a failed rebuild leaves the cache coherent. /// /// The rebuild itself is O(store) and must therefore not fall inside a /// request whose connection carries a deadline. `runServe` primes the cache /// before it accepts anything. A round that appends to the log does NOT /// rebuild before it returns — it deliberately leaves the snapshot stale /// (describing the pre-append log) rather than pay the O(store) rebuild inside /// its own reply deadline. The append-only stat gate above makes that /// stale snapshot self-correcting: the size no longer matches, so the next /// `baseSnapshot` rebuilds. That next call is the accept loop's `maintain` /// hook (see `ServeCtx.maintain`), run between connections with no connection /// open — so the deferred rebuild still lands outside every connection /// deadline, never on an innocent later request. Nothing here can bound the /// startup build itself; that is bounded by the store, and always was. /// /// THE CACHE KEY IS NOT THE SIZE ALONE, and since this snapshot began carrying /// the §3.3 shun table it cannot be. A stale snapshot used to cost stale CONTENT /// — a peer misses a newly committed artifact and the next round supplies it — /// which is self-correcting. It now also decides whether the SUPPRESSION table is /// re-read, and a missed re-read serves an artifact the operator shunned, with /// nothing failing and nothing logged. "Every writer only appends, so the length /// always grows" is true of `zit shun --defer` and `zit commit` today, but it is /// an assumption about all present and future writers rather than a checked /// property: a generation rewrite landing on the same byte length, or any /// out-of-band mutation, would keep the old mask forever. So the key is size AND /// mtime — an in-place rewrite that does not change the length still moves the /// mtime. Two scalar comparisons against one `stat` this function already /// performs, and they can only over-invalidate (which costs a rebuild), never /// under-invalidate on a write that reaches THIS INODE. /// /// WHAT THE CACHE KEY CANNOT COVER, THE PATH STAT DOES. `st` comes /// from an `fstat` on the held handle, so it detects every write reaching /// that inode and no rename ABOVE it: a `zit shun --force` (or `zit pack`) /// generation swap leaves the old inode's size and mtime frozen forever, /// with the shun row in a file this handle would never see. So the path is /// stat'ed beside the handle each round, and an inode mismatch reopens the /// handle (dropping the snapshot) before anything is answered — the round /// then serves the live generation. Failure on the path stat or the reopen /// keeps the old handle and answers from it: availability first, and the /// next round retries the swap detection. fn baseSnapshot(self: *ServeCtx, gpa: std.mem.Allocator) serve_mod.AnswerError!repostate.ArtifactStore { var st = self.file.stat(self.io) catch return error.ServerStorage; if (self.root_dir.statFile(self.io, repo_filename, .{})) |path_st| { if (path_st.inode != st.inode) { if (self.root_dir.openFile(self.io, repo_filename, .{})) |fresh_handle| { self.file.close(self.io); self.file.* = fresh_handle; self.dropSnapshot(); st = self.file.stat(self.io) catch return error.ServerStorage; } else |_| {} } } else |_| {} if (self.snapshot) |s| { if (st.size == self.snapshot_log_size and st.mtime.nanoseconds == self.snapshot_log_mtime_ns) return s.store(); } // Build the fresh index BEFORE dropping the old one: a failed rebuild // leaves the previous snapshot (and its size) intact, so the stat gate // keeps mismatching until a rebuild succeeds and stale content is never // served in the meantime. `openFromFile` fully frees itself on any error. const fresh = LogIndexStore.openFromFile(gpa, self.io, self.file.*, st.size) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; self.dropSnapshot(); self.snapshot = fresh; // Resolve the cluster index over the new snapshot, then drop // the cluster plaintext it memoised: the index keeps only names and bits. self.clusters = loadClusterIndex(gpa, self.io, self.root_dir, &fresh.log, fresh.store()) catch null; fresh.releaseTransient(); self.snapshot_log_size = st.size; self.snapshot_log_mtime_ns = st.mtime.nanoseconds; self.loads += 1; return fresh.store(); } /// The accept loop's between-connections maintenance hook (see /// `serve.Responder.maintainFn`). Runs after a connection has closed and /// before the next accept, so no connection is open to charge its O(store) /// work to. It re-primes the cache via the same stat-gated `baseSnapshot`: /// after a pure read round the stat is unchanged and this is one `stat`; after /// a round that appended (a push) the log grew, so this rebuilds the snapshot /// the pushing round deliberately left stale rather than pay inside its /// deadline. A failed rebuild is swallowed for the same reason /// `baseSnapshot` leaves the cache coherent on failure: `baseSnapshot` drops /// the old snapshot only after a fresh one is fully built and updates /// `snapshot_log_size` only on success, so a failed rebuild RETAINS the /// previous (stale) snapshot with its old size — the stat gate keeps /// mismatching until a rebuild succeeds, and stale content is never served in /// the meantime. The next round (or the next `maintain`) retries, still /// outside any earlier round's budget. Type-erased to match the `maintainFn` /// seam. fn maintain(ctx_ptr: *anyopaque, gpa: std.mem.Allocator) void { const self: *ServeCtx = @ptrCast(@alignCast(ctx_ptr)); const before = self.snapshot_log_size; if (self.baseSnapshot(gpa)) |_| {} else |_| {} // A push left the zero-parse image stale, and the next // protected push would re-derive it inside its own deadline. Derive it // here instead, between connections, where O(store) is allowed — and // only when the log actually grew, so a pull-heavy fronted server does // not pay an image load per connection. Only a fronted server has a // resolver to warm; a failure is retried by the round itself, which // derives on demand and fails closed on a fault. if (self.fronted and self.snapshot_log_size != before) { var repo: Repository = undefined; if (Repository.open(&repo, gpa, self.io, self.root_dir, .{ .mode = .fast_read })) |_| { repo.deinit(gpa); } else |_| {} } } }; /// The `serve.Responder` seam: snapshot the repository into an in-memory store, /// answer the framed request over it, and — when the request pushed new content — /// make that content durable in the served repository's on-disk log (sync §3.3). /// /// `respond` stays sans-IO: it only `put`s pushed artifacts into the snapshot /// store it is given. The durable write is this serve-layer's job and reuses the /// idempotent `persistStore` (a re-push of an already-stored hash appends nothing /// — content is addressed by hash), the very path clone/pull persist through. A /// pure gimme/pull request makes nothing new servable, so the overlay stays /// empty and the log is never reopened; only a genuine push touches disk. /// /// `respond` is answered over a per-request WRITE OVERLAY of the cached snapshot /// (`repostate.OverlayStore`), never over the snapshot itself: the responder /// interns phantoms and upgrades entries as it works, and letting those survive /// into the next round changes what that round serves. Reads fall through to the /// snapshot, so reuse costs nothing; writes are confined to the overlay, and the /// persist pass — which is idempotent over everything already in the log — can /// only append what the overlay holds. /// /// The pushed bytes were size-validated by `respond`'s request scan before they /// entered the store, so `persistStore` only ever writes well-formed content, and /// a mid-write fault surfaces before `commit`, never as a partial log append. fn serveAnswer(ctx_ptr: *anyopaque, gpa: std.mem.Allocator, request_container: []const u8, cookie: ?[]const u8, protect: ?[]const u8) serve_mod.AnswerError!serve_mod.XferReply { const ctx: *ServeCtx = @ptrCast(@alignCast(ctx_ptr)); return serveAnswerInner(ctx, gpa, request_container, cookie, protect) catch |e| { // NAME THE FAILURE. `serve.zig` collapses every AnswerError into one // bare status -- `ServerStorage`, `OutOfMemory` and `ResponseTooLarge` // all become 500 with no body and no log line -- so an operator (or a // bench harness) sees "HTTP 500" and has no way to learn which of three // unrelated faults produced it. This is the server-side half of naming // the status the client now reports; without both ends a 500 is a dead // end for anyone diagnosing from the outside. // // Single funnel on purpose: every `AnswerError` this responder can // produce passes through here, so a future cause is reported without a // second edit. Diagnostics only -- the error is re-raised unchanged and // the status the peer sees does not move. if (ctx.diag) |w| { w.print("zit serve: request failed: {s}\n", .{@errorName(e)}) catch {}; w.flush() catch {}; } return e; }; } fn serveAnswerInner(ctx: *ServeCtx, gpa: std.mem.Allocator, request_container: []const u8, cookie: ?[]const u8, protect: ?[]const u8) serve_mod.AnswerError!serve_mod.XferReply { const reply = try serveRound(ctx, gpa, request_container, cookie, protect); // Past this point nothing borrows the snapshot: the overlay is gone with // `serveRound`'s frame and the reply owns its own bytes. Dropping the // plaintext memo is bounded (it frees only what this round inflated), so it // is safe inside the deadline. // // A round that appended (a push) has left the snapshot stale — it still // describes the pre-push log. It is NOT rebuilt here: an O(store) rebuild // inside the reply's connection deadline is exactly the failure this avoids. The // rebuild is the accept loop's job, via `ServeCtx.maintain`, run after this // connection closes and before the next accept — outside every deadline. The // append-only stat gate in `baseSnapshot` makes the stale snapshot self- // correcting: the next `baseSnapshot` (in `maintain` or the next round) sees // the grown size and rebuilds, so no `appended` signal needs to be threaded // out of the round. // // A post-scan read fault this round mapped a held artifact to null. The §8.2 // clone pager skips such a rid and still reports the sequence complete // (NEXT 0), silently finishing a clone MISSING an artifact — a failure the // copy-based prime could not produce (a fault surfaced at prime, and a // retained-but-uninflatable container was still served verbatim). Read the // flag BEFORE releasing the round's memos (which clears it), then surface it // as a storage error so the client retries; the next round re-reads. const faulted = if (ctx.snapshot) |s| s.read_fault else false; ctx.releaseSnapshotPlaintext(); if (faulted) { gpa.free(reply.body); return error.ServerStorage; } return reply; } /// One round over a per-request write overlay. Split out of `serveAnswer` so the /// overlay — which borrows the cached snapshot — is provably destroyed before /// `serveAnswer` touches that snapshot's storage. A round that pushes new content /// persists it durably here; the snapshot it read from is left describing the /// pre-push log, and `baseSnapshot`'s append-only stat gate rebuilds it on the /// next call (in the accept loop's `maintain`, never inside this round's /// deadline), so this round signals nothing back to its caller. fn serveRound( ctx: *ServeCtx, gpa: std.mem.Allocator, request_container: []const u8, cookie: ?[]const u8, protect: ?[]const u8, ) serve_mod.AnswerError!serve_mod.XferReply { var overlay = repostate.OverlayStore.init(gpa, try ctx.baseSnapshot(gpa)); defer overlay.deinit(); // sync §3.3 L527-530 / §8.8 L1777-1779 — answer this round with THIS // snapshot's shun table. `ctx.server` is the caller's long-lived config and // stays const; the per-round copy carries the fields that are properties of // the snapshot rather than of the server, so the reply's phantom scan never // begs for an artifact this repository has shunned (a client `igot` for a // shunned hash still interns its phantom — §3.3 r24(1) L511-514 — it is just // never requested). // `ctx.baseSnapshot` above published it — the rebuild path assigns // `self.snapshot`, the cached path requires it non-null — so the null arm is // unreachable today. It is a CHECKED error rather than a fail-open `.none` // because "the filter is not skipped" must hold in a release build: a future // ctx that primes a store without publishing the snapshot, or an early-out // added above, must fail the round rather than silently serve with no shun // table and re-beg every shunned hash with nothing failing. // // submanifests §8.1/§7.2 — the same per-round freshness arms the send gate. // The long-lived `ctx.server` is a build identity (version, servercode, // project code) and outlives every snapshot; the seen-submanifest indicator, // the §8.1 emission setting, and the two settings this server only REPORTS // are facts about the CURRENT log and are re-read whenever the snapshot is. // Copying per round is what keeps them from drifting: a push that stores this // repository's first tree arms the gate for the next round rather than for // the next process, and a `zit submanifest enable`/`retune` against a served // repository is obeyed by the next round rather than the next process. These // read the snapshot unconditionally for the same reason the shun table does — // serving a stale default is the failure, not a fallback. var round_server = ctx.server.*; const snapshot = ctx.snapshot orelse return error.ServerStorage; round_server.shun = snapshot.shun.shunSet(); // §9.3 r28 — and the row view, for a `reqconfig /shun`. round_server.shun_rows = snapshot.shun_rows; // Sparse-cone RFC section 6.2 -- cone-scoped requests answer filtered. round_server.cone_classifier = snapshot.coneClassifier(); // Bounded announcement: the clusters carry what they name. if (ctx.clusters) |*c| { round_server.clustered = &c.clustered; round_server.cluster_names = c.cluster_names; } // This server's own content policy, re-read per round like every // other setting here, so `zit settings binary-glob ...` against a served // repository is obeyed by the next round rather than the next process. // // Resolved from the SERVER's settings, never from the artifacts arriving in // the push: a push carrying its own policy would otherwise arrive bearing // permission to land. The versioned tier is still consulted, and that is // deliberate -- it reads the server's own working tree, which a push does // not touch, so adopting a project's policy stays an act the operator takes // by updating, exactly as `zit trust` is for before-commit consent. var policy_arena = std.heap.ArenaAllocator.init(gpa); defer policy_arena.deinit(); if (ctx.amb) |amb| { // The repository tier comes from the snapshot's per-build // config read, NOT from `resolveSetting`, whose repository read opens // the store and re-scans every frame header. That scan per setting // per round is what made a 1.17M-artifact clone take half an hour. // The snapshot's (size, mtime) key preserves the per-round freshness // contract: a settings write grows the log and rebuilds the copy. const pa = policy_arena.allocator(); const repo_bin: ?[]const u8 = if (snapshot.policy_binary_glob.len != 0) snapshot.policy_binary_glob else null; const repo_crlf: ?[]const u8 = if (snapshot.policy_crlf_glob.len != 0) snapshot.policy_crlf_glob else null; const bin = resolveSettingWith(ctx.io, pa, ctx.root_dir, amb, .binary_glob, repo_bin) catch zit.settings.Resolved{ .value = "", .source = .default }; const crlf = resolveSettingWith(ctx.io, pa, ctx.root_dir, amb, .crlf_glob, repo_crlf) catch zit.settings.Resolved{ .value = "", .source = .default }; round_server.push_policy = .{ .binary_glob = bin.value, .crlf_glob = crlf.value }; } // Sparse-cone RFC section 5.3 -- a sparse origin refuses clones it does // not cover. Read per round like every other snapshot-freshness fact: an // `update --cone` against the served tree is obeyed by the next round. var origin_cone = zit.repocache.readStoreCone(gpa, ctx.io, ctx.root_dir); defer if (origin_cone) |*c| c.deinit(gpa); round_server.origin_cone = if (origin_cone) |*c| c else null; // sync §7.7 L1428-1432 — the other half of the same admission predicate, from // the same snapshot, so `ingest` never pairs this round's shun table with a // stale policy. round_server.hash_policy = snapshot.hash_policy; round_server.seen_submanifest = snapshot.seen_submanifest; round_server.submanifests_enabled = snapshot.submanifests_enabled; round_server.submanifest_k = snapshot.submanifest_k; round_server.submanifest_auto_retune = snapshot.submanifest_auto_retune; // The front's membership rule for THIS push, and the resolver it // needs. Both set together: a rule with no resolver fails closed in // `pushProtectFault`, which is the right answer to a misconfiguration but // not a state this code should be able to produce. The resolver opens the // repository lazily -- a round that pushes nothing protected never pays -- // and the view it opens reads the on-disk log, which every earlier round // already persisted, so a parent pushed last round resolves; a parent in // this round is resolved in-round by the rule itself. var resolver = RoundBranchResolver{ .ctx = ctx, .gpa = gpa }; defer resolver.deinit(); if (protect) |globs| if (globs.len != 0) { round_server.push_policy.protect_globs = globs; round_server.branch_of = resolver.resolver(); }; const reply = try serve_mod.handleXfer(gpa, &round_server, overlay.store(), request_container, cookie); errdefer gpa.free(reply.body); // sync §9.3 r33 — an AUTHORIZED pushed config card's settings, applied to // the served repository here because this is where the config lives; the // responder is sans-IO and only reports the decision. Unreachable today by // construction: `runServe` configures no credential, so `Server.hasAdmin` is // false for every requester and every push is rejected (the e2e pins that). // It is wired anyway because the alternative — a server that can AUTHORIZE a // push and then silently drop it — is the failure mode worth ruling out. if (reply.config_apply.any()) { updateConfig(gpa, ctx.io, ctx.root_dir, .{ .enable_submanifests = reply.config_apply.enable, .submanifest_k = reply.config_apply.k, .submanifest_auto_retune = reply.config_apply.auto_retune, }) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; } if (overlay.writtenCount() > 0) { // submanifests §7.1 — the durable half of the push-side duty; the // reply-visible half is `sync.ingest`, which phantomizes into the very // store the §3.2 walk reads. The gap COUNT is discarded on purpose: a // stateless server (§3.8/§8.9) has no operator to report to, and §7.1 // asks for a phantom, not a diagnostic. _ = persistStore(gpa, ctx.io, ctx.root_dir, overlay.store(), null, 0) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; } return reply; } /// `sync.BranchResolver` over the served repository's own view, /// opened on first use and only for the round. `.fast_read` takes the /// zero-parse image when it is fresh (`ServeCtx.maintain` keeps it so) and /// derives when it is not, so the answer is crosslink's own in either case -- /// which is the point: "which branch does this check-in resolve to" has one /// authority, and this is it, not a re-implementation over raw tags. const RoundBranchResolver = struct { ctx: *ServeCtx, gpa: std.mem.Allocator, repo: Repository = undefined, opened: bool = false, failed: bool = false, fn resolver(self: *RoundBranchResolver) zit.sync.BranchResolver { return .{ .ctx = self, .branchOfFn = branchOf, .mainBranchFn = mainBranch }; } fn deinit(self: *RoundBranchResolver) void { if (self.opened) self.repo.deinit(self.gpa); self.* = undefined; } /// Open once; a failure is sticky for the round so the rule fails closed /// on every question rather than retrying a broken read per artifact. fn open(self: *RoundBranchResolver) error{Unavailable}!*Repository { if (self.failed) return error.Unavailable; if (!self.opened) { Repository.open(&self.repo, self.gpa, self.ctx.io, self.ctx.root_dir, .{ .mode = .fast_read }) catch { self.failed = true; return error.Unavailable; }; self.opened = true; } return &self.repo; } fn branchOf(ctx_ptr: *anyopaque, hash: []const u8) error{Unavailable}!?[]const u8 { const self: *RoundBranchResolver = @ptrCast(@alignCast(ctx_ptr)); const repo = try self.open(); const rid = repo.lookup(hash) orelse return null; // `branchOf` is defined on check-ins; a P card naming a blob (or a // phantom) resolves to nothing rather than tripping its precondition. if (!repo.view.state.hasEvent(rid)) return null; return repo.branchOf(rid); } fn mainBranch(ctx_ptr: *anyopaque) error{Unavailable}![]const u8 { const self: *RoundBranchResolver = @ptrCast(@alignCast(ctx_ptr)); const repo = try self.open(); return repo.view.main_branch; } }; /// A serve snapshot that reads container bytes from the log ON DEMAND instead of /// copying the whole repository into RAM at prime. It owns a read-only /// `LogStore` — whose open scan already whole-frame-CRC'd every byte, so the /// single read stays; the second read `loadContentStore` performed does not — and /// a resident INDEX: the adapter-rid→log-rid map plus the lowercase-hex wire names /// the responder walks. `held()` answers from that index with zero I/O; `wire()` /// and `content()` read the one frame they are asked for and memoise it for the /// round; `releaseTransient` drops those memos between rounds, so a round's /// resident bytes are bounded by what it emitted, not by the store. /// /// Heap-allocated because it owns a `storage.FileStorage` whose `storageFile()` /// vtable points back into the struct — the box must not move once the log borrows /// it. `openFromFile` takes that ownership (the serve path); `open` borrows a /// caller-owned `StorageFile` (a test instrumenting read traffic). /// /// Adapter rids `1..=rids.len` name the log's non-config artifact frames in /// `ridsInPersistedOrder` — the SAME enumeration `loadContentStore` assigns, so /// adapter rid r is byte-for-byte the rid the old MemStore snapshot gave, which is /// what keeps the clone pager's persisted-order cursor (sync §8.2) stable. A frame /// named in the repository shun table keeps its slot in that enumeration and is /// MASKED rather than removed (see `masked`), so the shun table never renumbers /// a rid and §8.2 L1504-1506's "silently skipping gaps" carries the exclusion. Rids /// above `rids.len` are an appendix `MemStore` holding any hash a caller `intern`s /// or `put`s that the log does not carry. The serve path never writes the base /// (every write lands in the per-request `OverlayStore`), so the appendix stays /// empty in production; it exists so this store honours the whole `ArtifactStore` /// interface — composed over the existing `MemStore` rather than hand-rolled — /// instead of trapping on a write it will never see. const LogIndexStore = struct { gpa: std.mem.Allocator, /// Owned storage binding on the `openFromFile` path (the log borrows /// `&self.fs`); `undefined` on the `open` path, where the caller owns the /// storage and keeps it alive for this store's life. fs: storage.FileStorage, log: logstore.LogStore, /// Adapter rid a (1-based) → `rids[a-1]` = the log rid. Persisted order, config /// frames removed. rids: []logstore.Rid, /// Adapter rid a → `names[a-1]` = its lowercase-hex wire name (sync §3.3), /// backed by `name_arena`. The single site the binary-log digest becomes the /// wire name — the boundary `loadContentStore` held, kept to one place. names: [][]const u8, name_arena: std.heap.ArenaAllocator, by_name: std.StringHashMapUnmanaged(repostate.Rid) = .empty, /// Sparse-cone RFC section 6.2 -- single-entry classification memo for /// cone-scoped requests: the server is stateless per round, so without /// this every round of a cone-scoped clone would re-parse the store. /// Keyed by the cone digest; dropped with the snapshot, so a rebuilt /// snapshot can never serve a stale classification. cone_memo_digest: ?[32]u8 = null, cone_memo: ?SparseClassification = null, /// sync §9.5 L1988-1992 — adapter rid a → `masked[a-1]`: this frame's name /// is in the repository shun table, so the enumeration MASKS it. §9.5 asks for /// "igot for every eligible holding after the Section 3.3 shun … filters", and /// this array is where that filter lives: a masked entry answers `held` false, /// `content`/`wire` null, and its name is left out of `by_name`, so the server /// does not announce it (§3.2's walk gates on `held`), and an inbound `gimme` /// for it — which keys on `lookup` — lands on §3.3 L532-535's "If absent, /// silently skip" with no code on that path. /// /// THE CLONE HALF IS AN ERRATUM CANDIDATE, not a citation. Because the mask /// lives in the enumeration, a masked holding is also absent from the clone /// batch — but no draft clause says a clone server withholds one. §9.5's /// sentence is scoped to the `send-catalog` pragma's igot inventory; §3.4 /// L545-547 hands clone to §8.1-§8.4, and §8.2 L1504-1506 licenses only the /// MECHANISM ("silently skipping gaps"), never the policy. Applying §3.3's /// shun set to the clone reply is this implementation's extension, taken /// because the alternative — a repository under a takedown order handing the /// content to every new clone — is not a behaviour an operator would read the /// drafts' silence as licensing. Erratum candidate against sync §8.1/§8.2: /// state whether a clone server's batch is filtered by the §3.3 shun set. /// /// SCOPE, stated exactly: this masks what THIS BASE SNAPSHOT enumerates. A /// served round answers over a per-request `repostate.OverlayStore` wrapping the /// base, and its `putImpl` keys on `base.lookup` — which the mask answers null /// for — so a peer that PUSHES `file <masked name>` in the same round would /// create an overlay-local rid that genuinely holds those bytes and is /// announced by that round's §3.2 walk. That route is closed, but NOT here: it /// is closed by receive-side admission (§7.7 L1437-1440: "A non-clone file /// whose target is shunned is consumed and discarded after its payload and /// pending private marker; it emits no error … and makes no repository /// mutation."), which `scanCard` applies before the card can enter `scan.files` /// and `ingest` re-applies before the `put`, both reading `Server.shun` — the /// set `serveRound` builds from THIS SAME snapshot's table, so the two can /// never disagree about a name. Recorded here because the send-side filter /// genuinely cannot reach that path: what makes the guarantee hold is the pair, /// and removing either half reopens it. /// /// A MASK, not a removal, and §8.2 L1504-1506 is why: "SEQNO is a server /// blob-table RID. Iterate from supplied RID through current maximum, /// silently skipping gaps; clients MUST NOT infer count or size from range." /// Dropping the frame from the enumeration would renumber every survivor above /// it. The server is stateless (§8.9 L1783-1784: "Section 3.7 is authoritative; /// the stateless server signals no completion") — the client's SEQNO *is* the clone /// cursor — and a shun row appended between two rounds of one clone grows the /// log, which is exactly what trips `ServeCtx.baseSnapshot`'s append-only stat /// gate and rebuilds this index. Renumbering there would make a resuming /// client's cursor step over a survivor and still be told the sequence was /// complete: a silently short clone, the failure class `read_fault` exists to /// prevent. Keeping the slot and answering it as a §8.2 gap makes adapter-rid /// identity a pure function of the log, independent of the shun table. /// /// Filled in the SAME loop as `rids`/`names`/`content_memo`/`wire_memo`, off /// the same index — the FILL alignment is structural (one loop, one counter), /// not asserted. A mask derived from ANY second walk is the one way adapter rid /// r could come to name one artifact and mask another, and nothing about the log /// makes such a walk look wrong on inspection: it would agree on length and on /// set membership and differ only in which slot carries which answer. The /// LENGTHS are a separate question and the fill loop cannot speak to them; they /// are gated by a checked error at the end of `build` (see there). /// /// AND A PURE FUNCTION OF `shun.digests` — the other half of the same /// invariant, and the one a later filter is most likely to break. `wireImpl`'s /// delta-basis check asks the SET (`shun.digests.contains(d.baseHash())`) while /// `held`/`content`/`wire` by rid ask THIS ARRAY; the two agree only because the /// fill loop below defines `masked[a]` as exactly that membership. §9.5 /// L1988-1992 names FOUR filters — "igot for every eligible holding after the /// Section 3.3 shun, phantom, private, and peer-has filters" — and this array is /// the natural landing site for the other three. The day one of them widens /// `masked` beyond shun membership, the basis check silently stops following it /// and the server re-emits a DELTASRC it will not send. Widening this array /// therefore obliges the same change to resolve the basis THROUGH it (basis /// digest → adapter rid → `masked[rid - 1]`) rather than through the set. masked: []bool, /// Per-round transient memos over the persisted rids: the owned plaintext / /// wire container that `content()`/`wire()` read on demand, released each round /// by `releaseTransient`. content_memo: []?[]u8, wire_memo: []?WireHold, /// Hashes not in the log, interned/put by a caller. Empty on the serve path. appendix: repostate.MemStore, /// sync §3.3 L527-530 / §8.8 L1777-1779 — the served repository's shun table, /// re-loaded on every build. That is the freshness mechanism the rest of this /// index already rides: `ServeCtx.baseSnapshot` rebuilds whenever the log has /// grown, and a `zit shun` grows it, so a shun row written between two served /// rounds is honoured by the next round with no restart (§3.8's guarantee /// applied to the shun table rather than to check-ins). shun: RepoShunSet, /// sync §9.3 r28 — the same shun table as ROWS, for the `/shun` config group /// a peer requests. `shun` above answers "is this artifact suppressed" for /// the send filter; this carries the operator metadata (when, and why) that /// a peer needs to reconstruct the tombstone rather than merely obey it. /// /// The reason text borrows `self.log`'s own row storage (which this /// snapshot owns and outlives every round served from it), but the NAMES do /// not: the log keys shun rows by BINARY digest and the wire carries /// lowercase hex, so the hex is materialised once here rather than per /// round. `shun_hex` is its backing. shun_rows: []zit.sync.ShunRow, shun_hex: []u8, /// sync §7.7 L1428-1432 — the served repository's effective hash-policy, read /// off the same build as `shun` because it is the other half of the same /// receive-side admission predicate and must never be a build behind it. hash_policy: artifact.HashPolicy, /// Total plaintext this store has read on demand from the log, monotonic /// across `releaseTransient` — the work counter that makes "answering a round /// materialises only what it emits" testable (the log-backed twin of /// `MemStore.inflated_bytes`). Container reads are excluded, as they were /// there: this measures inflation, not verbatim retransmission. materialized_bytes: usize = 0, /// Set true when an on-demand `content()`/`wire()` read this round mapped a /// real error to null (a post-scan I/O fault or an OOM building the wire /// form) — NOT when a frame legitimately has no plaintext or no wire form. /// The vtable cannot signal an error, so the §8.2 clone pager would skip the /// null rid and still report the sequence complete, shipping a clone missing /// an artifact; `serveAnswer` reads this after the round and converts a set /// flag to `error.ServerStorage` so the client retries instead. Cleared with /// the round's memos by `releaseTransient`, so every round starts clean. read_fault: bool = false, /// draft-fossil-submanifests-00 §8.1 — this log's durable seen-submanifest /// indicator, read from the config frame of the SAME log generation this /// index describes. It arms the §7.2 send gate (`serveRound`), and it lives /// here rather than beside the long-lived `sync.Server` so it is re-read /// whenever the snapshot is: a push that lands the repository's first tree /// grows the log, the stat gate rebuilds the index, and the very next round /// serves that tree through an ARMED gate. Cached on a stale snapshot it /// would stay false for the life of the process. seen_submanifest: bool = false, /// draft-fossil-submanifests-00 §8.1 — this log's `enable-submanifests` /// setting, read from the same config frame and for the same reason: §7.2 /// makes the reply's `avoid-submanifests` pragma a function of it, so a /// stale copy would advertise the wrong policy for the life of the process. submanifests_enabled: bool = zit.sync.submanifest_emission_default, /// draft-fossil-submanifests-00 §6.3 / §6.4 — this log's pinned /// partition parameter and its auto-retune switch, read off the same config /// frame as the two above and for the same freshness reason: sync §9.3 /// serves them to a cloning peer, so an operator who re-pins k must be /// answered by the next round, not by the next process. submanifest_k: u8 = 0, submanifest_auto_retune: bool = false, /// The repository tier of the two push-policy settings, /// copied (owned) off the SAME per-build config read as the four facts /// above. Before this rode the snapshot, `serveAnswer` resolved the pair /// through a fresh `LogStore.openReadOnly` per setting per round -- a /// full frame-header scan of the log each time, which on a 1.17M-artifact /// store turned every round into ~11 seconds of dead re-scanning and a /// clone into half an hour. Freshness is unchanged: a settings /// write appends a config frame, the log grows, and the (size, mtime) /// snapshot key rebuilds this copy for the next round -- exactly the /// contract `seen_submanifest` above already relies on. An empty string /// is the repository having no opinion, as in `repositorySetting`. policy_binary_glob: []const u8 = "", policy_crlf_glob: []const u8 = "", /// A memoised wire form for one persisted rid: the owned §4.1 container and, /// for the delta form, the basis' owned hex name (DELTASRC) and the /// reconstructed target size (delta §3.2). Freed by `releaseTransient`. const WireHold = struct { container: []u8, delta: ?Delta = null, const Delta = struct { src_hex: []u8, target_size: u64 }; }; /// Build over a caller-owned `StorageFile` (the log borrows it; `fs` unused). fn open(gpa: std.mem.Allocator, storage_file: storage.StorageFile) logstore.OpenError!*LogIndexStore { const self = try gpa.create(LogIndexStore); errdefer gpa.destroy(self); self.gpa = gpa; self.fs = undefined; try self.build(storage_file); return self; } /// Build over an owned `FileStorage` bound to `file` (the serve path). `size` /// must equal `file`'s current length (`FileStorage.init`'s precondition). fn openFromFile(gpa: std.mem.Allocator, io: std.Io, file: File, size: u64) logstore.OpenError!*LogIndexStore { const self = try gpa.create(LogIndexStore); errdefer gpa.destroy(self); self.gpa = gpa; self.fs = storage.FileStorage.init(io, file, size); try self.build(self.fs.storageFile()); return self; } fn build(self: *LogIndexStore, storage_file: storage.StorageFile) logstore.OpenError!void { self.materialized_bytes = 0; // `self` came from `gpa.create` (undefined memory) and is filled field by // field, so the struct-declaration default `read_fault = false` never runs. // `serveAnswer` reads this flag on the FIRST round after a prime/rebuild, // before any `releaseTransient` clears it — leave it uninitialised and a // garbage-true value spuriously fails a healthy round. Zero it here. self.read_fault = false; // Same undefined-memory rule for the cone memo: garbage here reads as // a live classification and deinit frees wild pointers. self.cone_memo = null; self.cone_memo_digest = null; self.name_arena = std.heap.ArenaAllocator.init(self.gpa); errdefer self.name_arena.deinit(); self.by_name = .empty; errdefer self.by_name.deinit(self.gpa); self.appendix = repostate.MemStore.init(self.gpa); errdefer self.appendix.deinit(); self.log = try logstore.LogStore.openReadOnly(self.gpa, storage_file); errdefer self.log.deinit(self.gpa); // submanifests §8.1 — one config read per index build, never per round; // both settings come off the same frame. self.seen_submanifest = false; self.submanifests_enabled = zit.sync.submanifest_emission_default; self.submanifest_k = 0; self.submanifest_auto_retune = false; self.policy_binary_glob = ""; self.policy_crlf_glob = ""; if (try self.log.readConfig(self.gpa)) |*oc_const| { var oc = oc_const.*; defer oc.deinit(self.gpa); self.seen_submanifest = oc.config.seen_submanifest; self.submanifests_enabled = oc.config.enable_submanifests; self.submanifest_k = oc.config.submanifest_k; self.submanifest_auto_retune = oc.config.submanifest_auto_retune; self.policy_binary_glob = try self.gpa.dupe(u8, oc.config.binary_glob); errdefer self.gpa.free(self.policy_binary_glob); self.policy_crlf_glob = try self.gpa.dupe(u8, oc.config.crlf_glob); } // Function-scope: a failure in ANY later build step must free these // (a zero-length free of the "" default is a no-op). errdefer self.gpa.free(self.policy_binary_glob); errdefer self.gpa.free(self.policy_crlf_glob); // sync §3.3 L527-530 — the shun table, off this same build's open. Copied // rather than borrowed so the set has one shape on both the serve and the // client path (one hex→digest boundary, one lookup). self.shun = RepoShunSet.init(self.gpa); errdefer self.shun.deinit(); try self.shun.load(&self.log); // §9.3 r28 — the row view of that same table, off the same open, so the // group a peer requests can never be a build behind the filter. const rows = self.log.shunList(); var hex_bytes: usize = 0; for (rows) |r| hex_bytes += 2 * r.hash.len; self.shun_hex = try self.gpa.alloc(u8, hex_bytes); errdefer self.gpa.free(self.shun_hex); self.shun_rows = try self.gpa.alloc(zit.sync.ShunRow, rows.len); errdefer self.gpa.free(self.shun_rows); var hex_at: usize = 0; for (rows, self.shun_rows) |src_row, *dst| { const span = self.shun_hex[hex_at..][0 .. 2 * src_row.hash.len]; _ = std.fmt.bufPrint(span, "{x}", .{src_row.hash}) catch unreachable; hex_at += span.len; dst.* = .{ .hash = span, .epoch = src_row.epoch, .reason = src_row.reason }; } // sync §7.7 L1428-1432 — and the effective hash-policy off the same open. self.hash_policy = try effectiveHashPolicy(self.gpa, &self.log); const persisted = try self.log.ridsInPersistedOrder(self.gpa); defer self.gpa.free(persisted); // The `.config` test is a GUARD, not a live skip: a config frame is never // interned (`LogStore.scan` records only its offset, and `putConfig` appends // without touching `entries`), so `ridsInPersistedOrder` — which walks // `entries` — cannot yield one today. Kept because this counting loop and // the filling loop below must agree on `n` exactly, and because // `loadContentStore` states the same guard: the day a config frame becomes // interned, both stay correct instead of both silently naming it as an // artifact. var n: usize = 0; for (persisted) |lr| { if (self.log.kindOf(lr) != .config) n += 1; } self.rids = try self.gpa.alloc(logstore.Rid, n); errdefer self.gpa.free(self.rids); self.names = try self.gpa.alloc([]const u8, n); errdefer self.gpa.free(self.names); self.masked = try self.gpa.alloc(bool, n); errdefer self.gpa.free(self.masked); self.content_memo = try self.gpa.alloc(?[]u8, n); errdefer self.gpa.free(self.content_memo); self.wire_memo = try self.gpa.alloc(?WireHold, n); errdefer self.gpa.free(self.wire_memo); try self.by_name.ensureTotalCapacity(self.gpa, @intCast(n)); var a: usize = 0; for (persisted) |lr| { if (self.log.kindOf(lr) == .config) continue; // The one site the binary-log digest becomes the wire name (sync §3.3), // exactly as `loadContentStore` did — held to one place. var name_buf: [2 * digest_len]u8 = undefined; const wire_name = digestToWireName(self.log.hashOf(lr), &name_buf); const owned = try self.name_arena.allocator().dupe(u8, wire_name); self.rids[a] = lr; self.names[a] = owned; self.content_memo[a] = null; self.wire_memo[a] = null; // sync §9.5 L1988-1992 — the shun filter, applied to the enumeration // itself and off THIS iteration's index, so every parallel array plus // the mask derive from one loop and one counter (see `masked`). const is_masked = self.shun.digests.contains(self.log.hashOf(lr)); self.masked[a] = is_masked; // Hashes are unique per log entry (content-addressed intern), so no key // collides; adapter rid is a+1. A masked name is deliberately absent: // `lookup` answering null is what makes an inbound `gimme` for it hit // §3.3 L532-535's "If absent, silently skip" with no code of its own. if (!is_masked) self.by_name.putAssumeCapacity(owned, @intCast(a + 1)); a += 1; } // THE PARALLEL-ARRAY LENGTH GATE. `heldImpl`, `contentImpl` and `wireImpl` // bounds-test an incoming rid against `rids.len` ALONE and then index // `masked`/`content_memo`/`wire_memo` with it. Their alignment is what makes // adapter rid r name `rids[r-1]`; a length that disagreed would be an // out-of-bounds read, and in ReleaseFast/ReleaseSmall the slice bounds check // is gone too, so it would be a silent one: a garbage `masked` true withholds // a survivor (the silently short clone `read_fault` exists to prevent), a // garbage false serves an artifact the operator shunned (the §9.5 L1988-1992 // policy this whole filter exists to enforce). Both must be impossible in a // RELEASE build, where `std.debug.assert` compiles to nothing — so this is a // CHECKED error, paid once per build (five comparisons against an O(store) // walk), not an assertion. It also carries the fill loop's own `a == n`, which // is the same question asked of the counter rather than of the allocations. if (a != n or self.rids.len != n or self.names.len != n or self.masked.len != n or self.content_memo.len != n or self.wire_memo.len != n) return error.Corrupt; } /// Free every per-round transient memo, restoring the store to "index only, /// no materialised container bytes" — the shape between rounds. Every borrowed /// `content()`/`wire()` slice handed out this round is invalidated here, so it /// runs only once every borrow of the snapshot has ended (see /// `ServeCtx.releaseSnapshotPlaintext`). fn releaseTransient(self: *LogIndexStore) void { for (self.content_memo) |*slot| { if (slot.*) |b| self.gpa.free(b); slot.* = null; } for (self.wire_memo) |*slot| { if (slot.*) |wh| { self.gpa.free(wh.container); if (wh.delta) |d| self.gpa.free(d.src_hex); } slot.* = null; } // A round's fault is a property of that round's reads; clearing it here // (once every borrow of the round's memos has ended) lets a retry start // clean over the same snapshot. self.read_fault = false; } /// Total container/plaintext bytes currently memoised — a test/accounting /// probe. ~0 right after a prime (the whole point: no whole-store copy) and /// after `releaseTransient`; rises only as a round materialises what it emits. fn residentBytes(self: *const LogIndexStore) usize { var total: usize = 0; for (self.content_memo) |slot| { if (slot) |b| total += b.len; } for (self.wire_memo) |slot| { if (slot) |wh| total += wh.container.len; } return total; } fn deinit(self: *LogIndexStore) void { if (self.cone_memo) |*c| c.deinit(self.gpa); self.releaseTransient(); self.gpa.free(self.content_memo); self.gpa.free(self.wire_memo); self.gpa.free(self.rids); self.gpa.free(self.names); self.gpa.free(self.masked); self.by_name.deinit(self.gpa); self.name_arena.deinit(); self.appendix.deinit(); self.shun.deinit(); self.gpa.free(self.shun_rows); self.gpa.free(self.shun_hex); self.gpa.free(self.policy_binary_glob); self.gpa.free(self.policy_crlf_glob); self.log.deinit(self.gpa); } /// Free the store's internals and the heap box. `self` is invalid after. fn destroy(self: *LogIndexStore) void { const gpa = self.gpa; self.deinit(); gpa.destroy(self); } /// The ConeClassifier vtable target (sparse-cone RFC section 6.2). Any /// failure -- undecodable member, invalid cone, memo OOM -- answers null: /// the reply degrades to dense, never to an error (section 6.3). fn classifyCone( context: *anyopaque, members: []const []const u8, not_members: []const []const u8, ) ?*const std.DynamicBitSetUnmanaged { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (members.len == 0) return null; if (members.len + not_members.len > zit.cone.max_members) return null; var arena_state = std.heap.ArenaAllocator.init(self.gpa); defer arena_state.deinit(); const aa = arena_state.allocator(); var decoded: std.ArrayListUnmanaged([]const u8) = .empty; for (members) |member| { const buf = aa.alloc(u8, member.len) catch return null; const plain = artifact.decodeEscapes(member, buf); decoded.append(aa, plain) catch return null; } var decoded_not: std.ArrayListUnmanaged([]const u8) = .empty; for (not_members) |member| { const buf = aa.alloc(u8, member.len) catch return null; const plain = artifact.decodeEscapes(member, buf); decoded_not.append(aa, plain) catch return null; } var cone = zit.cone.Cone.init(self.gpa, decoded.items, decoded_not.items) catch return null; defer cone.deinit(self.gpa); const digest = cone.digest(self.gpa) catch return null; if (self.cone_memo_digest) |have| { if (std.mem.eql(u8, &have, &digest)) return &self.cone_memo.?.keep; } const classification = classifyKeepRids(self.gpa, self.store(), &cone) catch return null; if (self.cone_memo) |*old_memo| old_memo.deinit(self.gpa); self.cone_memo = classification; self.cone_memo_digest = digest; return &self.cone_memo.?.keep; } fn coneClassifier(self: *LogIndexStore) zit.sync.ConeClassifier { return .{ .context = @ptrCast(self), .vtable = &.{ .classify = classifyCone } }; } fn store(self: *LogIndexStore) repostate.ArtifactStore { return .{ .context = self, .vtable = &vtable }; } const vtable = repostate.ArtifactStore.VTable{ .intern = internImpl, .put = putImpl, .content = contentImpl, .hashOf = hashOfImpl, .lookup = lookupImpl, .count = countImpl, .wire = wireImpl, .retain = retainImpl, .held = heldImpl, }; fn wireFrameOf(wh: *const WireHold) repostate.WireFrame { if (wh.delta) |d| return .{ .container = wh.container, .delta = .{ .src_hash = d.src_hex, .target_size = d.target_size } }; return .{ .container = wh.container }; } fn countImpl(context: *anyopaque) usize { const self: *LogIndexStore = @ptrCast(@alignCast(context)); return self.rids.len + self.appendix.store().count(); } /// The adapter rid of a MASKED entry named `hash`, or null when this /// enumeration does not carry it under a mask. The mask deliberately keeps the /// name out of `by_name` (that absence IS §3.3 L532-535's "If absent, silently /// skip" on the gimme path), so the write doors below need this second, explicit /// question to avoid MINTING A SECOND IDENTITY for a name the enumeration /// already holds — see `putImpl`. A scan rather than a second name map, so /// "masked" has exactly one representation in this struct. /// /// THE TIME BOUND, in code rather than in prose. The walk is O(`rids.len`) per /// call and its callers are the vtable's write doors, whose call count belongs /// to a caller this struct does not bound: interning k names the log does not /// carry would otherwise pay k full walks plus a `std.mem.eql` per masked slot. /// An empty shun table is the case in which no slot can match, and it is also /// the overwhelmingly common one — a repository that has never shunned anything /// — so answering it in O(1) leaves the linear walk reachable only where a shun /// table exists to bound it. (Not reached on the serve path either way: every /// served write lands in the per-request `OverlayStore`.) fn maskedRidFor(self: *const LogIndexStore, hash: []const u8) ?repostate.Rid { if (self.shun.digests.count() == 0) return null; for (self.masked, 0..) |m, i| { if (!m) continue; if (std.mem.eql(u8, self.names[i], hash)) return @intCast(i + 1); } return null; } fn internImpl(context: *anyopaque, hash: []const u8) error{OutOfMemory}!repostate.Rid { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (self.by_name.get(hash)) |rid| return rid; // A MASKED name resolves to ITS OWN slot, not to a fresh appendix phantom: // the enumeration carries that frame at a fixed adapter rid (§8.2 L1504-1506 // gap), and it is already exactly what §3.3 L511-514 would create — a record // with the name and no content ("an igot HASH creates a phantom only when // HASH is absent"; `held` is false and `content`/`wire` are null here). A // second identity for one name would be the only way this store could hand // the walk two rids naming the same artifact. // // Not the mask's safety argument, which lives one layer up: this store never // sees a served `intern` at all — `respond` runs over an `OverlayStore` whose // `internImpl` probes `base.lookup` and, on the mask's null, interns into its // OWN round-local `added` store, which is discarded at round end. What keeps // that round's reply from begging for the name is `Server.shun` (§3.3 // L527-530), which `serveRound` sets from this same snapshot's table. if (self.maskedRidFor(hash)) |rid| return rid; return self.rids.len + try self.appendix.store().intern(hash); } fn lookupImpl(context: *anyopaque, hash: []const u8) ?repostate.Rid { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (self.by_name.get(hash)) |rid| return rid; if (self.appendix.store().lookup(hash)) |l| return self.rids.len + l; return null; } fn putImpl(context: *anyopaque, hash: []const u8, content_bytes: []const u8) error{OutOfMemory}!repostate.Rid { const self: *LogIndexStore = @ptrCast(@alignCast(context)); // A persisted hash is content-addressed and already carries these bytes, // so `put` is idempotent and `content(rid)` still reads them from the log // — the serve path never reaches this arm (writes go to the overlay). if (self.by_name.get(hash)) |rid| { // Parity with `MemStore.putImpl`: a repeat put must carry identical // bytes (a differing answer is a hash collision). Verified only under // runtime safety, and only when the log frame actually inflates — a // read fault leaves `content` null and the assert is skipped rather // than firing spuriously. Inflate through a THROWAWAY read (not // `contentImpl`), so this assertion path never memoises a container or // bumps `materialized_bytes`/`residentBytes` — the probes the round // tests assert on stay untouched by an assert that never runs on serve. if (std.debug.runtime_safety) { if (self.log.content(self.gpa, self.rids[@intCast(rid - 1)]) catch null) |existing| { defer self.gpa.free(existing); std.debug.assert(std.mem.eql(u8, existing, content_bytes)); } } return rid; } // A MASKED name is not in `by_name`, so without this arm a `put` of one // would fall through to the appendix and create a SECOND, HELD entry for it: // `held` true, `content` the pushed bytes, and `lookup` resolving the masked // name to it — the §3.2 walk would then announce the shunned name and the // clone pager would emit its `cfile`, undoing the whole §9.5 L1988-1992 // filter through a write. The mask outranks the write: the frame keeps its // one adapter rid and stays a §8.2 L1504-1506 gap. // // A DELIBERATE narrowing of the store contract's "after `put`, `content` is // readable" — `content(rid)` stays null here. The alternative is a name this // server has been told never to send becoming servable because a peer pushed // it back, and admitting a shunned target is a receive-side decision (§7.7 // L1437-1440: "consumed and discarded … makes no repository mutation"), not // something a store adapter may take by accident. // // The narrowing cannot be SIGNALLED — the vtable's error set is // `error{OutOfMemory}` — so what keeps it safe is that no caller is in a // position to be surprised by it. On serve that holds because `respond` runs // over a per-request `repostate.OverlayStore` and every served write lands // THERE, never on this base. That is a property of the serve path rather than // of this function, so it is PINNED BY A TEST at this layer ("a served round's // writes land in the round-local overlay — the base enumeration's write doors // are never reached") instead of resting on a claim this comment makes about // code two modules away. A caller that wires this store in as a WRITABLE store // is outside that proof and must read this arm before relying on `put`. if (self.maskedRidFor(hash)) |rid| return rid; return self.rids.len + try self.appendix.store().put(hash, content_bytes); } fn hashOfImpl(context: *anyopaque, rid: repostate.Rid) []const u8 { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (rid <= self.rids.len) return self.names[@intCast(rid - 1)]; return self.appendix.store().hashOf(rid - self.rids.len); } fn heldImpl(context: *anyopaque, rid: repostate.Rid) bool { const self: *LogIndexStore = @ptrCast(@alignCast(context)); // Every persisted (non-phantom, non-config) frame carries a form to serve // verbatim, so it is held — answered from the index with no frame read. // A MASKED frame (sync §9.5 L1988-1992) is the exception: it is a §8.2 // L1504-1506 gap, so it is not held and §3.2's walk never announces it. // This line is the one place the send side genuinely disagreed with the // log: `LogStore.content`/`wire` already read a shunned frame as ABSENT, // while this answered true from the index with no frame read — so the // inventory walk advertised an artifact no round could then send, and (with // the name still in `by_name`) `emitCloneReply`'s D3 basis proof could reach // a resolvable rid whose two forms were both null. `LogStore.shunned`'s doc // names exactly this disagreement as a SYNC decision to settle; §9.5's // "every eligible holding after the Section 3.3 shun … filters" settles it. if (rid <= self.rids.len) return !self.masked[@intCast(rid - 1)]; return self.appendix.store().held(rid - self.rids.len); } fn retainImpl(context: *anyopaque, rid: repostate.Rid, ingest: repostate.WireIngest) error{OutOfMemory}!void { const self: *LogIndexStore = @ptrCast(@alignCast(context)); // Advisory (sync §7.2): a persisted rid already carries a stored form, so // first-form-wins makes this a no-op — the log is the retained form. if (rid <= self.rids.len) return; return self.appendix.store().retain(rid - self.rids.len, ingest); } fn contentImpl(context: *anyopaque, rid: repostate.Rid) ?[]const u8 { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (rid > self.rids.len) return self.appendix.store().content(rid - self.rids.len); const i: usize = @intCast(rid - 1); // sync §9.5 L1988-1992 — a masked frame has nothing to hand out. Checked // before the memo so the mask cannot be bypassed by a slot some earlier // round filled, and it is a plain null: a policy gap, never a read fault. // // `LogStore.content` ALREADY answers null for a shunned entry (its own // rid-keyed read mask), so today this line changes no answer: it is a // structural guard, not covered behaviour. Measured, not assumed — deleting // this line leaves the whole suite green, because the read one line below // returns the same null. What IS covered, by "the adapter owns its content // mask", is the surrounding fact the guard exists to protect: the frame's // plaintext is still on disk and still readable through // `LogStore.contentUnmasked`, so the null is a policy refusal over present // bytes rather than an absence. That test is what would turn red the day the // two layers were consolidated into one. It is here // because the adapter must OWN its mask rather than inherit it. §7.7 // L1444-1449 wants a held-but-shunned artifact to remain usable as a delta // BASIS — "For an admitted target whose DELTASRC name is in the repository // shun table, the receiver does not apply admission independently to that // source: content still held locally MAY serve as the delta basis" — and the // only reason that works today is that basis resolution walks byte offsets // and never re-enters `content`. The moment that basis path is given an // explicitly unmasked door — the repair `LogStore.shunned`'s own doc // contemplates — the log's mask stops covering this one, and the send-side // exclusion must not evaporate with it. if (self.masked[i]) return null; if (self.content_memo[i]) |b| return b; // Read the frame on demand and memoise for the round. The vtable cannot // signal an error, so a read fault is recorded on `read_fault` and // returned as null; `serveAnswer` turns a set flag into // `error.ServerStorage` so the client retries rather than accepting a // silently short clone. A frame that legitimately has no plaintext is a // plain null and does NOT set the flag. // // THIS IS THE PRIMARY CORRUPTION CHANNEL IN SERVE, and it did not used // to be. `openReadOnly` once CRC-verified every payload while building // the snapshot, which meant a damaged store failed to open at all and // this catch saw only genuine post-scan faults -- transient EIO, or rot // arriving after a clean scan. Both open paths now scan header-only and // `readFrameAt` CRC-checks every frame it reads instead, so ordinary // stored corruption reaches serve HERE, on the read, rather than at // snapshot build. // // That is a better shape, not a worse one: one bad frame used to make // the server refuse to serve anything, and now it fails only the round // that touches the damage while every healthy artifact still moves. But // it means this arm is live traffic rather than a rare event, and // anything reasoning about how often it fires should start from that. const maybe = self.log.content(self.gpa, self.rids[i]) catch { self.read_fault = true; return null; }; const bytes = maybe orelse return null; self.content_memo[i] = bytes; self.materialized_bytes += bytes.len; return bytes; } fn wireImpl(context: *anyopaque, rid: repostate.Rid) ?repostate.WireFrame { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (rid > self.rids.len) return self.appendix.store().wire(rid - self.rids.len); const i: usize = @intCast(rid - 1); // sync §9.5 L1988-1992 — same gap as `contentImpl`, checked ahead of the // memo, and owned here for the same reason (see there). if (self.masked[i]) return null; if (self.wire_memo[i]) |*wh| return wireFrameOf(wh); // A frame read fault is a real error recorded on `read_fault` so // `serveAnswer` can fail the round; a `.raw`/phantom frame legitimately // has no wire form (the responder compresses `content()` instead) and // yields a plain null that does NOT set the flag. As at `contentImpl`, // this arm now sees ordinary stored corruption rather than only // post-scan faults -- no open path CRC-verifies payloads any more. var w = (self.log.wire(self.gpa, self.rids[i]) catch { self.read_fault = true; return null; }) orelse return null; switch (w) { .full => |c| { self.wire_memo[i] = .{ .container = c }; }, .delta => |d| { // sync §9.5 L1988-1992 — the delta form is servable only while its // basis is. `zit shun --defer` records the row without rewriting // the generation, so a SURVIVING frame here can be a delta rooted // on a MASKED one; emitting its `cfile` would ship a DELTASRC this // server never sends, leaving the peer a permanently dangling delta // whose basis it gimmes and is silently skipped on (§3.3 L532-535) // — corrupt rather than absent, and `emitCloneReply`'s D3 basis // lookup would trap under `runtime_safety` on the way. Withhold the // wire form instead; the responder's §7.2 compress-on-the-fly arm // then emits the FULL form, and `log.content` reconstructs through // the frame chain by BYTE OFFSET, which the name-level mask does // not reach. A plain null, not a read fault: policy, not I/O. // // Asked as the SAME predicate the fill loop applied — the raw digest // against the shun table — so "masked" has one spelling in this // struct. (`by_name`-absence would answer identically for a log // delta, whose basis is always a non-config frame in the // enumeration, but it would make the exclusion mean two things and // pay a hex encode plus a map lookup to say the second one.) if (self.shun.digests.contains(d.baseHash())) { w.deinit(self.gpa); return null; } // The basis precedes this frame in the log, so its wire name is the // hex of the base digest the wire read carried (delta §3.2 target // size alongside). var src_buf: [2 * digest_len]u8 = undefined; const src_hex = digestToWireName(d.baseHash(), &src_buf); const owned_src = self.gpa.dupe(u8, src_hex) catch { // Failing to produce the wire form here would silently drop a // held delta from the clone; record it so the round errors. w.deinit(self.gpa); self.read_fault = true; return null; }; self.wire_memo[i] = .{ .container = d.container, .delta = .{ .src_hex = owned_src, .target_size = d.target_size } }; }, } return wireFrameOf(&self.wire_memo[i].?); } }; /// Snapshot every stored artifact from the persisted log into `ms`, skipping /// config frames (not artifacts). A content-bearing frame becomes a held entry; /// a content-absent frame becomes a phantom — exactly the held/phantom split /// `respond` walks (sync §3.2/§3.3). /// /// The log keys frames by the RAW binary digest, but the sync wire names an /// artifact by that digest's lowercase hex (sync §3.3 — a `file`/`igot`/`gimme` /// HASH is 40/64 hex characters). The store the responder walks must therefore /// hold the hex name, or `respond` would emit non-hex bytes that break the /// peer's card tokenizer. Every hash crossing this boundary is hex-encoded once, /// here — the single place binary-log identity becomes wire identity. fn loadContentStore( gpa: std.mem.Allocator, log: *logstore.LogStore, ms: *repostate.MemStore, ) serve_mod.AnswerError!void { const rids = log.ridsInPersistedOrder(gpa) catch return error.OutOfMemory; defer gpa.free(rids); for (rids) |rid| { if (log.kindOf(rid) == .config) continue; var name_buf: [2 * digest_len]u8 = undefined; const name = digestToWireName(log.hashOf(rid), &name_buf); // sync §4.1/§7.2 — copy the stored container into the snapshot verbatim // so the clone serves it without inflating here and re-compressing at // serve. `wire()` is null only for a `.raw` frame or a phantom; those // fall back to the plaintext path below. A `.zlib` frame carries the // full form; a `.delta` frame carries the stored delta container plus // its basis name and reconstructed size (the §7.2 USIZE, delta §3.2). var maybe_wire = log.wire(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; if (maybe_wire) |*w| { defer w.deinit(gpa); const recorded: bool = switch (w.*) { .full => |container| blk: { _ = ms.putWire(name, container) catch return error.OutOfMemory; break :blk true; }, .delta => |d| blk: { // sync §3.3 L527-530 — the delta form is usable only while its // basis is. `zit shun --defer` records the row without // rewriting the generation, so a SURVIVING frame here can be a // delta rooted on a shunned one; that basis reads null through // this loop's own masked doors and interns as a PHANTOM, so // recording the delta would leave an entry the store calls held // (`wire != null`) whose `content()` can never resolve. §3.2's // igot walk would then announce a name no `gimme` can ever be // answered with, and the push would spin the round with nothing // to send. Fall through to the plaintext path instead: the // artifact itself is NOT shunned, so `log.content` reads it // through the frame chain by BYTE OFFSET — which the name-level // mask does not reach — and it is snapshotted as a held full // form. Exactly `LogIndexStore.wireImpl`'s masked-basis guard, // stated for the client's own holdings snapshot. const base = d.baseHash(); if (logstore.isCompleteDigest(base.len) and log.isShunned(base)) break :blk false; // The basis frame precedes this one in the log (delta_of < // offset), so it was already inserted at an earlier rid; its // wire name is the hex of the base's binary digest. var src_name_buf: [2 * digest_len]u8 = undefined; const src_name = digestToWireName(base, &src_name_buf); _ = ms.putWireDelta(name, d.container, src_name, d.target_size) catch return error.OutOfMemory; break :blk true; }, }; if (recorded) continue; } const maybe = log.content(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; if (maybe) |bytes| { defer gpa.free(bytes); _ = ms.store().put(name, bytes) catch return error.OutOfMemory; } else { _ = ms.store().intern(name) catch return error.OutOfMemory; } } } // --------------------------------------------------------------------------- // `zit clone URL DIR` — the interop milestone: a real client converging against // a real `zit serve` over HTTP. Create a fresh repo at DIR, open an HTTP // transport to URL, pump the sans-IO clone client (`sync.Client`) to §3.7 // convergence through `sync_driver.runSync`, then persist the received // artifacts into DIR's on-disk log so the clone is durable. Anonymous only. // --------------------------------------------------------------------------- /// §9.1 — a fixed placeholder repository identity emitted in the client's /// pull/push operation cards. Neither a fresh clone nor a pulled repository /// carries a persistent SERVERCODE/PROJECTCODE of its own yet (real per-repo /// sync identity is future work); the stateless server echoes the peer's codes /// and never gates on a configured project code (sync §9.1), so any well-formed /// 16-hex pair carries a content-only exchange. Shared by clone and pull. const sync_servercode = "0000000000000000"; const sync_projectcode = "0000000000000000"; /// §10.1 — the release integer the sync client announces as `pragma /// client-version`. A fixed build identity, matching `serve`'s own version /// constant style. (`announcedVersion` raises what actually goes on the wire — /// submanifests §7.2.) const sync_client_version: u32 = 22700; /// §10.1 r11/r12 — the DATED client-version pragma every zit run announces. /// /// The DATE and TIME are not decoration: §10.1 r11/r12 make the server's /// `server-version` reply conditional on them ("at least five tokens total"), /// and submanifests §7.2 gates every upload on "the remote peer's announced /// version". An undated pragma therefore leaves `Client.peer.version` at 0 for /// the whole session — below every threshold — so an armed client would refuse /// its own trees to a server that would have announced support if only it had /// been asked. Soliciting the reply is what makes the gate a peer-version gate /// rather than a permanent refusal. Fossil sends the dated form too (§A.4). const sync_client_version_info: zit.sync.VersionInfo = .{ .version = sync_client_version, .datetime = .{ .date = 20260715, .time = 0 }, }; /// An explicit bound on CONSECUTIVE ROUNDS THAT MOVE NO BYTES (TigerStyle: /// every loop is bounded). See `sync_driver.runSync` for why the bound counts /// stalls rather than rounds: a total-round cap multiplies out against the /// per-round batch size into an accidental ceiling on repository size, which a /// large healthy repository trips while converging perfectly. /// /// This value bounds only unproductive looping, so it is a statement about how /// long a peer may accomplish nothing — not about how much there is to /// transfer. §3.7's longest by-design stall is the unversioned three-round /// grace window ((d)(2)); config exchange (f) and private-content discovery (c) /// are each shorter. 16 clears every one of them several times over while still /// failing a stuck peer in well under a second of round trips. const max_stalled_sync_rounds: usize = 16; /// The canonical form of a clone URL for persistence: rebuilt from parsed /// components — scheme://[user@]host[:port]/path[?query] — so a userinfo /// password is excluded BY CONSTRUCTION: a component the rebuild never names /// cannot survive into the stored value, even if a future parser change adds /// fields (fossil url.c's canonical form works the same way). The username is /// kept, matching fossil. The REQUESTED URL is what canonicalizes, never the /// redirect-resolved one: an alias in front of a moving host is load-bearing, /// and the redirect chain re-resolves on every sync anyway. pub fn canonicalRemoteUrl(gpa: std.mem.Allocator, url: []const u8) error{ OutOfMemory, InvalidUrl }![]u8 { const uri = std.Uri.parse(url) catch return error.InvalidUrl; const bytesOf = struct { fn f(c: std.Uri.Component) []const u8 { return switch (c) { .raw => |r| r, .percent_encoded => |p| p, }; } }.f; var out: std.ArrayList(u8) = .empty; errdefer out.deinit(gpa); try out.appendSlice(gpa, uri.scheme); try out.appendSlice(gpa, "://"); if (uri.user) |u| { try out.appendSlice(gpa, bytesOf(u)); try out.append(gpa, '@'); } const host = uri.host orelse return error.InvalidUrl; try out.appendSlice(gpa, bytesOf(host)); if (uri.port) |p| try out.print(gpa, ":{d}", .{p}); const path = bytesOf(uri.path); if (path.len == 0) try out.append(gpa, '/') else try out.appendSlice(gpa, path); if (uri.query) |q| { try out.append(gpa, '?'); try out.appendSlice(gpa, bytesOf(q)); } return out.toOwnedSlice(gpa); } test "canonicalRemoteUrl: password excluded by construction, username kept" { const gpa = std.testing.allocator; const cases = [_]struct { in: []const u8, want: []const u8 }{ .{ .in = "http://user:secret@host/repo", .want = "http://user@host/repo" }, .{ .in = "http://user@host:8080/repo", .want = "http://user@host:8080/repo" }, .{ .in = "https://host", .want = "https://host/" }, .{ .in = "https://host/a/b?x=1#frag", .want = "https://host/a/b?x=1" }, }; for (cases) |c| { const got = try canonicalRemoteUrl(gpa, c.in); defer gpa.free(got); try std.testing.expectEqualStrings(c.want, got); try std.testing.expect(std.mem.indexOf(u8, got, "secret") == null); } try std.testing.expectError(error.InvalidUrl, canonicalRemoteUrl(gpa, "not a url")); } /// `zit clone URL DIR` — thin wiring over three deep modules: `transport` /// (HTTP/IO), `sync.Client` + `sync_driver.runSync` (sans-IO §3.7 loop), and /// `LogStore` (persistence). The URL is used verbatim (sync §4 r1 — the client /// never appends /xfer). All-or-nothing failures surface an honest exit-1 /// diagnostic; a bad URL scheme is rejected before any repo is created. pub fn runClone( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { return runCloneObserved(gpa, io, arena, verb_args, streams, null); } /// `runClone` with benchmark-owned phase attribution. Sync §8.2's project-code /// adoption remains in the observed config phase; a null observer is the /// ordinary CLI path, which reads no clock and owns no counters. pub fn runCloneObserved( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, observer: ?*const CloneObserver, ) cli.Exit { const exit = cloneObserved(gpa, io, arena, verb_args, streams, observer); if (exit != .ok) return exit; // A clone is followed by the `open` the operator would type next // (Fossil 2.x), run inside DIR by this same binary so the two verbs stay // one implementation; --no-open keeps the bare store. It runs only here, // after cloneObserved returned and released its exclusive store lock — // the child takes that same lock. The child opens the same cone the clone // elided to: members from the flags and from --cone-file (read here, // relative to this cwd) are passed explicitly. const parsed = switch (cli.parseCloneArgs(verb_args)) { .ok => |p| p, else => return exit, }; if (parsed.help or !parsed.open) return exit; var members = parsed.cone_members; var len = parsed.cone_len; var not_members = parsed.cone_not_members; var not_len = parsed.cone_not_len; if (parsed.cone_file) |cf| { if (cli.appendConeFile(arena, io, cf, &members, len, ¬_members, not_len)) |lens| { len = lens.members; not_len = lens.not_members; } } return autoOpen(gpa, io, arena, streams, parsed.dir, members[0..len], not_members[0..not_len]); } /// Spawn this same binary as `zit open` inside the freshly cloned DIR. fn autoOpen( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, dir: []const u8, cone: []const []const u8, cone_not: []const []const u8, ) cli.Exit { streams.out.flush() catch {}; streams.err.flush() catch {}; const exe = std.process.executablePathAlloc(io, gpa) catch { streams.err.writeAll("zit clone: cloned, but cannot locate zit to open the working tree; run 'zit open' inside the directory\n") catch {}; return .failure; }; defer gpa.free(exe); var argv = std.ArrayList([]const u8).empty; argv.append(arena, exe) catch return .failure; argv.append(arena, "open") catch return .failure; for (cone) |m| argv.append(arena, std.fmt.allocPrint(arena, "--cone={s}", .{m}) catch return .failure) catch return .failure; for (cone_not) |m| argv.append(arena, std.fmt.allocPrint(arena, "--cone-not={s}", .{m}) catch return .failure) catch return .failure; var child = std.process.spawn(io, .{ .argv = argv.items, .cwd = .{ .path = dir }, }) catch { streams.err.print("zit clone: cloned, but 'zit open' could not be started; run it inside '{s}'\n", .{dir}) catch {}; return .failure; }; const term = child.wait(io) catch return .failure; return switch (term) { .exited => |code| if (code == 0) .ok else .failure, else => .failure, }; } fn cloneObserved( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, observer: ?*const CloneObserver, ) cli.Exit { const parsed = switch (cli.parseCloneArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "clone", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "clone", .extra_argument, tok), .no_url => return cli.usageError(streams, "clone", .no_url, ""), .no_dir => return cli.usageError(streams, "clone", .no_dir, ""), .empty_value => |tok| return cli.usageError(streams, "clone", .empty_value, tok), .too_many_cones => |tok| return cli.usageError(streams, "clone", .too_many_cones, tok), }; if (parsed.help) { streams.out.writeAll(clone_help) catch {}; return .ok; } // Open the transport first: a malformed URL fails before any repo lands on // disk (nothing half-made to roll back). `open` performs no network I/O. observeClonePhase(observer, .transport, .begin); var tr = transport.Transport.open(io, gpa, parsed.url) catch |e| { observeClonePhase(observer, .transport, .end); return syncOpenFail(streams, "clone", parsed.url, e); }; observeClonePhase(observer, .transport, .end); defer tr.close(); // Create a fresh `.zit` at DIR by reusing the init sequence, quietly (its // own success line is suppressed; clone prints its own). A pre-existing repo // there is init's "already exists" error, propagated verbatim. observeClonePhase(observer, .initialize, .begin); const init_rc = runInit(gpa, io, arena, &.{ "-q", "--", parsed.dir }, streams); observeClonePhase(observer, .initialize, .end); if (init_rc != .ok) return init_rc; // Converge an in-memory store against the peer, then persist it into DIR. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); // §9.1 — the clone client's own instance identity, derived from the // destination it is creating (informational; a fresh instance is exactly // what a clone is). const clone_code = blk: { const cwd = (cli.currentPathAlloc(arena, io) catch return stateOom(streams, "clone")) orelse break :blk zit.sync.deriveServerCode(&.{parsed.dir}); const abs = std.fs.path.resolve(arena, &.{ cwd, parsed.dir }) catch return stateOom(streams, "clone"); break :blk zit.sync.deriveServerCode(&.{abs}); }; // Live per-round progress, on stderr and ONLY when stderr is a // terminal: an operator watching a slow clone sees each round land instead // of silence, while piped stderr (scripts, tests) stays byte-identical to // before. The line is \r-overwritten in place — never fossil's // percent-flood — and finished with one accounting line when the exchange // ends, either way it ends. const stderr_tty = std.Io.File.stderr().isTty(io) catch false; var progress: CloneProgress = .{ .err = streams.err }; const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null; observeClonePhase(observer, .exchange, .begin); var clone_members = parsed.cone_members; var clone_len = parsed.cone_len; var clone_not_members = parsed.cone_not_members; var clone_not_len = parsed.cone_not_len; if (parsed.cone_file) |cf| { const lens = cli.appendConeFile(arena, io, cf, &clone_members, clone_len, &clone_not_members, clone_not_len) orelse { streams.err.print("zit clone: cannot read --cone-file '{s}' (or it overflows the {d}-member cap)\n", .{ cf, zit.cone.max_members }) catch {}; return .failure; }; clone_len = lens.members; clone_not_len = lens.not_members; } // The destination log, open BEFORE the exchange: a dense clone // persists each round as it converges instead of holding the repository // resident, so the log has to exist while rounds are still arriving. A // failed exchange leaves DIR holding a partial, uncommitted `.zit` the // operator deletes with the directory — the same recovery as before, with // different debris. var target = std.Io.Dir.cwd().openDir(io, parsed.dir, .{}) catch return renderStorage(streams, parsed.dir, "cannot reopen the clone target directory"); defer target.close(io); var file = cli.openStoreAppend(io, target) catch return renderStorage(streams, parsed.dir, "cannot write the cloned repository"); defer file.close(io); const st = file.stat(io) catch return renderStorage(streams, parsed.dir, "cannot write the cloned repository"); var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"), }; defer log.deinit(gpa); // Stream the persist unless a cone is requested. A cone-scoped // clone must classify its keep set over the CONVERGED store (sparse-cone // RFC section 6.3), so it keeps the batched path; every dense clone drains // per round and releases what it wrote, bounding peak memory by the round // batch plus the deltas whose basis has not arrived (git index-pack's // shape, applied to fossil's reverse-delta wire). const cone_scoped = clone_len + clone_not_len != 0; // Every cluster this clone writes, for the index it creates. var clone_clusters: std.ArrayListUnmanaged([]const u8) = .empty; defer { for (clone_clusters.items) |n| gpa.free(n); clone_clusters.deinit(gpa); } var stream_pass: ?PersistPass = if (cone_scoped) null else PersistPass.init(gpa, &log, ms.store(), null, &ms, 0, &clone_clusters) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"), }; defer if (stream_pass) |*pass| pass.deinit(); const drain: ?sync_driver.RoundDrain = if (stream_pass) |*pass| .{ .ctx = pass, .run = PersistPass.drainErased } else null; var peer_error: PeerErrorCopy = .{}; var outcome = cloneRun(gpa, &tr, &ms, &clone_code, sink, drain, clone_members[0..clone_len], clone_not_members[0..clone_not_len], &peer_error) catch |e| { progress.finishLine(); observeClonePhase(observer, .exchange, .end); // A peer that stated its refusal is quoted; anything else keeps the // generic mapping (sparse-cone RFC section 5.3 diagnostic path). if (e == error.FatalProtocolError) { if (peer_error.text()) |text| { streams.err.print("zit clone: the peer refused: {s}\n", .{text}) catch {}; return .failure; } } return syncRunFail(streams, "clone", parsed.url, e, tr.last_status); }; // §9.3 r28 — the latched shun rows are the one thing this outcome owns. defer outcome.deinit(gpa); progress.finishLine(); if (stderr_tty) { streams.err.print( "clone exchange: {d} rounds, {d} protocol bytes received, {d} sent\n", .{ outcome.rounds, outcome.received_bytes, outcome.sent_bytes }, ) catch {}; } observeClonePhase(observer, .exchange, .end); // Sparse-cone RFC section 6.3: a cone-scoped clone filters LOCALLY -- // correct against every peer, including cone-ignorant ones. The keep set // is classified over the converged in-memory store before any frame is // written; nothing is discarded silently (the count prints below). var clone_cone: ?zit.cone.Cone = null; defer if (clone_cone) |*c| c.deinit(gpa); var keep_bits: ?SparseClassification = null; defer if (keep_bits) |*k| k.deinit(gpa); if (cone_scoped) { clone_cone = zit.cone.Cone.init(gpa, clone_members[0..clone_len], clone_not_members[0..clone_not_len]) catch |e| return cloneConeFail(streams, e); keep_bits = classifyKeepRids(gpa, ms.store(), &clone_cone.?) catch return stateOom(streams, "clone"); } const clone_keep: ?*const std.DynamicBitSetUnmanaged = if (keep_bits) |*k| &k.keep else null; observeClonePhase(observer, .persist, .begin); const persisted = blk: { if (stream_pass) |*pass| { // Streaming: the rounds already drained; settle any straggler the // last round interned, then the leftovers and the bookkeeping. const summary = res: { pass.drainNew() catch |e| break :res @as(PersistError!PersistSummary, e); break :res pass.finish(); }; break :blk summary; } break :blk persistStoreOpen(gpa, &log, ms.store(), clone_keep, 0, &clone_clusters); } catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"), }; const written = persisted.written; observeClonePhase(observer, .persist, .end); // §8.2 and submanifests §6.3/§8.1 — adopt every origin-owned setting in // one field-preserving overlay. Missing fields leave `init`'s values // untouched, and one combined frame avoids intermediate clone config state. observeClonePhase(observer, .config, .begin); const project_code = outcome.projectCode(); // Save the clone URL as the default remote (fossil's behavior), unless // --once. The canonical rebuild strips a userinfo password by construction; // `zit remote` remains the explicit override and the audit view. The // InvalidUrl arm is unreachable in practice — `transport.open` parsed this // same URL — and degrades to not-saving rather than failing a completed // clone. const saved_remote: ?[]const u8 = if (parsed.once) null else canonicalRemoteUrl(arena, parsed.url) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.InvalidUrl => null, }; const cloned_patch: ConfigPatch = .{ .project_code = if (project_code.len > 0) project_code else null, .remote_url = saved_remote, .enable_submanifests = outcome.submanifest.enable, .submanifest_k = outcome.submanifest.k, .submanifest_auto_retune = outcome.submanifest.auto_retune, }; if (cloned_patch.any()) { updateConfigOpen(gpa, &log, cloned_patch) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"), }; } // §9.3 r28 — adopt the origin's shun table. Written BEFORE the commit // below, so a clone is durable with its tombstones or not at all: a // repository that came up holding the content but not the record of its // suppression is the exact state this carriage exists to prevent. // // The wire carries hex; the log keys by binary digest. A name that will not // decode is skipped rather than fatal — the artifacts are already down, and // failing a finished clone over one unparseable row helps nobody. The count // is reported so a skip is never silent. var shun_adopted: usize = 0; var shun_skipped: usize = 0; for (outcome.shun_rows) |row| { var digest_buf: [32]u8 = undefined; if (row.hash.len != 40 and row.hash.len != 64) { shun_skipped += 1; continue; } const digest = digest_buf[0 .. row.hash.len / 2]; _ = std.fmt.hexToBytes(digest, row.hash) catch { shun_skipped += 1; continue; }; log.putShun(digest, row.reason, row.epoch) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), else => { shun_skipped += 1; continue; }, }; shun_adopted += 1; } if (shun_adopted != 0 or shun_skipped != 0) { streams.out.print("adopted {d} shun row{s} from the origin", .{ shun_adopted, plural(shun_adopted) }) catch {}; if (shun_skipped != 0) streams.out.print("; {d} unreadable and skipped", .{shun_skipped}) catch {}; streams.out.writeAll("\n") catch {}; } // Every persisted artifact and the optional combined config frame must be // durable before the cache observes this log or clone reports success. log.commit() catch return renderStorage(streams, parsed.dir, "cannot write the cloned repository"); refreshClusterSidecar(gpa, io, target, &log, clone_clusters.items, true) catch return stateOom(streams, "clone"); observeClonePhase(observer, .config, .end); // The committed writable log is still open, so reuse its derivation path // directly instead of reopening `.zit` solely to prime the disposable cache. // Say so first: on a large store this phase is minutes of derive, and an // operator watching a silent clone cannot tell transfer from derivation. streams.out.print("clone from {s}: transferred {d} artifact{s} in {d} round{s}; deriving caches\n", .{ parsed.url, written, plural(written), outcome.rounds, plural(outcome.rounds) }) catch {}; observeClonePhase(observer, .warm_cache, .begin); Repository.warmCacheFromLog(gpa, io, target, &log); observeClonePhase(observer, .warm_cache, .end); if (clone_cone) |*c| { // Policy lands with the disposable tier (RFC section 5.2 loss rule). zit.repocache.writeStoreCone(gpa, io, target, c); streams.out.print( "cloned {d} artifacts into {s} in {d} rounds ({d} elided by the cone)\n", .{ written, parsed.dir, outcome.rounds, persisted.elided }, ) catch {}; } else { streams.out.print("cloned {d} artifact{s} into {s} in {d} round{s}\n", .{ written, plural(written), parsed.dir, outcome.rounds, plural(outcome.rounds) }) catch {}; } if (project_code.len > 0) streams.out.print("project-code: {s}\n", .{project_code}) catch {}; reportUvSkipped(streams, "clone", outcome.uv_skipped); // submanifests §7.1/§7.3 — a clone converges on the cfile cursor, so it // copies the peer's holdings faithfully, INCLUDING an incomplete tree // closure the peer itself was carrying. The phantoms the persist hook minted // say so; report them, because a check-in whose file set cannot be produced // (§5.2) is not a converged clone, and "exit 0 with nothing said" is exactly // how a repository stays quietly broken. Reported AFTER the success line and // AFTER the repository is fully written: a partial clone is still progress, // and this is the same shape `pull` already uses for artifacts it did not // obtain. if (persisted.submanifest_gaps > 0) { streams.err.print( "zit clone: {d} sub-manifest references are missing; the cloned repository cannot enumerate every check-in\n", .{persisted.submanifest_gaps}, ) catch {}; return .failure; } return .ok; } /// Render an `open` diagnostic to stderr and fail. Best-effort on the write: a /// diagnostic that cannot be printed must not change the exit code. fn openDiag(s: cli.Streams, msg: []const u8) cli.Exit { s.err.print("zit open: {s}\n", .{msg}) catch {}; return .failure; } /// Map a materialization fault to its `open` diagnostic, one arm per /// `update.ApplyError` variant and NO `else`: a new variant must fail this /// compile and force a decision here, exactly as it does in `updateApplyFail`. /// /// Only `InvalidCheckinId` and `CheckinNotFound` are genuinely unreachable — /// `open` hands the engine a hash it just resolved out of this repository's own /// derived state, never a caller-supplied name — and even those get honest /// prose rather than a corruption claim. Everything else is reachable on an /// intact repository: the content/manifest arms describe a repository that /// really is damaged, while `SymlinkUnsupported` / `InvalidPath` / /// `ReservedPath` are raised while planning the TARGET manifest, so a clone of /// any repository containing a symlink or a path this platform cannot represent /// reaches them with nothing corrupt at all. Reporting those as corruption /// would be a false accusation against a healthy repository; the prose mirrors /// `updateApplyFail`'s. /// Decode a lowercase-hex artifact name to its raw digest for a presence probe. /// Null for anything that cannot be a stored name, which the caller reads as /// "not a name this store could hold" and therefore not a missing-content claim. fn openDecodeName(name: []const u8, out: *[logstore.max_hash_len]u8) ?[]const u8 { if (name.len == 0 or name.len % 2 != 0) return null; const bytes = name.len / 2; if (bytes > logstore.max_hash_len) return null; _ = std.fmt.hexToBytes(out[0..bytes], name) catch return null; return out[0..bytes]; } /// The tip resolved to a check-in whose file content this repository does not /// hold. Named as a CLASS rather than as a missing file, because the missing /// file is a symptom: the usual cause is that the store holds artifacts which /// parse as check-ins of a different repository, and no amount of looking at the /// named path explains that. /// /// This refuses. It never falls back to another check-in — picking a different /// tip because the real one is unusable would be inventing history, and the /// operator would have no way to know it happened. fn openForeignCheckin( s: cli.Streams, target: []const u8, branch: []const u8, missing: usize, first: []const u8, ) cli.Exit { s.err.print( "zit open: the tip of '{s}' is check-in {s}, whose file content this repository does not hold\n", .{ branch, target }, ) catch {}; s.err.print( "zit open: {d} file(s) are missing, starting with '{s}'\n", .{ missing, first }, ) catch {}; s.err.writeAll( "zit open: this happens when a repository stores artifacts that parse as check-ins of\n" ++ "zit open: ANOTHER repository -- imported mirror metadata is the common source -- so they\n" ++ "zit open: join the branch as check-ins whose blobs were never here. The working tree was\n" ++ "zit open: not modified. 'zit leaves' shows the branch's check-ins; 'zit update <check-in>'\n" ++ "zit open: materializes a specific one.\n", ) catch {}; return .failure; } fn openApplyFail(s: cli.Streams, err: update.ApplyError) cli.Exit { return switch (err) { // applyCheckin never emits this: only applyMerge's cross-cone guard // does, and this switch maps a checkin materialization. The unreachable // IS the assertion; a future emitter trips it in tests immediately. error.SparseMergeUnsupported => unreachable, error.FileContentMissing => blk: { var b: [512]u8 = undefined; break :blk openDiag(s, update.stageFaultReason(&b)); }, error.OutOfMemory => openDiag(s, "out of memory"), error.InvalidCheckinId => openDiag(s, "cannot read repository: the resolved check-in ID is not a valid artifact name"), error.CheckinNotFound => openDiag(s, "cannot read repository: the resolved check-in is not present"), error.CheckinContentMissing => openDiag(s, "cannot read repository: check-in content is missing"), error.AggregateChecksumMismatch => openDiag(s, "the check-in's R checksum disagrees with its content"), error.CheckinNameMismatch => openDiag(s, "cannot read repository: check-in content does not match its name"), error.TargetNotManifest => openDiag(s, "cannot read repository: the check-in is not a valid manifest"), error.BaselineNotFound => openDiag(s, "cannot read repository: the check-in baseline is not found"), error.BaselineContentMissing => openDiag(s, "cannot read repository: the check-in baseline content is missing"), error.BaselineNameMismatch => openDiag(s, "cannot read repository: the check-in baseline content does not match its name"), error.BaselineNotManifest => openDiag(s, "cannot read repository: the check-in baseline is not a valid manifest"), error.BaselineIsDelta => openDiag(s, "cannot read repository: the check-in baseline is not fully resolvable"), error.StateRecordInvalid => openDiag(s, "cannot read repository: " ++ update.state_record_invalid_reason), error.Corrupt => openDiag(s, "cannot read repository: repository is corrupt"), error.SymlinkUnsupported => openDiag(s, "the check-in contains an unsupported symlink entry"), error.InvalidPath => openDiag(s, "the check-in names an invalid file path"), error.ReservedPath => openDiag(s, "the check-in names a reserved file path"), error.WorktreeNotClean => openDiag(s, "working tree is not clean"), error.UnmanagedCollision => openDiag(s, "an untracked file or directory is in the way"), error.CurrentStateInvalid => openDiag(s, "current working-tree state is invalid"), error.Storage => openDiag(s, "cannot update working tree: input/output error"), error.TransactionUnavailable => openDiag(s, "cannot create a transaction directory"), error.RollbackFailed => openDiag(s, "open failed and could not be rolled back; the previous working-tree state is retained for recovery"), error.CleanupFailed => openDiag(s, "open committed, but the transaction directory could not be removed"), }; } /// `zit open` — the second half of `zit clone URL DIR`: turn a repository whose /// checkout is still UNBORN into a materialized working tree. Fossil's own UX is /// the same two steps (`fossil clone` then `fossil open`), and splitting them /// keeps `runClone` strictly repo-only. /// /// Thin wiring over the shipped machinery, deliberately: probe the checkout /// record, resolve the tip of the main branch out of the derived repo state, and /// hand materialization to the SAME `update.applyCheckin` engine every other /// working-tree move rides (journal-armed, transactional, rollback-safe). Only /// the tip resolution is open's own work, and it is load-bearing: `update` /// accepts nothing but a literal 40/64-hex check-in ID, so a freshly cloned tree /// has no way to name its own tip. /// /// It writes NO checkout record and NO managed set of its own. `init` — which /// `clone` delegates repo creation to — already wrote the unborn record; an /// absent managed set already reads as the empty set; and the update engine is /// the single writer of the born record and the managed set. Adding a second /// writer of that bootstrap state would only create two things to keep in sync. pub fn runOpen( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { // One demand-fetch retry, statically bounded (TigerStyle: no unbounded // recursion -- the second attempt runs with fetching disabled). // `zit open URL` is `zit clone URL` — cloned into a directory // named from the URL and opened there. for (verb_args) |tok| { if (std.mem.startsWith(u8, tok, "http://") or std.mem.startsWith(u8, tok, "https://")) { return runClone(gpa, io, arena, verb_args, streams); } if (tok.len > 0 and tok[0] != '-') break; } return openAttempt(gpa, io, arena, verb_args, streams, amb, true); } /// The `--store` bootstrap: validate both ends, then write the container. /// The current directory must not already be a checkout (either control /// file), and PATH must hold a readable `.zit` -- checked here so the /// operator hears "no store at PATH" instead of a discovery miss. The /// container write is `checkout.write`'s atomic splice plus one /// `store_location` section carrying the store root's ABSOLUTE path: a /// worktree is addressed from anywhere, so a relative pointer would go /// stale with every `cd`. const Bootstrap = union(enum) { created, repaired, failed: cli.Exit }; fn bootstrapDetached( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, store_arg: []const u8, ) Bootstrap { var cwd = std.Io.Dir.cwd(); if (cwd.statFile(io, repo_filename, .{})) |_| { streams.err.writeAll("zit open: this directory holds the repository itself; --store points a DETACHED tree elsewhere\n") catch {}; return .{ .failed = .failure }; } else |_| {} const repairing = if (cwd.statFile(io, checkout.filename, .{})) |_| true else |_| false; // PATH may name the store root or the `.zit` file itself; either way the // pointer stored is the DIRECTORY, absolute. const named_file = std.mem.endsWith(u8, store_arg, "/.zit") or std.mem.eql(u8, store_arg, ".zit"); const store_root_rel = if (named_file) (std.fs.path.dirname(store_arg) orelse ".") else store_arg; var store_root = cwd.openDir(io, store_root_rel, .{}) catch { streams.err.print("zit open: no directory at '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; defer store_root.close(io); _ = store_root.statFile(io, repo_filename, .{}) catch { streams.err.print("zit open: no repository ('.zit') under '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; const abs = store_root.realPathFileAlloc(io, ".", arena) catch { streams.err.print("zit open: cannot resolve '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; if (repairing) { // Re-open: the checkout exists and its pointer (or the store it // pointed at) went stale — a moved store strands its satellites, and // a stranded tree can hold uncommitted work, so the pointer must be // repairable in place. The one gate that matters: the NEW store must // contain this tree's current check-in, or the recorded tree state // would be a lie against the store it now names. "Same store at a // new address" passes; "some other repository" is refused. var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined; const current = checkout.read(io, cwd, &ck_buf) catch { streams.err.writeAll("zit open: cannot read the checkout state\n") catch {}; return .{ .failed = .failure }; }; if (current) |state| if (state.version.len != 0) { var digest_buf: [digest_len]u8 = undefined; const digest = hexToDigest(state.version, &digest_buf) orelse { streams.err.writeAll("zit open: the checkout state is corrupt\n") catch {}; return .{ .failed = .failure }; }; var store_file = store_root.openFile(io, repo_filename, .{}) catch { streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; defer store_file.close(io); const store_st = store_file.stat(io) catch { streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; var store_fs = storage.FileStorage.init(io, store_file, store_st.size); var store_log = logstore.LogStore.openReadOnly(gpa, store_fs.storageFile()) catch { streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; defer store_log.deinit(gpa); if (store_log.lookup(digest) == null) { streams.err.print( "zit open: the repository under '{s}' does not contain this tree's check-in '{s}'; refusing to re-point at a different repository\n", .{ store_root_rel, state.version }, ) catch {}; return .{ .failed = .failure }; } }; checkout.updateSections(io, cwd, &.{.{ .tag = .store_location, .payload = abs }}) catch { streams.err.writeAll("zit open: cannot write the checkout state\n") catch {}; return .{ .failed = .failure }; }; streams.out.print("checkout now points at '{s}'\n", .{abs}) catch {}; return .repaired; } checkout.write(io, cwd, .{ .version = "", .branch = "trunk", .merge_parent = "", .special_op = "", .manifest_flags = "" }) catch { streams.err.writeAll("zit open: cannot write the checkout state\n") catch {}; return .{ .failed = .failure }; }; checkout.updateSections(io, cwd, &.{.{ .tag = .store_location, .payload = abs }}) catch { cwd.deleteFile(io, checkout.filename) catch {}; streams.err.writeAll("zit open: cannot write the checkout state\n") catch {}; return .{ .failed = .failure }; }; return .created; } fn openAttempt( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, allow_fetch: bool, ) cli.Exit { const parsed = switch (cli.parseOpenArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "open", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "open", .extra_argument, tok), .repeated_option => |tok| return cli.usageError(streams, "open", .repeated_option, tok), .empty_value => |tok| return cli.usageError(streams, "open", .empty_value, tok), .bad_manifest => |tok| return cli.usageError(streams, "open", .bad_manifest, tok), .too_many_cones => |tok| return cli.usageError(streams, "open", .too_many_cones, tok), }; if (parsed.help) { streams.out.writeAll(open_help) catch {}; return .ok; } // `--store PATH` bootstraps a DETACHED checkout in the current // directory: an unborn checkout record (exactly what `init` writes) plus // the store pointer, and nothing else. Everything after this block is the // ordinary open — discovery finds the fresh container, `Repository.open` // follows the pointer, and the materialization below neither knows nor // cares that the store lives elsewhere. if (parsed.store) |store_arg| { switch (bootstrapDetached(gpa, io, arena, streams, store_arg)) { .created => {}, // A repair is complete in itself: the tree is already open at its // check-in, so the materialization below has nothing to do. .repaired => return .ok, .failed => |rc| return rc, } } // The full derive: resolving the main branch reads the tag tables, which the // `.fast_read` image path does not carry. var repo: Repository = undefined; var root_dir: std.Io.Dir = undefined; const rc = openRepo(gpa, io, arena, streams, "open", null, false, false, &repo, &root_dir); if (rc != .ok) return rc; defer root_dir.close(io); defer repo.deinit(gpa); // BORN-ness, not presence, is the "already open" signal. Every fresh // repository carries a checkout record — `init` writes an unborn one // (`version == ""`, branch `trunk`) as its step 8, and `clone` delegates to // `init` — so a presence-based gate would reject `zit clone && zit open` on // every clone, the primary path this verb exists to serve. Only a non-empty // version means a tree has actually been materialized against a check-in. var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined; const current = checkout.read(io, root_dir, &ck_buf) catch |e| switch (e) { error.Storage => return openDiag(streams, "cannot read the checkout state: input/output error"), error.Corrupt => return openDiag(streams, "the checkout state is corrupt"), }; if (current) |state| if (state.version.len != 0) { streams.err.print( "zit open: this working tree is already open at check-in '{s}'; use 'zit update' to change check-ins\n", .{state.version}, ) catch {}; return .failure; }; // The target: an explicit CHECKIN resolved like `zit info` (§8.3), or the // tip of the main branch (repo-state §9 current-trunk). Everything below — // preflight, materialization, the manifest opt-in — is target-generic. const main_branch = repo.mainBranch(); const tip = if (parsed.checkin) |name| repo.resolve(name, .checkin) catch |e| { switch (e) { error.NotFound => streams.err.print("zit open: check-in '{s}' not found\n", .{name}) catch {}, error.Ambiguous => streams.err.print("zit open: name '{s}' is ambiguous\n", .{name}) catch {}, error.OutOfMemory => return openDiag(streams, "out of memory"), } return .failure; } else // Switched, not an unbound `catch`: the success message below is an // exit-0 claim that the tree is already correct, and it is only honest // for `NotFound`. Naming the arm means a later Storage/OOM arm on // `currentTrunk` fails this compile instead of silently reporting a // genuine failure as success on a materialization verb. repo.state().currentTrunk(main_branch) catch |e| switch (e) { // A repository with no check-in on its main branch is ALREADY in its // correct end state — the unborn record and an empty tree. There is // nothing to materialize and nothing wrong, so this is reported, not // failed, and the record is left exactly as it was found. An // explicit CHECKIN never lands here: in an empty repository it // resolves to NotFound above, a failure, because the operator asked // for something specific and did not get it. error.NotFound => { streams.out.print( "open: no check-in on '{s}' yet; the working tree is already in its correct (empty) state\n", .{main_branch}, ) catch {}; return .ok; }, // Settling an equal-mtime tie by ancestry allocates (repo-state // `MtimeMaxPick`). A failure here is a genuine failure — reporting // it as the exit-0 "already correct" message would be the lie the // switch above exists to prevent. error.OutOfMemory => return openDiag(streams, "out of memory"), }; // Both borrow the derived view, which outlives the update below. const target = repo.hashOf(tip); const branch = repo.branchOf(tip); // An assertion, not a check: unlike `update`, whose target is user-supplied // and therefore validated, open's target comes out of this repository's own // derived state. A malformed name here is a defect in the derivation, not // bad input, so it belongs in the negative space rather than in a // diagnostic path no correct run can reach. std.debug.assert(artifact.validateName(target)); // Distinct from the open above, which shares `repoOpenFail`'s wording: this // one has already derived the view successfully and is opening the log a // SECOND time, read-only, for materialization. Reporting both identically // made a failure here indistinguishable from a failure to derive — and they // are different repositories' worth of problem. const log = repo.logStore(gpa) catch |e| { switch (e) { error.Corrupt => streams.err.print( "zit open: the repository derived cleanly but its log will not reopen for reading: corrupt\n", .{}, ) catch {}, else => return repoOpenFail(streams, "open", e), } return .failure; }; // Sparse-cone RFC section 4.1: `open --cone` materializes the first // checkout already scoped. Absent flag = dense, exactly the shipped // behavior; `open` has no --dense because a fresh checkout IS dense. var open_members = parsed.cone_members; var open_len = parsed.cone_len; var open_not_members = parsed.cone_not_members; var open_not_len = parsed.cone_not_len; if (parsed.cone_file) |cf| { const lens = cli.appendConeFile(arena, io, cf, &open_members, open_len, &open_not_members, open_not_len) orelse { streams.err.print("zit open: cannot read --cone-file '{s}' (or it overflows the {d}-member cap)\n", .{ cf, zit.cone.max_members }) catch {}; return .failure; }; open_len = lens.members; open_not_len = lens.not_members; } var open_cone: ?zit.cone.Cone = null; defer if (open_cone) |*c| c.deinit(gpa); const sparse: update.SparseOpt = if (open_len + open_not_len != 0) blk: { open_cone = zit.cone.Cone.init(gpa, open_members[0..open_len], open_not_members[0..open_not_len]) catch |e| return openConeFail(streams, e); break :blk .{ .set = &open_cone.? }; } else .inherit; // PREFLIGHT: does this repository actually hold the content of the check-in // it just resolved as the tip? // // The format is content-addressed and types artifacts by STRUCTURE, so a // stored blob whose bytes happen to be a valid check-in manifest IS a // check-in — repo-state §4.4 grants opacity only to artifacts a conformant // parser DECLINES to classify, and a foreign manifest classifies fine. A // repository that tracks a file containing one (a Fossil->Git mirror's // `manifest`, a test fixture, a documentation example) therefore holds // check-ins of a DIFFERENT repository, whose file content it never had. That // is conformant derivation, not corruption, and Fossil behaves the same way. // // What is not acceptable is spending several minutes materializing before // discovering it. `stagePlan` already refuses safely — nothing is written // until every byte is verified and staged — so this changes no outcome; it // moves the refusal earlier and lets it NAME the situation, which a bare // missing-content error cannot. // // Presence-only: `lookup` asks whether a name is held, reconstructing // nothing. Cost is one probe per file in the tip, against a derive that // already walked every artifact in the store. if (repo.files(gpa, tip)) |maybe_list| { if (maybe_list) |list_const| { var list = list_const; defer list.deinit(gpa); // Sparse-cone RFC section 5.2: absence has three classes, and // this preflight must not read a deliberately sparse store as the // foreign-check-in class. A missing blob whose path the STORE // cone does not cover is elided -- healthy. Elided files that the // requested materialization would need refuse by the elision // class (section 6.4; demand fetch is a later wave), never as // foreign, and never as corruption. var store_cone = zit.repocache.readStoreCone(gpa, io, root_dir); defer if (store_cone) |*c| c.deinit(gpa); var missing: usize = 0; var first_missing: ?[]const u8 = null; var elided: usize = 0; var first_elided: ?[]const u8 = null; var it = list.iterator(); while (it.next()) |f| { const h = f.hash orelse continue; var digest_buf: [logstore.max_hash_len]u8 = undefined; const raw = openDecodeName(h, &digest_buf) orelse continue; if (log.lookup(raw) == null) { if (store_cone) |*c| { if (!c.covers(f.filename)) { elided += 1; if (first_elided == null) first_elided = f.filename; continue; } } missing += 1; if (first_missing == null) first_missing = f.filename; } } if (missing != 0) return openForeignCheckin(streams, target, branch, missing, first_missing.?); if (elided != 0) { // The materialization cone decides whether elision matters: // an open scoped inside the held content proceeds; one that // needs elided bytes refuses, naming the class and the count. const wants_elided = blk: { switch (sparse) { .set => |c| { if (c) |request| { if (!request.covers(first_elided.?)) { var again = list.iterator(); var any = false; while (again.next()) |f| { const h2 = f.hash orelse continue; var db2: [logstore.max_hash_len]u8 = undefined; const raw2 = openDecodeName(h2, &db2) orelse continue; if (log.lookup(raw2) == null and request.covers(f.filename)) { any = true; break; } } break :blk any; } break :blk true; } break :blk true; // dense request needs everything }, .inherit => break :blk true, // no worktree cone yet on open } }; if (wants_elided) { // Sparse-cone RFC section 6.4, policy form: the requested // materialization IS the new coverage intent. Write it as // the store cone (null = dense), run an ordinary pull -- // whose suppression and receive filtering read the NEW // policy -- and reprobe. Anything the peer lacked stays a // transit phantom for later synchronization to heal. var url_buf: [logstore.max_config_str]u8 = undefined; const fetchable = allow_fetch and blk: { _ = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch break :blk false; break :blk true; }; if (!fetchable) { streams.err.print( "zit open: {d} file(s) of this check-in are elided by the store's sparse cone, starting with '{s}'\n" ++ "zit open: no saved remote to fetch them from; open inside the cone\n" ++ "zit open: ('zit open --cone=...'), set a remote ('zit remote URL'), or re-clone\n", .{ elided, first_elided.? }, ) catch {}; return .failure; } streams.err.print( "zit open: {d} file(s) are elided by the store's sparse cone; fetching from the saved remote\n", .{elided}, ) catch {}; const requested: ?*const zit.cone.Cone = switch (sparse) { .set => |c| c, .inherit => null, // a dense open drops the cone }; zit.repocache.writeStoreCone(gpa, io, root_dir, requested); const pull_exit = runPull(gpa, io, arena, &.{}, streams, null); if (pull_exit != .ok) return pull_exit; // Everything resolved so far (tip, branch, this log // handle) predates the fetch; re-attempt from scratch with // fetching disabled, so a peer that also lacked the // content produces the elided refusal rather than a loop. return openAttempt(gpa, io, arena, verb_args, streams, amb, false); } } } } else |_| { // A files() fault is not this preflight's to report: the engine below // reaches the same data and has the diagnostics for it. Falling through // keeps this advisory, never a new way to fail. } // `open` is the ONE verb that SETS the manifest opt-in. `--manifest` // is the operator's explicit choice and always wins; `--manifest=off` is // remembered as the refusal. Absent, the PROJECT's // `manifest` setting supplies it, which is fossil's model: a clone gets // build stamps because the project asked for them, not because each // developer remembered to. The tree is not materialized yet, so the // versioned tier is read out of the check-in itself, as fossil's // `db_get_versioned` does through `historical_blob`. // The verb's own `log` is already open, so the project setting is // resolved from the config it holds rather than through a second open. var open_config: ?logstore.OwnedConfig = log.readConfig(gpa) catch null; defer if (open_config) |*c| c.deinit(gpa); const open_cfg: ?*const logstore.Config = if (open_config) |*c| &c.config else null; const manifest: update.ManifestOpt = if (parsed.manifest) |m| .{ .set = m } else .{ .inherit_or = blk: { const r = checkinSetting(gpa, io, arena, &repo, tip, amb, open_cfg, .manifest) catch break :blk ""; break :blk r.value; }, }; // The check-in is named on failure because `open` RESOLVES its own target // (the tip of the configured main branch) rather than being handed one. A // diagnostic that omits it cannot distinguish "materializing the expected // check-in went wrong" from "resolved a different check-in than the operator // believes is the tip" — and the operator has no other way to see which. update.applyCheckinWith(gpa, io, root_dir, log, target, manifest, sparse, &repo, null) catch |e| { streams.err.print("zit open: materializing check-in {s} on {s}\n", .{ target, branch }) catch {}; return openApplyFail(streams, e); }; // The `empty-dirs` shape, laid over the tree the check-in // just produced. Read AFTER the materialization because a versioned // `.fossil-settings/empty-dirs` does not exist on disk before it; fossil's // checkout reads the same value out of the check-in it is installing, so // the two see the same list. { const dirs: []const u8 = blk: { const r = localSetting(io, arena, root_dir, amb, open_cfg, .empty_dirs) catch break :blk ""; break :blk r.value; }; update.ensureEmptyDirs(io, root_dir, dirs, streams.err); } // A successful open stamps this tree into the store's worktree // registry (best-effort; discovery data never gates the open). if (std.process.currentPathAlloc(io, gpa)) |own_abs| { defer gpa.free(own_abs); cli.stampWorktreeRegistry(gpa, io, root_dir, own_abs); } else |_| {} streams.out.print("opened {s} on {s}\n", .{ target, branch }) catch {}; return .ok; } /// Map a Cone.init refusal for `open`. Same wording discipline as update's /// mapper: states the rule, never says "missing" or "corrupt". fn openConeFail(s: cli.Streams, err: zit.cone.Error) cli.Exit { switch (err) { error.OutOfMemory => s.err.writeAll("zit open: out of memory\n") catch {}, error.InvalidMember => s.err.writeAll( "zit open: invalid --cone member: members are repository-relative paths using '/', with no '.', '..', or empty components\n", ) catch {}, error.ReservedMember => s.err.writeAll("zit open: a --cone member names a reserved control path\n") catch {}, error.Bounds => s.err.writeAll("zit open: --cone needs between 1 and 255 members (exclusions included), at least one inclusion\n") catch {}, error.DeadExclusion => s.err.writeAll("zit open: a --cone-not member no --cone member covers is a dead rule\n") catch {}, error.Contradiction => s.err.writeAll("zit open: one path given as both --cone and --cone-not\n") catch {}, } return .failure; } /// The fields a `updateConfig` overlay may change; a null field is left exactly /// as the stored config has it. Naming a single change site keeps the "copy the /// whole config, overlay one field" discipline in one place — a caller that /// means to change only the project code can never silently drop the credential /// or the saved remote (the class of bug the pre-`updateConfig` re-puts had). const ConfigPatch = struct { project_code: ?[]const u8 = null, login: ?zit.sync.Credential = null, remote_url: ?[]const u8 = null, /// draft-fossil-submanifests-00 §6.3/§8.1 and §6.4 — the three /// project-scope settings sync §9.3's scalar channel carries. Optional for /// the same reason every field here is: a peer that named only one of them /// must leave the other two exactly as stored. enable_submanifests: ?bool = null, submanifest_k: ?u8 = null, submanifest_auto_retune: ?bool = null, /// The autosync setting, stored inverted (true = off). Local, like /// `remote_url`; only the settings verb writes it. autosync_off: ?bool = null, ignore_glob: ?[]const u8 = null, /// The two signing settings, both LOCAL and both deliberately /// outside every config group: fossil classifies `pgp-command` sensitive /// (it is a command line a peer would otherwise get to choose) and puts /// `clearsign` in no group at all. clearsign: ?bool = null, pgp_command: ?[]const u8 = null, /// The rest of fossil's checkout settings at the repository tier. crnl_glob: ?[]const u8 = null, clean_glob: ?[]const u8 = null, empty_dirs: ?[]const u8 = null, encoding_glob: ?[]const u8 = null, dotfiles: ?[]const u8 = null, mimetypes: ?[]const u8 = null, attachment_size_limit: ?[]const u8 = null, /// The explicit-set bits, which are what make "this repository has no /// opinion" representable and therefore let the global tier be reached. autosync_set: ?bool = null, clearsign_set: ?bool = null, keep_glob: ?[]const u8 = null, manifest: ?[]const u8 = null, before_commit: ?[]const u8 = null, /// The two content-policy globs. Data settings, so a tracked /// `.zit-settings/` file outranks whatever is stored here. binary_glob: ?[]const u8 = null, crlf_glob: ?[]const u8 = null, /// The consent record. Empty string is a real value here (revoke), /// so it is written whenever the field is present, not when non-empty. before_commit_trust: ?[]const u8 = null, editor: ?[]const u8 = null, autosync_mode: ?[]const u8 = null, fn any(self: ConfigPatch) bool { return self.project_code != null or self.login != null or self.remote_url != null or self.enable_submanifests != null or self.submanifest_k != null or self.submanifest_auto_retune != null or self.autosync_off != null or self.ignore_glob != null or self.clearsign != null or self.pgp_command != null or self.autosync_set != null or self.clearsign_set != null or self.keep_glob != null or self.manifest != null or self.before_commit != null or self.before_commit_trust != null or self.binary_glob != null or self.crlf_glob != null or self.editor != null or self.autosync_mode != null; } }; /// Read the live config from an already-open log, overlay `patch`, and append /// the result while preserving every unnamed field. The caller owns durability, /// so a clone can combine its origin settings in one config frame and commit /// its complete result once. fn updateConfigOpen( gpa: std.mem.Allocator, log: *logstore.LogStore, patch: ConfigPatch, ) PersistError!void { var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse return error.Corrupt; // init always writes a config frame defer oc.deinit(gpa); // Copy every field, then overlay only the named ones. `next`'s string slices // borrow `oc.backing` / `patch`; both outlive the `putConfig` memcpy below. var next = oc.config; if (patch.project_code) |c| next.project_code = c; if (patch.login) |cred| { next.login_user = cred.user; // Point at the function-scoped `patch` storage, not the block-local // capture `cred`: `next` is read by `putConfig` after this block closes. next.login_secret_hex = &patch.login.?.secret_hex; } if (patch.remote_url) |u| next.remote_url = u; if (patch.enable_submanifests) |v| next.enable_submanifests = v; if (patch.submanifest_k) |v| next.submanifest_k = v; if (patch.submanifest_auto_retune) |v| next.submanifest_auto_retune = v; if (patch.autosync_off) |v| next.autosync_off = v; if (patch.ignore_glob) |v| next.ignore_glob = v; if (patch.clearsign) |v| next.clearsign = v; if (patch.pgp_command) |v| next.pgp_command = v; if (patch.autosync_set) |v| next.autosync_set = v; if (patch.clearsign_set) |v| next.clearsign_set = v; if (patch.keep_glob) |v| next.keep_glob = v; if (patch.manifest) |v| next.manifest = v; if (patch.before_commit) |v| next.before_commit = v; if (patch.before_commit_trust) |v| next.before_commit_trust = v; if (patch.binary_glob) |v| next.binary_glob = v; if (patch.crlf_glob) |v| next.crlf_glob = v; if (patch.editor) |v| next.editor = v; if (patch.autosync_mode) |v| next.autosync_mode = v; if (patch.crnl_glob) |v| next.crnl_glob = v; if (patch.clean_glob) |v| next.clean_glob = v; if (patch.empty_dirs) |v| next.empty_dirs = v; if (patch.encoding_glob) |v| next.encoding_glob = v; if (patch.dotfiles) |v| next.dotfiles = v; if (patch.mimetypes) |v| next.mimetypes = v; if (patch.attachment_size_limit) |v| next.attachment_size_limit = v; log.putConfig(next) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; } /// Open `dir/.zit` for a field-preserving config update, retaining the legacy /// wrapper for pull, sync, serve, and remote callers that do not already own a /// writable log. pub fn updateConfig( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, patch: ConfigPatch, ) PersistError!void { var file = cli.openStoreAppend(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; defer log.deinit(gpa); try updateConfigOpen(gpa, &log, patch); log.commit() catch return error.Storage; } /// §8.2 — persist the origin's project code into the freshly cloned repo at /// `dir/.zit`, preserving every other field `init` wrote. A no-op when the peer /// sent no code. Thin wrapper over `updateConfig` (the field-preserving RMW). fn storeClonedProjectCode( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, project_code: []const u8, ) PersistError!void { if (project_code.len == 0) return; // peer sent none; keep the local init code return updateConfig(gpa, io, dir, .{ .project_code = project_code }); } /// draft-fossil-submanifests-00 §6.3/§8.1 (and §6.4) over sync §9.3 /// — persist the origin's three sub-manifest settings into the freshly cloned /// repo at `dir/.zit`, preserving every other field `init` wrote. A no-op when /// the peer named none. Thin wrapper over `updateConfig`, the same shape /// `storeClonedProjectCode` has and for the same reason. /// /// One-way and clone-only, deliberately. A fresh clone has nothing of its own /// to conflict with, so adoption needs no merge; an EXISTING repository would, /// and §9.3's channel resolves conflicts by MTIME, which zit's config frame does /// not store (see `sync.submanifest_config_mtime`). Refreshing settings on a /// later pull is therefore a real conflict-resolution feature with a config /// schema change behind it, not plumbing this wrapper can be widened into — /// `runPull`/`runPush` deliberately have no seam here. fn storeClonedSubmanifestSettings( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, cfg: zit.sync.SubmanifestConfig, ) PersistError!void { if (!cfg.any()) return; // peer sent none; keep the local init defaults return updateConfig(gpa, io, dir, .{ .enable_submanifests = cfg.enable, .submanifest_k = cfg.k, .submanifest_auto_retune = cfg.auto_retune, }); } /// Drives a modern-clone client (§3.8 CLONE_ROUND0 → EXCHANGE) to convergence /// over `tr`, filling `ms` with the peer's artifacts; returns the round count. /// Sans-IO on this side of `tr.exchange`: the state machine only moves /// `[]const u8` bytes. Its closed error set is the driver's own causes unioned /// with the transport's, so `runClone` switches the whole surface exhaustively. /// The result of a clone run: the round count plus the origin's project code /// (§8.2), copied out of the sans-IO client so the caller can persist the /// shared identity after the client is torn down. const CloneOutcome = struct { rounds: usize, uv_skipped: u64 = 0, /// Uncompressed protocol bytes moved (`RunStats` totals), for the /// end-of-clone accounting line. sent_bytes: usize, received_bytes: usize, project_code_buf: [zit.sync.max_projectcode_len]u8, project_code_len: usize, /// submanifests §6.3/§8.1 over sync §9.3 — the origin's three project-scope /// settings, learned from the cleanup round's config card. Copied out of the /// sans-IO client for the same reason the project code is: the client is /// torn down before the caller persists anything. Owns no memory (three /// optional scalars), so no buffer twin is needed. submanifest: zit.sync.SubmanifestConfig, /// §9.3 r28 — the origin's shun table, learned from the same cleanup round. /// Unlike the settings above this OWNS memory (a row count the wire /// decides), so `deinit` is not optional: the client that parsed these rows /// is torn down before the caller writes anything durable. /// /// Carrying the shun table is not a nicety. fossil's shun table is its /// takedown mechanism; a clone that drops it will, on its next sync with /// any third party, re-accept exactly the content an operator was ordered /// to remove. shun_rows: []zit.sync.OwnedShunRow = &.{}, fn projectCode(self: *const CloneOutcome) []const u8 { return self.project_code_buf[0..self.project_code_len]; } fn deinit(self: *CloneOutcome, gpa: std.mem.Allocator) void { for (self.shun_rows) |row| { gpa.free(row.hash); gpa.free(row.reason); } gpa.free(self.shun_rows); self.shun_rows = &.{}; } }; /// The stderr live-progress sink a TTY clone runs with. Each completed /// round overwrites one line in place (\r, no newline); `finishLine` closes it /// with a newline exactly once, so whatever stderr carries next starts clean. /// Flushed per round — a progress line that sits in a buffer until the clone /// ends reports nothing. const CloneProgress = struct { err: *std.Io.Writer, emitted: bool = false, fn emit(ctx: *anyopaque, info: sync_driver.RoundInfo) void { const self: *CloneProgress = @ptrCast(@alignCast(ctx)); self.emitted = true; self.err.print( "\rround {d}: {d} bytes received", .{ info.round, info.total_received }, ) catch return; self.err.flush() catch {}; } fn finishLine(self: *CloneProgress) void { if (!self.emitted) return; self.emitted = false; self.err.writeAll("\n") catch return; self.err.flush() catch {}; } }; /// Map a Cone.init refusal for clone. Same wording discipline as the other /// verbs' mappers. fn cloneConeFail(s: cli.Streams, err: zit.cone.Error) cli.Exit { switch (err) { error.OutOfMemory => s.err.writeAll("zit clone: out of memory\n") catch {}, error.InvalidMember => s.err.writeAll( "zit clone: invalid --cone member: members are repository-relative paths using '/', with no '.', '..', or empty components\n", ) catch {}, error.ReservedMember => s.err.writeAll("zit clone: a --cone member names a reserved control path\n") catch {}, error.Bounds => s.err.writeAll("zit clone: --cone needs between 1 and 255 members (exclusions included), at least one inclusion\n") catch {}, error.DeadExclusion => s.err.writeAll("zit clone: a --cone-not member no --cone member covers is a dead rule\n") catch {}, error.Contradiction => s.err.writeAll("zit clone: one path given as both --cone and --cone-not\n") catch {}, } return .failure; } /// A bounded copy of a peer error-card text, taken before the aborted sync /// client goes out of scope so the CLI can quote the refusal. const PeerErrorCopy = struct { buf: [zit.sync.peer_error_cap]u8 = undefined, len: usize = 0, fn set(self: *PeerErrorCopy, source: ?[]const u8) void { const text_in = source orelse return; std.debug.assert(text_in.len <= self.buf.len); @memcpy(self.buf[0..text_in.len], text_in); self.len = text_in.len; } fn text(self: *const PeerErrorCopy) ?[]const u8 { if (self.len == 0) return null; return self.buf[0..self.len]; } }; fn cloneRun( gpa: std.mem.Allocator, tr: *transport.Transport, ms: *repostate.MemStore, self_servercode: []const u8, sink: ?sync_driver.RoundSink, drain: ?sync_driver.RoundDrain, cone_members: []const []const u8, cone_not_members: []const []const u8, peer_error_out: *PeerErrorCopy, ) sync_driver.RunError(*transport.Transport)!CloneOutcome { var client = zit.sync.Client{ .state = zit.sync.initialState(.clone), .operation = .clone, .self_servercode = self_servercode, .project_code = sync_projectcode, .version = sync_client_version_info, // Sparse-cone RFC section 6.1: request server-side filtering; the // local persist filter stays as the section 6.3 belt regardless. .cone_members = cone_members, .cone_not_members = cone_not_members, }; defer client.deinit(gpa); const stats = sync_driver.runSyncProgress(gpa, &client, ms.store(), tr, max_stalled_sync_rounds, sink, drain) catch |e| { // A refusing peer's words outlive the aborted client (bounded copy) // so the caller renders them instead of a generic protocol error. peer_error_out.set(client.peerErrorText()); return e; }; // §8.2 — the peer's project code, learned from the clone reply's `push` // card, copied out before the client is deinitialized. var outcome = CloneOutcome{ .rounds = stats.rounds, .uv_skipped = client.uv_skipped, .sent_bytes = stats.sent_bytes, .received_bytes = stats.received_bytes, .project_code_buf = undefined, .project_code_len = 0, // §9.3 — the origin's sub-manifest settings, latched from the config // card the §8.4 cleanup round asked for. .submanifest = client.peerSubmanifestConfig(), }; const pc = client.peerProjectCode(); @memcpy(outcome.project_code_buf[0..pc.len], pc); outcome.project_code_len = pc.len; // §9.3 r28 — copy the latched shun rows out of the client, which the // `defer` above tears down on return. const latched = client.peerShunRows(); if (latched.len != 0) { var rows = try gpa.alloc(zit.sync.OwnedShunRow, latched.len); var filled: usize = 0; errdefer { for (rows[0..filled]) |r| { gpa.free(r.hash); gpa.free(r.reason); } gpa.free(rows); } for (latched, rows) |src_row, *dst| { const h = try gpa.dupe(u8, src_row.hash); errdefer gpa.free(h); const r = try gpa.dupe(u8, src_row.reason); dst.* = .{ .hash = h, .epoch = src_row.epoch, .reason = r }; filled += 1; } outcome.shun_rows = rows; } return outcome; } /// The closed error set of `persistStore`: memory, a storage write fault, or a /// corrupt/oversized peer artifact name that will not decode to a log key. const PersistError = error{ OutOfMemory, Storage, Corrupt }; /// The disposition of one artifact in `persistStore`'s ordered pass. const PersistOutcome = union(enum) { /// A non-phantom frame for this artifact now exists in the log — written /// here, or already present from a prior cascade / a pull re-persist. persisted, /// A reverse-delta whose basis is a content-bearing store entry not YET /// written to the log. It is deferred on the basis wire-hash and re-driven /// once that basis lands, so its `delta_of` offset points strictly backward. deferred: []const u8, }; /// What one `persistStore` sweep did: the artifacts it made durable, and the /// sub-manifest references (draft-fossil-submanifests-00 §7.1) it phantomized /// and could not resolve from this repository's own holdings. const PersistSummary = struct { /// Content-bearing artifacts in the store — the count every caller reports. written: usize, /// §7.1 — distinct S references still standing as phantoms when the sweep /// ended: this repository holds an artifact whose tree closure is open. /// Zero for every repository that carries no S card, which is every /// repository predating this draft. submanifest_gaps: usize, /// Sparse-cone RFC section 6.3 -- artifacts deliberately not persisted /// under the clone's cone. Zero for every dense clone. elided: usize = 0, }; /// draft-fossil-submanifests-00 §7.1 — the S references one persist sweep saw. /// /// Rids, not names: `LogStore` entries are append-only, so an interned rid stays /// valid for the sweep, and whether a reference is still a GAP can only be /// decided once the sweep has ended — a sub-manifest may be persisted after the /// tree manifest that names it, and `putZlib` upgrades the phantom minted for it /// in place. Deciding per S card instead would report a gap for every reference /// whose artifact happens to sit later in rid order. const SubManifestRefs = struct { rids: std.AutoArrayHashMapUnmanaged(logstore.Rid, void) = .empty, /// Delta-dialect memory: this sweep stored a B-card manifest, so the /// `seen_delta_manifest` config indicator is written alongside `seen`'s /// on the same false->true edge discipline. seen_delta: bool = false, /// submanifests §8.1 — set the first time this sweep stores an artifact /// inside §7.2's gated set: "When the first S-bearing artifact is stored, an /// implementation SHOULD record a local seen-submanifest indicator". Both /// §7.2 shapes count, leaf sub-manifests included — see /// `phantomizeSubManifestRefs` for why the literal S-card-only reading would /// leave the gate blind to exactly the artifacts that carry a tree's files. /// It is also the durable gate on the §7.1 phantom REBUILD (see /// `persistStore`), which is why it tracks artifacts seen, not gaps left. seen: bool = false, fn deinit(self: *SubManifestRefs, gpa: std.mem.Allocator) void { self.rids.deinit(gpa); self.* = undefined; } /// Distinct references that never became a content-bearing frame. fn unresolved(self: *const SubManifestRefs, log: *const logstore.LogStore) usize { var n: usize = 0; for (self.rids.keys()) |rid| { if (log.isPhantom(rid)) n += 1; } std.debug.assert(n <= self.rids.count()); return n; } }; /// draft-fossil-submanifests-00 §7.1 — "Storing a structural artifact that /// carries S cards whose referenced sub-manifests are absent MUST create a /// phantom for each missing reference, with the referencing artifact's privacy /// classification, exactly as an absent delta source creates a source phantom." /// /// This is §7.1's SECOND storing boundary. The receive path /// (`sync.Client.absorbSubManifestRefs`) covers artifacts a pull absorbs; it /// cannot cover a clone, which stores every artifact wire-backed and never /// inflates one (so it has no plaintext to scan), nor a server-side push /// ingest, which absorbs nothing at all. Both of those reach the durable store /// only HERE — and here the plaintext is already in hand, reconstructed for the /// §7.7 verify, so the whole duty costs one `sBearingCandidate` scan of bytes /// this function was handed anyway. The clone's O(wire) receive bound is /// untouched: no container is inflated that §7.7 was not already inflating. /// /// A clone from a peer whose OWN closure is incomplete is exactly the case the /// receive path cannot see: the cursor delivers every artifact the peer holds, /// and the peer does not hold the sub-manifests. Without this, such a clone /// records its gap nowhere and never asks for the leaves. /// /// Privacy: zit's phantom is an interned entry with no content and carries no /// per-entry privacy at either boundary, so an S-reference phantom is minted /// exactly as an absent delta source's is — which is what §7.1 asks for. fn phantomizeSubManifestRefs( gpa: std.mem.Allocator, log: *logstore.LogStore, content: []const u8, seen: *SubManifestRefs, ) PersistError!void { // §8.1's indicator, over BOTH §7.2 shapes rather than over S cards alone. // // §8.1 says "when the first S-bearing artifact is stored"; a LEAF // sub-manifest carries F cards and no S card (§4.4), so the literal reading // records nothing for a repository holding leaves. That reading cannot be // right: this indicator is what arms the §7.2 transfer gate, §7.2 gates // leaves by TYPE, and §8's whole reason for the transfer layer is that a // pre-S reader can only hold a leaf as "unreadable ... opaque content". §8.2 // reads the same way — "a repository containing an S-bearing artifact" is a // repository holding any part of a tree, and a repository that holds a tree's // leaves is exactly that. So the indicator answers `transferGatedArtifact`. // // Cheap where it runs: the predicate rejects any byte string whose first two // bytes are not a card letter followed by SP or LF — every file blob, in two // comparisons. A structural artifact pays one allocation-free walk of its // card lines, beside the full-artifact rehash §7.7 already charges here. if (!seen.seen and repostate.transferGatedArtifact(content)) seen.seen = true; // The delta-dialect twin: remember that this repository stores delta // manifests, so `zit commit` keeps the dialect (config `seen_delta`). if (!seen.seen_delta and repostate.deltaManifestArtifact(content)) seen.seen_delta = true; var refs: std.ArrayListUnmanaged([]const u8) = .empty; defer refs.deinit(gpa); // Validate-then-apply lives in the shared extractor: a malformed card run // mints nothing, so a half-read artifact never leaves a half-minted phantom // set. `content` outlives `refs`, which borrows its hash tokens. if (!try repostate.collectSubManifestRefs(gpa, content, &refs)) return; for (refs.items) |name| { var buf: [digest_len]u8 = undefined; // §3.5 admits a 40- or 64-hex name and `decodeWireName` decodes both; a // name too wide for this repository's digest cannot key a frame here, so // it is a corrupt reference rather than a gap. const digest = decodeWireName(name, &buf) orelse return error.Corrupt; const rid = log.intern(digest) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, }; try seen.rids.put(gpa, rid, {}); } } /// Persist one artifact `rid` into `log`, choosing its stored form, and verify /// §7.7 integrity (the reconstructed content MUST hash to its wire name) BEFORE /// any durable write. A retained delta whose basis is already a held log frame /// under the read depth bound is stored VERBATIM as a `.delta`; a retained delta /// whose basis is a not-yet-written content-bearing store entry is DEFERRED /// (when `allow_defer`) so the basis frame is written first (reverse-delta /// ordering); every other case — no wire form, a full container, or an /// absent / phantom / over-deep basis — lands as the full form, never a /// phantom-based or unreadable delta. Idempotent: an artifact already holding a /// non-phantom frame returns `.persisted` without a re-write or re-verification /// (§7.7 was proven when it was first persisted). /// Record a cluster's wire name as it is written (see /// `persistArtifact`'s `clusters_out`). The dupe is owned by the list. fn noteCluster(gpa: std.mem.Allocator, clusters_out: ?*std.ArrayListUnmanaged([]const u8), name: []const u8, content: []const u8) error{OutOfMemory}!void { const list = clusters_out orelse return; if (!clusterindex.isCluster(content)) return; const owned = try gpa.dupe(u8, name); errdefer gpa.free(owned); try list.append(gpa, owned); } fn persistArtifact( gpa: std.mem.Allocator, store: repostate.ArtifactStore, log: *logstore.LogStore, rid: repostate.Rid, allow_defer: bool, /// Streaming persist: defer even on a basis the store has not /// SEEN yet — mid-exchange, an unseen basis is usually still in flight, so /// falling back to a full frame here would write different bytes than the /// converged pass. The final leftover drain runs with `allow_defer` off, /// which restores the full-form fallback for a basis that truly never /// arrived. Always false on the batched paths, whose store IS converged. defer_unseen: bool, /// A bounded plaintext memo shared across `persistStore`'s whole sweep. Each /// artifact reconstructs its own plaintext exactly once here — for the §7.7 /// verify — and primes it under its just-written frame's offset, so a later /// reverse-delta naming this frame as basis reuses that plaintext instead of /// re-walking (re-inflating, re-applying) the chain. Bounded (LRU, fixed byte /// + entry caps): a cache miss just costs a fresh reconstruction, never a /// wrong answer, so the stored bytes are identical with or without it. cache: *logstore.PlaintextCache, /// submanifests §7.1 — the sweep-wide S-reference record. Every plaintext /// this function reconstructs for the §7.7 verify is offered to it, so the /// phantom duty attaches to STORING and not to any one arrival path. srefs: *SubManifestRefs, /// submanifests §7.1/§8.1 — REBUILD the S-reference phantom set for an /// artifact that is ALREADY a durable frame. A phantom is an interned entry /// with no frame (`LogStore.intern`: "Phantoms never reach disk"), so the /// §7.1 phantoms this repository minted on a previous run are gone the /// moment the process exits; the gap has to be re-derived from the durable /// artifacts that name it. Without this an open tree closes itself on the /// next run: the tree manifest is already persisted, the early return skips /// its scan, and the repository reports converged while it still cannot /// enumerate that check-in. /// /// Passed as a parameter rather than read here because the decision is the /// repository's, not this artifact's: §8.1's durable seen-submanifest /// indicator is false for every repository predating this draft, so the /// rescan — and its one inflate per already-persisted artifact — is /// unreachable for all of them and the persist sweep is what it always was. rescan_persisted: bool, /// Where to record the wire name of every cluster this call /// writes, so the persist keeps the cluster index warm without a rescan. clusters_out: ?*std.ArrayListUnmanaged([]const u8), ) PersistError!PersistOutcome { const name = store.hashOf(rid); var digest_buf: [digest_len]u8 = undefined; const digest = decodeWireName(name, &digest_buf) orelse return error.Corrupt; if (log.lookup(digest)) |ex| { if (!log.isPhantom(ex)) { if (rescan_persisted) { // Read through the sweep's own memo, so an artifact this sweep // already reconstructed costs nothing to rescan. const plain = (log.contentCached(gpa, ex, cache) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse return .persisted; defer gpa.free(plain); try phantomizeSubManifestRefs(gpa, log, plain, srefs); } return .persisted; } } // A genuine phantom carries nothing durable — skip it (never a drop of held // content). `held` classifies without inflating a wire-backed entry. if (!store.held(rid)) return .persisted; // §7.7 L1465/L1469-1471 — the repository transaction-commit hook. Recompute // the artifact hash over the reconstructed bytes and abort BEFORE any frame is // written on a mismatch (a lying container inflates but does not name its // content). The verify plaintext is reconstructed against the LOG basis (a // read bounded by `max_delta_chain`) — NOT the in-memory wire chain, which, // now a clone stores every artifact wire-backed, is the full reverse-delta // history and would exceed the read depth bound. `persistStore` writes basis // frames before the deltas that name them (reverse-delta order), so a delta's // basis is already a log frame here. if (store.wire(rid)) |wf| { if (wf.delta) |d| { var base_buf: [digest_len]u8 = undefined; const base_digest = decodeWireName(d.src_hash, &base_buf) orelse return error.Corrupt; const maybe_base = log.lookup(base_digest); if (maybe_base == null or log.isPhantom(maybe_base.?)) { // The basis is not yet a log frame. Defer on it (reverse-delta // order: the newer basis frame must be written first) when it is a // held store entry that WILL be persisted. if (allow_defer) { if (store.lookup(d.src_hash)) |bs_rid| { if (store.held(bs_rid)) return .{ .deferred = d.src_hash }; } if (defer_unseen) return .{ .deferred = d.src_hash }; } // Basis absent and not deferrable (§3.3): fall back to FULL form // from the entry's OWN plaintext — a put+retain entry carries it // directly (never a phantom-based delta). In a converged clone this // is unreachable (every delta's basis is a written frame, handled // below); a wire-delta entry whose basis truly never arrived cannot // reconstruct — a hard error, never a fabricated artifact. const content = store.content(rid) orelse return error.Corrupt; if (!zit.artifact.verify(name, content)) return error.Corrupt; // submanifests §7.1 — the plaintext §7.7 just verified is the same // bytes the phantom duty is stated over; scan it here rather than // forcing a second inflate anywhere else. try phantomizeSubManifestRefs(gpa, log, content, srefs); try noteCluster(gpa, clusters_out, name, content); const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } const base_rid = maybe_base.?; const base_off = log.offsetOf(base_rid); // Reuse the basis plaintext memoised when the basis frame was // persisted — reverse-delta order writes every basis before the // delta that names it, so the memo is normally warm. Only a miss // (first use, or an LRU eviction) reconstructs it by walking the // chain; either way `base_plain` is owned and freed below. const base_plain = if (cache.get(base_off)) |hit| blk: { // §7.7 positive space: a memo entry is the exact plaintext that // named its wire hash at the basis's own persist, so the reused // bytes still hash to the basis name (debug-only assertion; the // hash is never recomputed on the hot path). std.debug.assert(zit.artifact.verify(d.src_hash, hit)); break :blk gpa.dupe(u8, hit) catch return error.OutOfMemory; } else blk: { const p = (log.contentCached(gpa, base_rid, cache) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse return error.Corrupt; // `contentCached` does not memoise a top-level full-form basis; // prime it here so sibling reverse-deltas on the same basis reuse // this reconstruction instead of repeating it. try cache.put(gpa, base_off, p); break :blk p; }; defer gpa.free(base_plain); const raw = zit.container.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(raw); const content = zit.delta.apply(gpa, base_plain, raw) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(content); if (!zit.artifact.verify(name, content)) return error.Corrupt; // submanifests §7.1 — the plaintext §7.7 just verified is the same // bytes the phantom duty is stated over; scan it here rather than // forcing a second inflate anywhere else. try phantomizeSubManifestRefs(gpa, log, content, srefs); try noteCluster(gpa, clusters_out, name, content); // Store the delta VERBATIM iff its basis is a held frame under the READ // depth bound; an over-deep basis falls back to full form (never an // unreadable chain). `delta_of` is the basis's backward offset — // strictly lower than this new frame, the invariant `putDeltaContainer` // asserts. if (log.depthOf(base_rid) < logstore.max_delta_chain) { const new_rid = log.putDeltaContainer(digest, base_rid, wf.container) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the just-verified plaintext so an older reverse-delta // naming this frame as basis reuses it (no chain re-walk). try cache.put(gpa, log.offsetOf(new_rid), content); } else { const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; try cache.put(gpa, log.offsetOf(new_rid), content); } return .persisted; } // §7.2 full form — decode to verify, then store the container verbatim. const content = zit.container.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(content); if (!zit.artifact.verify(name, content)) return error.Corrupt; // submanifests §7.1 — the plaintext §7.7 just verified is the same // bytes the phantom duty is stated over; scan it here rather than // forcing a second inflate anywhere else. try phantomizeSubManifestRefs(gpa, log, content, srefs); try noteCluster(gpa, clusters_out, name, content); const new_rid = log.putContainer(digest, wf.container) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the just-verified full-form plaintext so a reverse-delta naming // this frame as basis reuses it instead of re-inflating the container. try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } // §4.1 — no retained wire (a raw pull `put`): the plaintext is memoised in the // store, no chain to walk. Verify and compress it; the hash names the // UNCOMPRESSED bytes, so compression is address-transparent. const content = store.content(rid) orelse return error.Corrupt; if (!zit.artifact.verify(name, content)) return error.Corrupt; // submanifests §7.1 — the plaintext §7.7 just verified is the same // bytes the phantom duty is stated over; scan it here rather than // forcing a second inflate anywhere else. try phantomizeSubManifestRefs(gpa, log, content, srefs); try noteCluster(gpa, clusters_out, name, content); const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the plaintext so a reverse-delta naming this frame as basis reuses it. try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } /// Evidence seam: the persist drain's deferred waiting set — the /// reverse-deltas resident because their basis frame has not been written yet — /// at its peak, as a frame count and as wire container bytes. This is the /// number the streaming-persist decision hinges on: it is exactly what a /// per-round drain would have to keep (or spill) instead of the whole store. /// Off unless `ZIT_PERSIST_WAITING` asks for it at exit, and it changes /// nothing else. pub var persist_waiting_peak: u64 = 0; /// Evidence seam: what the last sync run moved over the wire and /// what its cluster index cost — bounded announcement is a claim about bytes /// per round, and this is where it is measured. `ZIT_SYNC_STATS` at exit. pub var sync_rounds: u64 = 0; pub var sync_sent_bytes: u64 = 0; pub var sync_received_bytes: u64 = 0; pub var cluster_index_names: u64 = 0; fn recordSyncStats(stats: sync_driver.RunStats) void { sync_rounds = stats.rounds; sync_sent_bytes = stats.sent_bytes; sync_received_bytes = stats.received_bytes; } pub var persist_waiting_peak_bytes: u64 = 0; /// Persists every content-bearing artifact in `store` into an already-open /// writable log. The caller owns durability, allowing clone to append artifacts /// and its combined config overlay before one final commit. fn persistStoreOpen( gpa: std.mem.Allocator, log: *logstore.LogStore, store: repostate.ArtifactStore, keep: ?*const std.DynamicBitSetUnmanaged, start_after: repostate.Rid, clusters_out: ?*std.ArrayListUnmanaged([]const u8), ) PersistError!PersistSummary { var pass = try PersistPass.init(gpa, log, store, keep, null, start_after, clusters_out); defer pass.deinit(); try pass.drainNew(); return pass.finish(); } /// The persist drain as a RESUMABLE pass: `drainNew` persists every /// content-bearing artifact interned since the last call, and `finish` settles /// the leftovers and the sub-manifest bookkeeping. The batched shape /// (`persistStoreOpen`) is init + one drainNew + finish; a streaming clone /// calls drainNew after every round instead, with `mem` set so each persisted /// artifact's resident bytes are released the moment its frame is durable — /// git index-pack's shape (stream to disk, defer a delta whose base is unseen, /// resolve when it lands) applied to fossil's reverse-delta wire. const PersistPass = struct { gpa: std.mem.Allocator, log: *logstore.LogStore, store: repostate.ArtifactStore, keep: ?*const std.DynamicBitSetUnmanaged, /// Set on the streaming path: the concrete store behind `store`, so a /// persisted rid's bytes can be released. Null keeps every byte resident /// until the caller drops the store — the batched behaviour. mem: ?*repostate.MemStore, /// One bounded plaintext memo for the whole pass: a basis reconstructed for /// its own §7.7 verify is reused by every reverse-delta that names it, so a /// delta family reconstructs each basis ~once rather than once per /// referencing delta. Bounded (LRU byte + entry caps) — never the whole repo. cache: logstore.PlaintextCache, /// submanifests §7.1 — one record for the whole pass, so a reference is /// judged a gap only after every artifact this store carries has landed. srefs: SubManifestRefs = .{}, rescan: bool, waiting: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(repostate.Rid)) = .empty, ready: std.ArrayListUnmanaged(repostate.Rid) = .empty, written: usize = 0, elided: usize = 0, /// The first rid `drainNew` has not seeded yet — the resume cursor. next_seed: repostate.Rid = 1, /// See `persistArtifact`; null records nothing. clusters_out: ?*std.ArrayListUnmanaged([]const u8) = null, /// Streaming only: rids the cursor passed while they were /// phantoms. Fossil interns a delta's unseen DELTASRC as a phantom the /// moment the delta arrives; the content lands at that SAME rid in a later /// round, behind the cursor. Each drainNew re-checks these first and /// drives the ones that have since gained content; whatever is left at /// `finish` is a true phantom, skipped exactly as the batched pass skips /// it. Empty on the batched path, whose store is already converged. revisit: std.ArrayListUnmanaged(repostate.Rid) = .empty, // Evidence-seam running totals for the peak counters above. waiting_count: u64 = 0, waiting_bytes: u64 = 0, fn init( gpa: std.mem.Allocator, log: *logstore.LogStore, store: repostate.ArtifactStore, keep: ?*const std.DynamicBitSetUnmanaged, mem: ?*repostate.MemStore, // Begin the walk AFTER this rid, so a pull persists only the // artifacts it received (which the holdings store appends beyond its // log-backed prefix) instead of re-walking every already-durable // holding. Zero means start at rid 1 (clone/push/serve: the whole // store IS the new content). start_after: repostate.Rid, clusters_out: ?*std.ArrayListUnmanaged([]const u8), ) PersistError!PersistPass { // submanifests §8.1/§7.1 — THE REBUILD-ON-OPEN INVARIANT, stated once here. // // §7.1's phantoms are not durable and are not made durable: a phantom is an // interned entry with no frame, exactly as an absent delta source's is // ("exactly as an absent delta source creates a source phantom"), and // `LogStore` writes none of either kind ("Phantoms never reach disk"). The // `.zit` on-disk format version is frozen (`logstore.version`), so minting a // durable phantom record would reject every repository already written. // // What makes the in-memory phantoms CORRECT across a restart is this // invariant: the S-reference gap set is a pure function of the durable // content-bearing frames, and this sweep re-derives it in full on every run. // A repository whose §8.1 indicator says it has ever stored an S reference // rescans its already-persisted artifacts (`rescan_persisted`); one that has // not — every repository predating this draft — rescans nothing and pays // nothing. The indicator itself IS durable, which is the whole trick: one // config byte survives the restart instead of a phantom table. const rescan = blk: { var oc = log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; if (oc) |*c| { defer c.deinit(gpa); break :blk c.config.seen_submanifest; } break :blk false; }; return .{ .gpa = gpa, .log = log, .store = store, .keep = keep, .mem = mem, .cache = logstore.PlaintextCache.init(), .rescan = rescan, .next_seed = start_after + 1, .clusters_out = clusters_out, }; } /// The type-erased `RoundDrain` entry: persist everything the round that /// just folded in has interned. fn drainErased(ctx: *anyopaque) sync_driver.DrainError!void { const self: *PersistPass = @ptrCast(@alignCast(ctx)); return self.drainNew(); } fn deinit(self: *PersistPass) void { self.cache.deinit(self.gpa); self.srefs.deinit(self.gpa); var it = self.waiting.valueIterator(); while (it.next()) |list| list.deinit(self.gpa); self.waiting.deinit(self.gpa); self.ready.deinit(self.gpa); self.revisit.deinit(self.gpa); } /// Drive one rid through `persistArtifact`, drain the waiters its frame /// unblocks, and — on the streaming path — release everything persisted. fn drive(self: *PersistPass, first: repostate.Rid) PersistError!void { const gpa = self.gpa; try self.ready.append(gpa, first); while (self.ready.pop()) |rid| { switch (try persistArtifact(gpa, self.store, self.log, rid, true, self.mem != null, &self.cache, &self.srefs, self.rescan, self.clusters_out)) { .persisted => { // `rid`'s frame now exists: re-drive any reverse-deltas that // deferred on it as their (now-written) basis. if (self.waiting.fetchRemove(self.store.hashOf(rid))) |kv| { var list = kv.value; defer list.deinit(gpa); for (list.items) |w| { self.waiting_count -= 1; self.waiting_bytes -= if (self.store.wire(w)) |wf| wf.container.len else 0; } try self.ready.appendSlice(gpa, list.items); } if (self.mem) |m| m.releasePersisted(rid); }, .deferred => |base_hash| { const gop = try self.waiting.getOrPut(gpa, base_hash); if (!gop.found_existing) gop.value_ptr.* = .empty; try gop.value_ptr.append(gpa, rid); // Evidence seam: a deferred rid is exactly what // streaming persist keeps resident; meter it. self.waiting_count += 1; self.waiting_bytes += if (self.store.wire(rid)) |wf| wf.container.len else 0; if (self.waiting_count > persist_waiting_peak) persist_waiting_peak = self.waiting_count; if (self.waiting_bytes > persist_waiting_peak_bytes) persist_waiting_peak_bytes = self.waiting_bytes; }, } } } /// Persist every content-bearing artifact interned since the last call. /// /// Reverse-delta persist ordering (§7.2/§8.2). Fossil stores an OLDER /// artifact as a delta of a NEWER one, so a delta cfile can land in the store /// at a LOWER rid than the basis it points at, while the log's `delta_of` is a /// BACKWARD offset that REQUIRES the basis frame to precede the delta. Persist /// therefore drains a worklist: a delta whose basis is not yet a frame is /// deferred (keyed by the basis wire-hash) and re-driven the instant that /// basis lands. Each content-bearing artifact seeds the drain once in rid /// order; each deferred delta is re-driven exactly once, so the pass is O(n) /// and terminates — a chain bottoms out at fossil's full newest artifact, and /// content addressing forbids a cycle. fn drainNew(self: *PersistPass) PersistError!void { // Revived phantoms first: their frames may be the bases this round's // deltas are about to defer on. var i: usize = 0; while (i < self.revisit.items.len) { const rid = self.revisit.items[i]; if (self.store.held(rid)) { _ = self.revisit.swapRemove(i); self.written += 1; try self.drive(rid); } else i += 1; } const n = self.store.count(); while (self.next_seed <= n) : (self.next_seed += 1) { const seed = self.next_seed; if (!self.store.held(seed)) { // Residual phantom: not durable. Mid-exchange it may yet gain // content at this very rid — remember it for the next drain. if (self.mem != null) try self.revisit.append(self.gpa, seed); continue; } if (self.keep) |k| { // Sparse-cone RFC section 6.3: elision writes nothing -- the frame // simply never lands, and rebuild-on-open re-interns the dangling // references as phantoms the cone classifies. A kept delta whose // basis is elided falls to the leftover drain below and stores // FULL FORM, so elision can never orphan held content. if (!k.isSet(seed)) { self.elided += 1; continue; } } self.written += 1; // count each content-bearing artifact exactly once, at its rid try self.drive(seed); } } /// Settle the leftovers and the sub-manifest bookkeeping; the pass is done. fn finish(self: *PersistPass) PersistError!PersistSummary { const gpa = self.gpa; const log = self.log; // Defensive drain: a deferred basis is itself content-bearing and therefore // persisted above (draining its waiters), so `waiting` is normally empty // here. Should a basis nonetheless be missing, its waiters land as full form // rather than be dropped — every content-bearing artifact stays durable. var leftover = self.waiting.iterator(); while (leftover.next()) |e| { for (e.value_ptr.items) |rid| { switch (try persistArtifact(gpa, self.store, log, rid, false, false, &self.cache, &self.srefs, self.rescan, self.clusters_out)) { .persisted => if (self.mem) |m| m.releasePersisted(rid), .deferred => unreachable, // allow_defer=false never defers } } } // submanifests §7.1 — decided once, at the end: a reference is a gap only // if nothing in this whole store turned out to name it. const gaps = self.srefs.unresolved(log); // submanifests §8.1 — "When the first S-bearing artifact is stored, an // implementation SHOULD record a local seen-submanifest indicator so // operational surfaces can warn before the setting is enabled or a pre-S // client is granted access." Written once, on the false->true edge, so a // steady-state sync appends no config frame; it is also what arms the // rebuild above on every subsequent run of this repository. if ((self.srefs.seen or self.srefs.seen_delta) and !self.rescan) { var oc = log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; var cfg: logstore.Config = if (oc) |c| c.config else .{ .hash_policy = 0, .default_branch = "", .project_id = "" }; defer if (oc) |*c| c.deinit(gpa); const already = (cfg.seen_submanifest or !self.srefs.seen) and (cfg.seen_delta_manifest or !self.srefs.seen_delta); if (!already) { if (self.srefs.seen) cfg.seen_submanifest = true; if (self.srefs.seen_delta) cfg.seen_delta_manifest = true; log.putConfig(cfg) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; } } return .{ .written = self.written, .submanifest_gaps = gaps, .elided = self.elided }; } }; /// Opens `dir/.zit` for the legacy persistence callers, preserving their /// artifact result and commit boundary while routing the write through the /// already-open-log implementation. fn persistStore( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, store: repostate.ArtifactStore, keep: ?*const std.DynamicBitSetUnmanaged, start_after: repostate.Rid, ) PersistError!PersistSummary { var file = cli.openStoreAppend(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; defer log.deinit(gpa); // A log with no entries at open is being CREATED by this persist, // so the clusters it records are the whole index; otherwise they extend one. const fresh = log.entryCount() == 0; var clusters: std.ArrayListUnmanaged([]const u8) = .empty; defer { for (clusters.items) |n| gpa.free(n); clusters.deinit(gpa); } const summary = try persistStoreOpen(gpa, &log, store, keep, start_after, &clusters); log.commit() catch return error.Storage; try refreshClusterSidecar(gpa, io, dir, &log, clusters.items, fresh); return summary; } /// Keep the cluster index warm across a persist: adopt the stored /// index (its key still validates — this persist only appended), append the /// clusters just written, re-key to the committed log, store. With no valid /// index to extend, only a persist that created the log writes one; any /// other store waits for the next sync open, which builds it by one walk. /// Advisory on storage (a failed write leaves the sidecar as it was); honest /// on memory, so the persist's OOM contract holds through it. fn refreshClusterSidecar( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *const logstore.LogStore, written: []const []const u8, fresh: bool, ) error{OutOfMemory}!void { var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); const a = arena.allocator(); var names: std.ArrayListUnmanaged([]const u8) = .empty; var adopted = false; if (try zit.repocache.loadClusterIndex(gpa, io, dir)) |blob| { defer gpa.free(blob); if (try clusterindex.parse(a, blob, log)) |p| { for (p.names) |n| try names.append(a, try a.dupe(u8, n)); adopted = true; } } if (!adopted and !fresh) return; for (written) |w| try names.append(a, w); const blob = (try clusterindex.serialize(gpa, log, log.entryCount(), names.items)) orelse return; defer gpa.free(blob); zit.repocache.storeClusterIndex(io, dir, blob); } /// The cluster index for one sync open: adopt the sidecar when its /// key validates, walk only the tail it has not seen (or everything, once, on /// a miss), store the refreshed sidecar, and resolve it against `store`. /// Nothing here is an error beyond memory: any refusal is a rebuild, and a /// store the walk cannot read simply stays unindexed — unbounded, never wrong. fn loadClusterIndex( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *logstore.LogStore, store: repostate.ArtifactStore, ) error{OutOfMemory}!clusterindex.Resolved { var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); const a = arena.allocator(); var names: std.ArrayListUnmanaged([]const u8) = .empty; var seen: std.StringHashMapUnmanaged(void) = .empty; defer seen.deinit(gpa); var after: repostate.Rid = 0; var adopted = false; if (try zit.repocache.loadClusterIndex(gpa, io, dir)) |blob| { defer gpa.free(blob); if (try clusterindex.parse(a, blob, log)) |p| { for (p.names) |n| { const owned = try a.dupe(u8, n); if ((try seen.getOrPut(gpa, owned)).found_existing) continue; try names.append(a, owned); } after = @intCast(p.n_rids); adopted = true; } } // Phantoms never reach disk, so an index written after a persist may count // entries this scan does not see; clamping never skips a real frame the // writer had not already recorded. const n: repostate.Rid = @intCast(log.entryCount()); if (after > n) after = n; if (after < n) { var walked: std.ArrayListUnmanaged([]const u8) = .empty; try clusterindex.walkClusters(gpa, a, log, after, &walked); for (walked.items) |w| { if ((try seen.getOrPut(gpa, w)).found_existing) continue; try names.append(a, w); } } if (!adopted or after < n) { if (try clusterindex.serialize(gpa, log, n, names.items)) |blob| { defer gpa.free(blob); zit.repocache.storeClusterIndex(io, dir, blob); } } cluster_index_names = names.items.len; return clusterindex.resolve(gpa, store, names.items); } // --------------------------------------------------------------------------- // `zit pack` — repack the store into a compact new generation. // // On-disk storage is UNSPECIFIED by the RFCs (they fix the wire format and // content-addressed naming, §4.1 L889-892/L918-919 and artifact §3.1 L318-326), so pack is free to reorganize // the log as long as every artifact's exact content and name survive and the // §7.2 verbatim wire bytes are still servable. It re-ingests every artifact, // in the source's offset order, into a FRESH log through `putArtifact` with a // lineage base hint — a check-in against its parent manifest, a file blob // against the same path's blob in the parent check-in (the commit path's own // §4.1/§7.2 compaction heuristic, applied offline). Offset order plus the // strictly-backward `delta_of` invariant means every base is already written // before the delta that names it, so no worklist is needed (unlike // `persistStore`, whose input is rid-ordered). // // The new log is written to a temporary file beside `.zit`, fsynced, and then // atomically renamed over `.zit`: a crash anywhere before the rename leaves the // original byte-identical (the temp is discarded next run). The `.zit-cache` // invalidates itself on the tail change and rebuilds on the next read verb. // --------------------------------------------------------------------------- /// pack's core store-to-store transform lives in the `zit` module (pack.zig) /// so the DST/fuzz harnesses drive the REAL pass in-process; the CLI wiring /// below (temp file, atomic rename, stats rendering) stays here. These aliases /// keep every call site and test reading /// `packLog`/`PackError`/`PackStats`/`Filter`/`shunFilter`. const PackError = zit.pack.PackError; const PackStats = zit.pack.PackStats; const packLog = zit.pack.packLog; const Filter = zit.pack.Filter; const shunFilter = zit.pack.shunFilter; /// The name of the transient new-generation file the rewriting verbs share, /// beside `.zit`. A fixed name (never a random one) so a temp left by a crashed /// run is reused rather than accumulating, and so `extras.zig` can name it in /// the single never-tracked predicate. Because it IS fixed and TWO verbs now /// reach for it, `rewriteAndSwap` holds an exclusive advisory lock on it for the /// whole rewrite — see there. const pack_tmp_filename = extras.pack_tmp_filename; /// `zit pack` — repack the current repository's store into a compact new /// generation and atomically swap it in. Thin CLI wiring over `packLog`: parse /// (no positional), discover the checkout root, open `.zit` read-only and /// capture its tail key, stream the packed log into a temp file, fsync, re-check /// the source tail (the no-repo-lock concurrency guard), then rename the temp /// over `.zit`. On any failure the temp is deleted and the original store is /// intact. /// /// The integrity guarantee this rewrite depends on is UNCHANGED, but it is no /// longer collected at open. `openReadOnly` used to whole-frame-CRC-verify every /// frame while scanning, so a corrupt source aborted before a byte was written — /// an eager pre-flight. It scans header-only now. The check that matters happens /// on the reads instead: `packLog` pulls every artifact through `srcContent`, /// which propagates `error.Corrupt` rather than skipping the frame, and /// `readFrameAt` CRC-checks every frame it reads. So a rotted payload still /// aborts the pack with the temp deleted and `.zit` untouched — it is caught in /// the copy loop rather than at open, and the operator sees the same outcome. /// This ordering is load-bearing for a verb that RENAMES OVER the original: if /// `srcContent` were ever changed to answer null on a corrupt read, the rewrite /// would silently omit the damaged artifact and then delete the only copy of it. /// The packed-generation memo's byte shape: magic, then the four /// `LogKey` fields little-endian — 32 bytes exactly. No self-checksum: a /// damaged memo can only fail the equality below, and the cost of a miss is /// one real pack. const packed_key_magic = "zPK1"; const packed_key_len = 4 + 8 + 8 + 8 + 4; fn serializePackedKey(buf: *[packed_key_len]u8, key: logstore.LogStore.LogKey) []const u8 { @memcpy(buf[0..4], packed_key_magic); std.mem.writeInt(u64, buf[4..12], key.end, .little); std.mem.writeInt(u64, buf[12..20], key.frame_count, .little); std.mem.writeInt(u64, buf[20..28], key.last_frame_offset, .little); std.mem.writeInt(u32, buf[28..32], key.last_header_crc, .little); return buf[0..packed_key_len]; } fn parsePackedKey(blob: []const u8) ?logstore.LogStore.LogKey { if (blob.len != packed_key_len or !std.mem.eql(u8, blob[0..4], packed_key_magic)) return null; return .{ .end = std.mem.readInt(u64, blob[4..12], .little), .frame_count = std.mem.readInt(u64, blob[12..20], .little), .last_frame_offset = std.mem.readInt(u64, blob[20..28], .little), .last_header_crc = std.mem.readInt(u32, blob[28..32], .little), }; } /// Record the just-swapped generation's key so the next identity-filter /// pack over an unchanged log is a no-op, and refresh the scan-index sidecar /// the rename invalidated — the open below pays the scan either way, so the /// next read verb should not pay it again. Best-effort throughout: any failure /// leaves the memo absent, which costs a real pack, never an answer. fn recordPackedGeneration(gpa: std.mem.Allocator, io: std.Io, store_dir: std.Io.Dir) void { var file = store_dir.openFile(io, repo_filename, .{}) catch return; defer file.close(io); const st = file.stat(io) catch return; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch return; defer log.deinit(gpa); const key = log.logKey() catch return; var buf: [packed_key_len]u8 = undefined; zit.repocache.storePackedKey(io, store_dir, serializePackedKey(&buf, key)); refreshSettingLogIndex(gpa, io, store_dir, &log); } pub fn runPack( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { var drop_elided = false; var pack_args: std.ArrayListUnmanaged([]const u8) = .empty; defer pack_args.deinit(gpa); for (verb_args) |tok| { if (std.mem.eql(u8, tok, "--drop-elided")) { drop_elided = true; } else { pack_args.append(gpa, tok) catch return stateOom(streams, "pack"); } } const help = switch (cli.parseNoPositionalArgs(pack_args.items)) { .ok => |a| a.help, .unknown_option => |tok| return cli.usageError(streams, "pack", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "pack", .extra_argument, tok), }; if (help) { streams.out.writeAll(pack_help) catch {}; return .ok; } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "pack")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // pack is store-scoped end to end: the reads, the pack-tmp lock // and the final rename must all land beside the STORE, so a detached // worktree resolves its pointer once here and the whole flow inherits it. if (cli.storeDirOf(io, root_dir) catch return repoReadFail(streams, "pack")) |resolved| { root_dir.close(io); root_dir = resolved; } // Open the source read-only: `openReadOnly` scans and whole-frame-CRC-checks // every frame, so a corrupt source aborts here (no temp written). The tail // key pins the log we packed for the pre-rename concurrency re-check. // // LIMIT OF THE RECLAIM'S DAMAGE TOLERANCE, stated where it is imposed: this // pre-flight runs before `packLog` sees the filter, so it covers the shunned // frames too. The reclaim contract's filter-before-reconstruction ordering // rescues a shunned artifact whose CONTENT no longer hashes to its name (and // one that will not decode) — but NOT one whose frame CHECKSUM is damaged, // which fails here with the reclaim never attempted. `pack_help` says so in // those terms; narrowing the pre-flight to kept frames would trade a // whole-store integrity gate for a rarer recovery path and is not this // slice's call to make. var src_file = root_dir.openFile(io, repo_filename, .{}) catch return repoReadFail(streams, "pack"); defer src_file.close(io); const src_st = src_file.stat(io) catch return repoReadFail(streams, "pack"); var src_fs = storage.FileStorage.init(io, src_file, src_st.size); var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| { switch (e) { error.OutOfMemory => streams.err.writeAll("zit pack: out of memory\n") catch {}, else => streams.err.print("zit pack: cannot read repository: {s}\n", .{reasonOf(e)}) catch {}, } return .failure; }; defer src.deinit(gpa); const src_key = src.logKey() catch return packFail(streams, error.Corrupt); // The no-op gate: a log whose key equals the one the last pack // recorded has had NO append since (a commit, a shun row, and a pull all // grow the log), so there is nothing to compact and nothing to reclaim // that the last pack did not already handle. Gated to the identity // shape: a `--drop-elided` run composes a cone filter the memo knows // nothing about, so it always runs. if (!drop_elided) { if (zit.repocache.tryLoadPackedKey(gpa, io, root_dir)) |blob| { defer gpa.free(blob); if (parsePackedKey(blob)) |recorded| { if (std.meta.eql(recorded, src_key)) { streams.out.writeAll("already packed; nothing to do\n") catch {}; return .ok; } } } } // `zit pack` is where the repository's deferred shuns are RECLAIMED: every // artifact the source's shun table names is dropped from the new generation // instead of relocated into it (the pack filter). The tombstones themselves // carry forward, so the name keeps reading as absent and a later sync will // not re-fetch what the operator removed. pack names no NEW tombstone — that // is `zit shun`'s job — so `extra_shuns` is empty. // Sparse-cone RFC section 5.4 -- narrow-then-compact, the spec's ONLY // deletion: an explicit `pack --drop-elided` drops what the store cone // elides (blobs referenced exclusively out-of-cone), composed with the // ordinary shun reclaim. Refused without a store cone: with no policy on // record there is nothing "elided" to drop, and a bare pack must never // guess one. var elided_dropper: ?ElidedDropper = null; defer if (elided_dropper) |*d| d.deinit(gpa); if (drop_elided) { var cone = zit.repocache.readStoreCone(gpa, io, root_dir) orelse { streams.err.writeAll( "zit pack: --drop-elided needs a store cone and this store has none\n" ++ "zit pack: narrow first ('zit update --cone=...'), or drop the flag\n", ) catch {}; return .failure; }; elided_dropper = ElidedDropper.build(gpa, src_fs.storageFile(), &cone) orelse { cone.deinit(gpa); streams.err.writeAll("zit pack: cannot classify the store for --drop-elided\n") catch {}; return .failure; }; cone.deinit(gpa); } const pack_filter: Filter = if (elided_dropper) |*d| d.composed(shunFilter(&src)) else shunFilter(&src); const stats = rewriteAndSwap( gpa, io, root_dir, &src, src_key, pack_filter, &.{}, ) catch |e| switch (e) { error.TailChanged => return tailChangedFail(streams, "pack"), error.RewriteBusy => return rewriteBusyFail(streams, "pack"), else => |pe| return packFail(streams, pe), }; renderPackStats(streams.out, stats); // Only the identity-filter shape records the memo: a --drop-elided pack's // outcome depends on a cone the memo does not capture, so the next plain // pack must run for real. if (!drop_elided) recordPackedGeneration(gpa, io, root_dir); return .ok; } /// `rewriteAndSwap`'s causes: everything `packLog` can fail with, plus the two /// aborts that are not failures of the rewrite at all and must not be reported as /// if they were. `TailChanged` means the source log is intact and NEWER than the /// snapshot we packed; `RewriteBusy` means another rewrite already owns the temp. /// In both the original store is untouched and the operator's next move is /// obvious, whereas every `PackError` arm means the new generation could not be /// built. pub const RewriteError = PackError || error{ TailChanged, RewriteBusy }; /// The new-generation rewrite `pack` and `shun` share: create the fixed-name /// readable temp beside `.zit`, stream `packLog(src → temp)` through `filter` /// (recording `extra_shuns` in the new generation only), fsync, re-check the /// source tail against `src_key`, then atomically rename the temp over `.zit`. /// Returns the pack's own stats; on every failing path the temp is deleted and /// the original store is byte-identical. /// /// ONE durable transition — the rename — and that is why the two verbs share /// this body rather than each having its own: the property is a property of this /// sequence, so a second copy would be a second place for it to stop holding. /// `src` is already open (both callers must read it before they can decide what /// to filter); the pre-rename check deliberately reads NEITHER it nor the /// caller's file handle. /// /// THAT IS THE GUARD, and it takes a fresh HANDLE, not merely a fresh stat. /// `storage.FileStorage` caches the append watermark it was constructed with and /// `size()` returns that cached value without a stat (src/storage.zig /// `sizeImpl`), so re-checking the tail through the caller's existing view /// compares `key.end` against the very number the key was derived from — it /// cannot fail, and the concurrency guard is then a no-op that reads as a guard. /// The caller's HANDLE fails the same way against the other concurrent writer: /// once a competing rewrite has renamed its generation over `.zit`, that handle /// names an ORPHAN inode whose size and tail are frozen, so it agrees with the /// key forever — and this run would then rename a snapshot taken before the /// winner existed over the winner, silently un-shunning what it just shunned. /// Reopening `.zit` BY NAME under the lock is what makes both an append (`zit /// commit`) and a rename (a racing `pack`/`shun --force`) detectable, and — for /// `shun` — is what makes "an aborted run leaves NO tombstone" something more /// than an assertion about code that never runs. Pinned by the two "a run the /// concurrency guard aborts leaves .zit untouched and records NO tombstone" /// tests, one per concurrent-writer shape. Same argument, same shape, as /// `shunAppendRows`. /// /// MUTUAL EXCLUSION, and why it lives here rather than in either verb. The temp /// has a FIXED name, so two overlapping rewrites would each truncate the other's /// in-flight new generation and then fail the §7.7 reconstruct-and-rehash pass on /// frames that had become holes — reported as `error.Corrupt`, i.e. two healthy /// processes telling the operator their repository is corrupt. A cron'd `zit pack` /// overlapping an operator's `zit shun --force` is an entirely ordinary way to hit /// that, and the shun is the run that must not silently not happen. So the temp is /// opened under an EXCLUSIVE, NON-BLOCKING advisory lock: the second arrival gets /// `error.RewriteBusy` and says so, instead of a corruption claim. The lock is /// held by the open file description, so a crashed run releases it — which is why /// this is a lock rather than an `exclusive` create, whose leftover would block /// every future rewrite. Truncation is therefore ours to do, AFTER the lock is /// held (`truncate` in the open would race the lock acquisition). /// /// Parameterised by DATA only — a drop predicate and a tombstone list — never by /// a policy callback. Which names those are, what to tell the operator, and /// whether to refuse at all stay entirely in the callers; nothing here knows /// which verb it is running, which is why it needs no verb label. /// /// `pub` for one reason: `src/sim/shun_sweep.zig` drives THIS FUNCTION under a /// fault-injecting `std.Io`. The property the sweep exists to pin — one durable /// transition, so a crash leaves either the old store or the new one — is a /// property of this sequence and of nothing else, so the sweep has to reach the /// sequence. It deliberately does NOT drive `runPack`/`runShun`: both begin at /// `cli.discoverRoot`, which reads the process's current directory, and a sweep /// that depends on ambient cwd is a sweep whose failures do not replay. pub fn rewriteAndSwap( gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, src: *logstore.LogStore, src_key: logstore.LogStore.LogKey, filter: Filter, extra_shuns: []const logstore.ShunRecord, ) RewriteError!PackStats { // New-generation file: a readable temp beside `.zit` (createFileAtomic opens // write-only, but pack reads its own delta bases back, so we roll our own // readable temp and rename it). Opened under the exclusive non-blocking lock // this function's doc explains; `WouldBlock` is another rewrite in progress, // not a storage failure. Cleaned up on every path except the successful // rename — and on the busy path we must NOT delete it, since it belongs to the // run that holds the lock. var tmp_file = root_dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = false, .lock = .exclusive, .lock_nonblocking = true, }) catch |e| switch (e) { error.WouldBlock => return error.RewriteBusy, else => return error.Storage, }; var renamed = false; // UNLINK BEFORE CLOSE, not after. `close` releases the advisory lock, and the // lock is what makes this path ours; unlinking after it would let a second // rewrite create and lock a fresh temp at the same name in between, which this // run's `deleteFile` would then pull out from under it — leaving that run to // write a complete new generation into an unlinked inode and fail its own // rename with ENOENT, the "healthy process told the operator something is // wrong" outcome the lock exists to prevent. Matches `shunAppendRows`. defer { if (!renamed) root_dir.deleteFile(io, pack_tmp_filename) catch {}; tmp_file.close(io); } // Discard whatever a prior crashed run left, now that the lock makes us the // only writer. `LogStore.open` below is handed watermark 0 and writes from // offset 0, so trailing bytes from a longer stale temp would survive past the // new generation's end — a checked truncation, not an optimisation. tmp_file.setLength(io, 0) catch return error.Storage; var dst_fs = storage.FileStorage.init(io, tmp_file, 0); var dst = try logstore.LogStore.open(gpa, dst_fs.storageFile(), src.hash_kind); const stats = packLog(gpa, src, &dst, filter, extra_shuns) catch |e| { dst.deinit(gpa); return e; }; dst.deinit(gpa); // `packLog` already fsynced; deinit only frees memory // No repo lock exists: if the source log changed under us since we scanned // it, abort rather than clobber a concurrent write with our stale snapshot. // O(1) (one stat + one header read); the original stays intact. For `shun` // this is also what makes an aborted run leave NO tombstone behind: the rows // were written into the temp this path is about to delete. // // A FRESH HANDLE, a fresh stat and a fresh storage view, for the reason given // in the doc above: the caller's view answers `size()` from a cached // watermark, so asking it would always agree with the key — and the caller's // HANDLE would agree just as unconditionally once a competing rewrite has // renamed its generation over `.zit`, since that leaves the handle on an // orphan inode whose size and tail can never move again. Reopening by NAME is // what makes that case an abort; the caller's handle only ever catches an // append to the same inode. var check_file = root_dir.openFile(io, repo_filename, .{}) catch return error.Storage; defer check_file.close(io); const now_st = check_file.stat(io) catch return error.Storage; var check_fs = storage.FileStorage.init(io, check_file, now_st.size); if (!logstore.LogStore.validateTail(check_fs.storageFile(), src_key)) return error.TailChanged; // The single durable transition: atomically replace `.zit` with the packed // log. A crash before this leaves the original untouched; after it, the // complete new log. There is no window where `.zit` is partial. root_dir.rename(pack_tmp_filename, root_dir, repo_filename, io) catch return error.Storage; renamed = true; return stats; } /// The no-repo-lock concurrency guard's diagnostic, general over `verb` because /// both rewriting verbs abort on it identically. Not a crash and not corruption: /// the wording leads with what the operator most needs to know. fn tailChangedFail(s: cli.Streams, verb: []const u8) cli.Exit { s.err.print( "zit {s}: repository changed during {s}; aborted (original is intact)\n", .{ verb, verb }, ) catch {}; return .failure; } /// The temp-lock diagnostic, general over `verb` for the same reason: both /// rewriting verbs contend for one fixed-name temp. Says what is happening and /// what to do, and — critically — does NOT say "corrupt", which is what an /// unguarded collision used to produce. fn rewriteBusyFail(s: cli.Streams, verb: []const u8) cli.Exit { s.err.print( "zit {s}: another repository rewrite is in progress; try again when it finishes\n", .{verb}, ) catch {}; return .failure; } /// Render pack's failure `err` as `zit pack: <reason>` on stderr, exit 1. The /// original store is always intact when this runs (every write went to the /// temp, which the caller's `defer` deletes). fn packFail(s: cli.Streams, err: PackError) cli.Exit { s.err.print("zit pack: {s}\n", .{reasonOf(err)}) catch {}; return .failure; } /// One human line summarizing a repack. "already packed" only when the run /// changed NOTHING an operator would care about — the log did not shrink AND /// nothing was reclaimed (the store was already compact; pack still verified every /// frame, its integrity-scrub value). A reclaim is never described as "already /// packed" even when the log grew, which it legitimately can: dropping an artifact /// that others were deltified against moves its content into one of them, so a /// byte count is exactly the wrong signal for whether work happened. /// /// The reclaim clause appears only when the run actually dropped something: a /// repository with no deferred shuns is the common case and "0 shunned reclaimed" /// on every pack would be noise, while a run that DID remove bytes must say so — /// dropping content silently is the one outcome an operator must never have to /// infer from a byte count. `pack_help` states the clause the same conditional way. /// /// `artifacts` is labelled "walked" rather than "packed" because `PackStats` /// counts SOURCE frames: on a reclaiming run `artifacts - dropped` is what the new /// generation holds, and an unlabelled first number would read as the packed /// count. Split from `runPack` so the exact wording is unit tested without a /// filesystem. fn renderPackStats(out: *std.Io.Writer, stats: PackStats) void { const unchanged = stats.bytes_after >= stats.bytes_before and stats.dropped == 0; const verb = if (unchanged) "already packed" else "packed"; out.print("{s}: {d} artifacts walked, {d} re-deltaed, ", .{ verb, stats.artifacts, stats.deltas }) catch {}; if (stats.dropped > 0) out.print("{d} shunned reclaimed, ", .{stats.dropped}) catch {}; out.print("{d} -> {d} bytes\n", .{ stats.bytes_before, stats.bytes_after }) catch {}; } // --------------------------------------------------------------------------- // `zit shun` — record artifacts in this repository's shun table and, by default, // remove their bytes in the same operation. // // NO DRAFT SPECIFIES THIS VERB. sync §3.3 (docs/rfcs/ // draft-fossil-sync-protocol-00.txt L518-519) defines the repository shun table // as "a persistent, repository-local set keyed by complete artifact name", // L520-521 grants that "Local administration MAY add or remove names", and // L527-530 makes the set mask the phantom scan. That is the whole of it: no // draft names a command, an option, a report, a refusal, or the operation that // removes a shunned artifact's bytes. Everything below — the refuse-by-default // policy, the `--force` gate, the unconditional current-checkout stop, the // per-type diagnostics — is this implementation's decision, and the choices are // stated at the point that makes them rather than attributed to a section. // // The one advisory the plan of record cites for sub-manifests cannot be // implemented or quoted here: there is no `submanifest` variant in // `artifact.ArtifactType` on this branch (it runs manifest=1 .. forum=8) and no // sub-manifest draft in docs/rfcs/. `shunConsequence`'s switch is total over // that enum, so the day a ninth type lands the compiler demands its diagnostic // — including the wider-blast-radius warning a sub-manifest deserves. // --------------------------------------------------------------------------- const NameSet = zit.pack.NameSet; /// What a `zit shun` run will do about one hash the operator named. Split into /// three groups by consequence, because the runner acts on the group, not the /// member: the three `blocking` values stop the whole invocation before anything /// durable happens; `remove`/`absent` each need a new tombstone; `reclaim` needs /// no row but still has bytes to remove; `already` needs nothing at all. const ShunDisposition = enum { /// Held here: the rewrite drops its bytes and records its tombstone. remove, /// Already in the set from an earlier `--defer`, but the BYTES ARE STILL HERE: /// no new row to write, and yet the removal the operator just asked for has /// genuinely not happened. Grouping this with `already` is what would make /// `zit shun --force H` after `zit shun --defer --force H` print "nothing to /// do" and exit 0 with the content still readable — the worst sentence this /// verb could produce, on the one path where an operator is acting under an /// order. So it is its own disposition: no tombstone, but a rewrite. reclaim, /// Not held here: the tombstone is recorded and nothing is reclaimed. A /// pre-emptive shun is meaningful and free — sync §3.3 L527-530 filters the /// phantom scan on the set, so a name recorded before the artifact ever /// arrives is never requested. absent, /// Already in the set AND already gone: a true no-op. `putShun` is idempotent /// as set membership, so the durable row keeps the epoch and reason it was /// written with. already, /// The manifest of the current checkout. REFUSED, with no `--force` /// override — the one hard stop. current_checkout, /// Not a lowercase-hex artifact-name prefix at all. not_a_name, /// An abbreviated prefix matching no stored artifact. Refused rather than /// recorded: the shun table is keyed by COMPLETE name, so a prefix cannot be /// completed and a row keyed on one would name nothing. unresolved, /// A prefix matching more than one stored artifact. ambiguous, /// Whether this disposition stops the whole invocation before any durable /// write. A total switch so a value added later cannot silently join the /// permissive group. fn blocks(self: ShunDisposition) bool { return switch (self) { .current_checkout, .not_a_name, .unresolved, .ambiguous => true, .remove, .reclaim, .absent, .already => false, }; } /// Whether this disposition needs a new tombstone row. fn records(self: ShunDisposition) bool { return switch (self) { .remove, .absent => true, .reclaim, .already, .current_checkout, .not_a_name, .unresolved, .ambiguous => false, }; } /// Whether this disposition has BYTES for the rewrite to drop. Separate from /// `records` precisely because the two are not the same question: `absent` /// records without removing and `reclaim` removes without recording. fn reclaims(self: ShunDisposition) bool { return switch (self) { .remove, .reclaim => true, .absent, .already, .current_checkout, .not_a_name, .unresolved, .ambiguous => false, }; } }; /// One hash the operator named, resolved. `name` is the full lowercase-hex /// artifact name — or, for the three unresolvable dispositions, the operator's /// own token, which is all there is to name. `digest` is the raw binary key the /// shun table and the pack filter both use, empty when there is none. `kind` is /// null for an opaque blob (an artifact that does not parse as a structural one), /// which is a first-class answer here exactly as it is for `zit artifact`. const ShunTarget = struct { name: []const u8, digest: []const u8 = &.{}, kind: ?artifact.ArtifactType = null, disposition: ShunDisposition, }; /// The report label for a target's type. Structural artifacts use §5.1 Table 3's /// external name (`artifact.ArtifactType.externalName`), the same vocabulary /// `zit artifact` prints, so an operator sees one word per type across the CLI; /// a blob is "file", matching `zit artifact`'s "type: file". fn shunTypeLabel(kind: ?artifact.ArtifactType) []const u8 { const t = kind orelse return "file"; return t.externalName(); } /// The consequence of removing an artifact of this type, in one clause. /// /// A TOTAL switch on `artifact.ArtifactType`, deliberately: the alternative is a /// default arm, and a type added later would then get no diagnostic at all while /// the operator was told the removal was safe. The six types that all lose the /// same thing share one arm rather than repeating one sentence six times. /// /// NO COUNTS, and that is a deliberate reduction from the intended report shape. /// "N descendant check-ins become unresolvable" needs a public descendants /// accessor over `repostate`'s child map plus a BFS, and "src/secret.key, in 6 /// check-ins" needs a reverse F-card index that exists nowhere in this tree — /// producing either means decoding every manifest in the repository on the /// refusal path, which is supposed to be the cheap path. The clause without the /// count states the same consequence; the count is worth revisiting only /// alongside the index that would make it cheap. fn shunConsequence(kind: ?artifact.ArtifactType) []const u8 { const t = kind orelse return "file content; every check-in listing it loses that file"; return switch (t) { .manifest => "a check-in; its descendants become unresolvable", .cluster => "sync housekeeping only; regenerable", .submanifest => "part of a check-in's file list; every check-in whose tree names it stops enumerating", .control, .wiki, .ticket, .attachment, .event, .forum => "structural content; what it renders stops rendering", }; } /// English plural suffix for `n`. Used only in these reports. fn plural(n: usize) []const u8 { return if (n == 1) "" else "s"; } /// The `-es` plural suffix, for the one noun these reports pluralize that does /// not take a bare `-s`: "hash" / "hashes". A separate function rather than a /// caller-supplied string so no call site can spell it "hashs". fn pluralEs(n: usize) []const u8 { return if (n == 1) "" else "es"; } /// Render the hard stops: one line per blocking target, then nothing happens. /// Every arm of the switch prints, including the non-blocking ones — a renderer /// with an `unreachable` arm would be undefined behavior in `ReleaseFast`, which /// is a poor trade for a line of output nobody should ever see. fn renderShunBlockers(err: *std.Io.Writer, targets: []const ShunTarget) void { for (targets) |t| switch (t.disposition) { .current_checkout => { err.print("zit shun: {s} is the check-in this checkout is on; refusing\n", .{t.name}) catch {}; err.writeAll("run 'zit update' to a different check-in first\n") catch {}; }, .not_a_name => err.print("zit shun: '{s}' is not an artifact name\n", .{t.name}) catch {}, .unresolved => err.print( "zit shun: no artifact matches '{s}'; give the complete name to shun a hash this repository does not hold\n", .{t.name}, ) catch {}, .ambiguous => err.print("zit shun: hash '{s}' is ambiguous\n", .{t.name}) catch {}, .remove, .reclaim, .absent, .already => {}, }; } /// Whether `src` still holds reclaimable BYTES under `digest` — an interned, /// non-phantom frame. This is exactly `packLog`'s own `dropped` predicate (it /// counts non-phantom rids the filter drops), so "would be removed" here and "was /// removed" there cannot disagree. No decode, so the refusal path stays cheap. fn holdsBytes(src: *const logstore.LogStore, digest: []const u8) bool { const rid = src.lookup(digest) orelse return false; return !src.isPhantom(rid); } /// How many rows already in `src`'s shun table still hold bytes that this run's /// rewrite would drop, EXCLUDING the ones `targets` names (those are counted as /// `.reclaim` targets and listed by name, so counting them here too would double /// them). This is the clause the refusal — the consent gate — would otherwise be /// silent about: `renderShunStats` is honest that `dropped` can exceed `recorded`, /// but that honesty arrives after the deletion. fn deferredStillHeld(src: *const logstore.LogStore, targets: []const ShunTarget) usize { var n: usize = 0; rows: for (src.shunList()) |row| { if (!holdsBytes(src, row.hash)) continue; for (targets) |t| { if (std.mem.eql(u8, t.digest, row.hash)) continue :rows; } n += 1; } return n; } /// Render the default refusal: what this run would do, then stop. The register is /// `clean`'s (`cleanRefuse` in cmd/tracking.zig) — the counts and the word /// "refusing" in ONE line, so an operator learns nothing happened before reading /// six lines of inventory, then the flag that proceeds, and never a prompt. /// /// THREE counts, because they answer three different questions and only the first /// used to be stated. How many rows become permanent (`recording` — including the /// pre-emptive ones, which remove nothing). How much content this invocation's own /// targets destroy (`removing` — which is NOT a subset of `recording`: a hash /// shunned earlier with `--defer` records nothing and still has bytes to lose). /// And `deferred_held`, the content the same rewrite reclaims from rows an EARLIER /// run recorded — collateral the operator never named, which the success line /// already admits to and the consent gate must not hide. /// /// `defer_reclaim` IS A PARAMETER BECAUSE THE TWO RUNS ARE DIFFERENT OPERATIONS, /// and this block's whole job is to state the operation accurately before an /// irreversible act. A `--defer` run removes nothing NOW: describing it as "would /// be permanently removed" overstates it, the `deferred_held` collateral clause is /// about a rewrite that is not going to happen, and — worst — a suggestion of /// `zit shun --force` would have an operator who copies it escalate a deferred /// tombstone into a full-store reclaim of the whole union set, including collateral /// the run never named. The suggested command is therefore always the one the /// operator actually asked for, plus `--force`. fn renderShunRefusal( err: *std.Io.Writer, targets: []const ShunTarget, deferred_held: usize, defer_reclaim: bool, ) void { var recording: usize = 0; var removing: usize = 0; for (targets) |t| { if (t.disposition.records()) recording += 1; if (t.disposition.reclaims()) removing += 1; } err.writeAll("zit shun: ") catch {}; if (recording == 0) { err.writeAll("no new hash would be recorded") catch {}; } else { err.print("{d} hash{s} would be shunned in this repository", .{ recording, pluralEs(recording) }) catch {}; } if (removing == 0) { err.writeAll("; nothing you named holds content to remove") catch {}; } else if (defer_reclaim) { // Named, still held, and deliberately NOT reclaimed by this run. "stays // reconstructible" rather than merely "unreclaimed": the mask is on the // name, so the plaintext is still materialisable through any surviving // delta, which is precisely what an operator under a takedown order would // otherwise assume they had already prevented. err.print( "; the bytes of {d} artifact{s} stay on disk and stay reconstructible until 'zit pack'", .{ removing, plural(removing) }, ) catch {}; } else { err.print("; {d} artifact{s} would be permanently removed", .{ removing, plural(removing) }) catch {}; } // Collateral of the REWRITE, so it is silent on the deferred path: there is no // rewrite in a `--defer` run and those rows keep their bytes untouched. if (deferred_held != 0 and !defer_reclaim) { err.print( "; the same rewrite also removes {d} artifact{s} shunned earlier and not yet reclaimed", .{ deferred_held, plural(deferred_held) }, ) catch {}; } err.writeAll("; refusing without --force:\n\n") catch {}; for (targets) |t| switch (t.disposition) { .remove => err.print(" {s} {s} {s}\n", .{ t.name, shunTypeLabel(t.kind), shunConsequence(t.kind), }) catch {}, .reclaim => if (defer_reclaim) { err.print(" {s} {s} already shunned; content still here and stays\n", .{ t.name, shunTypeLabel(t.kind), }) catch {}; } else { err.print(" {s} {s} already shunned; content still here and would be removed\n", .{ t.name, shunTypeLabel(t.kind), }) catch {}; }, .absent => err.print(" {s} recorded only (not present in this repository)\n", .{t.name}) catch {}, .already => err.print(" {s} already shunned; content already gone\n", .{t.name}) catch {}, // Unreachable in practice (`renderShunBlockers` ran and the caller // exited), but printed rather than asserted: see `renderShunBlockers`. .current_checkout, .not_a_name, .unresolved, .ambiguous => err.print( " {s} cannot be shunned\n", .{t.name}, ) catch {}, }; err.writeAll( \\ \\This is irreversible: zit undo does not recover a shun, and the content \\cannot be re-fetched from a peer. \\ ) catch {}; err.print("run 'zit shun {s}--force <HASH>...' to proceed\n", .{ if (defer_reclaim) "--defer " else "", }) catch {}; } /// One human line summarizing a completed reclaiming shun, in `renderPackStats`'s /// register. `dropped` can EXCEED `shuns_recorded`, because the same rewrite also /// reclaims any shun that was deferred earlier — an operator must be told that /// rather than left to infer it from a byte count. Split out so the exact wording /// is unit tested without a filesystem. /// /// EVERY NUMBER HERE COMES FROM THE PACK, none from the caller's intent — which is /// the whole reason `PackStats.shuns_recorded` exists. The rows this run meant to /// write and the rows the new generation actually holds are two different facts, /// and the one an operator is owed after a takedown is the second. `packLog`'s /// row-count gate makes them equal or fails the pack; if that gate were ever /// weakened, this line would under-report rather than over-report. fn renderShunStats(out: *std.Io.Writer, stats: PackStats) void { // `dropped` is counted among the source frames walked, so this subtraction // cannot underflow. Asserted anyway, and precisely because it is a tautology: // it is the only unchecked arithmetic in this verb's reporting, and an // underflow would print an absurd remaining count rather than trap. std.debug.assert(stats.dropped <= stats.artifacts); out.print("shunned: {d} hash{s} recorded, {d} artifact{s} removed, {d} -> {d} remaining, {d} -> {d} bytes\n", .{ stats.shuns_recorded, pluralEs(stats.shuns_recorded), stats.dropped, plural(stats.dropped), stats.artifacts, stats.artifacts - stats.dropped, stats.bytes_before, stats.bytes_after, }) catch {}; } /// `--defer`'s success line. It does NOT promise a byte count: nothing on this /// path measured one, and the only honest number would require doing the rewrite /// the operator just declined. /// /// `recorded == 0` with `holding != 0` is the re-`--defer` of a hash already /// deferred: no row to add, and yet content still to lose. Saying "0 hashes /// recorded" there would read as a no-op on a repository that still holds the /// bytes, so that case gets its own clause naming what is left. fn renderShunDeferred(out: *std.Io.Writer, recorded: usize, holding: usize) void { if (recorded == 0) { out.print( "shunned: no new hash to record; {d} artifact{s} still hold{s} content; run 'zit pack' to reclaim the space\n", .{ holding, plural(holding), if (holding == 1) "s" else "" }, ) catch {}; return; } out.print( "shunned: {d} hash{s} recorded; run 'zit pack' to reclaim the space\n", .{ recorded, pluralEs(recorded) }, ) catch {}; } /// The success line for a run with tombstones to write and NOTHING to reclaim — /// the purely pre-emptive shun. It takes the append path rather than the rewrite /// (see `runShun`), so there is no byte count to report and, more to the point, /// nothing was removed: a line ending in a store that had GROWN would be the /// wrong thing to print under the word "shunned". fn renderShunRecordedOnly(out: *std.Io.Writer, recorded: usize) void { out.print( "shunned: {d} hash{s} recorded; none is held here, so nothing was removed\n", .{ recorded, pluralEs(recorded) }, ) catch {}; } /// `--list`: the shun table in persisted order, one row per line — full artifact /// name, the UTC date it was recorded, and the reason when one was given. The /// date comes from `cli.formatWriterTimestamp`'s first ten bytes, so this shares /// the CLI's one epoch formatter rather than growing a second. fn renderShunList(out: *std.Io.Writer, rows: []const logstore.ShunRecord) void { if (rows.len == 0) { out.writeAll("the shun table is empty\n") catch {}; return; } for (rows) |row| { var hex: [2 * digest_len]u8 = undefined; var ts: [19]u8 = undefined; const date = cli.formatWriterTimestamp(row.epoch, &ts)[0..10]; out.print("{s} {s}", .{ digestToWireName(row.hash, &hex), date }) catch {}; if (row.reason.len != 0) out.print(" {s}", .{row.reason}) catch {}; out.writeAll("\n") catch {}; } } /// Render shun's failure `err` as `zit shun: <reason>` on stderr, exit 1. On /// every path that reaches this the store is byte-identical: either nothing was /// written yet, or every write went to the temp the rewrite's `defer` deletes. fn shunFail(s: cli.Streams, err: anyerror) cli.Exit { s.err.print("zit shun: {s}\n", .{reasonOf(err)}) catch {}; return .failure; } /// The two shun-row rejections (`BadKey`, `ReasonTooLong`) that are unreachable by /// construction here — every digest passed `isCompleteDigest` and every reason /// passed the length gate — given a REAL phrase rather than left to `reasonOf`, /// whose `else` arm renders both as "input/output error". Being unreachable is /// exactly why the phrase matters: if one ever fires it is a key-width or /// reason-length bug, and telling the operator their disk failed would send them /// looking at hardware. Nothing durable has happened on either path. fn shunRowBug(s: cli.Streams) cli.Exit { s.err.writeAll("zit shun: internal error: a shun row was rejected as malformed; nothing was changed\n") catch {}; return .failure; } /// `shunAppendRows`' failure, with its two unreachable-by-construction arms routed /// to `shunRowBug` instead of through `reasonOf`'s "input/output error" catch-all. /// A torn append is truncated by the log's own recovery, so a failure here leaves /// the shun simply absent, never half present. /// /// `TailChanged` and `RewriteBusy` join the set for the reason `RewriteError` /// carries them: neither is a failure of the append at all, both leave the store /// exactly as it was, and both have the same operator-facing meaning here as they /// do on the rewrite path — which is why they route to the SAME two diagnostics /// rather than growing a second pair of sentences for the append. const ShunAppendError = logstore.OpenError || logstore.ShunError || logstore.CommitError || error{ TailChanged, RewriteBusy }; fn shunAppendFail(s: cli.Streams, err: ShunAppendError) cli.Exit { return switch (err) { error.OutOfMemory => stateOom(s, "shun"), error.BadKey, error.ReasonTooLong => shunRowBug(s), error.TailChanged => tailChangedFail(s, "shun"), error.RewriteBusy => rewriteBusyFail(s, "shun"), else => shunFail(s, err), }; } /// Canonicalize a `zit shun` positional to a lowercase-hex artifact-name prefix, /// or null when it is not one. The same band `zit artifact` accepts (4..64 hex), /// deliberately: an operator pastes the same token into both verbs. fn canonicalizeShunHex(arg: []const u8, out: *[2 * logstore.max_hash_len]u8) ?[]const u8 { if (arg.len < 4 or arg.len > out.len) return null; for (arg, 0..) |c, i| { const lc = std.ascii.toLower(c); if (!((lc >= '0' and lc <= '9') or (lc >= 'a' and lc <= 'f'))) return null; out[i] = lc; } return out[0..arg.len]; } /// Decode a COMPLETE lowercase-hex artifact name (40 or 64 hex, the two forms /// artifact-format §3.2 defines) into `out`, returning the used prefix, or null /// when `hex` is not one of those two widths. The width, not `hash_kind`, decides /// — the same rule `logstore.isCompleteDigest` enforces on the write end, so /// nothing here interprets a digest. fn shunHexToDigest(hex: []const u8, out: *[logstore.max_hash_len]u8) ?[]const u8 { if (hex.len != 40 and hex.len != 64) return null; const n = hex.len / 2; std.debug.assert(n <= out.len); _ = std.fmt.hexToBytes(out[0..n], hex) catch return null; return out[0..n]; } /// Append `rows` to the live store and fsync — `--defer`'s whole effect, and the /// path a `--force` run with nothing to reclaim takes too. One frame per row plus /// one `commit`, so a crash mid-append leaves a torn tail the log's existing /// recovery truncates: the shun is then simply absent, never half present. /// Deliberately NOT the rewrite path: nothing here is atomic across rows, and /// nothing needs to be, because each row is independently meaningful. /// /// IT CARRIES BOTH OF `rewriteAndSwap`'S CONCURRENCY PROTECTIONS, for reasons that /// are this path's own and not a symmetry argument. /// /// The LOCK — the same exclusive non-blocking lock on the same fixed /// `pack_tmp_filename`, so an append and a rewrite exclude each other. Without it a /// `zit pack` (or another `zit shun --force`) that renames its new generation over /// `.zit` after our `openFile` leaves us appending to an ORPHANED INODE: the row /// lands nowhere anyone will ever read, and this run still reports success. On the /// `--defer` path that means the deferred shun silently does not exist while the /// help text has just told the operator to run `zit pack` to finish it. A cron'd /// pack overlapping an operator's shun is an entirely ordinary way to hit that. /// /// The TAIL RE-VALIDATION against `src_key` — because the caller classified its /// targets against a snapshot taken before the lock existed, and both success /// lines this path can print (`renderShunRecordedOnly`'s "none is held here, so /// nothing was removed", `renderShunDeferred`'s count of what still holds content) /// are claims about STORE STATE, not about the argument list. A `zit commit` /// landing a named hash in that window makes the first of them false: the /// tombstone would be recorded, exit 0, and an operator under a takedown order /// told nothing was held while the bytes were live. Re-checking the tail on the /// very handle the append will go through — under the lock, so no rename can have /// intervened — turns that into an abort. `error.TailChanged` here means nothing /// was written: the check precedes the first `putShun`. /// /// The check reads the REOPENED handle rather than the caller's, and that is the /// stronger of the two: a rewrite that completed just before we took the lock has /// left the caller's handle pointing at an orphan whose tail still matches its own /// key, while `.zit` by name is now a different generation. Validating the handle /// we are about to WRITE is what makes that case an abort instead of an append /// driven by a stale snapshot. fn shunAppendRows( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, src_key: logstore.LogStore.LogKey, rows: []const logstore.ShunRecord, ) ShunAppendError!void { var lock_file = dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = false, .lock = .exclusive, .lock_nonblocking = true, }) catch |e| switch (e) { error.WouldBlock => return error.RewriteBusy, else => return error.Storage, }; // Ours for the duration and removed on every path: an append leaves no new // generation behind, so a zero-length temp outliving the run would be litter. // The lock lives in the open file description, so a crash releases it. defer { dir.deleteFile(io, pack_tmp_filename) catch {}; lock_file.close(io); } var file = cli.openStoreAppend(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); // A FRESH handle and a FRESH stat, for the same reason `rewriteAndSwap` takes // them: the caller's view answers `size()` from the cached watermark its key // was derived from, so asking it could never disagree. if (!logstore.LogStore.validateTail(fs.storageFile(), src_key)) return error.TailChanged; // The header of the existing repo carries the hash kind; `open` reads it from // the non-empty file, so the passed value is a don't-care (0). var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer log.deinit(gpa); for (rows) |row| try log.putShun(row.hash, row.reason, row.epoch); try log.commit(); } /// `zit shun` — record one or more artifacts in this repository's shun table and, /// unless `--defer`, remove their bytes in the same operation. /// /// POLICY, decided here and not by any draft: refuse by default and name the /// consequence; require `--force` for BOTH paths, because `--defer` also appends /// an irreversible tombstone and the read mask hides the content the instant it /// lands; never prompt; refuse the current checkout's own check-in with no /// override, because that is the one target whose removal breaks the tree the /// operator is standing in and `zit update` is a cheap way out. /// /// CRASH SAFETY. `--force` performs exactly ONE durable transition. When there is /// content to remove that is the rename inside `rewriteAndSwap`, and the new /// tombstones are written into the NEW generation (`packLog`'s `extra_shuns`), /// never appended to `.zit` first — two durable transitions would expose a state /// where the tombstone exists and the content still does. It also means an aborted /// run leaves no tombstone at all: the rows were in the temp that the abort /// deletes. Pinned by "runShun: a --force run the temp lock turns away records NO /// tombstone". /// /// When NOTHING is held — every named hash is pre-emptive — the single durable /// transition is instead the append plus commit, and the rewrite is skipped /// altogether. It is not load-bearing there: the rule exists to stop a tombstone /// outliving its content, and with no content to remove the append is already /// crash-safe under the log's torn-tail recovery. Running a whole-store rewrite to /// reclaim nothing would cost O(store) plus transient room for both generations and /// end with a report whose byte count had GROWN. /// /// The rename is visible to a running `zit serve` DESPITE its held handle: /// serve's `baseSnapshot` stats the path beside the handle each round and /// reopens on an inode change, so the round after this swap serves the /// rewritten generation — shunned content gone — with no restart. pub fn runShun( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseShunArgs(arena, verb_args) catch return stateOom(streams, "shun")) { .ok => |a| a, .unknown_option => |tok| return cli.usageError(streams, "shun", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "shun", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(shun_help) catch {}; return .ok; } if (!parsed.list and parsed.hashes.len == 0) { return cli.usageError(streams, "shun", .no_hash, ""); } // A reason the durable row cannot hold is refused BEFORE anything is opened, // and refused rather than truncated: the note on a shun is often the legal // reference the operator will be asked for, and silently keeping a prefix of // it is worse than making them shorten it themselves. // // A CONTROL BYTE IS REFUSED THE SAME WAY, and this is the reason `--reason` is // validated at all rather than merely bounded. `--list` is the only view of a // durable, irreversible, un-removable record, and `renderShunList` prints one // row per line: a newline inside a reason forges a row there that is // indistinguishable from a real one. Reasons arrive from scripts and from pasted // takedown text as often as they are typed, so this is not a hand-typed-only // hazard. Rejecting at input rather than escaping at render keeps the fix to one // place and keeps the stored bytes exactly what the operator supplied. NO DRAFT // SPEAKS TO THIS and `logstore.ShunError` has no arm for it — a local decision, // taken the same way the length gate was: refuse rather than mangle. if (parsed.reason) |r| { if (r.len > logstore.max_shun_reason) { streams.err.print( "zit shun: --reason is {d} bytes; the recorded note is at most {d}\n", .{ r.len, logstore.max_shun_reason }, ) catch {}; return .failure; } for (r) |c| { if (c >= 0x20 and c != 0x7f) continue; streams.err.writeAll( "zit shun: --reason contains a control byte; the recorded note must be a single line of text\n", ) catch {}; return .failure; } } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "shun")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // shun is store-scoped end to end: the reads, the pack-tmp lock // and the final rename must all land beside the STORE, so a detached // worktree resolves its pointer once here and the whole flow inherits it. if (cli.storeDirOf(io, root_dir) catch return repoReadFail(streams, "shun")) |resolved| { root_dir.close(io); root_dir = resolved; } // Open the source read-only, exactly as `pack` does, and for the same // reasons — including the same caveat: this open no longer CRC-verifies // payloads (it scans header-only), so a corrupt source aborts during the // rewrite's own reads rather than here. Nothing is written either way. The // tail key pins the log for the pre-rename re-check. See `runPack`'s doc // comment for why that ordering matters to a verb that renames over `.zit`. var src_file = root_dir.openFile(io, repo_filename, .{}) catch return repoReadFail(streams, "shun"); defer src_file.close(io); const src_st = src_file.stat(io) catch return repoReadFail(streams, "shun"); var src_fs = storage.FileStorage.init(io, src_file, src_st.size); var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "shun"), else => |re| return shunFail(streams, re), }; defer src.deinit(gpa); if (parsed.list) { renderShunList(streams.out, src.shunList()); return .ok; } const src_key = src.logKey() catch return shunFail(streams, error.Corrupt); // The current checkout's own check-in, for the one refusal with no override. // A `.zit-checkout` we cannot parse is a hard failure, not an empty answer: // the guard would silently become vacuous and the operator would be allowed // to delete the check-in they are standing on. var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined; const ck_state = checkout.read(io, root_dir, &ck_buf) catch |e| return main.checkoutFail(streams, "shun", e); const ck_version: []const u8 = if (ck_state) |s| s.version else ""; // Classified, then DEDUPLICATED BY RESOLVED DIGEST — which is why it cannot be // done in the parser. `zit shun --force H <prefix-of-H>` is one artifact named // twice, and an operator pasting a takedown list beside an abbreviation from // `zit log` cannot see that it is. Left folded in, the duplicate builds two // identical rows, `putShun` collapses them, and `packLog`'s row-count gate — // whose documented precondition is that `extra_shuns` rows are DISTINCT — fires // `error.Corrupt`: the verb would tell the operator their repository is corrupt, // on the one occasion they can least afford to doubt it. Deduping here rather // than in `packLog` keeps that gate a gate. O(n^2) over an argument list. const target_buf = arena.alloc(ShunTarget, parsed.hashes.len) catch return stateOom(streams, "shun"); var target_count: usize = 0; tokens: for (parsed.hashes) |tok| { const t = classifyShunTarget(gpa, arena, &src, ck_version, tok) catch |e| return shunFail(streams, e); // Only a resolved target can be a duplicate: the unresolvable dispositions // carry the operator's own token and no digest, and two different bad tokens // both deserve their own diagnostic line. if (t.digest.len != 0) { for (target_buf[0..target_count]) |prev| { if (std.mem.eql(u8, prev.digest, t.digest)) continue :tokens; } } target_buf[target_count] = t; target_count += 1; } const targets = target_buf[0..target_count]; // THE GUARD RUNS BEFORE ANY DURABLE WRITE — the ordering is the point, not an // implementation detail. A refused run must leave the store byte-identical, // which it cannot do if a tombstone was recorded while deciding. for (targets) |t| { if (t.disposition.blocks()) { renderShunBlockers(streams.err, targets); return .failure; } } // Two counts, and they are independent: `absent` records without removing, // `reclaim` removes without recording. Either one alone is work to do. var recorded: usize = 0; var reclaiming: usize = 0; for (targets) |t| { if (t.disposition.records()) recorded += 1; if (t.disposition.reclaims()) reclaiming += 1; } if (recorded == 0 and reclaiming == 0) { // Every named hash was already in the set AND its content already gone: a // genuine no-op, so it needs no `--force` and it is not a failure. Reaching // this on a hash whose bytes SURVIVED is the defect `.reclaim` exists to // prevent — it would report "nothing to do" over content the operator had // just been ordered to remove. streams.out.print( "shunned: nothing to do; {d} hash{s} already shunned\n", .{ targets.len, pluralEs(targets.len) }, ) catch {}; return .ok; } if (!parsed.force) { renderShunRefusal(streams.err, targets, deferredStillHeld(&src, targets), parsed.defer_reclaim); return .failure; } // ONE owned name set drives both the drop predicate and the tombstone list, // so a name cannot be recorded without also being dropped. It is the UNION of // the source's already-deferred shuns and the hashes named now: `shunFilter` // covers only the first half, and half a drop set would relocate into the new // generation exactly the bytes this run reports as removed. var set: NameSet = .{}; defer set.deinit(gpa); for (src.shunList()) |row| set.add(gpa, row.hash) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "shun"), // A persisted row whose key is not a complete digest is a corrupt store, // and that is the one place `BadKey` here can come from. error.BadKey => return shunFail(streams, error.Corrupt), }; const row_buf = arena.alloc(logstore.ShunRecord, recorded) catch return stateOom(streams, "shun"); var n: usize = 0; for (targets) |t| { if (!t.disposition.records()) continue; set.add(gpa, t.digest) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "shun"), error.BadKey => return shunRowBug(streams), }; row_buf[n] = .{ .hash = t.digest, .epoch = amb.now_epoch, .reason = parsed.reason orelse "" }; n += 1; } // SLICED, not asserted-and-passed-whole. `recorded` and `n` count the same pure // predicate over the same unmutated `targets`, so they cannot differ today — but // `row_buf` is arena memory and the FULL buffer used to be what went to // `packLog` and `shunAppendRows`. Had they ever differed, `row_buf[n..]` would // be undefined `ShunRecord`s whose `hash` is a garbage slice, handed to // `putShun` on the way to a durable irreversible write — in `ReleaseFast`, where // the assert below does not exist. Taking the slice makes the mismatch // unrepresentable and leaves the assert as documentation of the tautology. const rows = row_buf[0..n]; std.debug.assert(rows.len == recorded); // Nothing to reclaim: take the append path even without `--defer`. See the // crash-safety note in the doc above — the rewrite is not what makes this safe, // and running it would cost a whole-store copy to remove nothing. if (reclaiming == 0) { shunAppendRows(gpa, io, root_dir, src_key, rows) catch |e| return shunAppendFail(streams, e); renderShunRecordedOnly(streams.out, rows.len); return .ok; } if (parsed.defer_reclaim) { // `rows` is empty when every target was already tombstoned and only its // bytes remain; skip the append rather than fsync for nothing. if (rows.len != 0) { shunAppendRows(gpa, io, root_dir, src_key, rows) catch |e| return shunAppendFail(streams, e); } renderShunDeferred(streams.out, rows.len, reclaiming); return .ok; } // THE CURRENT-CHECKOUT STOP, RE-ASKED OF THE DROP SET ITSELF — one hashmap // lookup, at the point of consequence. `classifyShunTarget` enforces it over the // NAMED TOKENS, but `set` is the union of those with every row already in the // shun table, and the rewrite deletes the union. A table that already contained // this check-in's manifest would therefore have `--force` delete the check-in the // operator is standing in while the per-token guard never fired — the refusal // line would count it only as an anonymous integer of earlier deferred // collateral. That state is not constructible through zit today (`--defer` goes // through the same per-token guard, and S1's mask makes `zit update` to a shunned // check-in fail), which is exactly why the invariant belongs here as well: the // property `shun_help` promises is about what gets deleted, not about the // argument list. if (ck_version.len != 0) { var ck_raw: [logstore.max_hash_len]u8 = undefined; if (shunHexToDigest(ck_version, &ck_raw)) |ck_digest| { if (set.contains(ck_digest)) { renderShunBlockers(streams.err, &.{.{ .name = ck_version, .digest = ck_digest, .disposition = .current_checkout, }}); return .failure; } } } const stats = rewriteAndSwap( gpa, io, root_dir, &src, src_key, set.filter(), rows, ) catch |e| switch (e) { error.TailChanged => return tailChangedFail(streams, "shun"), error.RewriteBusy => return rewriteBusyFail(streams, "shun"), else => |pe| return shunFail(streams, pe), }; // Reported from the PACK's observation of the new generation, not from // `rows.len`: see `renderShunStats`. They agree or `packLog`'s row-count gate // failed the pack, which is why this is an assert rather than a branch. std.debug.assert(stats.shuns_recorded == rows.len); renderShunStats(streams.out, stats); return .ok; } /// Resolve one operator token against `src` into a `ShunTarget`. Impure only in /// that it reads the store: no policy, no output, no durable write, so the /// runner's whole decision is a fold over the results. /// /// RESOLUTION RULE. Every token is offered to prefix resolution FIRST, at every /// width in `canonicalizeShunHex`'s band — the same band and the same resolver /// `zit artifact` uses (cmd/history.zig), which is what makes "paste the same /// token into both verbs" true rather than merely intended. Only when nothing /// matches does a COMPLETE name (40 or 64 hex — artifact-format §3.2's two forms) /// fall back to being used literally, because a hash this repository does not hold /// is precisely the pre-emptive case worth recording; an ABBREVIATED prefix that /// matches nothing is refused, since the shun table is keyed by complete name /// (sync §3.3 L518-519), a prefix cannot be completed, and a row keyed on one would /// read as a removal while naming nothing. /// /// TRYING RESOLUTION FIRST IS THE LOAD-BEARING PART, not an optimisation. Treating /// a complete-WIDTH token as a complete NAME without asking the store means a /// 40-hex abbreviation of a 64-hex artifact (and a 64-hex abbreviation in a sha1 /// repository) records a tombstone naming nothing, reports success, and leaves the /// content the operator was ordered to remove still readable — the `unresolved` /// refusal exists to stop exactly that, and a width-first rule walks past it. /// It also un-hides the current-checkout guard, which cannot fire on a token that /// never resolved to the check-in's name. fn classifyShunTarget( gpa: std.mem.Allocator, arena: std.mem.Allocator, src: *logstore.LogStore, ck_version: []const u8, tok: []const u8, ) (logstore.ContentError || error{OutOfMemory})!ShunTarget { var canon_buf: [2 * logstore.max_hash_len]u8 = undefined; const canon = canonicalizeShunHex(tok, &canon_buf) orelse return .{ .name = tok, .disposition = .not_a_name }; var raw_buf: [logstore.max_hash_len]u8 = undefined; var digest: []const u8 = undefined; if (src.resolveHexPrefix(canon) catch |e| switch (e) { error.Ambiguous => return .{ .name = tok, .disposition = .ambiguous }, }) |rid| { digest = src.hashOf(rid); } else if (shunHexToDigest(canon, &raw_buf)) |d| { digest = d; } else { return .{ .name = tok, .disposition = .unresolved }; } std.debug.assert(logstore.isCompleteDigest(digest.len)); const owned_digest = try arena.dupe(u8, digest); var hex_buf: [2 * digest_len]u8 = undefined; const name = try arena.dupe(u8, digestToWireName(owned_digest, &hex_buf)); // THE CURRENT-CHECKOUT STOP IS CHECKED FIRST, ahead of the shun table. An // already-tombstoned check-in whose bytes survive is otherwise `.reclaim`, and // reclaiming it would delete the check-in the operator is standing on — the one // thing this verb refuses outright. `--defer` goes through this same guard, so // the state cannot arise through zit today; the ordering is what keeps that // true if it ever does. if (std.mem.eql(u8, name, ck_version)) return .{ .name = name, .digest = owned_digest, .disposition = .current_checkout, }; const already = src.isShunned(owned_digest); const rid = src.lookup(owned_digest) orelse return .{ .name = name, .digest = owned_digest, .disposition = if (already) .already else .absent, }; // THROUGH THE MAINTENANCE READER, deliberately. `content` applies the shun // mask, so an already-shunned artifact whose bytes are still on disk would read // as absent and be reported as nothing left to do — which is the whole defect // `.reclaim` exists to prevent. Shunning is maintenance, not an artifact read. // // A phantom is an announced name with no frame: there are no bytes to reclaim, // so it is the `absent` case with a rid attached. const content = (try src.contentUnmasked(gpa, rid, null)) orelse return .{ .name = name, .digest = owned_digest, .disposition = if (already) .already else .absent, }; defer gpa.free(content); // The type verdict, through the same door `zit artifact` uses: a structural // artifact classifies (`parseWindow` runs §5's `classify` internally), and // anything that does not parse is an opaque blob — a first-class answer, not // an error. A blob is by far the common shun target. const held: ShunDisposition = if (already) .reclaim else .remove; var m = artifact.parseWindow(gpa, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return .{ .name = name, .digest = owned_digest, .kind = null, .disposition = held }, }; defer m.deinit(); return .{ .name = name, .digest = owned_digest, .kind = m.kind, .disposition = held }; } /// Maps a transport-open failure (bad URL) to the `verb` diagnostic + exit 1. /// Shared by clone and pull — the only difference is the verb label. fn syncOpenFail(s: cli.Streams, verb: []const u8, url: []const u8, err: transport.TransportError) cli.Exit { switch (err) { error.InvalidUrl => s.err.print("zit {s}: '{s}' is not a valid URL\n", .{ verb, url }) catch {}, error.UnsupportedScheme => s.err.print("zit {s}: unsupported URL scheme in '{s}' (use http or https)\n", .{ verb, url }) catch {}, error.OutOfMemory => s.err.print("zit {s}: out of memory\n", .{verb}) catch {}, else => s.err.print("zit {s}: cannot open '{s}'\n", .{ verb, url }) catch {}, } return .failure; } /// Maps a sync run (transport + driver) failure to the `verb` diagnostic + exit /// 1. Exhaustive over the driver's own causes and the transport's, so the whole /// closed error set is accounted for (no `else`). Shared by clone and pull. /// `status` is the transport's `last_status` -- the HTTP status that produced an /// `error.ProtocolError`, or null for every other cause. Named in the message /// because "an unexpected HTTP status" sends an investigation nowhere: 500, 413, /// 400 and 417 are four different bugs and the operator could not tell which one /// they had hit. /// Tolerate-and-report — one stderr line naming what the session /// declined: unversioned files a fossil peer offered that zit has no store /// for. Zero is silent; the count makes the data non-transfer VISIBLE, which /// is the entire posture (the alternative was an abort nobody chose). fn reportUvSkipped(s: cli.Streams, verb: []const u8, count: u64) void { if (count == 0) return; s.err.print( "zit {s}: {d} unversioned file(s) not transferred (unsupported)\n", .{ verb, count }, ) catch {}; } fn syncRunFail(s: cli.Streams, verb: []const u8, url: []const u8, err: sync_driver.RunError(*transport.Transport), status: ?std.http.Status) cli.Exit { const reason: []const u8 = switch (err) { error.OutOfMemory => "out of memory", // Not "did not converge": the guard counts rounds that moved no bytes, // so this fires only when the peer kept the exchange open while // delivering nothing. A slow or very large transfer never reaches it. error.RoundLimitExceeded => "the peer kept the exchange open without transferring anything", error.FatalProtocolError => "the peer sent a fatal protocol error", error.RequestTooLarge => "the sync request exceeded the transport limit", error.ReplyTruncated, error.ReplyMalformed => "the peer sent a malformed reply", // The streaming drain's own faults, surfaced through the // driver: writing the destination log failed, or a received artifact // did not verify against its name. error.Storage => "cannot write the cloned repository", error.Corrupt => "a received artifact failed verification", error.ReplyTooLarge => "the peer's reply exceeded the size limit", error.RedirectLimit => "too many HTTP redirects", error.InsecureRedirect => "the peer redirected HTTPS to plaintext HTTP", error.RedirectNotAllowed => "the peer redirected on a transport that forbids it", error.MissingLocation => "the peer sent a redirect with no location", error.InvalidUrl => "a redirect target was not a valid URL", error.UnsupportedScheme => "a redirect named an unsupported URL scheme", error.ProtocolError => blk: { if (status) |st| { s.err.print("zit {s}: the peer answered HTTP {d} ({s})\n", .{ verb, @intFromEnum(st), st.phrase() orelse "unrecognized" }) catch {}; return .failure; } break :blk "the peer answered with an unexpected HTTP status"; }, error.ConnectionFailed => { s.err.print("zit {s}: cannot connect to {s}\n", .{ verb, url }) catch {}; return .failure; }, }; s.err.print("zit {s}: {s}\n", .{ verb, reason }) catch {}; return .failure; } /// The causes `resolveRemoteUrl` can surface: a store read failure, OOM, or the /// repository having no URL to use (`NoRemote` — neither an explicit arg nor a /// saved remote). Closed set so the caller switches exhaustively. const RemoteResolveError = error{ OutOfMemory, Storage, NoRemote }; /// sync §4/§6 — resolve the peer URL for a sync-family verb: an explicit `arg` /// (non-empty) wins; otherwise the saved remote (`config.remote_url`) is used; /// a repository with neither is a usage error (`NoRemote`). The saved URL is /// copied into `buf` so it outlives the read-only config frame. `buf` need only /// live until `transport.Transport.open` (which dupes the URL). An out-of-spec /// stored URL longer than the config-field cap is treated as none. fn resolveRemoteUrl( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, arg: []const u8, buf: *[logstore.max_config_str]u8, ) RemoteResolveError![]const u8 { if (arg.len > 0) return arg; var file = cli.openStoreRead(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.NoRemote; defer oc.deinit(gpa); const saved = oc.config.remote_url; if (saved.len == 0 or saved.len > buf.len) return error.NoRemote; @memcpy(buf[0..saved.len], saved); return buf[0..saved.len]; } /// sync §4/§6 — the usage error (exit 2) when a sync-family `verb` is given no /// URL and the repository has no saved remote to fall back to. Names the fix. /// Shared by pull, push, and sync so the wording lives in one place. fn noRemoteError(s: cli.Streams, verb: []const u8) cli.Exit { s.err.print("zit {s}: no repository URL given and no saved remote\n", .{verb}) catch {}; s.err.writeAll("set one with 'zit remote URL', or pass the URL explicitly\n") catch {}; return .usage; } // --------------------------------------------------------------------------- // `zit pull URL` — incremental sync into an EXISTING repository. The sibling of // clone: it reuses the same transport + `runSync` + persist machinery, but opens // the cwd's `.zit` (never creating one), announces what it already holds so the // peer sends only the missing artifacts (sync §3.2/§3.3 igot/gimme), and drives // to §3.7 termination. Receive-only (§3.3 r24: pull obtains but never uploads). // --------------------------------------------------------------------------- /// `zit pull URL` — open the current repository, converge it toward the peer at /// URL, and persist the newly received artifacts. Thin wiring over the same deep /// modules clone uses: `transport` (HTTP/IO), `sync.Client` + `runSync` (sans-IO /// §3.7 loop), and `LogStore` (persistence). With URL omitted the saved remote /// (`zit remote`) is used; a repo with neither is a usage error. Anonymous only. pub fn runPull( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, repo_override: ?[]const u8, ) cli.Exit { const parsed = switch (cli.parsePullArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "pull", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "pull", .extra_argument, tok), // An omitted URL is not an error: it falls back to the saved remote below. .no_url => cli.PullArgs{ .url = "" }, }; if (parsed.help) { streams.out.writeAll(pull_help) catch {}; return .ok; } // Locate the current repository before any network I/O: a pull outside a repo // is a usage failure, not a wasted connection. const root = (cli.resolveRoot(arena, io, repo_override) catch return stateOom(streams, "pull")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), error.NoRemote => return noRemoteError(streams, "pull"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "pull", url, e); defer tr.close(); // The client's holdings view: igot names, held/phantom, shun mask, and // hash-policy off the log's frame index, with content lazy (the // old whole-store `MemStore` snapshot held the repository's plaintext // resident for the run). var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), }; defer holdings.deinit(io); var admission = holdings.admission(); // Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once // and suppress spontaneous requests for elided phantoms. Both the // classification and the wrapper outlive the run below. var sync_cone: ?zit.cone.Cone = null; defer if (sync_cone) |*c| c.deinit(gpa); var sync_classification: ?SparseClassification = null; defer if (sync_classification) |*c| c.deinit(gpa); var sync_elided: RepoElidedSet = undefined; if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| { sync_cone = c; sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch return stateOom(streams, "pull"); sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() }; admission.elided = sync_elided.shunSet(); admission.cone_members = sync_cone.?.members; admission.cone_not_members = sync_cone.?.not_members; } // §9.1 — announce this repository's adopted project code in the pull card; // a real peer refuses a PROJECTCODE mismatch, so the placeholder cannot be // sent against fossil. The buffer outlives the run (the client borrows it). // The cluster index: what the held clusters already announce // (never announced again individually) and what they promised that this // repository never collected (seeded as phantoms, so the run asks for it). // The clusters' plaintext is memoised by the holdings adapter while it // resolves; drop it — the index keeps only names and bits. var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "pull"); defer clusters.deinit(); holdings.index.releaseTransient(); var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), }; // Live per-round progress on a terminal, the same sink clone uses. const stderr_tty = std.Io.File.stderr().isTty(io) catch false; var progress: CloneProgress = .{ .err = streams.err }; const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null; defer progress.finishLine(); const outcome = pullRun(gpa, &tr, holdings.store(), identity, admission, sink, &clusters) catch |e| return syncRunFail(streams, "pull", url, e, tr.last_status); // Sparse-cone RFC section 6.3 applies to EVERY receive: a dense peer // volunteers content the client never announced, so the filter runs at // persist. Reclassified over the post-run store, so a brand-new commit's // blobs classify by their own F cards -- new in-cone content persists, // new out-of-cone content elides with the rest. var post_classification: ?SparseClassification = null; defer if (post_classification) |*c| c.deinit(gpa); if (sync_cone) |*c| { post_classification = classifyKeepRids(gpa, holdings.store(), c) catch return stateOom(streams, "pull"); } // Persist only what this pull received. The holdings store appends // received artifacts beyond its log-backed prefix, so the walk begins at that // boundary and never re-visits the already-durable holdings. A pull that // received nothing persists nothing: a no-op pull was O(store) here (every // holding read once from disk to be early-returned), 196 s on a 905k store. const persisted = persistStore(gpa, io, root_dir, holdings.store(), if (post_classification) |*c| &c.keep else null, holdings.logBackedCount()) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), error.Corrupt => return renderStorage(streams, root.abs, "the peer sent a malformed artifact name"), }; // The walk started past the holdings, so `written` is exactly the received // count (each received artifact is persisted once). const received = persisted.written; // The peer named artifacts this run never obtained. Everything is persisted // — a partial pull is still progress — but the exit status says so, because // "exit 0 with nothing said" is indistinguishable from a converged sync and // leaves a repository quietly out of date. This holds whether `received` is // zero or not: the invariant is convergence, not motion. // A promise a held cluster made that this peer could not keep is // reported, not failed on: the run asked on its own initiative (no peer // igot set the expectation) and the repository IS converged against this // peer. Fossil asks for these every round, silently; zit says so once. if (outcome.cluster_gaps > 0) { streams.err.print( "zit pull: {d} artifact{s} named by held clusters could not be obtained from this peer\n", .{ outcome.cluster_gaps, plural(outcome.cluster_gaps) }, ) catch {}; } if (outcome.unresolved > 0) { streams.err.print( "zit pull: the peer holds {d} artifacts this pull did not obtain\n", .{outcome.unresolved}, ) catch {}; return .failure; } // submanifests §7.1/§7.3 — the same reasoning one level down. A pull can // converge against the peer (nothing left to gimme) while the tree closure // this repository now holds is still open, because the peer's own closure // was open: §7.1's "at most d+1 pull rounds" bounds a tree whose levels are // ARRIVING, not one whose leaves nobody has. The phantoms the persist hook // minted are that distinction, so report them rather than call it converged. // // §7.3/§9.4 — two counts, because they answer different questions and the // draft names both: `submanifest_gaps` is the durable store's missing-hash // count, `deferred_trees` is the session's DEPENDENT-keyed record ("enough // to rediscover that manifest"). A run reporting the first without the // second tells the operator something is missing but not what it breaks. if (persisted.submanifest_gaps > 0 or outcome.deferred_trees > 0) { // Two sentences, not one, because the two counts have different // LIFETIMES and a single sentence reads as one fact. `submanifest_gaps` // is a property of the durable store and survives a restart; // `deferred_trees` is this SESSION's §9.4 record and is zero for a run // that received no S card — including every re-run against an already // open closure. Printed together they said "N references are missing; 0 // check-ins cannot be enumerated", which is exactly backwards. if (persisted.submanifest_gaps > 0) { streams.err.print( "zit pull: {d} sub-manifest references are missing; the check-ins that reference them cannot be enumerated\n", .{persisted.submanifest_gaps}, ) catch {}; } if (outcome.deferred_trees > 0) { streams.err.print( "zit pull: {d} check-ins received this session are deferred until they arrive\n", .{outcome.deferred_trees}, ) catch {}; } return .failure; } if (received == 0) { streams.out.print("pull from {s}: already in sync\n", .{url}) catch {}; } else { streams.out.print("pull from {s}: received {d} artifact{s} in {d} round{s}\n", .{ url, received, plural(received), outcome.rounds, plural(outcome.rounds) }) catch {}; reportUvSkipped(streams, "pull", outcome.uv_skipped); } return .ok; } /// Drives a pull client (§3.2/§3.3) to convergence over `tr`, filling `ms` with /// the artifacts the peer holds and this side lacked; returns the round count. /// The client announces `ms`'s holdings via igot and requests the rest via /// gimme (receive-only — §3.3 r24: pull obtains content but never uploads). /// Sans-IO on this side of `tr.exchange`. Its closed error set is the driver's /// own causes unioned with the transport's, so `runPull` switches it exhaustively. fn pullRun( gpa: std.mem.Allocator, tr: *transport.Transport, holdings: repostate.ArtifactStore, identity: RepoSyncIdentity, admission: RepoAdmission, sink: ?sync_driver.RoundSink, clusters: ?*const clusterindex.Resolved, ) sync_driver.RunError(*transport.Transport)!PullOutcome { var client = syncClient(.pull, &identity, null, admission); defer client.deinit(gpa); if (clusters) |c| { client.clustered = &c.clustered; try client.seedClusterPhantoms(gpa, holdings, c.missing); } const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null); recordSyncStats(stats); return .{ .rounds = stats.rounds, .unresolved = client.unresolvedPhantoms(holdings), .cluster_gaps = client.clusterGaps(holdings), // submanifests §7.3/§9.4 — the DEPENDENT-keyed half of the verdict: how // many received check-ins still have a recorded S-reference gap. .deferred_trees = client.deferredTrees(), .uv_skipped = client.uv_skipped, }; } /// What one pull run reports: the round count, and how many artifacts the peer /// announced (§3.3 r24 — by igot, or as a member of a cluster it sent) that this /// side still lacks when §3.7 reached its clean terminal. A nonzero `unresolved` /// is a pull that stopped short of what it was told existed; the caller must not /// report success for it, however well-formed every card on the wire was. /// submanifests §7.3/§9.4 — `deferred_trees` is the §9.4 record read back: the /// count of DEPENDENTS (check-ins) whose processing this session deferred on a /// sub-manifest that never arrived. `unresolved` counts missing HASHES; a /// dependent-keyed count is what §9.4 requires be recorded and is what an /// operator can act on ("which check-in is broken", not "how many blobs"). const PullOutcome = struct { rounds: usize, unresolved: usize, deferred_trees: usize = 0, uv_skipped: u64 = 0, cluster_gaps: usize = 0 }; /// sync §3.3 L527-530 / §8.8 L1777-1779 — this repository's shun table as the /// `zit.sync.ShunSet` the phantom scan asks. Owns a copy of every shunned /// DIGEST, taken while the caller already has the log open, so the set outlives /// the handle it was read through. /// /// FRESHNESS, stated exactly. This is a SNAPSHOT, not a live view: `load` reads /// `shunList` once and the set answers from that copy thereafter. On the SERVE /// path that is per-round freshness, because `LogIndexStore` rebuilds — and so /// re-loads this — whenever the stat gate sees the log grow, which a `zit shun` /// append does; a shun written between two served rounds is honoured by the next /// one. On the CLIENT path it is a PROCESS snapshot: `ClientHoldings` builds /// its index once at run start and never rebuilds mid-run, so a shun applied /// mid-run lands on the next process, not the next round. That is a /// deliberate limit, recorded here rather than implied away — the protocol layer /// re-probes every scan (`sync.ShunSet`), so nothing else has to change if this /// ever grows a re-read. /// /// This is the ONE place a wire NAME becomes a log DIGEST for shun purposes — /// the boundary `LogIndexStore.names` already declares is held to one place. The /// width/hex validation is a CHECKED rejection rather than an assert, because an /// assert is nothing in ReleaseFast: `logstore.LogStore.isShunned`'s caller /// contract (see its doc) binds this boundary to reject a non-complete name /// BEFORE decoding it, so the width is gated on the NAME, ahead of the decode. /// The sync-facing elided-set: answers RFC section 5.2 rule 3 over a /// classification computed once at run start. Same total-function contract /// as `RepoShunSet`; borrows both the classification and the store, so both /// must outlive the run. /// `pack --drop-elided`'s filter half (sparse-cone RFC section 5.4): built /// once over the SOURCE log, drops exactly the classification's elide set. /// A digest that does not resolve in the index is kept -- unknown provenance /// is never deleted. const ElidedDropper = struct { index: *LogIndexStore, classification: SparseClassification, inner: zit.pack.Filter, fn build(gpa: std.mem.Allocator, src: storage.StorageFile, cone: *const zit.cone.Cone) ?ElidedDropper { const index = LogIndexStore.open(gpa, src) catch return null; const classification = classifyKeepRids(gpa, index.store(), cone) catch { index.deinit(); return null; }; return .{ .index = index, .classification = classification, .inner = zit.pack.Filter.keep_all }; } fn drop(ctx: *anyopaque, digest: []const u8) bool { const self: *ElidedDropper = @ptrCast(@alignCast(ctx)); if (self.inner.dropFn(self.inner.ctx, digest)) return true; // shun reclaim first var name_buf: [2 * logstore.max_hash_len]u8 = undefined; const name = digestToWireName(digest[0..digest.len], &name_buf); const rid = self.index.store().lookup(name) orelse return false; return self.classification.elided(rid); } fn composed(self: *ElidedDropper, inner: zit.pack.Filter) zit.pack.Filter { self.inner = inner; return .{ .ctx = @ptrCast(self), .dropFn = drop }; } fn deinit(self: *ElidedDropper, gpa: std.mem.Allocator) void { self.classification.deinit(gpa); self.index.destroy(); self.* = undefined; } }; const RepoElidedSet = struct { classification: *const SparseClassification, store: repostate.ArtifactStore, fn contains(context: *anyopaque, name: []const u8) bool { const self: *const RepoElidedSet = @ptrCast(@alignCast(context)); const rid = self.store.lookup(name) orelse return false; return self.classification.elided(rid); } fn shunSet(self: *RepoElidedSet) zit.sync.ShunSet { return .{ .context = @ptrCast(self), .vtable = &.{ .contains = contains } }; } }; const RepoShunSet = struct { gpa: std.mem.Allocator, /// Keys are owned binary digests; the value side carries nothing — this is a /// set. Empty is the common case (no repository has a shun row until an /// operator writes one), and an empty map allocates nothing. digests: std.StringHashMapUnmanaged(void) = .empty, fn init(gpa: std.mem.Allocator) RepoShunSet { return .{ .gpa = gpa }; } /// Copy every shun row's digest out of `log`. `shunList` borrows the store's /// own bytes, which die with the handle they were read through, so each key /// is duped. Re-loading over a populated set replaces it wholesale, which is /// what a `LogIndexStore` rebuild wants. fn load(self: *RepoShunSet, log: *const logstore.LogStore) error{OutOfMemory}!void { self.clear(); const rows = log.shunList(); // State the bound where it is relied on rather than narrowing usize into // the map's u32 capacity argument by an unchecked cast: a truncated // reservation would let `getOrPutAssumeCapacity` below write past the // capacity it assumes, which is silent corruption in ReleaseFast instead // of the `error.OutOfMemory` a shun table that large has already earned. // This is the one such count an operator grows by direct action. const row_count = std.math.cast(u32, rows.len) orelse return error.OutOfMemory; try self.digests.ensureTotalCapacity(self.gpa, row_count); for (rows) |row| { // Dupe BEFORE inserting. `row.hash` belongs to the store and dies // with the handle, so the key must be owned — and inserting the // borrowed slice first and replacing it after leaves a BORROWED key // in an owned-key map on the one error this function can return. // Every unwind path then runs `clear`, which frees the store's own // live digest buffer: a double free at `log.deinit`, and a // use-after-free of `shun_entries[i].hash` while the store is open. const owned = try self.gpa.dupe(u8, row.hash); // Capacity is already reserved for every row, so this cannot fail. const gop = self.digests.getOrPutAssumeCapacity(owned); // Two rows naming the same digest collapse to one entry, and the // second dupe is surplus — say so rather than leaking it. if (gop.found_existing) self.gpa.free(owned) else gop.key_ptr.* = owned; } } fn clear(self: *RepoShunSet) void { var it = self.digests.keyIterator(); while (it.next()) |k| self.gpa.free(k.*); self.digests.clearAndFree(self.gpa); } fn deinit(self: *RepoShunSet) void { self.clear(); } fn shunSet(self: *RepoShunSet) zit.sync.ShunSet { return .{ .context = self, .vtable = &vtable }; } const vtable: zit.sync.ShunSet.VTable = .{ .contains = containsWireName }; /// Exact-key lookup on the digest a COMPLETE wire name decodes to. A name /// whose WIDTH is not one of artifact-format §3.2 L347-350's two forms is /// rejected before the decode, which is the order `isShunned`'s caller /// contract fixes; a correct-width token that is not hex fails the decode. /// Neither can be a key any shun row carries (`putShun` rejects a /// non-complete key with `error.BadKey`, and the open-time scan fails /// `error.Corrupt` on one), so `false` is the exact answer. Never a prefix /// match, never a truncate, never a pad: a prefix must not be able to /// masquerade as a shun-table hit. /// /// An UPPERCASE-hex spelling of a shunned name decodes to the same digest /// and therefore answers TRUE. artifact-format §3.3 L370-377 makes such a /// token invalid but explicitly lets an implementation accept it at name /// validation and fail it at verification, and this direction is the safe /// one: it over-suppresses a name that can never verify against content /// anyway. No spelling of a shunned name can BYPASS the filter, which is the /// only direction §3.3 L527-530 cares about. fn containsWireName(context: *anyopaque, name: []const u8) bool { const self: *RepoShunSet = @ptrCast(@alignCast(context)); if (self.digests.count() == 0) return false; if (name.len != 2 * logstore.sha1_digest_len and name.len != 2 * logstore.sha3_256_digest_len) return false; var digest_buf: [digest_len]u8 = undefined; const digest = decodeWireName(name, &digest_buf) orelse return false; std.debug.assert(logstore.isCompleteDigest(digest.len)); return self.digests.contains(digest); } }; /// Opens `dir/.zit` read-only and snapshots its stored artifacts into `ms` — the /// held/phantom split the pull client announces via `igot` (sync §3.2/§3.3) — /// and its shun table into `shun` (sync §3.3 L527-530), both off the one open. /// Built by `loadContentStore`, which is the CLIENT's builder and not `serve`'s: /// `serve` builds a `LogIndexStore` instead, whose §8.2 slot-preserving mask, /// masked write doors and `read_fault` channel this snapshot has no use for. The /// two share the `digestToWireName` encoding and the held/phantom split, and /// nothing else — a filter added to one does NOT reach the other, which is why /// the §3.3 masked-basis guard is stated in both. The /// read-only handle is closed before `persistStore` reopens the log read-write, /// which is why `shun` owns its keys rather than borrowing the store's. /// sync §7.7 L1428-1432 — this repository's EFFECTIVE hash-policy value, as the /// §7.7 target-admission predicate needs it: read from the repository's own /// stored config, never from peer state or a card field. /// /// The stored byte is opaque to storage (`logstore.Config.hash_policy`), so two /// cases have no defined policy: a repository carrying no config frame at all, /// and a byte naming no artifact-format §7.1 Table 7 ordinal. Neither is /// shun-sha1 — the ONLY distinction §7.7 draws — so both answer `.auto` and the /// receive path admits, rather than failing a whole sync over a field this /// clause reads for one boolean. That is a read-side default, not a repair: the /// stored byte is left exactly as found. /// /// A READ FAULT IS NOT ONE OF THOSE TWO CASES. `readConfig` also returns /// `Storage` (the frame would not read) and `Corrupt` (the frame is not a config /// frame, or does not parse), and both mean the policy is UNKNOWN rather than /// undefined. Answering `.auto` there would silently disable the shun-sha1 half /// of the §7.7 predicate and admit SHA1-named content this repository is /// configured to refuse, with nothing failing — the same fail-open `serveRound` /// refuses for the shun-table half ("must fail the round rather than silently /// serve with no shun table"). So they propagate, exactly as `RepoShunSet.load` /// does three lines above the serve-side call site. fn effectiveHashPolicy(gpa: std.mem.Allocator, log: *logstore.LogStore) error{ OutOfMemory, Storage }!artifact.HashPolicy { var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, // A config frame that will not read or will not parse is a storage-level // failure of this repository, not a policy answer. error.Storage, error.Corrupt => return error.Storage, }) orelse return .auto; defer oc.deinit(gpa); return std.enums.fromInt(artifact.HashPolicy, oc.config.hash_policy) orelse .auto; } /// sync §3.3 L527-530 / §7.7 L1434-1437 — the two repository-local inputs to the /// receive-side filters, as ONE value. They are halves of one question (§7.7's /// combined target predicate reads both; §3.3's phantom scan reads the table), /// they are always read off the same open, and a client wired with one but not /// the other silently applies half a filter with nothing failing. Travelling as /// a pair is what makes that unrepresentable: `syncClient` takes this struct, so /// there is exactly one place either field is handed to a `Client`. /// /// `shun` borrows the caller's `RepoShunSet`, which must outlive the run. const RepoAdmission = struct { shun: zit.sync.ShunSet, hash_policy: artifact.HashPolicy, /// RFC section 5.2 rule 3 -- `.none` for every dense store. elided: zit.sync.ShunSet = .none, /// RFC section 6.1 -- the store cone's raw members, borrowed from the /// run-lifetime cone; empty for a dense store. The client encodes them /// as `pragma cone MEMBER` cards so a cone-aware peer filters replies. cone_members: []const []const u8 = &.{}, /// The store cone's exclusion members (`pragma cone-not MEMBER`). cone_not_members: []const []const u8 = &.{}, }; /// The ONE construction of a sync `Client` from repository state. `pullRun`, /// `pushRun` and `syncRun` differ only in the operation tag and whether a /// credential is named, so everything else — the servercode, the version, and /// both halves of `RepoAdmission` — is stated here once. Three copies of this /// literal is how a repository filter comes to be wired into two verbs and not /// the third. fn syncClient( op: zit.sync.Operation, identity: *const RepoSyncIdentity, credential: ?zit.sync.Credential, admission: RepoAdmission, ) zit.sync.Client { return .{ .state = zit.sync.initialState(op), .operation = op, .self_servercode = &identity.server_code, .project_code = identity.project_code, .version = sync_client_version_info, // sync §6 — a named credential makes `buildRequest` emit the §6.1 login // card; null leaves the session anonymous. .credential = credential, // sync §3.3 L527-530 — the per-round phantom scan's shun filter. .shun = admission.shun, .elided = admission.elided, .cone_members = admission.cone_members, .cone_not_members = admission.cone_not_members, // sync §7.7 L1428-1432 — this repository's effective hash-policy, the // SHA1-name half of the receive-side target-admission predicate. .hash_policy = admission.hash_policy, // submanifests §8.1/§7.2 — the durable indicator and the §8.1 emission // setting together arm the §7.2 upload gate. A pull uploads nothing, so // both are inert there; they are set for every operation so no caller has // to remember which ones upload, which is the same reason the credential // and the admission pair live here rather than at three call sites. .seen_submanifest = identity.seen_submanifest, .submanifests_enabled = identity.enable_submanifests, }; } /// The sync client's holdings view, log-backed and lazy. The clients /// used to snapshot every stored artifact into a `MemStore` (`loadRepoHoldings`) /// before the first wire byte, which resident-loaded the whole store's /// plaintext per pull/push/sync — 3.7 GB for a 147,920-artifact store, an OOM /// kill at tcl scale — just to announce igot names. This seats the client on /// the serve side's `LogIndexStore` instead: names, held/phantom, the /// shun mask, the hash-policy, and the config facts come off the frame index; /// content and wire forms materialize per-rid on demand; received artifacts /// land in the index's appendix `MemStore` and are the only new bytes /// resident. `persistStore` then walks the same store — every pre-held rid /// early-outs on the append log's own `lookup`, so persist stays flat too. /// /// The read handle stays open across the run and the persist; readers take no /// lock (`cli.openStoreRead`), so the append side's exclusive lock is not /// contested by it. const ClientHoldings = struct { file: File, index: *LogIndexStore, fn open(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) error{ OutOfMemory, Storage }!ClientHoldings { var file = cli.openStoreRead(io, dir) catch return error.Storage; errdefer file.close(io); const st = file.stat(io) catch return error.Storage; const index = LogIndexStore.openFromFile(gpa, io, file, st.size) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; return .{ .file = file, .index = index }; } fn store(self: *ClientHoldings) repostate.ArtifactStore { return self.index.store(); } /// The number of already-durable, log-backed artifacts. Received /// artifacts are appended beyond this, so persist begins here and never /// re-walks the holdings a pull did not change. fn logBackedCount(self: *ClientHoldings) repostate.Rid { return @intCast(self.index.rids.len); } /// The §3.3/§7.7 admission pair, borrowed from the index — which outlives /// the run exactly as the caller-owned `RepoShunSet` used to. fn admission(self: *ClientHoldings) RepoAdmission { return .{ .shun = self.index.shun.shunSet(), .hash_policy = self.index.hash_policy }; } fn deinit(self: *ClientHoldings, io: std.Io) void { self.index.destroy(); self.file.close(io); } }; /// sync §9.1 — the repository's stored project code, copied into `buf`, or the /// zero placeholder when the repo carries none (a repo predating the stored /// code) or an out-of-spec-length one. `init` derives a project code and a /// clone adopts the origin's (`storeClonedProjectCode`), so a real repo /// announces its adopted code here; the pull/push client emits it verbatim as /// the operation card's PROJECTCODE, and a real peer (fossil §9.1, ABNF /// `pull-card = "pull" SP servercode SP projectcode`) refuses a mismatch — so a /// zero placeholder would be rejected. Bounded by `max_projectcode_len`: a /// longer stored field is out of spec and never truncated into the wire. fn repoProjectCode( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, buf: *[zit.sync.max_projectcode_len]u8, ) error{ OutOfMemory, Storage }![]const u8 { return (try repoSyncIdentity(gpa, io, dir, "test-fixture-repo", buf)).project_code; } /// The durable config facts every sync-family run needs before it can build /// a peer: §9.1's project code and draft-fossil-submanifests-00 §8.1's /// seen-submanifest indicator and emission setting. Read together because they /// come out of the same config frame — a second open to fetch another field /// would be one more way for them to disagree about which frame they came from. const RepoSyncIdentity = struct { /// §9.1 — the adopted project code, or the zero placeholder. project_code: []const u8, /// submanifests §8.1 — has this repository ever stored a §7.2-gated /// artifact? False for every repository predating this draft, and the /// second thing that arms the §7.2 send gate (see `Server.seen_submanifest`). seen_submanifest: bool, /// submanifests §8.1 — the emission setting. The client twin of /// `serveContext`'s server-side refresh: it is the OTHER half of the §7.2 /// arming disjunction (`Client.mayHoldSubmanifests`). Today it is /// belt-and-braces, because any repository actually holding a gated artifact /// also has `seen_submanifest` set — the two are written in one `log.commit` /// with the first tree. It is threaded anyway so the field is never read /// stale: leaving it at its `false` default would silently disarm the gate /// for any future surface that commits mid-session, where the setting is on /// but the indicator has not been written yet. enable_submanifests: bool, /// §9.1 — this instance's own SERVERCODE, derived from the checkout's /// absolute path (`deriveServerCode`); informational on the wire and /// never adopted from a peer. By value so the client's borrow has no /// lifetime to manage beyond the identity itself. server_code: zit.sync.HashHex, }; fn repoSyncIdentity( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, abs: []const u8, buf: *[zit.sync.max_projectcode_len]u8, ) error{ OutOfMemory, Storage }!RepoSyncIdentity { var file = cli.openStoreRead(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return .{ .project_code = sync_projectcode, .seen_submanifest = false, .enable_submanifests = zit.sync.submanifest_emission_default, .server_code = zit.sync.deriveServerCode(&.{abs}), }; defer oc.deinit(gpa); const seen = oc.config.seen_submanifest; const enabled = oc.config.enable_submanifests; const server_code = zit.sync.deriveServerCode(&.{abs}); const code = oc.config.project_code; if (code.len == 0 or code.len > buf.len) return .{ .project_code = sync_projectcode, .seen_submanifest = seen, .enable_submanifests = enabled, .server_code = server_code, }; @memcpy(buf[0..code.len], code); return .{ .project_code = buf[0..code.len], .seen_submanifest = seen, .enable_submanifests = enabled, .server_code = server_code, }; } /// The number of content-bearing (non-phantom) artifacts in `store` — the held /// set (§3.3 r24). A pull measures its yield as the held count after convergence /// minus the count before, so "already up to date" is an exact zero, never a guess. fn contentCount(store: repostate.ArtifactStore) usize { // `held`, never `content`: on the log-backed client store a // `content()` probe MATERIALIZES the rid's plaintext — asking it of every // rid re-inflates the whole repository to produce a number, which is the // exact whole-store cost this store exists to remove. `held` answers from // the index, and `persistStore` counts `written` by the same predicate, // so the `written >= held_before` arithmetic stays aligned. return store.heldCount(); } // --------------------------------------------------------------------------- // `zit push URL` — the mirror of pull: UPLOAD this repository's artifacts into an // EXISTING served repo. It opens the cwd's `.zit`, announces what it holds so the // peer requests only what it lacks (sync §3.2/§3.3 igot/gimme), and uploads the // missing artifacts as `file` cards, driving to §3.7 termination. The peer's // `zit serve` persists what it receives; nothing changes on this side (§3.3 r24). // Anonymous only. // --------------------------------------------------------------------------- /// What one push run reports: the number of artifacts uploaded and the round /// count it took. `sent` reads the client's own upload counter, so it is the /// exact count of `file` cards emitted — never an estimate. /// submanifests §7.2 — `refused` counts artifacts the §7.2 gate declined to /// upload because the peer's announced version predates sub-manifest support. /// Carried alongside `sent` so a run that withheld an artifact can never be /// reported as a success: the draft rules out silent omission. const PushOutcome = struct { rounds: usize, sent: usize, refused: usize = 0, uv_skipped: u64 = 0 }; /// `zit push URL` — open the current repository, announce its holdings to the /// peer at URL, and upload every artifact the peer lacks until §3.7 termination. /// Thin wiring over the same deep modules pull uses: `transport` (HTTP/IO), /// `sync.Client` + `runSync` (sans-IO §3.7 loop). With URL omitted the saved /// remote (`zit remote`) is used; a repo with neither is a usage error. pub fn runPush( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parsePushArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "push", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "push", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(push_help) catch {}; return .ok; } // sync §6.3 — a login name is a whitespace-free token (the shared-secret // derivation embeds it between "/" separators). Reject a malformed --user // as a usage failure rather than reaching the derivation's assertion. if (parsed.user) |u| { if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) { streams.err.writeAll("zit push: --user must not contain whitespace\n") catch {}; return .failure; } } // Locate the current repository before any network I/O: a push outside a repo // is a usage failure, not a wasted connection. const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "push")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), error.NoRemote => return noRemoteError(streams, "push"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "push", url, e); defer tr.close(); // The client's holdings view — log-backed and lazy, the igot // announcement plus the upload source: the peer requests only what it // lacks, and only those artifacts' content materializes. var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), }; defer holdings.deinit(io); var admission = holdings.admission(); // Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once // and suppress spontaneous requests for elided phantoms. Both the // classification and the wrapper outlive the run below. var sync_cone: ?zit.cone.Cone = null; defer if (sync_cone) |*c| c.deinit(gpa); var sync_classification: ?SparseClassification = null; defer if (sync_classification) |*c| c.deinit(gpa); var sync_elided: RepoElidedSet = undefined; if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| { sync_cone = c; sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch return stateOom(streams, "push"); sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() }; admission.elided = sync_elided.shunSet(); } // §9.1 — announce this repository's adopted project code in the push card // (same identity gate as pull; a real peer refuses a mismatch). The buffer // outlives the run — the client borrows it. // The cluster index: what the held clusters already announce // (never announced again individually) and what they promised that this // repository never collected (seeded as phantoms, so the run asks for it). // The clusters' plaintext is memoised by the holdings adapter while it // resolves; drop it — the index keeps only names and bits. var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "push"); defer clusters.deinit(); holdings.index.releaseTransient(); var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), }; // sync §6 — resolve the login credential (null for an anonymous push). The // password plaintext is confined to `pw_buf`/argv/env; only the derived // 40-hex secret (§6.3) is signed with and — on success — persisted. var pw_buf: [max_password_len]u8 = undefined; // Defense-in-depth: once the 40-hex secret is derived the plaintext in // pw_buf is spent, so scrub it on every exit from this scope — success and // error alike — rather than leave it resident until the stack is reused. defer std.crypto.secureZero(u8, &pw_buf); var user_buf: [logstore.max_config_str]u8 = undefined; const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, parsed, amb, &pw_buf, &user_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), error.NoPassword => { streams.err.writeAll("zit push: no password supplied for --user\n") catch {}; return .failure; }, }; var peer_error: PeerErrorCopy = .{}; const stderr_tty = std.Io.File.stderr().isTty(io) catch false; var progress: CloneProgress = .{ .err = streams.err }; const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null; defer progress.finishLine(); const outcome = pushRun(gpa, &tr, holdings.store(), identity, if (resolved) |r| r.credential else null, admission, &peer_error, sink, &clusters) catch |e| { // A peer that stated its refusal is quoted rather than flattened into // the generic mapping — the whole point of a server-side refusal is that the operator learns // WHICH file and WHICH rule without a second round trip. if (e == error.FatalProtocolError) { if (peer_error.text()) |text| { streams.err.print("zit push: the peer refused: {s}\n", .{text}) catch {}; return .failure; } } return syncRunFail(streams, "push", url, e, tr.last_status); }; // DECISION-3 — persist the derived secret (never the plaintext) at 0600 only // after the peer accepted the authenticated push, and only when it was // freshly derived: a reused stored secret is already durable. if (resolved) |r| { if (r.persist) { storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), else => return repoReadFail(streams, "push"), }; } } // submanifests §7.2 — "the sender MUST refuse to transmit that artifact, // with an explicit error -- never by converting, downgrading, or silently // omitting it." A client cannot put an `error` card in a REQUEST (it is // outside the §3.9 strict-accept request vocabulary), so the explicit error // is this: the operator is told exactly what was withheld and why, and the // run fails. Reported before the success line, because there was none. if (outcome.refused > 0) { streams.err.print( "zit push: {d} artifacts were withheld: {s}\n", .{ outcome.refused, zit.sync.submanifest_peer_version_error_text }, ) catch {}; return .failure; } if (outcome.sent == 0) { streams.out.print("push to {s}: already in sync\n", .{url}) catch {}; } else { streams.out.print("push to {s}: sent {d} artifact{s} in {d} round{s}\n", .{ url, outcome.sent, plural(outcome.sent), outcome.rounds, plural(outcome.rounds) }) catch {}; reportUvSkipped(streams, "push", outcome.uv_skipped); } return .ok; } /// The upper bound on an interactively-typed or `$ZIT_PASSWORD` password. A /// login password far shorter than this in practice; the fixed bound keeps the /// no-echo read on the stack with no unbounded allocation (TigerStyle). const max_password_len: usize = 1024; /// sync §6.3 — a resolved push credential and whether its secret was freshly /// derived (so it should be persisted after a successful push). const ResolvedCredential = struct { credential: zit.sync.Credential, persist: bool, }; /// The causes `resolvePushCredential` can surface: store read failure, OOM, or /// a `--user` with no obtainable password (`NoPassword`). Closed set so the /// caller switches exhaustively. const ResolveError = error{ OutOfMemory, Storage, NoPassword }; /// sync §6.3/DECISION-3 — turn `push`'s `--user`/`--password` into a signing /// credential. An empty `--user` is an anonymous push (null result). An /// ABSENT `--user` falls back to the login a previous authenticated push /// stored: that credential is reused as-is and this branch never /// prompts — there is no operator at a keyboard by definition, since it is /// how autosync pushes — so with nothing stored the push is anonymous. That is /// fossil's shape: a remembered login authenticates every later sync. /// Otherwise the 40-hex secret is resolved, in order, from: a stored credential /// for this exact login (reused verbatim, already durable), `--password`, /// `$ZIT_PASSWORD`, or a no-echo terminal prompt. A freshly derived secret is /// flagged `persist`; the plaintext lives only in `pw_buf`/argv/env and is /// never returned or stored. `user_buf` holds a stored login's name for the /// returned credential to borrow; it must outlive the credential. fn resolvePushCredential( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, streams: cli.Streams, project_code: []const u8, parsed: cli.PushArgs, amb: cli.Ambient, pw_buf: *[max_password_len]u8, user_buf: *[logstore.max_config_str]u8, ) ResolveError!?ResolvedCredential { const user = parsed.user orelse { const stored = (try storedLogin(gpa, io, dir)) orelse return null; @memcpy(user_buf[0..stored.user_len], stored.user()); return .{ .credential = .{ .user = user_buf[0..stored.user_len], .secret_hex = stored.secret_hex }, .persist = false, }; }; if (user.len == 0) return null; // an empty --user is an anonymous push std.debug.assert(!zit.sync.containsFossilWhitespace(user)); // pre-validated in runPush // A stored secret for this exact login is reused as-is: no re-derive, no // prompt, no re-persist. Only consulted when no explicit password is given. if (parsed.password == null and amb.push_password == null) { if (try storedSecretFor(gpa, io, dir, user)) |sec| { return .{ .credential = .{ .user = user, .secret_hex = sec }, .persist = false }; } } // Otherwise derive from a plaintext password. Precedence: --password, then // $ZIT_PASSWORD, then an interactive no-echo prompt. const password: []const u8 = parsed.password orelse amb.push_password orelse (readPasswordNoEcho(streams, user, pw_buf) orelse return error.NoPassword); const secret = zit.sync.deriveSharedSecret(project_code, user, password); return .{ .credential = .{ .user = user, .secret_hex = secret }, .persist = true }; } /// The login a successful authenticated push left in this repo's config /// (DECISION-3): the name and its 40-hex secret, both copied out — the config /// frame is freed here — so the value outlives the read. const StoredLogin = struct { user_buf: [logstore.max_config_str]u8, user_len: usize, secret_hex: [40]u8, fn user(self: *const StoredLogin) []const u8 { return self.user_buf[0..self.user_len]; } }; /// The stored login, or null when none is stored. Read-only. fn storedLogin( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, ) error{ OutOfMemory, Storage }!?StoredLogin { var file = cli.openStoreRead(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return null; defer oc.deinit(gpa); if (oc.config.login_secret_hex.len != 40) return null; const name = oc.config.login_user; if (name.len == 0 or name.len > logstore.max_config_str) return null; var out: StoredLogin = .{ .user_buf = undefined, .user_len = name.len, .secret_hex = undefined }; @memcpy(out.user_buf[0..name.len], name); @memcpy(&out.secret_hex, oc.config.login_secret_hex[0..40]); return out; } /// DECISION-3 — the 40-hex secret stored for `user` in this repo's config, or /// null when none is stored or it belongs to a different login. fn storedSecretFor( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, user: []const u8, ) error{ OutOfMemory, Storage }!?[40]u8 { const stored = (try storedLogin(gpa, io, dir)) orelse return null; if (!std.mem.eql(u8, stored.user(), user)) return null; return stored.secret_hex; } /// DECISION-3 — persist the derived login credential into the repo config, /// preserving every other field, then tighten the repository file to 0600 so /// the stored secret is owner-only at rest. The permission tightening is /// best-effort: the secret is already durable, so a filesystem that refuses /// chmod does not fail the push (perms + FDE are the boundary, not the sole /// guarantee — per DECISION-3). NEVER writes the plaintext password. fn storePushCredential( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, credential: zit.sync.Credential, ) PersistError!void { std.debug.assert(credential.secret_hex.len == 40); // The field-preserving RMW writes the credential without disturbing the // project code or a saved remote (a bare `putConfig` here would drop them). try updateConfig(gpa, io, dir, .{ .login = credential }); // Owner-only now that the file carries a credential (best-effort; see above). var file = cli.openStoreAppend(io, dir) catch return; defer file.close(io); file.setPermissions(io, File.Permissions.fromMode(0o600)) catch {}; } /// CLI/IO seam — prompt on stderr and read one password line from the terminal /// with echo disabled, returning the trimmed line in `buf` (never its newline), /// or null on a read failure or an empty line. Echo is restored on every exit /// path; when stdin is not a terminal (`tcgetattr` fails) the line is read with /// no termios toggle. Not exercised by the automated gates (they pass /// `--password`/`$ZIT_PASSWORD`) — this is the interactive path. fn readPasswordNoEcho( streams: cli.Streams, user: []const u8, buf: *[max_password_len]u8, ) ?[]const u8 { streams.err.print("Password for {s}: ", .{user}) catch {}; streams.err.flush() catch {}; const fd = std.posix.STDIN_FILENO; const restore: ?std.posix.termios = blk: { var t = std.posix.tcgetattr(fd) catch break :blk null; const saved = t; t.lflag.ECHO = false; std.posix.tcsetattr(fd, .FLUSH, t) catch break :blk null; break :blk saved; }; defer if (restore) |saved| { std.posix.tcsetattr(fd, .FLUSH, saved) catch {}; streams.err.writeAll("\n") catch {}; streams.err.flush() catch {}; }; const n = std.posix.read(fd, buf) catch return null; if (n == 0) return null; var line = buf[0..n]; // Trim a single trailing CRLF/LF; interior bytes (including spaces) are kept. if (line.len > 0 and line[line.len - 1] == '\n') line = line[0 .. line.len - 1]; if (line.len > 0 and line[line.len - 1] == '\r') line = line[0 .. line.len - 1]; if (line.len == 0) return null; return line; } /// Drives a push client (§3.2/§3.3) to convergence over `tr`, uploading the /// artifacts the peer lacks; returns the upload count and round count. The client /// announces `ms`'s holdings via igot and uploads what the peer gimme's (§3.3 /// r24: push sends content but never obtains it). Sans-IO on this side of /// `tr.exchange`. Its closed error set is the driver's own causes unioned with /// the transport's, so `runPush` switches it exhaustively. fn pushRun( gpa: std.mem.Allocator, tr: *transport.Transport, holdings: repostate.ArtifactStore, identity: RepoSyncIdentity, credential: ?zit.sync.Credential, admission: RepoAdmission, /// A server that REFUSED states why in its error card. Copied out /// here because the client owns the buffer and is deinit'd on the way out, /// so the caller cannot reach it afterwards; the clone path does the same /// for the sparse-origin refusal. peer_error_out: *PeerErrorCopy, sink: ?sync_driver.RoundSink, clusters: ?*const clusterindex.Resolved, ) sync_driver.RunError(*transport.Transport)!PushOutcome { // A push client still RECEIVES — §3.3 answers the server's gimme-driven // `file` cards in the same session — so `admission` is load-bearing here too. var client = syncClient(.push, &identity, credential, admission); defer client.deinit(gpa); if (clusters) |c| { client.clustered = &c.clustered; try client.seedClusterPhantoms(gpa, holdings, c.missing); } errdefer peer_error_out.set(client.peerErrorText()); const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null); recordSyncStats(stats); // Read the upload counter before `deinit` runs (the return expression is // evaluated first): the exact number of `file` cards this session emitted. return .{ .rounds = stats.rounds, .sent = client.content_sent, .refused = client.submanifests_refused, .uv_skipped = client.uv_skipped }; } // --------------------------------------------------------------------------- // `zit sync [URL]` — BIDIRECTIONAL convergence with a peer in ONE session. // Unlike pull-then-push, a single `operation=.sync` client announces igot, // requests gimme, AND uploads on one §3.7 exchange (§3.2 round structure: a sync // request carries both push and pull each round) — not a third copy of the driver // loop over two transport sessions. // Opens the cwd's `.zit`, resolves the URL (arg → saved remote), may // authenticate (sync uploads), persists what it receives like pull, and reports // both directions. A saved `zit remote` lets the URL be omitted. // --------------------------------------------------------------------------- /// What one sync run reports: uploads (`sent`) and the round count. Received is /// measured by the caller as the persisted-count growth (as pull does), so it is /// not carried here. /// submanifests §7.2 — see `PushOutcome.refused`. const SyncOutcome = struct { rounds: usize, sent: usize, refused: usize = 0, uv_skipped: u64 = 0 }; pub fn runSync( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseSyncArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "sync", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "sync", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(sync_help) catch {}; return .ok; } // sync §6.3 — a login name is whitespace-free (mirrors push). Reject a // malformed --user as usage before the secret derivation's assertion. if (parsed.user) |u| { if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) { streams.err.writeAll("zit sync: --user must not contain whitespace\n") catch {}; return .failure; } } // Locate the current repository before any network I/O. const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "sync")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.NoRemote => return noRemoteError(streams, "sync"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "sync", url, e); defer tr.close(); // The client's holdings view — log-backed and lazy: the igot // announcement, the upload source, and the baseline for the received // count. The held set can only grow (§3.3 storage only adds content). var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), }; defer holdings.deinit(io); var admission = holdings.admission(); // Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once // and suppress spontaneous requests for elided phantoms. Both the // classification and the wrapper outlive the run below. var sync_cone: ?zit.cone.Cone = null; defer if (sync_cone) |*c| c.deinit(gpa); var sync_classification: ?SparseClassification = null; defer if (sync_classification) |*c| c.deinit(gpa); var sync_elided: RepoElidedSet = undefined; if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| { sync_cone = c; sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch return stateOom(streams, "sync"); sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() }; admission.elided = sync_elided.shunSet(); admission.cone_members = sync_cone.?.members; admission.cone_not_members = sync_cone.?.not_members; } const held_before = contentCount(holdings.store()); // §9.1 — announce this repository's adopted project code (a real peer refuses // a mismatch). The buffer outlives the run — the client borrows it. // The cluster index: what the held clusters already announce // (never announced again individually) and what they promised that this // repository never collected (seeded as phantoms, so the run asks for it). // The clusters' plaintext is memoised by the holdings adapter while it // resolves; drop it — the index keeps only names and bits. var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "sync"); defer clusters.deinit(); holdings.index.releaseTransient(); var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), }; // sync §6 — resolve the login credential (null for anonymous). A sync // uploads, so it may authenticate exactly as push does; the plaintext lives // only in `pw_buf`/argv/env and is scrubbed on every exit from this scope. var pw_buf: [max_password_len]u8 = undefined; defer std.crypto.secureZero(u8, &pw_buf); var user_buf: [logstore.max_config_str]u8 = undefined; const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf, &user_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.NoPassword => { streams.err.writeAll("zit sync: no password supplied for --user\n") catch {}; return .failure; }, }; const stderr_tty = std.Io.File.stderr().isTty(io) catch false; var progress: CloneProgress = .{ .err = streams.err }; const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null; defer progress.finishLine(); const outcome = syncRun(gpa, &tr, holdings.store(), identity, if (resolved) |r| r.credential else null, admission, sink, &clusters) catch |e| return syncRunFail(streams, "sync", url, e, tr.last_status); // Sparse-cone RFC section 6.3 applies to EVERY receive: a dense peer // volunteers content the client never announced, so the filter runs at // persist. Reclassified over the post-run store, so a brand-new commit's // blobs classify by their own F cards -- new in-cone content persists, // new out-of-cone content elides with the rest. var post_classification: ?SparseClassification = null; defer if (post_classification) |*c| c.deinit(gpa); if (sync_cone) |*c| { post_classification = classifyKeepRids(gpa, holdings.store(), c) catch return stateOom(streams, "sync"); } const persisted = persistStore(gpa, io, root_dir, holdings.store(), if (post_classification) |*c| &c.keep else null, 0) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.Corrupt => return renderStorage(streams, root.abs, "the peer sent a malformed artifact name"), }; const written = persisted.written; std.debug.assert(written >= held_before); const received = written - held_before; // DECISION-3 — persist a freshly derived secret only after the peer accepted // the authenticated session, and only when it was freshly derived (mirrors push). if (resolved) |r| { if (r.persist) { storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), else => return repoReadFail(streams, "sync"), }; } } // submanifests §7.2 — "the sender MUST refuse to transmit that artifact, // with an explicit error -- never by converting, downgrading, or silently // omitting it." A client cannot put an `error` card in a REQUEST (it is // outside the §3.9 strict-accept request vocabulary), so the explicit error // is this: the operator is told exactly what was withheld and why, and the // run fails. Reported before the success line, because there was none. if (outcome.refused > 0) { streams.err.print( "zit sync: {d} artifacts were withheld: {s}\n", .{ outcome.refused, zit.sync.submanifest_peer_version_error_text }, ) catch {}; return .failure; } // submanifests §7.1/§7.3 — see `runPull`: convergence against the peer is // not the same fact as a complete tree closure. if (persisted.submanifest_gaps > 0) { streams.err.print( "zit sync: {d} sub-manifest references are missing; some check-ins cannot be enumerated\n", .{persisted.submanifest_gaps}, ) catch {}; return .failure; } if (outcome.sent == 0 and received == 0) { streams.out.print("sync with {s}: already in sync\n", .{url}) catch {}; } else { streams.out.print("sync with {s}: sent {d}, received {d} in {d} rounds\n", .{ url, outcome.sent, received, outcome.rounds }) catch {}; reportUvSkipped(streams, "sync", outcome.uv_skipped); } return .ok; } /// Drives a bidirectional sync client (§3.2 round structure — both directions on /// ONE §3.7 session) to convergence over `tr`; returns the upload count and round /// count. /// The received count is the caller's persisted-count growth. Sans-IO on this /// side of `tr.exchange`; its closed error set is switched exhaustively by /// `runSync`. The sibling of `pullRun`/`pushRun`, differing only in the /// operation tag (`.sync`, which drives igot + gimme + upload on one session). fn syncRun( gpa: std.mem.Allocator, tr: *transport.Transport, holdings: repostate.ArtifactStore, identity: RepoSyncIdentity, credential: ?zit.sync.Credential, admission: RepoAdmission, sink: ?sync_driver.RoundSink, clusters: ?*const clusterindex.Resolved, ) sync_driver.RunError(*transport.Transport)!SyncOutcome { var client = syncClient(.sync, &identity, credential, admission); defer client.deinit(gpa); if (clusters) |c| { client.clustered = &c.clustered; try client.seedClusterPhantoms(gpa, holdings, c.missing); } const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null); recordSyncStats(stats); return .{ .rounds = stats.rounds, .sent = client.content_sent, .refused = client.submanifests_refused, .uv_skipped = client.uv_skipped }; } // --------------------------------------------------------------------------- // `zit remote [URL | --unset]` — read, set, or clear the single default peer URL // (sync §4/§6). Bare prints the saved remote; a URL saves it; --unset clears it. // A local, non-synced config value (`config.remote_url`) — anonymous only, no // credential in the URL — that lets pull/push/sync omit their URL argument. Not // a git-style named-remote registry: one value, matching the single-value config. // --------------------------------------------------------------------------- /// `zit settings [NAME [VALUE]]` — the local-settings surface, holding /// exactly one setting today: `autosync`. Bare lists; NAME prints; NAME VALUE /// sets. The value lives in the config frame like `remote_url` — local, never /// synced — inverted in storage (the flag records OFF) so absence means the /// fossil default, on. pub fn runSettings( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseSettingsArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "settings", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "settings", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(settings_help) catch {}; return .ok; } const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "settings")) orelse { main.reportNoRoot(streams, io, "settings", amb.repo_override); return .failure; }; var root_dir = root.dir; defer root_dir.close(io); const zset = zit.settings; // The tier the operator named. `--global` selects WHERE a write lands and // which stored value a bare NAME reports; it never changes the fact that a // listing reports the value in EFFECT. const tier: Tier = if (parsed.global) .global else .repository; if (parsed.name) |name| { const setting = zset.Setting.byName(name) orelse { var names_buf: [256]u8 = undefined; streams.err.print( "zit settings: unknown setting '{s}' (known: {s})\n", .{ name, zset.knownNames(&names_buf) }, ) catch {}; return .failure; }; if (parsed.value) |value| { return writeSetting(gpa, io, arena, root_dir, streams, amb, setting, value, tier); } return showSetting(gpa, io, arena, root_dir, streams, amb, setting, parsed.changed, parsed.value_only); } inline for (comptime std.enums.values(zset.Setting)) |setting| { const exit = showSetting(gpa, io, arena, root_dir, streams, amb, setting, parsed.changed, parsed.value_only); if (exit != .ok) return exit; } return .ok; } /// Write one setting into the tier the operator named. /// /// Validation happens BEFORE either tier is touched, so a rejected value never /// leaves a half-written global file or a config frame carrying a value the /// reader will refuse. fn writeSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, streams: cli.Streams, amb: cli.Ambient, setting: zit.settings.Setting, value: []const u8, tier: Tier, ) cli.Exit { const zset = zit.settings; switch (setting.kind()) { .boolean => if (zset.parseBool(value) == null and !(setting == .autosync and std.mem.eql(u8, value, "pullonly"))) { streams.err.print( "zit settings: {s} takes 'on' or 'off'{s}\ntry 'zit settings --help'\n", .{ setting.name(), if (setting == .autosync) " (or 'pullonly')" else "" }, ) catch {}; return .usage; }, .string => if (tier == .repository and value.len > logstore.max_config_str) { // Refusing beats truncating: a truncated glob list silently stops // protecting files, and a truncated command silently runs a // different program. if (setting == .ignore_glob) { streams.err.print( "zit settings: ignore-glob is limited to {d} bytes; use the versioned {s} file for longer lists\n", .{ logstore.max_config_str, zit.ignore.versioned_path }, ) catch {}; } else { streams.err.print( "zit settings: {s} is limited to {d} bytes\n", .{ setting.name(), logstore.max_config_str }, ) catch {}; } return .failure; }, } if (tier == .global) { writeGlobalSetting(gpa, io, arena, amb, setting, value) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "settings"), error.NoGlobalHome => { streams.err.writeAll( "zit settings: no global settings location (set $ZIT_CONFIG, $XDG_CONFIG_HOME or $HOME)\n", ) catch {}; return .failure; }, else => { streams.err.writeAll("zit settings: cannot write the global settings file\n") catch {}; return .failure; }, }; streams.out.print("{s} {s}\n", .{ setting.name(), value }) catch {}; // A global write that is shadowed locally changes nothing an operator // can see, which is precisely the confusion the provenance work exists // to end -- so say so at the moment it happens. const eff = resolveSetting(gpa, io, arena, root_dir, amb, setting) catch return .ok; if (eff.source != .global) { streams.err.print( "note: the {s} tier still supplies {s} here, so this global value is not in effect\n", .{ eff.source.label(), setting.name() }, ) catch {}; } _ = noteVersionedShadow(io, root_dir, streams, setting, eff); if (setting == .ignore_glob) noteGitignorePresent(io, root_dir, streams); return .ok; } const patch = repositoryPatch(setting, value) orelse { streams.err.print( "zit settings: {s} cannot be stored in this repository yet\n", .{setting.name()}, ) catch {}; return .failure; }; updateConfig(gpa, io, root_dir, patch) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "settings"), else => return repoReadFail(streams, "settings"), }; streams.out.print("{s} {s}\n", .{ setting.name(), value }) catch {}; // The value just written IS the repository tier, so the shadow question is // answerable without re-opening the store. const r = resolveSettingWith(io, arena, root_dir, amb, setting, value) catch return .ok; _ = noteVersionedShadow(io, root_dir, streams, setting, r); if (setting == .ignore_glob) noteGitignorePresent(io, root_dir, streams); return .ok; } /// The config-frame patch for a repository-tier write, or null for a setting /// the frame does not yet carry. fn repositoryPatch(setting: zit.settings.Setting, value: []const u8) ?ConfigPatch { const on = zit.settings.parseBool(value); return switch (setting) { // Both booleans record the explicit-set bit alongside the value, which // is what makes "explicitly the default" distinguishable from untouched. .autosync => if (std.mem.eql(u8, value, "pullonly")) .{ .autosync_off = false, .autosync_set = true, .autosync_mode = "pullonly" } else .{ .autosync_off = !(on orelse return null), .autosync_set = true, .autosync_mode = "" }, .editor => .{ .editor = value }, .clearsign => .{ .clearsign = on orelse return null, .clearsign_set = true }, .ignore_glob => .{ .ignore_glob = value }, .pgp_command => .{ .pgp_command = value }, .keep_glob => .{ .keep_glob = value }, .manifest => .{ .manifest = value }, .before_commit => .{ .before_commit = value }, .binary_glob => .{ .binary_glob = value }, .crlf_glob => .{ .crlf_glob = value }, .crnl_glob => .{ .crnl_glob = value }, .clean_glob => .{ .clean_glob = value }, .empty_dirs => .{ .empty_dirs = value }, .encoding_glob => .{ .encoding_glob = value }, // Boolean-valued, stored as the operator's own text: `settings // dotfiles yes` round-trips as `yes`, and `settings.isTruth` is the // one thing that reads it. .dotfiles => .{ .dotfiles = value }, .mimetypes => .{ .mimetypes = value }, .attachment_size_limit => .{ .attachment_size_limit = value }, }; } /// Read-modify-write the operator-scope settings file, creating its directory /// when absent. Read-modify-write rather than append so a second edit of the /// same setting replaces the first instead of leaving a dead line above it. fn writeGlobalSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, amb: cli.Ambient, setting: zit.settings.Setting, value: []const u8, ) !void { const zset = zit.settings; var path_buf: [1024]u8 = undefined; const g = zset.globalPath(amb.environ, &path_buf) orelse return error.NoGlobalHome; const cwd = std.Io.Dir.cwd(); const existing = cwd.readFileAlloc(io, g.path, arena, .limited(zset.max_versioned_bytes)) catch ""; const next = try zset.globalSet(gpa, existing, setting, value); defer gpa.free(next); if (g.dir.len != 0) cwd.createDirPath(io, g.dir) catch {}; var f = cwd.createFile(io, g.path, .{ .truncate = true }) catch return error.Storage; defer f.close(io); f.writePositionalAll(io, next, 0) catch return error.Storage; } /// Which stored tier a `zit settings` invocation addresses. const Tier = enum { repository, global }; /// Print one setting as `NAME VALUE` plus the tier that supplied it. /// /// The provenance column is the point. zit has four places a value can /// live, and a display that showed only the value would let an operator edit /// the shadowed tier -- twice -- while the effective answer never moved. The /// note is on stderr so `zit settings NAME` remains a clean value on stdout for /// a script. fn showSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, streams: cli.Streams, amb: cli.Ambient, setting: zit.settings.Setting, only_changed: bool, value_only: bool, ) cli.Exit { const r = resolveSetting(gpa, io, arena, root_dir, amb, setting) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "settings"), else => return repoReadFail(streams, "settings"), }; if (only_changed and r.source == .default) return .ok; // --changed if (value_only) { streams.out.print("{s}\n", .{r.value}) catch {}; // --value } else { streams.out.print("{s} {s}\n", .{ setting.name(), r.value }) catch {}; } // Only say where it came from when that is not the obvious answer: naming // the tier for every default would bury the two lines that matter. One // fact, one line: when a versioned file is shadowing a stored value the // note SAYS so and replaces the provenance line, and when `.no-warn` // silences that, the provenance line still names the file that is live. if (!setting.versionable()) noteVersionedIgnored(io, root_dir, streams, setting); if (noteVersionedShadow(io, root_dir, streams, setting, r)) return .ok; if (r.source == .versioned) { const dir = r.versioned_dir orelse .zit; streams.err.print( "note: {s} is set in {s}/{s}\n", .{ setting.name(), dir.dirName(), setting.name() }, ) catch {}; } else if (r.source != .default and r.source != .repository) { streams.err.print( "note: {s} is set in the {s} tier\n", .{ setting.name(), r.source.label() }, ) catch {}; } return .ok; } /// Gather every tier's answer and apply `settings.resolve`. The reads live here /// -- the model module stays sans-IO so precedence is a pure, tested function. /// Public so the COMMIT path resolves signing settings through the same /// precedence a listing reports. They must never disagree: a `zit settings` /// that says `clearsign on` while the commit quietly does not sign is worse /// than having no global tier at all. pub fn resolveSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, amb: cli.Ambient, setting: zit.settings.Setting, ) !zit.settings.Resolved { const repository = try repositorySetting(gpa, io, arena, root_dir, setting); return resolveSettingWith(io, arena, root_dir, amb, setting, repository); } /// The effective `crlf-glob`, with fossil's older `crnl-glob` as the /// fallback SPELLING of the same setting. fossil reads /// `db_get("crlf-glob", db_get("crnl-glob",""))` (checkin.c), so `crnl-glob` /// is consulted only when no tier supplies `crlf-glob` at all -- a stored /// empty `crlf-glob` is a value and wins, exactly as it does for every other /// glob list. /// /// The COMMIT gate only. The wire twin (`sync.PushPolicy`, built from the /// serve-side snapshot) keeps reading `crlf-glob` alone: fossil's server never /// gates content, so there is no parity to mirror there. /// `config` is the caller's already-read frame, as for `localSetting`: the /// commit gate holds the store's write lock and reads its config once, so both /// spellings answer from that one frame rather than re-opening the store twice. pub fn resolveCrlfGlob( io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, amb: cli.Ambient, config: ?*const logstore.Config, ) error{OutOfMemory}!zit.settings.Resolved { const crlf = try localSetting(io, arena, root_dir, amb, config, .crlf_glob); if (crlf.source != .default) return crlf; return localSetting(io, arena, root_dir, amb, config, .crnl_glob); } /// `resolveSetting` with the REPOSITORY tier supplied by the caller instead of /// read here. `repositorySetting` opens the store and pays a full frame-header /// scan to reach one config frame -- fine for a one-shot verb, ruinous in the /// serve loop, where it turned every round of a large-store clone into seconds /// of dead re-scanning. The server passes the values its per-round /// snapshot already read; every other tier is a cheap local read and stays /// here, so the resolution order cannot fork between the two paths. pub fn resolveSettingWith( io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, amb: cli.Ambient, setting: zit.settings.Setting, repository: ?[]const u8, ) !zit.settings.Resolved { const zset = zit.settings; var c: zset.Candidates = .{}; // The versioned file is read ONLY for a versionable setting. `resolve` // enforces the same rule, so this is belt and braces on the boundary that // keeps a cloned repository from choosing what this machine executes. if (setting.versionable()) { // fossil's spelling first, zit's as the fallback -- a mirrored // Fossil repository's own policy is honoured without adopting zit's // directory name, and when both exist the fossil file wins. // Two buffers: the candidates are materialized before the loop, and a // shared buffer would leave the first slice pointing at the second's // bytes. var fossil_buf: [128]u8 = undefined; var zit_buf: [128]u8 = undefined; const candidates = .{ setting.fossilVersionedPath(&fossil_buf), setting.versionedPath(&zit_buf) }; const dirs = [_]zset.VersionedDir{ .fossil, .zit }; inline for (candidates, 0..) |maybe_path, i| { if (c.versioned == null) { if (maybe_path) |path| { if (root_dir.readFileAlloc(io, path, arena, .limited(zset.max_versioned_bytes))) |text| { // fossil's `db_get_versioned` strips `#` lines // and trims before anything reads the value, so every // versionable setting -- in either directory -- gets // the same treatment from the one reader. c.versioned = zset.stripCommentLines(text); c.versioned_dir = dirs[i]; } else |_| {} } } } } return resolveSettingTiers(io, arena, amb, setting, &c, repository); } /// `resolveSettingWith` with the versioned tier read from /// the check-in `rid` instead of the working tree: what `open` needs, since /// its tree does not exist yet. Same two file names in the same order, the /// same comment stripping, the same tiers below; fossil's `db_get_versioned` /// reads the file out of the check-in through `historical_blob` too. fn checkinSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, repo: *Repository, rid: Repository.Rid, amb: cli.Ambient, config: ?*const logstore.Config, setting: zit.settings.Setting, ) !zit.settings.Resolved { const zset = zit.settings; var c: zset.Candidates = .{}; if (setting.versionable()) { var fossil_buf: [128]u8 = undefined; var zit_buf: [128]u8 = undefined; const candidates = .{ setting.fossilVersionedPath(&fossil_buf), setting.versionedPath(&zit_buf) }; const dirs = [_]zset.VersionedDir{ .fossil, .zit }; inline for (candidates, 0..) |maybe_path, i| { if (c.versioned == null) { if (maybe_path) |path| { if (checkinFileText(gpa, arena, repo, rid, path)) |text| { c.versioned = zset.stripCommentLines(text); c.versioned_dir = dirs[i]; } } } } } const repository = if (config) |cf| try configValue(arena, cf, setting) else null; return resolveSettingTiers(io, arena, amb, setting, &c, repository); } /// The content of `path` in check-in `rid`, copied into `arena` and bounded /// like a versioned file on disk; null when the check-in has no such file or /// it cannot be read (the tiers below then answer, as for an unreadable file). fn checkinFileText(gpa: std.mem.Allocator, arena: std.mem.Allocator, repo: *Repository, rid: Repository.Rid, path: []const u8) ?[]u8 { const card = (repo.fileAt(gpa, rid, path) catch return null) orelse return null; defer Repository.freeFileCard(gpa, card); const hash = card.hash orelse return null; const bytes = (repo.blob(gpa, hash) catch return null) orelse return null; defer gpa.free(bytes); if (bytes.len > zit.settings.max_versioned_bytes) return null; return arena.dupe(u8, bytes) catch null; } /// The environment, repository and global tiers on top of an already-read /// versioned candidate -- the tail both readers above share, so the order the /// tiers are consulted in cannot fork between them. fn resolveSettingTiers( io: std.Io, arena: std.mem.Allocator, amb: cli.Ambient, setting: zit.settings.Setting, c: *zit.settings.Candidates, repository: ?[]const u8, ) !zit.settings.Resolved { if (setting.envVar()) |name| c.environment = amb.environ.getPosix(name); c.repository = repository; c.global = try globalSetting(io, arena, amb, setting); return zit.settings.resolve(setting, c.*); } /// Serialize and atomically store the content log's scan index (the same /// best-effort refresh `Repository.logStore` performs): a failure costs the /// next open one scan, nothing else. fn refreshSettingLogIndex(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *const logstore.LogStore) void { const idx = log.serializeIndex(gpa) catch return; defer gpa.free(idx); zit.repocache.storeLogIndex(io, dir, idx); } /// Open a read-only log over `sf`, adopting the persisted scan index when the /// checkout's store cache holds a fresh one — the CLI pattern, shared so /// every read verb stops paying the full frame-header scan (diff and /// status carried an O(store) floor of ~0.2s per invocation on a /// 148k-artifact store from exactly this). A refused index (absent, stale, /// corrupt) falls back to the scan and refreshes the sidecar, so the next /// open is cheap again. The sidecar lives with the STORE, never a worktree, /// so the cache directory resolves through the checkout's store pointer. pub fn openLogAdoptingIndex( gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, sf: storage.StorageFile, /// Whether a scan (or a tail-extended adoption) may persist the refreshed /// sidecar. `.read_only` exists for verbs PINNED as tree-untouching — /// extras' B-extras-16 requires that a run create nothing, `.zit-cache` /// included — which still adopt a sidecar someone else already wrote. sidecar: enum { refresh_on_scan, read_only }, ) logstore.OpenError!logstore.LogStore { var resolved = checkout.storeDirOf(io, root_dir) catch null; defer if (resolved) |*r| r.close(io); const cache_dir = if (resolved) |r| r else root_dir; indexed: { const idx = zit.repocache.tryLoadLogIndex(gpa, io, cache_dir) orelse break :indexed; // An adopted store TAKES `idx` and frees it in `deinit` -- its // entry hashes are slices of it. Every refusal path leaves it ours. var taken = false; defer if (!taken) gpa.free(idx); var extended = false; const adopted = logstore.LogStore.openReadOnlyIndexed(gpa, sf, idx, &extended) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => break :indexed, }; var log = adopted orelse break :indexed; taken = true; if (extended and sidecar == .refresh_on_scan) refreshSettingLogIndex(gpa, io, cache_dir, &log); return log; } var log = try logstore.LogStore.openReadOnly(gpa, sf); // The scan just paid for the product the index persists. if (sidecar == .refresh_on_scan) refreshSettingLogIndex(gpa, io, cache_dir, &log); return log; } /// Open this checkout's store read-only for a one-shot config read, adopting /// the persisted scan index when the cache holds a fresh one (the CLI /// half). The plain `openReadOnly` walks every frame header, which `open` and /// `update` were paying once per setting on every materialization. A /// refused index (absent, stale, corrupt) falls back to the scan and refreshes /// the sidecar, as `Repository.logStore` does. The sidecar lives with the /// STORE, never a worktree, so the cache directory is resolved through the /// checkout's store pointer first. const OpenedStoreLog = struct { file: std.Io.File, fs: storage.FileStorage, log: logstore.LogStore, fn close(self: *OpenedStoreLog, gpa: std.mem.Allocator, io: std.Io) void { self.log.deinit(gpa); self.file.close(io); gpa.destroy(self); } }; /// Heap-allocated: the log holds a pointer into `fs`, so the record must /// never move after `storageFile()` is taken. fn openStoreLogForRead(gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir) !*OpenedStoreLog { var file = cli.openStoreRead(io, root_dir) catch return error.Storage; errdefer file.close(io); const st = file.stat(io) catch return error.Storage; const out = gpa.create(OpenedStoreLog) catch return error.OutOfMemory; errdefer gpa.destroy(out); out.* = .{ .file = file, .fs = storage.FileStorage.init(io, file, st.size), .log = undefined }; out.log = openLogAdoptingIndex(gpa, io, root_dir, out.fs.storageFile(), .refresh_on_scan) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; return out; } /// This repository's stored answer, or null when it has no opinion. The /// booleans consult their explicit-set bit: without it a repository would /// always have an opinion and the global tier could never be reached. fn repositorySetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, setting: zit.settings.Setting, ) !?[]const u8 { const opened = try openStoreLogForRead(gpa, io, root_dir); defer opened.close(gpa, io); var oc = (opened.log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return null; defer oc.deinit(gpa); return configValue(arena, &oc.config, setting); } /// One config frame's answer for one setting, or null when the frame has no /// opinion. PURE: the caller supplies the frame, so a verb that has already /// read `Config` once resolves every name it needs without a second store /// open (now the rule rather than the exception). The arena /// dupe is the only way this can fail. pub fn configValue( arena: std.mem.Allocator, config: *const logstore.Config, setting: zit.settings.Setting, ) error{OutOfMemory}!?[]const u8 { return switch (setting) { .autosync => if (config.autosync_set) (if (config.autosync_off) "off" else if (std.mem.eql(u8, config.autosync_mode, "pullonly")) "pullonly" else "on") else null, .editor => if (config.editor.len == 0) null else try arena.dupe(u8, config.editor), .clearsign => if (config.clearsign_set) (if (config.clearsign) "on" else "off") else null, // A stored empty string is "no opinion" for these: the config frame has // no separate absent marker for its string fields, and an operator who // wants the empty value spells it by clearing, which is the same thing. .ignore_glob => if (config.ignore_glob.len == 0) null else try arena.dupe(u8, config.ignore_glob), .pgp_command => if (config.pgp_command.len == 0) null else try arena.dupe(u8, config.pgp_command), .keep_glob => if (config.keep_glob.len == 0) null else try arena.dupe(u8, config.keep_glob), .manifest => if (config.manifest.len == 0) null else try arena.dupe(u8, config.manifest), .before_commit => if (config.before_commit.len == 0) null else try arena.dupe(u8, config.before_commit), .binary_glob => if (config.binary_glob.len == 0) null else try arena.dupe(u8, config.binary_glob), .crlf_glob => if (config.crlf_glob.len == 0) null else try arena.dupe(u8, config.crlf_glob), .crnl_glob => if (config.crnl_glob.len == 0) null else try arena.dupe(u8, config.crnl_glob), .clean_glob => if (config.clean_glob.len == 0) null else try arena.dupe(u8, config.clean_glob), .empty_dirs => if (config.empty_dirs.len == 0) null else try arena.dupe(u8, config.empty_dirs), .encoding_glob => if (config.encoding_glob.len == 0) null else try arena.dupe(u8, config.encoding_glob), .dotfiles => if (config.dotfiles.len == 0) null else try arena.dupe(u8, config.dotfiles), .mimetypes => if (config.mimetypes.len == 0) null else try arena.dupe(u8, config.mimetypes), .attachment_size_limit => if (config.attachment_size_limit.len == 0) null else try arena.dupe(u8, config.attachment_size_limit), }; } /// Open this checkout's store, read its config ONCE, and hand back the owned /// result; null on any failure, exactly as every reader of an optional /// repository value degrades today. `.read_only`, because the listing verbs /// are pinned tree-untouching (B-extras-16) and must never mint a sidecar. /// The `OwnedConfig` owns its backing bytes, so the log and its file are /// closed here and the config outlives them. pub fn readRepositoryConfig( gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, ) ?logstore.OwnedConfig { var file = cli.openStoreRead(io, root_dir) catch return null; defer file.close(io); const st = file.stat(io) catch return null; var fs = zit.storage.FileStorage.init(io, file, st.size); // Indexed open: this config read rides every extras/clean/ // addremove listing, each of which was paying its own full header scan. var log = openLogAdoptingIndex(gpa, io, root_dir, fs.storageFile(), .read_only) catch return null; defer log.deinit(gpa); return log.readConfig(gpa) catch null; } /// Resolve one setting from a config the caller already holds. The whole job /// is to make "resolve a name from the frame in my hand" one call, so a verb /// that reads `Config` once can answer for every name it needs. /// /// The error set is written out rather than inferred: `resolveSettingWith`'s /// own failures are all `catch return null`, so `configValue`'s arena dupe is /// the only thing left that can fail. The day a read there stops degrading, /// the compiler names it at this boundary. pub fn localSetting( io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, amb: cli.Ambient, config: ?*const logstore.Config, setting: zit.settings.Setting, ) error{OutOfMemory}!zit.settings.Resolved { const repository = if (config) |c| try configValue(arena, c, setting) else null; return resolveSettingWith(io, arena, root_dir, amb, setting, repository); } /// The operator-scope answer, or null when there is no global file or it says /// nothing about this setting. A missing file is never an error: no global /// tier is the state of every installation that never set one. fn globalSetting( io: std.Io, arena: std.mem.Allocator, amb: cli.Ambient, setting: zit.settings.Setting, ) !?[]const u8 { const zset = zit.settings; var path_buf: [1024]u8 = undefined; const g = zset.globalPath(amb.environ, &path_buf) orelse return null; const text = std.Io.Dir.cwd().readFileAlloc(io, g.path, arena, .limited(zset.max_versioned_bytes)) catch return null; const found = zset.globalLookup(text, setting) orelse return null; return found; } /// The local settings `zit settings` edits, as one table rather than an /// if-chain per name. Every one of them is repo-LOCAL and never crosses the /// wire: `autosync` and `ignore-glob` because they describe this working /// copy's habits, `clearsign` and `pgp-command` because fossil classifies the /// latter SENSITIVE and puts the former in no config group. The settings /// that DO propagate are a separate mechanism — versioned `.zit-settings/` /// files and the sync §9.3 config groups — not this verb. const LocalSetting = enum { autosync, ignore_glob, clearsign, pgp_command, const known_names = "autosync, ignore-glob, clearsign, pgp-command"; const Kind = enum { boolean, string }; fn wireName(self: LocalSetting) []const u8 { return switch (self) { .autosync => "autosync", .ignore_glob => "ignore-glob", .clearsign => "clearsign", .pgp_command => "pgp-command", }; } fn byName(name: []const u8) ?LocalSetting { inline for (comptime std.enums.values(LocalSetting)) |s| { if (std.mem.eql(u8, name, s.wireName())) return s; } return null; } fn kind(self: LocalSetting) Kind { return switch (self) { .autosync, .clearsign => .boolean, .ignore_glob, .pgp_command => .string, }; } /// This setting's stored value as display text, borrowed from `current`. fn read(self: LocalSetting, current: Settings) []const u8 { return switch (self) { // autosync is stored as the DEVIATION, so the flag reads inverted. .autosync => if (current.autosync_off) "off" else "on", .clearsign => if (current.clearsign) "on" else "off", .ignore_glob => current.ignore_glob, .pgp_command => current.pgp_command, }; } /// The patch that sets this setting to `value`, or null when `value` is /// not one this setting accepts. fn patch(self: LocalSetting, value: []const u8) ?ConfigPatch { switch (self) { .autosync, .clearsign => { const on = if (std.mem.eql(u8, value, "on")) true else if (std.mem.eql(u8, value, "off")) false else return null; // Inverted in storage for autosync (absent = fossil's default, // on); plain for clearsign (absent = fossil's default, off). return switch (self) { .autosync => .{ .autosync_off = !on }, .clearsign => .{ .clearsign = on }, else => unreachable, }; }, .ignore_glob, .pgp_command => { // The config frame caps every string field; refusing beats // truncating, which would silently change what runs or what // is ignored. if (value.len > logstore.max_config_str) return null; return switch (self) { .ignore_glob => .{ .ignore_glob = value }, .pgp_command => .{ .pgp_command = value }, else => unreachable, }; }, } } fn reportBadValue(self: LocalSetting, streams: cli.Streams, value: []const u8) void { switch (self.kind()) { .boolean => streams.err.print( "zit settings: {s} takes 'on' or 'off'\ntry 'zit settings --help'\n", .{self.wireName()}, ) catch {}, .string => { _ = value; if (self == .ignore_glob) { streams.err.print( "zit settings: ignore-glob is limited to {d} bytes; use the versioned {s} file for longer lists\n", .{ logstore.max_config_str, zit.ignore.versioned_path }, ) catch {}; } else { streams.err.print( "zit settings: {s} is limited to {d} bytes\n", .{ self.wireName(), logstore.max_config_str }, ) catch {}; } }, } } fn print(self: LocalSetting, streams: cli.Streams, value: []const u8) void { streams.out.print("{s} {s}\n", .{ self.wireName(), value }) catch {}; } }; /// Every local setting's value, read in ONE store open. Four separate readers /// would open, scan and close the log four times for a bare `zit settings`. const Settings = struct { autosync_off: bool = false, clearsign: bool = false, ignore_glob: []const u8 = "", pgp_command: []const u8 = "", }; /// Backing store for the two string settings a `Settings` borrows: the config /// frame's own buffer dies with the read, so the values are copied out. const SettingsBuf = [2 * logstore.max_config_str]u8; fn readSettings( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, buf: *SettingsBuf, ) !Settings { var file = cli.openStoreRead(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return .{}; defer oc.deinit(gpa); const glob = oc.config.ignore_glob; const pgp = oc.config.pgp_command; std.debug.assert(glob.len + pgp.len <= buf.len); @memcpy(buf[0..glob.len], glob); @memcpy(buf[glob.len..][0..pgp.len], pgp); return .{ .autosync_off = oc.config.autosync_off, .clearsign = oc.config.clearsign, .ignore_glob = buf[0..glob.len], .pgp_command = buf[glob.len..][0..pgp.len], }; } /// fossil's posture: when the working tree carries the versioned /// ignore file, the sweeps use IT, and a display of the local setting must /// say so — a value that reads as live while being shadowed is how an /// operator edits the wrong one twice. fn noteVersionedShadow( io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams, setting: zit.settings.Setting, r: zit.settings.Resolved, ) bool { const shadowed = r.shadowed orelse return false; const dir = r.versioned_dir orelse return false; if (noWarnPresent(io, root_dir, setting)) return false; streams.err.print( "note: {s}/{s} overrides the {s} setting in the {s} tier; create {s}/{s}.no-warn to silence\n", .{ dir.dirName(), setting.name(), setting.name(), shadowed.label(), dir.dirName(), setting.name() }, ) catch {}; return true; } /// Whether either settings directory holds `<name>.no-warn`. /// /// EITHER silences the note, whichever directory holds the winning file: /// fossil has one directory and cannot pin this, and zit picks the permissive /// rule because the two directories are one setting with two spellings -- a /// silence the operator asked for should not evaporate because a later commit /// chose the other spelling. fn noWarnPresent(io: std.Io, root_dir: std.Io.Dir, setting: zit.settings.Setting) bool { inline for (comptime std.enums.values(zit.settings.VersionedDir)) |d| { var buf: [160]u8 = undefined; if (std.fmt.bufPrint(&buf, "{s}/{s}.no-warn", .{ d.dirName(), setting.name() })) |path| { if (root_dir.statFile(io, path, .{ .follow_symlinks = false })) |_| return true else |_| {} } else |_| {} } return false; } /// One line for a versioned file that cannot supply its /// setting. `autosync` describes this machine's relationship to the network, /// so no tracked file may set it; fossil ignores such a file silently, and zit /// says so once rather than leaving the operator to wonder why their committed /// value does nothing. fn noteVersionedIgnored( io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams, setting: zit.settings.Setting, ) void { std.debug.assert(!setting.versionable()); inline for (comptime std.enums.values(zit.settings.VersionedDir)) |d| { var buf: [160]u8 = undefined; if (std.fmt.bufPrint(&buf, "{s}/{s}", .{ d.dirName(), setting.name() })) |path| { if (root_dir.statFile(io, path, .{ .follow_symlinks = false })) |_| { // Two reasons a file cannot supply a setting, and the note says // which: network posture belongs to the machine, and an // executable-tier value a clone could set would be remote code // execution. const why: []const u8 = switch (setting.tier()) { .local => "a local setting", .executable => "never supplied by a tracked file", .data, .consented => unreachable, // versionable; not this path }; streams.err.print( "note: {s} is versioned and ignored; {s} is {s}\n", .{ path, setting.name(), why }, ) catch {}; } else |_| {} } else |_| {} } } /// `.zitignore` is not a settings file -- it is a second DIALECT that applies /// alongside whatever `ignore-glob` resolves to, so its presence is /// worth one line but is not a shadowing. fn noteGitignorePresent(io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams) void { if (root_dir.statFile(io, zit.ignore.gitignore_path, .{ .follow_symlinks = false })) |_| { streams.err.print( "note: {s} is present; its patterns apply in addition to the ignore-glob setting\n", .{zit.ignore.gitignore_path}, ) catch {}; } else |_| {} } /// The autosync hook: a best-effort pull (before commit/update/merge) /// or push (after a successful commit) against the saved remote. Silent no-op /// when autosync is off or no remote is saved (a nag would punish the /// single-repository workflow); any failure is ONE warning line and the /// caller proceeds — autosync must never turn a working offline operation /// into an error (fossil's posture). The exchange itself is the real verb /// runner, so behavior, diagnostics, and persistence never fork from a /// hand-run `zit pull`/`zit push`. pub const AutosyncOp = enum { pull, push }; pub fn autosyncPull(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, verb: []const u8) void { autosync(gpa, io, arena, streams, verb, .pull, null); } pub fn autosyncPush(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, verb: []const u8, amb: cli.Ambient) void { autosync(gpa, io, arena, streams, verb, .push, amb); } fn autosync( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, verb: []const u8, op: AutosyncOp, amb: ?cli.Ambient, ) void { { const root = (cli.discoverRoot(arena, io) catch return) orelse return; var root_dir = root.dir; defer root_dir.close(io); // Once per invocation: the pull hook runs before the push one on // every verb that has both, so this is the first hook to fire. if (op == .pull) noteVersionedIgnored(io, root_dir, streams, .autosync); const off = readAutosyncOff(gpa, io, root_dir) catch return; if (off) return; // `autosync pullonly`: pull as usual, never push. if (op == .push) { if (amb) |a| { const r = resolveSetting(gpa, io, arena, root_dir, a, .autosync) catch return; if (std.mem.eql(u8, r.value, "pullonly")) return; } } var url_buf: [logstore.max_config_str]u8 = undefined; _ = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch return; } // The exchange reports on STDERR only: the hosting verb owns stdout (a // commit's contract is ONE check-in line there), and autosync is ambient // narration, not the verb's answer. const side_streams: cli.Streams = .{ .out = streams.err, .err = streams.err }; const rc = switch (op) { .pull => runPull(gpa, io, arena, &.{}, side_streams, null), .push => runPush(gpa, io, arena, &.{}, side_streams, amb.?), }; if (rc != .ok) { streams.err.print( "zit {s}: autosync {s} failed; continuing locally\n", .{ verb, @tagName(op) }, ) catch {}; } } /// The stored autosync deviation, read off the config frame. Storage faults /// propagate; a repository with no config frame reads as the default (on). fn readAutosyncOff(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !bool { const opened = try openStoreLogForRead(gpa, io, dir); defer opened.close(gpa, io); var oc = (opened.log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return false; defer oc.deinit(gpa); return oc.config.autosync_off; } pub fn runRemote( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, repo_override: ?[]const u8, ) cli.Exit { const parsed = switch (cli.parseRemoteArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "remote", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "remote", .extra_argument, tok), .not_a_url => |tok| { streams.err.print("zit remote: '{s}' is not an http:// or https:// URL\nzit remote: no subcommands: 'zit remote' shows the saved remote, 'zit remote URL' sets it, 'zit remote --unset' clears it\n", .{tok}) catch {}; return .usage; }, }; if (parsed.help) { streams.out.writeAll(remote_help) catch {}; return .ok; } const root = (cli.resolveRoot(arena, io, repo_override) catch return stateOom(streams, "remote")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); if (parsed.unset) { // Overlay remote_url with empty — clears the saved value, every other // config field preserved by the field-preserving RMW. updateConfig(gpa, io, root_dir, .{ .remote_url = "" }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), else => return repoReadFail(streams, "remote"), }; streams.out.writeAll("remote cleared\n") catch {}; return .ok; } if (parsed.url.len > 0) { // A URL over the config-field cap cannot be stored (the putConfig assert); // reject it as usage rather than trip the assertion. if (parsed.url.len > logstore.max_config_str) { streams.err.print("zit remote: URL too long (max {d} bytes)\n", .{logstore.max_config_str}) catch {}; return .failure; } updateConfig(gpa, io, root_dir, .{ .remote_url = parsed.url }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), else => return repoReadFail(streams, "remote"), }; streams.out.print("remote set to {s}\n", .{parsed.url}) catch {}; return .ok; } // Bare `zit remote` — print the saved remote (reusing resolveRemoteUrl's read // with an empty arg; NoRemote means none is saved). var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), error.Storage => return repoReadFail(streams, "remote"), error.NoRemote => { streams.out.writeAll("no remote set\n") catch {}; return .ok; }, }; streams.out.print("{s}\n", .{url}) catch {}; return .ok; } test "serve §3.3: a pushed file card is persisted durably into the served repository's log" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A fresh, empty on-disk repository (header only) — the "server" a peer pushes // new content into. Built through the real LogStore, not seeded from fossil. const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.commit(); } // A pushed full-content file card for a 64-hex (SHA3-256 width) artifact the // server lacks (sync §7.1: `file HASH SIZE\n` + payload; §4.1 zlib container). // The name is the artifact's real SHA3-256 (computed by our own hasher, not // fossil) so the server's §7.7 commit-hook recompute accepts it. const push_body = "pushed artifact bytes\n"; // 22 bytes var name_buf: [64]u8 = undefined; const push_hash = zit.artifact.artifactName(.sha3_256, push_body, &name_buf); const request_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ push_hash, push_body.len, push_body }); defer gpa.free(request_plain); const request = try zit.container.encode(gpa, request_plain, .default); defer gpa.free(request); var digest_buf: [digest_len]u8 = undefined; const push_digest = decodeWireName(push_hash, &digest_buf).?; const server = zit.sync.Server{ .version = serve_mod.server_version }; // Answer the push twice through the real `serve.Responder` seam over a SINGLE // open handle (§3.8): the second answer re-reads the current on-disk log from // the same handle and sees the first push's frame, so it is an idempotent // re-push (content is addressed by hash — no dup frame). A stale startup // snapshot would instead re-append the artifact; this handle is never reopened. { var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); for (0..2) |_| { const reply = try serveAnswer(&ctx, gpa, request, null, null); gpa.free(reply.body); } } // THE PROOF: re-open the log from disk in a fresh handle; the pushed artifact // is present with its exact content — durably stored, not discarded — and // appears exactly once (the idempotent re-push added no second frame). var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var fs = storage.FileStorage.init(io, file, st.size); var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile()); defer log.deinit(gpa); const rid = log.lookup(push_digest) orelse return error.PushedArtifactMissing; const stored = (try log.content(gpa, rid)).?; defer gpa.free(stored); try std.testing.expectEqualStrings(push_body, stored); const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var artifacts: usize = 0; for (rids) |r| { if (log.kindOf(r) != .config) artifacts += 1; } try std.testing.expectEqual(@as(usize, 1), artifacts); } // --- D4: client verbatim retention + the §7.7 persist hook ------------------ /// Create a fresh, empty (header-only, committed) `.zit` in `dir` so a /// `persistStore` test can persist into it, exactly like a just-cloned repo. fn d4SeedEmptyRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.commit(); } /// Reopen the persisted `.zit` in `dir` for reading. `fs_out` is a caller-owned /// `FileStorage` that MUST outlive the returned `LogStore` (its `storageFile()` /// vtable borrows `fs_out`); the caller closes `fs_out.file` and deinits `log`. fn d4OpenLog(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, fs_out: *storage.FileStorage) !logstore.LogStore { const file = try dir.openFile(io, repo_filename, .{ .mode = .read_write }); const st = try file.stat(io); fs_out.* = storage.FileStorage.init(io, file, st.size); return try logstore.LogStore.open(gpa, fs_out.storageFile(), 0); } fn d4SeedConfiguredRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.putConfig(.{ .hash_policy = 9, .default_branch = "stable", .project_id = "project", .project_code = "init-code", .login_user = "operator", .login_secret_hex = "0" ** 40, .remote_url = "https://example.test/remote", .seen_submanifest = true, .enable_submanifests = false, .submanifest_k = 3, .submanifest_auto_retune = false, .submanifest_settled_cost_shape = [_]u8{0x42} ** 32, }); try seed.commit(); } test "with no --user the resolver reuses the stored login, never prompts, and an empty --user still means anonymous" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedConfiguredRepo(gpa, io, tmp.dir); // stores login "operator" / "0"*40 var sink: [256]u8 = undefined; var w = Io.Writer.fixed(&sink); const s: cli.Streams = .{ .out = &w, .err = &w }; const amb: cli.Ambient = .{ .now_epoch = 0, .user_env = null }; var pw_buf: [max_password_len]u8 = undefined; var user_buf: [logstore.max_config_str]u8 = undefined; // Absent --user: the stored login stands in, verbatim, and nothing is // written to either stream — this is the branch autosync takes, and a // prompt here would block a commit. const r = (try resolvePushCredential(gpa, io, tmp.dir, s, "init-code", .{ .url = "" }, amb, &pw_buf, &user_buf)) orelse return error.AnonymousDespiteStoredLogin; try std.testing.expectEqualStrings("operator", r.credential.user); try std.testing.expectEqualStrings("0" ** 40, &r.credential.secret_hex); try std.testing.expect(!r.persist); try std.testing.expectEqual(@as(usize, 0), w.buffered().len); // An explicit empty --user is the way to push anonymously regardless. try std.testing.expect((try resolvePushCredential(gpa, io, tmp.dir, s, "init-code", .{ .url = "", .user = "" }, amb, &pw_buf, &user_buf)) == null); // Nothing stored — no secret, and separately a secret with no name, which // is not a login either — absent --user is anonymous, and still no prompt. for ([_][2][]const u8{ .{ "", "" }, .{ "", "0" ** 40 } }) |login| { var bare = std.testing.tmpDir(.{}); defer bare.cleanup(); { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try bare.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.putConfig(.{ .hash_policy = 9, .default_branch = "trunk", .project_id = "project", .project_code = "init-code", .login_user = login[0], .login_secret_hex = login[1], .remote_url = "", .seen_submanifest = false, .enable_submanifests = false, .submanifest_k = 0, .submanifest_auto_retune = false, .submanifest_settled_cost_shape = [_]u8{0} ** 32, }); try seed.commit(); } try std.testing.expect((try resolvePushCredential(gpa, io, bare.dir, s, "init-code", .{ .url = "" }, amb, &pw_buf, &user_buf)) == null); try std.testing.expectEqual(@as(usize, 0), w.buffered().len); } } test "sync clone persistence: open-log artifacts and one combined config overlay preserve the wrapper result" { const gpa = std.testing.allocator; const io = std.testing.io; var wrapped_dir = std.testing.tmpDir(.{}); defer wrapped_dir.cleanup(); var open_dir = std.testing.tmpDir(.{}); defer open_dir.cleanup(); try d4SeedConfiguredRepo(gpa, io, wrapped_dir.dir); try d4SeedConfiguredRepo(gpa, io, open_dir.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const content = "one content-bearing clone artifact"; var name_buf: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, content, &name_buf); const container = try zit.container.encode(gpa, content, .default); defer gpa.free(container); const rid = try ms.store().put(name, content); try ms.store().retain(rid, .{ .full = container }); const patch: ConfigPatch = .{ .project_code = "origin-code", .enable_submanifests = true, .submanifest_k = 5, .submanifest_auto_retune = true, }; const wrapped = try persistStore(gpa, io, wrapped_dir.dir, ms.store(), null, 0); try updateConfig(gpa, io, wrapped_dir.dir, patch); var open_fs: storage.FileStorage = undefined; var open_log = try d4OpenLog(gpa, io, open_dir.dir, &open_fs); defer open_fs.file.close(io); defer open_log.deinit(gpa); const opened = try persistStoreOpen(gpa, &open_log, ms.store(), null, 0, null); try updateConfigOpen(gpa, &open_log, patch); try open_log.commit(); try std.testing.expectEqual(wrapped.written, opened.written); try std.testing.expectEqual(wrapped.submanifest_gaps, opened.submanifest_gaps); try std.testing.expectEqual(@as(u64, 3), (try open_log.logKey()).frame_count); var wrapped_fs: storage.FileStorage = undefined; var wrapped_log = try d4OpenLog(gpa, io, wrapped_dir.dir, &wrapped_fs); defer wrapped_fs.file.close(io); defer wrapped_log.deinit(gpa); var wrapped_cfg = (try wrapped_log.readConfig(gpa)).?; defer wrapped_cfg.deinit(gpa); var opened_cfg = (try open_log.readConfig(gpa)).?; defer opened_cfg.deinit(gpa); try std.testing.expectEqualDeep(wrapped_cfg.config, opened_cfg.config); try std.testing.expectEqualStrings("origin-code", opened_cfg.config.project_code); try std.testing.expectEqualStrings("operator", opened_cfg.config.login_user); try std.testing.expectEqualStrings("https://example.test/remote", opened_cfg.config.remote_url); try std.testing.expect(opened_cfg.config.seen_submanifest); try std.testing.expect(opened_cfg.config.enable_submanifests); try std.testing.expectEqual(@as(u8, 5), opened_cfg.config.submanifest_k); try std.testing.expect(opened_cfg.config.submanifest_auto_retune); try std.testing.expectEqualDeep([_]u8{0x42} ** 32, opened_cfg.config.submanifest_settled_cost_shape.?); var digest_buf: [digest_len]u8 = undefined; const stored = (try open_log.content(gpa, open_log.lookup(decodeWireName(name, &digest_buf).?).?)).?; defer gpa.free(stored); try std.testing.expectEqualStrings(content, stored); } test "sync clone config: absent origin settings append no frame and preserve init values" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedConfiguredRepo(gpa, io, tmp.dir); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); const patch: ConfigPatch = .{}; try std.testing.expect(!patch.any()); if (patch.any()) try updateConfigOpen(gpa, &log, patch); try log.commit(); try std.testing.expectEqual(@as(u64, 1), (try log.logKey()).frame_count); var cfg = (try log.readConfig(gpa)).?; defer cfg.deinit(gpa); try std.testing.expectEqual(@as(u8, 9), cfg.config.hash_policy); try std.testing.expectEqualStrings("stable", cfg.config.default_branch); try std.testing.expectEqualStrings("project", cfg.config.project_id); try std.testing.expectEqualStrings("init-code", cfg.config.project_code); try std.testing.expectEqualStrings("operator", cfg.config.login_user); try std.testing.expectEqualStrings("https://example.test/remote", cfg.config.remote_url); try std.testing.expect(cfg.config.seen_submanifest); try std.testing.expect(!cfg.config.enable_submanifests); try std.testing.expectEqual(@as(u8, 3), cfg.config.submanifest_k); try std.testing.expect(!cfg.config.submanifest_auto_retune); try std.testing.expectEqualDeep([_]u8{0x42} ** 32, cfg.config.submanifest_settled_cost_shape.?); } test "streaming persist writes the byte-identical log and releases every persisted entry" { const gpa = std.testing.allocator; const io = std.testing.io; // Two stores fed the same reverse-delta chain (older-as-delta-of-newer, // fossil's wire shape), one persisted batched and one streamed in // increments with release. The logs must match byte for byte — streaming // reorders nothing — and the streamed store must end empty of payload: // every entry held, none resident. const links = 6; var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| gpa.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(gpa, "streamed chain body: one long shared prefix kept across every revision -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } const seed = struct { fn one(a: std.mem.Allocator, s: repostate.ArtifactStore, i: usize, bs: *const [links][]u8, ns: *const [links][64]u8, nl: *const [links]usize) !void { const rid = try s.put(ns[i][0..nl[i]], bs[i]); if (i == links - 1) { const c = try zit.container.encode(a, bs[i], .default); defer a.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(a, bs[i + 1], bs[i]); defer a.free(dr); const c = try zit.container.encode(a, dr, .default); defer a.free(c); try s.retain(rid, .{ .delta = .{ .src = ns[i + 1][0..nl[i + 1]], .container = c, .target_size = bs[i].len } }); } } }.one; // Batched reference. var tmp_a = std.testing.tmpDir(.{}); defer tmp_a.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_a.dir); var ms_a = repostate.MemStore.init(gpa); defer ms_a.deinit(); for (0..links) |i| try seed(gpa, ms_a.store(), i, &bodies, &names, &nlen); { var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); const sum = try persistStoreOpen(gpa, &log, ms_a.store(), null, 0, null); try std.testing.expectEqual(@as(usize, links), sum.written); try log.commit(); } // Streamed: seed in two "rounds", draining after each, releasing as it goes. var tmp_b = std.testing.tmpDir(.{}); defer tmp_b.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_b.dir); var ms_b = repostate.MemStore.init(gpa); defer ms_b.deinit(); { var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp_b.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var pass = try PersistPass.init(gpa, &log, ms_b.store(), null, &ms_b, 0, null); defer pass.deinit(); for (0..3) |i| try seed(gpa, ms_b.store(), i, &bodies, &names, &nlen); try pass.drainNew(); // rids 1..3 are deltas of unseen bases: all defer for (3..links) |i| try seed(gpa, ms_b.store(), i, &bodies, &names, &nlen); try pass.drainNew(); // the full form lands and the whole chain drains const sum = try pass.finish(); try std.testing.expectEqual(@as(usize, links), sum.written); try std.testing.expectEqual(@as(usize, 0), sum.submanifest_gaps); try log.commit(); } // Every streamed entry is now frame-backed and released: held, no bytes. for (1..links + 1) |rid_us| { const rid: repostate.Rid = @intCast(rid_us); try std.testing.expect(ms_b.store().held(rid)); try std.testing.expect(ms_b.store().wire(rid) == null); try std.testing.expect(ms_b.store().content(rid) == null); } // The two logs are byte-identical: streaming changed when frames were // written, never what or in what order. const a_bytes = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 20)); defer gpa.free(a_bytes); const b_bytes = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 20)); defer gpa.free(b_bytes); try std.testing.expectEqualSlices(u8, a_bytes, b_bytes); } test "a basis interned as a phantom before the cursor passed it is revisited and persisted" { const gpa = std.testing.allocator; const io = std.testing.io; // The fossil-peer streaming shape: a delta cfile arrives naming an unseen // DELTASRC, which interns the basis as a PHANTOM at a low rid; the basis // content lands at that same rid in a later round, behind the drain's // cursor. The pass must come back for it — a drain that only moves forward // leaves the basis unwritten and its waiters unresolvable. var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s_store = ms.store(); const base_body = "phantom-first basis body: the long shared prefix every revision keeps\n"; const delta_body = "phantom-first basis body: the long shared prefix every revision keeps, edited\n"; var base_name: [64]u8 = undefined; const bn = zit.artifact.artifactName(.sha3_256, base_body, &base_name).len; var delta_name: [64]u8 = undefined; const dn = zit.artifact.artifactName(.sha3_256, delta_body, &delta_name).len; // Round 1: the basis is interned as a phantom (rid 1), then the delta // arrives as content (rid 2). const base_rid = try s_store.intern(base_name[0..bn]); const delta_rid = try s_store.put(delta_name[0..dn], delta_body); { const dr = try zit.delta.create(gpa, base_body, delta_body); defer gpa.free(dr); const c = try zit.container.encode(gpa, dr, .default); defer gpa.free(c); try s_store.retain(delta_rid, .{ .delta = .{ .src = base_name[0..bn], .container = c, .target_size = delta_body.len } }); } var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var pass = try PersistPass.init(gpa, &log, s_store, null, &ms, 0, null); defer pass.deinit(); try pass.drainNew(); // cursor passes the phantom; the delta defers on it // Round 2: the basis content lands at the SAME rid, behind the cursor. { const got = try s_store.put(base_name[0..bn], base_body); try std.testing.expectEqual(base_rid, got); const c = try zit.container.encode(gpa, base_body, .default); defer gpa.free(c); try s_store.retain(base_rid, .{ .full = c }); } try pass.drainNew(); // the revisit sweep drives it, unblocking the delta const sum = try pass.finish(); try std.testing.expectEqual(@as(usize, 2), sum.written); try log.commit(); // Both artifacts are durable frames, and both entries were released. var d: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(base_name[0..bn], &d).?) != null); try std.testing.expect(log.lookup(decodeWireName(delta_name[0..dn], &d).?) != null); try std.testing.expect(s_store.wire(base_rid) == null and s_store.held(base_rid)); try std.testing.expect(s_store.wire(delta_rid) == null and s_store.held(delta_rid)); } test "sync persistStoreOpen: malformed artifact retains the wrapper's corrupt error and writes no frame" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const content = "content that the wire name does not actually name\n"; const lying_name = "0" ** 64; const rid = try ms.store().put(lying_name, content); const container = try zit.container.encode(gpa, content, .default); defer gpa.free(container); try ms.store().retain(rid, .{ .full = container }); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); try std.testing.expectError(error.Corrupt, persistStoreOpen(gpa, &log, ms.store(), null, 0, null)); try log.commit(); var digest_buf: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null); } test "sync §7.7: persistStore recomputes the artifact hash and aborts on a lying container, writing no frame" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A received full cfile whose wire NAME does not name its content (§7.7 — // a clone skips the transfer-layer pre-storage check, so a lying container // reaches the persist commit hook). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const content = "content that the wire name does not actually name\n"; const lying_name = "0" ** 64; // not SHA3-256(content) const rid = try s.put(lying_name, content); const container = try zit.container.encode(gpa, content, .default); defer gpa.free(container); try s.retain(rid, .{ .full = container }); // §7.7 L1469-1471 — recompute and abort before ANY frame is written. try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, ms.store(), null, 0)); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var digest_buf: [digest_len]u8 = undefined; const digest = decodeWireName(lying_name, &digest_buf).?; try std.testing.expect(log.lookup(digest) == null); // no frame landed } test "sync §7.2/§4.1: persistStore stores received full and delta wire forms verbatim; a re-serve preserves the delta container byte-for-byte" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Model a clone reply in §8.2 RID order: a full base cfile then a delta // cfile against it. Names are the real SHA3-256 so the §7.7 hook accepts. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const base = "the base artifact with a long shared run so the delta stays a compact copy payload distinct in size from the reconstructed target"; const target = base ++ " plus a small trailing mutation"; var bn: [64]u8 = undefined; var tn: [64]u8 = undefined; const base_name = zit.artifact.artifactName(.sha3_256, base, &bn); const target_name = zit.artifact.artifactName(.sha3_256, target, &tn); const base_container = try zit.container.encode(gpa, base, .default); defer gpa.free(base_container); const base_rid = try s.put(base_name, base); try s.retain(base_rid, .{ .full = base_container }); const raw = try zit.delta.create(gpa, base, target); defer gpa.free(raw); const delta_container = try zit.container.encode(gpa, raw, .default); defer gpa.free(delta_container); const ts = try zit.delta.targetSize(raw); const delta_rid = try s.put(target_name, target); try s.retain(delta_rid, .{ .delta = .{ .src = base_name, .container = delta_container, .target_size = ts } }); const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 2), written.written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The target persisted as a `.delta` frame that reconstructs to the original. var tdig: [digest_len]u8 = undefined; const target_digest = decodeWireName(target_name, &tdig).?; const trid = log.lookup(target_digest).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(trid)); const got = (try log.content(gpa, trid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(target, got); // Clone-of-clone: a re-serve snapshot emits the SAME delta container bytes // (§7.2 retransmitted verbatim) — the compactness survives the second hop. var serve_ms = repostate.MemStore.init(gpa); defer serve_ms.deinit(); try loadContentStore(gpa, &log, &serve_ms); const serve_rid = serve_ms.store().lookup(target_name).?; const wf = serve_ms.store().wire(serve_rid).?; try std.testing.expect(wf.delta != null); try std.testing.expectEqualSlices(u8, delta_container, wf.container); try std.testing.expectEqual(@as(u64, target.len), wf.delta.?.target_size); } /// A tree manifest naming `subs` (submanifests §3.6), plus the §6.2 Z card, so /// the fixture is a well-formed card sequence rather than a bare S run. fn treeManifestBytes(gpa: std.mem.Allocator, subs: []const []const u8) ![]u8 { var body: std.ArrayListUnmanaged(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, "C tree\nD 2026-07-27T12:00:00.000\n"); // §3.4 — the S run ascends by filename; `a`, `b`, ... keeps it legal. for (subs, 0..) |h, i| { try body.print(gpa, "S {c}.txt {s}\n", .{ @as(u8, 'a') + @as(u8, @intCast(i)), h }); } try body.appendSlice(gpa, "U u\n"); var digest: [16]u8 = undefined; std.crypto.hash.Md5.hash(body.items, &digest, .{}); return std.fmt.allocPrint(gpa, "{s}Z {x}\n", .{ body.items, &digest }); } test "submanifests §7.1: the persist commit hook phantomizes every absent S reference" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Two leaves that were never received — the state a clone from a peer whose // OWN closure is incomplete lands in. Nothing on the receive path can see // this: a clone stores wire-backed and inflates nothing. const leaf_a = "F a.txt " ++ ("1" ** 64) ++ "\n"; const leaf_b = "F b.txt " ++ ("2" ** 64) ++ "\n"; var an: [64]u8 = undefined; var bn: [64]u8 = undefined; const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an); const name_b = zit.artifact.artifactName(.sha3_256, leaf_b, &bn); const tree = try treeManifestBytes(gpa, &.{ name_a, name_b }); defer gpa.free(tree); var tn: [64]u8 = undefined; const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const rid = try s.put(tree_name, tree); const container = try zit.container.encode(gpa, tree, .default); defer gpa.free(container); try s.retain(rid, .{ .full = container }); // §7.1: "Storing a structural artifact that carries S cards whose referenced // sub-manifests are absent MUST create a phantom for each missing reference." const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), summary.written); try std.testing.expectEqual(@as(usize, 2), summary.submanifest_gaps); } test "persist scoped past the holdings writes nothing and re-walks only received" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); // Five full-form blobs (the already-durable holdings), named by true hash. inline for (.{ "alpha", "bravo", "charlie", "delta", "echo" }) |body| { var nm: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, body, &nm); const rid = try s.put(name, body); const c = try zit.container.encode(gpa, body, .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } const first = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 5), first.written); // A pull that received nothing: scope past all five holdings. The walk // never begins, so nothing is persisted and nothing is re-visited (this // was O(store) — every holding read once from disk to be discarded). const noop = try persistStore(gpa, io, tmp.dir, ms.store(), null, 5); try std.testing.expectEqual(@as(usize, 0), noop.written); // Three artifacts arrive (rids 6..8); scope past the original five. Only the // new three are persisted — a mirror's incremental pull no longer re-walks // the millions it already holds. inline for (.{ "foxtrot", "golf", "hotel" }) |body| { var nm: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, body, &nm); const rid = try s.put(name, body); const c = try zit.container.encode(gpa, body, .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } const incr = try persistStore(gpa, io, tmp.dir, ms.store(), null, 5); try std.testing.expectEqual(@as(usize, 3), incr.written); // A full re-persist (start 0) early-returns every artifact and reaches the // same eight-artifact total, so scoping removed work, never content. const full = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 8), full.written); } test "submanifests §7.1/§8.1: the S-reference gap survives a RESTART, rebuilt from the durable indicator" { // §7.1's phantom is an interned entry with no frame, "exactly as an absent // delta source creates a source phantom" — and `LogStore.intern` states the // consequence outright: "Phantoms never reach disk." So a restart loses // them, and the only thing that can make them correct across one is a // REBUILD from the durable content-bearing frames. // // This test is that invariant: run 1 mints the gap, the process ends, run 2 // re-derives the identical gap over a store loaded from `.zit` alone. Remove // the rebuild (`rescan_persisted`) and run 2 reports ZERO gaps — a // repository that quietly declares an unenumerable check-in converged. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); const leaf_a = "F a.txt " ++ ("1" ** 64) ++ "\n"; const leaf_b = "F b.txt " ++ ("2" ** 64) ++ "\n"; var an: [64]u8 = undefined; var bn: [64]u8 = undefined; const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an); const name_b = zit.artifact.artifactName(.sha3_256, leaf_b, &bn); const tree = try treeManifestBytes(gpa, &.{ name_a, name_b }); defer gpa.free(tree); var tn: [64]u8 = undefined; const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn); // RUN 1 — the tree lands; neither sub-manifest is held. { var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(tree_name, tree); const first = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), first.written); try std.testing.expectEqual(@as(usize, 2), first.submanifest_gaps); } // RESTART — nothing survives but the bytes in `.zit`. var ms2 = repostate.MemStore.init(gpa); defer ms2.deinit(); { var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // §8.1 — the indicator IS durable: "record a local seen-submanifest // indicator". It is the one byte the rebuild rests on. var oc = (try log.readConfig(gpa)) orelse return error.NoConfig; defer oc.deinit(gpa); try std.testing.expect(oc.config.seen_submanifest); // …and the phantoms are NOT durable, asserted rather than assumed — // this is the premise the whole rebuild exists to answer. var adig: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(name_a, &adig).?) == null); // The snapshot a fresh process starts every verb from. try loadContentStore(gpa, &log, &ms2); } // Only the tree came back: the store a second run works from holds no trace // of the gap. try std.testing.expectEqual(@as(usize, 1), ms2.store().count()); // RUN 2 — the gap is re-derived, identically, from the durable frames. const second = try persistStore(gpa, io, tmp.dir, ms2.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), second.written); try std.testing.expectEqual(@as(usize, 2), second.submanifest_gaps); } test "submanifests §8.1: a repository predating this draft records no indicator and rescans nothing" { // The compatibility half of the rebuild: the indicator is false, so the // rescan is unreachable and a second persist over the same store is exactly // the sweep it always was — no config frame appended, no gap invented. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); const flat = "C msg\nD 2026-07-27T12:00:00.000\nF a.txt " ++ ("1" ** 64) ++ "\nU u\n"; var fn_buf: [64]u8 = undefined; const flat_name = zit.artifact.artifactName(.sha3_256, flat, &fn_buf); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(flat_name, flat); _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); const again = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 0), again.submanifest_gaps); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // No indicator was written at all: `d4SeedEmptyRepo` stores no config, and // nothing above gave it a reason to start one. try std.testing.expect((try log.readConfig(gpa)) == null); } test "submanifests §7.1: a persisted tree whose sub-manifests are held reports no gap" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); const leaf_a = "F a.txt " ++ ("1" ** 64) ++ "\n"; var an: [64]u8 = undefined; const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an); const tree = try treeManifestBytes(gpa, &.{name_a}); defer gpa.free(tree); var tn: [64]u8 = undefined; const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); // The sub-manifest is stored at a LOWER rid than the tree in one direction // and a higher one in the other; the gap verdict must not depend on which, // which is why it is taken once, after the whole sweep. _ = try s.put(tree_name, tree); _ = try s.put(name_a, leaf_a); const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 2), summary.written); try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps); // The §7.1 phantom was minted for the leaf BEFORE the leaf's own frame was // written (the tree persists first), so this asserts the phantom-upgrade // path storage actually takes: the frame must read back byte-for-byte, not // merely exist. A §7.1 duty that corrupted a frame would be worse than an // unmet one. var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var ldig: [digest_len]u8 = undefined; const leaf_digest = decodeWireName(name_a, &ldig).?; const lrid = log.lookup(leaf_digest).?; try std.testing.expect(!log.isPhantom(lrid)); const got = (try log.content(gpa, lrid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(leaf_a, got); } test "submanifests §7.1: an artifact predating this draft costs the persist hook no gap and no phantom" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // The backwards-compatibility guarantee as a test: a flat manifest carries // no S card, so §7.1's duty over it is empty and the persisted bytes and // frame count are exactly what they were before this rule existed. const flat = "C msg\nD 2026-07-27T12:00:00.000\nF a.txt " ++ ("1" ** 64) ++ "\nU u\n"; var fn_buf: [64]u8 = undefined; const flat_name = zit.artifact.artifactName(.sha3_256, flat, &fn_buf); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(flat_name, flat); const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), summary.written); try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var artifacts: usize = 0; for (rids) |r| { if (log.kindOf(r) != .config) artifacts += 1; } try std.testing.expectEqual(@as(usize, 1), artifacts); } test "submanifests §7.1: a malformed S run mints no phantom at the persist hook" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // §3.2: a third token on S is rejected. Validate-then-apply means the FIRST // (well-formed) S card mints nothing either — a half-read artifact never // leaves a half-minted phantom set behind. const bad = "C tree\nS a.txt " ++ ("1" ** 64) ++ "\nS b.txt " ++ ("2" ** 64) ++ " extra\nU u\n"; var bn: [64]u8 = undefined; const bad_name = zit.artifact.artifactName(.sha3_256, bad, &bn); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(bad_name, bad); const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), summary.written); try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps); } test "sync §3.3: persistStore falls back to full form when a retained delta's basis is absent, never a phantom-based delta" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Only the delta is present; its basis was never received/stored (a legal // §3.3 outcome that must not persist a delta over a phantom base). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const base = "an absent basis whose long shared run would make a compact delta if it were here at all"; const target = base ++ " mutated once"; var bn: [64]u8 = undefined; var tn: [64]u8 = undefined; const base_name = zit.artifact.artifactName(.sha3_256, base, &bn); const target_name = zit.artifact.artifactName(.sha3_256, target, &tn); const raw = try zit.delta.create(gpa, base, target); defer gpa.free(raw); const dc = try zit.container.encode(gpa, raw, .default); defer gpa.free(dc); const ts = try zit.delta.targetSize(raw); const rid = try s.put(target_name, target); try s.retain(rid, .{ .delta = .{ .src = base_name, .container = dc, .target_size = ts } }); const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), written.written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The target persisted as a FULL (.zlib) frame — the fallback, not a delta. var tdig: [digest_len]u8 = undefined; const target_digest = decodeWireName(target_name, &tdig).?; const trid = log.lookup(target_digest).?; try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(trid)); const got = (try log.content(gpa, trid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(target, got); // No phantom basis frame was minted for the absent DELTASRC. var bdig: [digest_len]u8 = undefined; const base_digest = decodeWireName(base_name, &bdig).?; try std.testing.expect(log.lookup(base_digest) == null); } test "sync §7.7/§3.3: persistStore stores a retained delta chain up to the read bound and falls back to full form beyond it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const cap: usize = logstore.max_delta_chain; // the READ bound (32) const n: usize = cap + 2; // links 0..cap+1: link cap+1's basis sits at depth cap // A chain grown one byte per link so each delta is tiny. Link i's basis is // link i-1 (§8.2 RID order); the persist path stores a `.delta` iff the base // depth is under `max_delta_chain`, so link cap+1 falls back to full form. var content: std.ArrayList(u8) = .empty; defer content.deinit(gpa); try content.appendSlice(gpa, "chain-root-artifact-with-a-long-shared-run-for-compact-forward-deltas"); var prev_name: [64]u8 = undefined; var prev_name_len: usize = 0; var deep_name: [64]u8 = undefined; // link cap (deepest stored delta) var deep_len: usize = 0; var last_name: [64]u8 = undefined; // link cap+1 (the fallback) var last_len: usize = 0; var i: usize = 0; while (i < n) : (i += 1) { try content.append(gpa, 'a' + @as(u8, @intCast(i % 26))); const cur = content.items; const prev = cur[0 .. cur.len - 1]; var nbuf: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, cur, &nbuf); const rid = try s.put(name, cur); if (i == 0) { const c = try zit.container.encode(gpa, cur, .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const raw = try zit.delta.create(gpa, prev, cur); defer gpa.free(raw); const c = try zit.container.encode(gpa, raw, .default); defer gpa.free(c); const tsz = try zit.delta.targetSize(raw); try s.retain(rid, .{ .delta = .{ .src = prev_name[0..prev_name_len], .container = c, .target_size = tsz } }); } @memcpy(prev_name[0..name.len], name); prev_name_len = name.len; if (i == cap) { @memcpy(deep_name[0..name.len], name); deep_len = name.len; } if (i == n - 1) { @memcpy(last_name[0..name.len], name); last_len = name.len; } } _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // Link cap is the deepest `.delta` the write bound allows; link cap+1 fell // back to a full `.zlib` frame yet still reconstructs (never a Corrupt read). var ddig: [digest_len]u8 = undefined; const deep_rid = log.lookup(decodeWireName(deep_name[0..deep_len], &ddig).?).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(deep_rid)); var ldig: [digest_len]u8 = undefined; const last_rid = log.lookup(decodeWireName(last_name[0..last_len], &ldig).?).?; try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(last_rid)); const got = (try log.content(gpa, last_rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(content.items, got); } test "sync §7.2/§7.4/§3.3: a dangling reverse-delta cfile persists as a verbatim .delta frame that re-serves byte-identically" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Fossil's clone stream: a reverse delta (the OLDER revision stored as a // delta of the NEWER one) arrives BEFORE its basis, so it is dangling. The // client retains the §4.1 container through the pending queue and, when the // basis lands, persists it as a compact `.delta` — not a full re-compression. const newer = "artifact NEWER: a long shared body kept verbatim so the reverse delta reconstructing the older revision stays a compact copy op"; const older = "artifact NEWER: a long shared body kept verbatim so the reverse delta reconstructing the older revision stays a compact copy op minus tail"; var nn: [64]u8 = undefined; var on: [64]u8 = undefined; const newer_name = zit.artifact.artifactName(.sha3_256, newer, &nn); const older_name = zit.artifact.artifactName(.sha3_256, older, &on); const newer_container = try zit.container.encode(gpa, newer, .default); defer gpa.free(newer_container); const raw = try zit.delta.create(gpa, newer, older); // OLDER = delta ∘ NEWER defer gpa.free(raw); const delta_container = try zit.container.encode(gpa, raw, .default); defer gpa.free(delta_container); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var client = zit.sync.Client{ .operation = .clone, .state = .clone_round0 }; defer client.deinit(gpa); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, "clone_seqno 0\npush 1111111111111111 aa87ea41b0bb05ba528f7a3bdba700d7e23d6318\n"); // Reverse (dangling) order: the delta cfile precedes its basis cfile. const dh = try std.fmt.allocPrint(gpa, "cfile {s} {s} {d} {d}\n", .{ older_name, newer_name, older.len, delta_container.len }); defer gpa.free(dh); try body.appendSlice(gpa, dh); try body.appendSlice(gpa, delta_container); try body.append(gpa, '\n'); const bh = try std.fmt.allocPrint(gpa, "cfile {s} {d} {d}\n", .{ newer_name, newer.len, newer_container.len }); defer gpa.free(bh); try body.appendSlice(gpa, bh); try body.appendSlice(gpa, newer_container); try body.append(gpa, '\n'); _ = try client.consumeReply(gpa, ms.store(), body.items); const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 2), written.written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // §3.3 — the older revision persisted as a `.delta`, not a full re-compression, // and reconstructs the original bytes (delta ∘ basis). var od: [digest_len]u8 = undefined; const older_rid = log.lookup(decodeWireName(older_name, &od).?).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(older_rid)); const got = (try log.content(gpa, older_rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(older, got); // §7.2 — a clone-of-clone re-serve re-emits the SAME delta container byte-for-byte. var serve_ms = repostate.MemStore.init(gpa); defer serve_ms.deinit(); try loadContentStore(gpa, &log, &serve_ms); const srid = serve_ms.store().lookup(older_name).?; const wf = serve_ms.store().wire(srid).?; try std.testing.expect(wf.delta != null); try std.testing.expectEqualSlices(u8, delta_container, wf.container); try std.testing.expectEqualStrings(newer_name, wf.delta.?.src_hash); } test "sync §7.2/§3.3: persistStore stores a whole reverse-delta chain, ordering every basis frame before the delta that points at it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A reverse-delta chain in fossil's storage shape: revision i is stored as a // delta of the NEWER revision i+1, the newest stored full. The store rids run // oldest..newest (put order), so EVERY delta's basis sits at a HIGHER rid than // the delta — the worst case for physical ordering. persist's worklist must // still write each basis frame before its delta so `delta_of` points backward; // naive rid-order would degrade all but the oldest link to full form. const links = 6; var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| gpa.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(gpa, "chain body: one long shared prefix kept across every revision for compact reverse deltas -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } // put oldest..newest (store rids 1..links); older i is a reverse delta of newer i+1. for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == links - 1) { const c = try zit.container.encode(gpa, bodies[i], .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(gpa, bodies[i + 1], bodies[i]); // older from newer defer gpa.free(dr); const c = try zit.container.encode(gpa, dr, .default); defer gpa.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } }); } } _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The newest link is full; every older link is a `.delta` whose basis frame is // physically earlier (a LOWER log rid = a lower offset), and all reconstruct. for (0..links) |i| { var d: [digest_len]u8 = undefined; const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?; const rgot = (try log.content(gpa, rid)).?; defer gpa.free(rgot); try std.testing.expectEqualStrings(bodies[i], rgot); if (i == links - 1) { try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(rid)); } else { try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(rid)); var bd: [digest_len]u8 = undefined; const brid = log.lookup(decodeWireName(names[i + 1][0..nlen[i + 1]], &bd).?).?; try std.testing.expect(brid < rid); // basis frame precedes the reverse-delta } } } test "sync §7.7: persistStore verifies a retained delta's reconstruction and refuses a lying delta container, writing no frame" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A retained delta whose stored plaintext does NOT hash to its wire name (a // clone skips the transfer-layer pre-storage check, §7.7 r6, so a lying // delta reaches the persist commit hook). The §7.7 recompute runs BEFORE the // wire-form branch, so a delta-form entry is rejected exactly like a full one. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const content = "content that the wire name does not actually name, delivered as a delta\n"; const lying_name = "0" ** 64; // not SHA3-256(content) const some_basis = "1" ** 64; // an absent basis; verify fails before it is even consulted const raw = try zit.delta.create(gpa, "a plausible basis blob", content); defer gpa.free(raw); const dc = try zit.container.encode(gpa, raw, .default); defer gpa.free(dc); const rid = try s.put(lying_name, content); try s.retain(rid, .{ .delta = .{ .src = some_basis, .container = dc, .target_size = content.len } }); try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, ms.store(), null, 0)); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var digest_buf: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null); // no frame landed } fn persistReverseChainNoLeak(allocator: std.mem.Allocator) !void { const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(allocator, io, tmp.dir); // A small reverse-delta chain built on the (failing) allocator so injected // OOM is exercised in both the store build and the persist worklist. var ms = repostate.MemStore.init(allocator); defer ms.deinit(); const s = ms.store(); const links = 4; var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| allocator.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(allocator, "reverse chain oom: one shared body across revisions -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == links - 1) { const c = try zit.container.encode(allocator, bodies[i], .default); defer allocator.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(allocator, bodies[i + 1], bodies[i]); defer allocator.free(dr); const c = try zit.container.encode(allocator, dr, .default); defer allocator.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } }); } } _ = try persistStore(allocator, io, tmp.dir, ms.store(), null, 0); } test "sync §7.2/§3.3: persistStore drains the reverse-delta worklist without leaking under allocation failure" { try std.testing.checkAllAllocationFailures(std.testing.allocator, persistReverseChainNoLeak, .{}); } /// Drives `persistArtifact` over a FORWARD delta chain (each link a delta of the /// immediately older, already-persisted lower-rid link) in rid order, so a test /// owns the shared basis memo and can read its hit counter. Every basis precedes /// its delta, so `allow_defer=false` never defers and no worklist is needed — /// mirrors `persistStore`'s open/commit around a plain rid-order drive. fn persistForwardChain( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, ms: *repostate.MemStore, cache: *logstore.PlaintextCache, ) !void { var file = try cli.openStoreAppend(io, dir); defer file.close(io); const st = try file.stat(io); var fs = storage.FileStorage.init(io, file, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer log.deinit(gpa); const store = ms.store(); const n = store.count(); var srefs: SubManifestRefs = .{}; defer srefs.deinit(gpa); var rid: repostate.Rid = 1; while (rid <= n) : (rid += 1) { switch (try persistArtifact(gpa, ms.store(), &log, rid, false, false, cache, &srefs, false, null)) { .persisted => {}, .deferred => unreachable, // forward chain: no basis is ever absent } } try log.commit(); } test "sync §7.2/§7.4: the basis memo reuses each basis (hits) and reconstructs a byte-identical store to the uncached path" { const gpa = std.testing.allocator; const io = std.testing.io; const links = 8; // A forward delta chain: link 0 is full, every later link is a delta that // reconstructs its body FROM the older body (its basis a lower, already- // persisted rid) — the case where the memo turns each basis reconstruction // into a lookup for the delta built on it. Store owns its copies (retain // dupes), but the bodies/names back the reconstructed-content assertions. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| gpa.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(gpa, "forward chain shared prefix across every revision -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == 0) { const c = try zit.container.encode(gpa, bodies[i], .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(gpa, bodies[i - 1], bodies[i]); // newer body FROM its older basis defer gpa.free(dr); const c = try zit.container.encode(gpa, dr, .default); defer gpa.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i - 1][0..nlen[i - 1]], .container = c, .target_size = bodies[i].len } }); } } // (A) a normal memo; (B) a zero-budget memo that can hold nothing — the exact // uncached reconstruction. Same store, two freshly-seeded repos. var tmp_a = std.testing.tmpDir(.{}); defer tmp_a.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_a.dir); var cache_a = logstore.PlaintextCache.init(); defer cache_a.deinit(gpa); try persistForwardChain(gpa, io, tmp_a.dir, &ms, &cache_a); var tmp_b = std.testing.tmpDir(.{}); defer tmp_b.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_b.dir); var cache_b = logstore.PlaintextCache.init(); cache_b.max_bytes = 0; // never memoises: forces a fresh reconstruction each time defer cache_b.deinit(gpa); try persistForwardChain(gpa, io, tmp_b.dir, &ms, &cache_b); // The memo reused every basis exactly once (one hit per delta link); the // zero-budget run reused nothing — the two ways the same bytes get built. try std.testing.expectEqual(@as(u64, links - 1), cache_a.hits); try std.testing.expectEqual(@as(u64, 0), cache_b.hits); // Byte-identical store: the memo changes only the work, never the bytes. const bytes_a = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(bytes_a); const bytes_b = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(bytes_b); try std.testing.expectEqualSlices(u8, bytes_b, bytes_a); // Every link reconstructs to its exact body (§7.7 chain correctness). var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); for (0..links) |i| { var d: [digest_len]u8 = undefined; const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?; const got = (try log.content(gpa, rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(bodies[i], got); } } test "sync §9.1: repoProjectCode returns the stored code, else the zero placeholder" { const gpa = std.testing.allocator; const io = std.testing.io; const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); const seed = struct { fn withCode(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, code: []const u8) !void { var f = try dir.createFile(i, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(i); var fs = storage.FileStorage.init(i, f, 0); var log = try logstore.LogStore.open(g, fs.storageFile(), hk); defer log.deinit(g); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = code }); try log.commit(); } }; // (a) a repo whose config stores a real 40-hex project code: it is returned // verbatim — the code the pull/push card announces (§9.1), never the zeros // fossil would refuse. var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const code = "a1b2c3d4" ** 5; // 40 hex chars try seed.withCode(gpa, io, tmp.dir, hash_kind, code); var buf: [zit.sync.max_projectcode_len]u8 = undefined; const got = try repoProjectCode(gpa, io, tmp.dir, &buf); try std.testing.expectEqualStrings(code, got); try std.testing.expect(!std.mem.eql(u8, got, sync_projectcode)); // (b) a repo predating the stored code (empty project_code): the placeholder // is the honest fallback (no adopted identity to announce yet). var tmp2 = std.testing.tmpDir(.{}); defer tmp2.cleanup(); try seed.withCode(gpa, io, tmp2.dir, hash_kind, ""); var buf2: [zit.sync.max_projectcode_len]u8 = undefined; const got2 = try repoProjectCode(gpa, io, tmp2.dir, &buf2); try std.testing.expectEqualStrings(sync_projectcode, got2); } test "sync §9.1: the pull request card carries the adopted project code, not the placeholder" { const gpa = std.testing.allocator; // The pull client, constructed exactly as `pullRun` does, with a stored code. const code = "a1b2c3d4" ** 5; // a real (non-zero) 40-hex project code var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = sync_servercode, .project_code = code, .version = .{ .version = sync_client_version }, }; defer client.deinit(gpa); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var buf: std.ArrayListUnmanaged(u8) = .empty; defer buf.deinit(gpa); const request = try client.buildRequest(gpa, ms.store(), &buf); // §9.1 ABNF `pull-card = "pull" SP servercode SP projectcode`: the adopted // code MUST be the card's PROJECTCODE (a real peer refuses a mismatch). const expected = "pull " ++ sync_servercode ++ " " ++ ("a1b2c3d4" ** 5) ++ "\n"; try std.testing.expect(std.mem.indexOf(u8, request, expected) != null); // And the all-zero placeholder is NOT the project code the card announces. const rejected = "pull " ++ sync_servercode ++ " " ++ sync_projectcode ++ "\n"; try std.testing.expect(std.mem.indexOf(u8, request, rejected) == null); } test "serve §3.8: a per-request store-load on one serve handle sees a later append (no restart)" { const gpa = std.testing.allocator; const io = std.testing.io; const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const append = struct { fn artifactFrame(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, digest: []const u8, bytes: []const u8) !void { var f = dir.openFile(i, repo_filename, .{ .mode = .read_write }) catch try dir.createFile(i, repo_filename, .{ .read = true }); defer f.close(i); const st = try f.stat(i); var fs = storage.FileStorage.init(i, f, st.size); var log = try logstore.LogStore.open(g, fs.storageFile(), hk); defer log.deinit(g); _ = try log.put(digest, bytes); try log.commit(); } }; // Seed the repo holding exactly one content artifact (a 32-byte raw digest). const digest_a = "a" ** digest_len; try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_a, "content a\n"); // Open ONE read handle — the handle `serve` holds for its whole lifetime. var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); // First per-request read: one content artifact. var index1 = try LogIndexStore.openFromFile(gpa, io, handle, (try handle.stat(io)).size); defer index1.destroy(); const n1 = contentCount(index1.store()); try std.testing.expectEqual(@as(usize, 1), n1); // A SEPARATE writer appends a second artifact while the handle stays open — // exactly a `zit commit` running alongside a live `zit serve`. const digest_b = "b" ** digest_len; try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_b, "content b\n"); // Second per-request read on the SAME handle sees the new artifact: proof the // server re-reads the current on-disk log per request (§3.8). A startup // snapshot would still report n1 — no restart is needed here. var index2 = try LogIndexStore.openFromFile(gpa, io, handle, (try handle.stat(io)).size); defer index2.destroy(); try std.testing.expectEqual(n1 + 1, contentCount(index2.store())); } /// A test-only `storage.StorageFile` that wraps another and tallies the bytes /// every `pread` actually returned — the instrument for "how many times does a /// prime read the log?". Every other vtable method delegates verbatim, so a log /// opened over it behaves identically while its read traffic is measured. const CountingStorage = struct { inner: storage.StorageFile, read_bytes: usize = 0, fn storageFile(self: *CountingStorage) storage.StorageFile { return .{ .context = self, .vtable = &vtable }; } const vtable = storage.StorageFile.VTable{ .append = appendImpl, .pread = preadImpl, .sync = syncImpl, .size = sizeImpl, .truncate = truncateImpl, }; fn appendImpl(ctx: *anyopaque, bytes: []const u8) storage.StorageError!void { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); return self.inner.append(bytes); } fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); const n = try self.inner.pread(offset, buf); self.read_bytes += n; return n; } fn syncImpl(ctx: *anyopaque) storage.StorageError!void { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); return self.inner.sync(); } fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); return self.inner.size(); } fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); return self.inner.truncate(len); } }; /// Seed a `.zit` under `dir` holding a full+delta+manifest+config mix (via /// `packSeedDeltaStore`) and return its committed byte size — the fixture the /// serve-prime read-traffic and index-equivalence tests both build over. fn seedServeRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !u64 { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer log.deinit(gpa); var tip: [64]u8 = undefined; try packSeedDeltaStore(gpa, &log, &tip); return log.size(); } test "serve prime reads the served log at most once (no second full-container copy)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); var fs = storage.FileStorage.init(io, handle, file_size); var counting = CountingStorage{ .inner = fs.storageFile() }; // Build the serve base snapshot over the counting storage and measure how // many bytes priming read. `openReadOnly`'s scan already whole-frame-CRC'd // every byte once (the warm-start CRC concern); an index prime adds NOTHING on top, so // total prime reads stay at roughly one pass over the log. The copy prime // (`loadContentStore`) read every container a SECOND time — this bound fails // for it, which is exactly the cost the log-offset index removes. var index = try LogIndexStore.open(gpa, counting.storageFile()); defer index.destroy(); try std.testing.expect(counting.read_bytes <= file_size + file_size / 2); } test "log-backed index answers count/hashOf/held/content/wire identically to the MemStore snapshot" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); // Reference: the whole-container MemStore snapshot `loadContentStore` builds. var ref_handle = try tmp.dir.openFile(io, repo_filename, .{}); defer ref_handle.close(io); var ref_fs = storage.FileStorage.init(io, ref_handle, file_size); var ref_log = try logstore.LogStore.openReadOnly(gpa, ref_fs.storageFile()); defer ref_log.deinit(gpa); var ref = repostate.MemStore.init(gpa); defer ref.deinit(); try loadContentStore(gpa, &ref_log, &ref); const rs = ref.store(); // Subject: the log-backed index over the same file. var idx_handle = try tmp.dir.openFile(io, repo_filename, .{}); defer idx_handle.close(io); var index = try LogIndexStore.openFromFile(gpa, io, idx_handle, file_size); defer index.destroy(); const is = index.store(); try std.testing.expectEqual(rs.count(), is.count()); try std.testing.expect(rs.count() > 0); var rid: repostate.Rid = 1; while (rid <= rs.count()) : (rid += 1) { // Identity, held-ness, and lookup round-trip. try std.testing.expectEqualStrings(rs.hashOf(rid), is.hashOf(rid)); try std.testing.expectEqual(rs.held(rid), is.held(rid)); try std.testing.expectEqual(@as(?repostate.Rid, rid), is.lookup(rs.hashOf(rid))); // Plaintext byte-for-byte. const rc = rs.content(rid); const ic = is.content(rid); try std.testing.expectEqual(rc == null, ic == null); if (rc) |rb| try std.testing.expectEqualSlices(u8, rb, ic.?); // Wire form: container bytes, and the delta arm's DELTASRC + target size. const rw = rs.wire(rid); const iw = is.wire(rid); try std.testing.expectEqual(rw == null, iw == null); if (rw) |rwf| { try std.testing.expectEqualSlices(u8, rwf.container, iw.?.container); try std.testing.expectEqual(rwf.delta == null, iw.?.delta == null); if (rwf.delta) |rd| { try std.testing.expectEqualStrings(rd.src_hash, iw.?.delta.?.src_hash); try std.testing.expectEqual(rd.target_size, iw.?.delta.?.target_size); } } } } test "appendix honours intern/put/retain for hashes the log does not carry" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); var index = try LogIndexStore.openFromFile(gpa, io, handle, file_size); defer index.destroy(); const s = index.store(); // The serve base is contractually read-only — every write lands in the // per-request OverlayStore (ticket constraint 3), so these appendix arms never // fire in production. They exist so the store honours the whole ArtifactStore // interface instead of trapping on a write; exercise them directly here so that // completeness is not dead-untested surface. const persisted = s.count(); try std.testing.expect(persisted > 0); // A hash the log cannot hold (all-0xAB digest, hex-named as the wire does). var d1: [digest_len]u8 = undefined; @memset(&d1, 0xAB); var h1_buf: [2 * digest_len]u8 = undefined; const h1 = digestToWireName(&d1, &h1_buf); try std.testing.expectEqual(@as(?repostate.Rid, null), s.lookup(h1)); // intern mints an appendix phantom ABOVE the persisted rids: not held (no // content yet), count grows by one, identity round-trips, and it is idempotent. const r1 = try s.intern(h1); try std.testing.expect(r1 > persisted); try std.testing.expectEqual(r1, try s.intern(h1)); try std.testing.expectEqual(@as(?repostate.Rid, r1), s.lookup(h1)); try std.testing.expectEqualStrings(h1, s.hashOf(r1)); try std.testing.expect(!s.held(r1)); try std.testing.expectEqual(@as(?[]const u8, null), s.content(r1)); try std.testing.expectEqual(persisted + 1, s.count()); // put upgrades that phantom in place: held flips true, content reads back, and // the persisted count is unchanged (an off-log hash never touches the index). const body = "off-log body bytes"; try std.testing.expectEqual(r1, try s.put(h1, body)); try std.testing.expect(s.held(r1)); try std.testing.expectEqualSlices(u8, body, s.content(r1).?); try std.testing.expectEqual(persisted + 1, s.count()); // retain on a PERSISTED rid is the advisory no-op (sync §7.2 first-form-wins; // the log IS the retained form): it returns without error and ignores ingest. try s.retain(1, .{ .full = "ignored — a persisted rid keeps its log form" }); // None of the appendix traffic left persisted-round memos resident. try std.testing.expectEqual(@as(usize, 0), index.residentBytes()); } test "index materialises only the container a round reads and releases it between rounds" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); var index = try LogIndexStore.openFromFile(gpa, io, handle, file_size); defer index.destroy(); const s = index.store(); // Right after the prime the index holds ZERO container/plaintext bytes — the // whole point: it did not copy the store into RAM. try std.testing.expectEqual(@as(usize, 0), index.residentBytes()); // Find one wire-bearing rid (a zlib/delta frame). var target: ?repostate.Rid = null; var rid: repostate.Rid = 1; while (rid <= s.count()) : (rid += 1) { if (s.wire(rid) != null) { target = rid; break; } } try std.testing.expect(target != null); // Reading it materialised only its container; drop the round's memos. try std.testing.expect(index.residentBytes() > 0); index.releaseTransient(); try std.testing.expectEqual(@as(usize, 0), index.residentBytes()); // A single round's residency is EXACTLY the one container it emitted — not // the whole store's containers (which the old snapshot held resident). const one = s.wire(target.?).?; try std.testing.expectEqual(one.container.len, index.residentBytes()); } /// A test-only `storage.StorageFile` that reads faithfully until `fail_reads` is /// flipped, then fails every `pread` — the instrument for a fault that strikes /// AFTER `openReadOnly` has scanned and whole-frame-CRC'd the log, exactly the /// post-scan bitrot / transient EIO the on-demand serve reads must not swallow /// silently. Every other method delegates verbatim. const FaultAfterScanStorage = struct { inner: storage.StorageFile, fail_reads: bool = false, fn storageFile(self: *FaultAfterScanStorage) storage.StorageFile { return .{ .context = self, .vtable = &vtable }; } const vtable = storage.StorageFile.VTable{ .append = appendImpl, .pread = preadImpl, .sync = syncImpl, .size = sizeImpl, .truncate = truncateImpl, }; fn appendImpl(ctx: *anyopaque, bytes: []const u8) storage.StorageError!void { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); return self.inner.append(bytes); } fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); if (self.fail_reads) return error.Io; return self.inner.pread(offset, buf); } fn syncImpl(ctx: *anyopaque) storage.StorageError!void { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); return self.inner.sync(); } fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); return self.inner.size(); } fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); return self.inner.truncate(len); } }; test "a post-scan read fault on a held rid is recorded, not silently swallowed" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); var fs = storage.FileStorage.init(io, handle, file_size); var faulting = FaultAfterScanStorage{ .inner = fs.storageFile() }; // Open succeeds: the scan CRCs the whole log BEFORE any fault is armed. var index = try LogIndexStore.open(gpa, faulting.storageFile()); defer index.destroy(); const s = index.store(); // From here every on-demand container/plaintext read faults (post-scan EIO). faulting.fail_reads = true; // Walk the store as the §8.2 clone pager does: a held rid whose wire() AND // content() both return null is one the pager silently skips while still // reporting the sequence complete — a clone finishing MISSING an artifact. var silently_omitted = false; var rid: repostate.Rid = 1; while (rid <= s.count()) : (rid += 1) { if (!s.held(rid)) continue; if (s.wire(rid) == null and s.content(rid) == null) silently_omitted = true; } // The fault genuinely makes a held rid unemittable (the bug's precondition)... try std.testing.expect(silently_omitted); // ...and the store RECORDED it, so the serve layer can convert the round to an // error instead of completing a broken clone. A copy-based prime could not // reach this state: a read fault surfaced at prime, not silently at serve. try std.testing.expect(index.read_fault); } test "serveAnswer converts a recorded read fault into a storage error (no silent short clone)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var hex_buf: [2 * digest_len]u8 = undefined; try serveSeedRepo(gpa, io, tmp.dir, 4, &hex_buf); const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const clone_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 1\n", .{}); defer gpa.free(clone_plain); const clone_req = try zit.container.encode(gpa, clone_plain, .default); defer gpa.free(clone_req); // A healthy round builds the cached snapshot and completes normally. const ok = try serveAnswer(&ctx, gpa, clone_req, null, null); gpa.free(ok.body); // Simulate a post-scan read fault having struck THIS round — the test above // proves the store records exactly this. serveAnswer MUST refuse to hand back // a reply built while an artifact read faulted: otherwise the §8.2 pager's // silent skip ships a clone missing an artifact and reports it complete. ctx.snapshot.?.read_fault = true; try std.testing.expectError(error.ServerStorage, serveAnswer(&ctx, gpa, clone_req, null, null)); // The flag is cleared with the round's memos, so an immediately retried round // over the now-healthy snapshot succeeds again (a transient fault self-heals). const recovered = try serveAnswer(&ctx, gpa, clone_req, null, null); gpa.free(recovered.body); } /// Build a full-form 3-check-in chain over an evolving file into `log`, plus a /// live config frame; returns the tip check-in's 64-hex name. Every blob and /// manifest lands as a FULL frame — pack's re-delta target. fn packSeedFullStore(gpa: std.mem.Allocator, log: *logstore.LogStore, tip_out: *[64]u8) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); var root_name: [64]u8 = undefined; var mid_name: [64]u8 = undefined; const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const b1 = try packSeedBlob(log, v1); try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "f.txt", &b1, &root_name); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); const b2 = try packSeedBlob(log, v2); try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "f.txt", &b2, &mid_name); const v3 = try packEvolvingBlob(gpa, 3); defer gpa.free(v3); const b3 = try packSeedBlob(log, v3); try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &mid_name, "f.txt", &b3, tip_out); try log.commit(); } /// Store `content` under its sha3-256 name, delta-encoded against `base` when /// one is given (the evolving blobs delta to a fraction, so the frame lands as a /// `.delta`), and return its rid. Models a fossil-clone store that RETAINED /// compact wire deltas rather than full frames. fn packSeedDeltaBlob(log: *logstore.LogStore, content: []const u8, base: ?logstore.Rid) !logstore.Rid { var digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(content, &digest, .{}); return log.putArtifact(&digest, content, base); } /// Build a clone-shaped store whose successive blobs are RETAINED as `.delta` /// frames (the compact wire form), under DISTINCT filenames so pack's /// same-path-blob lineage hint finds NO base for them — exactly the case where /// the old pack reconstructed every delta to a full frame and inflated the /// store. `.zit` therefore holds real delta frames whose bases pack cannot /// re-derive, so a correct pack must preserve them verbatim. Returns the tip /// check-in's 64-hex name. fn packSeedDeltaStore(gpa: std.mem.Allocator, log: *logstore.LogStore, tip_out: *[64]u8) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); var root_name: [64]u8 = undefined; var mid_name: [64]u8 = undefined; const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const r1 = try packSeedDeltaBlob(log, v1, null); // first version: full frame (base of the chain) var b1: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v1, &b1); try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "f1.txt", &b1, &root_name); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); const r2 = try packSeedDeltaBlob(log, v2, r1); // delta of v1 var b2: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v2, &b2); try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "f2.txt", &b2, &mid_name); const v3 = try packEvolvingBlob(gpa, 3); defer gpa.free(v3); _ = try packSeedDeltaBlob(log, v3, r2); // delta of v2 var b3: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v3, &b3); try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &mid_name, "f3.txt", &b3, tip_out); try log.commit(); } /// Seed a store holding a wire delta that pack CANNOT beat by re-deltification, /// so ONLY the §6.1 monotonic-shrink guard keeps the store from regressing. `B` /// is a one-line edit of `A`, stored as a COMPACT zit delta against `A` — the /// smallest form pack could ever reproduce. But `max_base_candidates` unrelated /// filler blobs sit between `A` and `B`, so pack's bounded backward candidate /// window (§5.2) saturates on the fillers and never reaches `A`; with no lineage /// hint either (bare blobs, no F-card path), pack's best re-encoding of `B` is a /// full zlib frame STRICTLY LARGER than the stored A-based delta. The pre-guard /// pack replaced the delta with that larger full frame (the +33% clone /// regression in miniature); the guard keeps the existing smaller frame. Fills /// A's and B's 64-hex names. fn packSeedUnbeatableWireDelta( gpa: std.mem.Allocator, log: *logstore.LogStore, a_name: *[64]u8, b_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // A: B's ideal base, stored full (zlib). const a = try packEvolvingBlob(gpa, 1); defer gpa.free(a); var a_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(a, &a_digest, .{}); const a_rid = try log.putZlib(&a_digest, a); _ = zit.artifact.artifactName(.sha3_256, a, a_name); // `max_base_candidates` unrelated fillers between A and B: they saturate B's // backward candidate window so A is out of reach when pack re-encodes B. var f: usize = 0; while (f < logstore.max_base_candidates) : (f += 1) { var fbuf: std.ArrayList(u8) = .empty; defer fbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try fbuf.print(gpa, "filler {d} unrelated row {d}\n", .{ f, i }); var fdig: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(fbuf.items, &fdig, .{}); _ = try log.putZlib(&fdig, fbuf.items); } // B: a one-line edit of A, stored as a COMPACT zit delta against A (the // smallest form pack could reproduce — and only against A, which pack's // window cannot reach here). const b = try packEvolvingBlob(gpa, 2); defer gpa.free(b); var b_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{}); _ = zit.artifact.artifactName(.sha3_256, b, b_name); const raw = try zit.delta.create(gpa, a, b); defer gpa.free(raw); const container = try zit.container.encode(gpa, raw, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&b_digest, a_rid, container); try log.commit(); } /// Assert the §6.1 per-frame monotonicity: NO artifact's packed frame is larger /// on disk than its source frame. Pack keeps the smaller of {existing frame, new /// encoding} for every artifact, so a repack is non-worsening frame by frame. /// This holds universally over THESE fixtures because their chains stay below the /// read cap; the one path that can enlarge a frame — `copySrcFrame`'s §3.3 /// deep-chain fallback re-storing a kept delta as full zlib — needs a base at /// `max_delta_chain` depth, which the seeds never build. fn packExpectNoFrameLarger(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void { const src_rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); for (src_rids) |sr| { const dr = dst.lookup(src.hashOf(sr)) orelse return error.PackedArtifactMissing; try std.testing.expect(dst.frameSpanOf(dr) <= src.frameSpanOf(sr)); } } /// Count the `.delta` frames among a store's persisted artifacts. fn packDeltaFrameCount(gpa: std.mem.Allocator, log: *logstore.LogStore) !usize { const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var n: usize = 0; for (rids) |r| { if (log.kindOf(r) == .delta) n += 1; } return n; } /// Assert that every artifact in `src` reconstructs byte-identically in `dst` /// and the two artifact SETS are equal (no drop, no addition). fn packExpectContentEqual(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void { const src_rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); const dst_rids = try dst.ridsInPersistedOrder(gpa); defer gpa.free(dst_rids); try std.testing.expectEqual(src_rids.len, dst_rids.len); for (src_rids) |sr| { const digest = src.hashOf(sr); const dr = dst.lookup(digest) orelse return error.PackedArtifactMissing; const a = (try src.content(gpa, sr)).?; defer gpa.free(a); const b = (try dst.content(gpa, dr)).?; defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); } } test "pack: the packed store's artifact set + every reconstructed content byte-equal the original (config preserved)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // No drop: the packed content-frame count equals the source's, and every // artifact reconstructs byte-for-byte. try std.testing.expectEqual(src.entries.items.len, stats.artifacts); try packExpectContentEqual(gpa, &src, &dst); // The live config survives as exactly one frame with identical fields. var soc = (try src.readConfig(gpa)).?; defer soc.deinit(gpa); var doc = (try dst.readConfig(gpa)).?; defer doc.deinit(gpa); try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy); try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch); try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id); try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code); } test "pack re-deltas full-form frames (delta count rises, .zit shrinks) with byte-identical content" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // The source is all full-form: not one delta frame. { const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); for (rids) |r| try std.testing.expect(src.kindOf(r) != .delta); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // The two child blobs (v2 against v1, v3 against v2) re-delta against the // same path's parent blob, so the delta count rises and the store shrinks — // with content preserved exactly. try std.testing.expect(stats.deltas >= 2); try std.testing.expect(stats.bytes_after < stats.bytes_before); try packExpectContentEqual(gpa, &src, &dst); } test "pack preserves wire deltas — a store with delta frames packs to <= source size (regression for the 8.65x inflation bug)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedDeltaStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // The source genuinely holds retained wire deltas (the compact form pack // must not discard) whose bases pack's lineage hint cannot re-derive. const src_deltas = try packDeltaFrameCount(gpa, &src); try std.testing.expect(src_deltas >= 2); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // MONOTONIC: the packed store is never larger than the source. The old pack // reconstructed each delta to a full zlib frame and inflated the store; a // correct pack preserves the compact deltas verbatim. try std.testing.expect(stats.bytes_after <= stats.bytes_before); // Every source delta survives AS a delta frame — none reconstructed to full // (the optional re-delta may add further deltas, so the total only grows). { const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); for (rids) |sr| { if (src.kindOf(sr) != .delta) continue; const dr = dst.lookup(src.hashOf(sr)).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr)); } } try std.testing.expect(try packDeltaFrameCount(gpa, &dst) >= src_deltas); // No drop, every reconstruction byte-exact. try packExpectContentEqual(gpa, &src, &dst); } test "pack copies a delta frame verbatim re-linked to its base's new rid, content byte-identical" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedDeltaStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // For every delta frame in the source, the packed store holds it as a delta // whose stored container is byte-identical (verbatim re-serve) and whose base // names the SAME artifact (re-linked to the base's new rid). Content matches. const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); var checked: usize = 0; for (rids) |sr| { if (src.kindOf(sr) != .delta) continue; const digest = src.hashOf(sr); const dr = dst.lookup(digest).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr)); var sw = (try src.wire(gpa, sr)).?; defer sw.deinit(gpa); var dw = (try dst.wire(gpa, dr)).?; defer dw.deinit(gpa); // Verbatim container bytes and the same base hash (re-link preserves the // logical base while its rid/offset differs in the packed generation). try std.testing.expectEqualSlices(u8, sw.delta.container, dw.delta.container); try std.testing.expectEqualSlices(u8, sw.delta.baseHash(), dw.delta.baseHash()); const a = (try src.content(gpa, sr)).?; defer gpa.free(a); const b = (try dst.content(gpa, dr)).?; defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); checked += 1; } try std.testing.expect(checked >= 2); } /// The rid naming the 64-hex `name` in `log`, or null when absent (test-only /// name→digest→rid convenience over `lookup`). fn packRidByName(log: *logstore.LogStore, name: *const [64]u8) ?logstore.Rid { var d_buf: [digest_len]u8 = undefined; const d = hexToDigest(name, &d_buf) orelse return null; return log.lookup(d); } /// Seed a store modeling a peer-supplied (verbatim wire) delta whose sender /// chose a POOR base. `A` (a large file) and `C` (an unrelated large blob) are /// stored full; `B` (a one-line edit of `A`) is stored as a hand-crafted §4.1 /// delta container against `C` — a large, suboptimal delta the sender picked. /// `A` precedes `B` chronologically and is B's ideal base, so pack's best-of-N /// re-deltification (§6.1) must re-base `B` onto `A` rather than trust the /// sender's `C`. Fills the 64-hex names of A/B/C. fn packSeedSuboptimalWireDelta( gpa: std.mem.Allocator, log: *logstore.LogStore, a_name: *[64]u8, b_name: *[64]u8, c_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // A: the ideal base for B (near-identical predecessor), stored full. const a = try packEvolvingBlob(gpa, 1); defer gpa.free(a); a_name.* = try packSeedBlob(log, a); // C: an unrelated large blob (a poor base for B), stored full. var cbuf: std.ArrayList(u8) = .empty; defer cbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try cbuf.print(gpa, "unrelated content row {d}\n", .{i}); const c = cbuf.items; c_name.* = try packSeedBlob(log, c); const c_rid = packRidByName(log, c_name).?; // B: a one-line edit of A. delta(A,B) is tiny; delta(C,B) is large. const b = try packEvolvingBlob(gpa, 2); defer gpa.free(b); _ = zit.artifact.artifactName(.sha3_256, b, b_name); var b_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{}); // Hand-craft the verbatim wire delta against the POOR base C — models a peer // that chose C as base, retained at rest by `putDeltaContainer`. const raw = try zit.delta.create(gpa, c, b); defer gpa.free(raw); const container = try zit.container.encode(gpa, raw, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&b_digest, c_rid, container); try log.commit(); } test "pack §6.1: a verbatim wire delta is re-deltified against the size-minimizing base, not the sender's" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var a_name: [64]u8 = undefined; var b_name: [64]u8 = undefined; var c_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSuboptimalWireDelta(gpa, &wlog, &a_name, &b_name, &c_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source really holds B as a delta whose base is C (the // sender's suboptimal choice) — the exact frame the old pack copied verbatim. { const sb = packRidByName(&src, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb)); var sw = (try src.wire(gpa, sb)).?; defer sw.deinit(gpa); var cbin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&cbin, &c_name); try std.testing.expectEqualSlices(u8, &cbin, sw.delta.baseHash()); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // After pack, B is still a delta, but its base is now A (the size-minimizing // candidate) — the sender's base C is no longer trusted at rest (§6.1/§3). const db = packRidByName(&dst, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db)); var dw = (try dst.wire(gpa, db)).?; defer dw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash()); // Every artifact still reconstructs byte-for-byte and the store shrank (a // tiny A-based delta replaced the large C-based one). try packExpectContentEqual(gpa, &src, &dst); try std.testing.expect(stats.bytes_after < stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); try std.testing.expectEqual(src.size(), stats.bytes_before); } /// Seed the §6.2 same-path scenario: two versions of ONE tracked file whose ideal /// base (the earlier version) sits OUTSIDE pack's bounded chronological window. /// `v1` of "tracked.txt" is stored full, then `max_base_candidates` unrelated /// filler blobs saturate the backward candidate window, then `v2` (a one-line /// edit of `v1`) is stored full. With ONLY the chronological window (step §6.1) /// pack cannot reach `v1` from `v2` — the fillers fill every slot — so `v2` stays /// a full frame and the store does not shrink. The mlink-analog same-path walk /// (§6.2) reads both check-in manifests, learns tracked.txt: [v1, v2], and offers /// `v1` as `v2`'s priority candidate, so pack re-deltifies `v2` onto `v1`. Both /// check-ins are FULL frames (pack's re-delta target); fills v1/v2's 64-hex names. fn packSeedSamePathOutOfWindow( gpa: std.mem.Allocator, log: *logstore.LogStore, v1_name: *[64]u8, v2_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // v1 of tracked.txt (full), and the root check-in that names it. const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); v1_name.* = try packSeedBlob(log, v1); var root_name: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &root_name); // `max_base_candidates` unrelated fillers between v1 and v2: they saturate // v2's bounded backward window so v1 is out of chronological reach. var f: usize = 0; while (f < logstore.max_base_candidates) : (f += 1) { var fbuf: std.ArrayList(u8) = .empty; defer fbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try fbuf.print(gpa, "samepath filler {d} unrelated row {d}\n", .{ f, i }); _ = try packSeedBlob(log, fbuf.items); } // v2 of tracked.txt (a one-line edit of v1), stored FULL, and its check-in. const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); v2_name.* = try packSeedBlob(log, v2); var tip_name: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "tracked.txt", v2_name, &tip_name); try log.commit(); } test "pack §6.2: a same-path predecessor beyond the chronological window becomes v2's base (store shrinks)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var v1_name: [64]u8 = undefined; var v2_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSamePathOutOfWindow(gpa, &wlog, &v1_name, &v2_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source holds v2 as a NON-delta (full) frame — the case // step §6.1's chronological window alone cannot improve (v1 is out of reach). { const sv2 = packRidByName(&src, &v2_name).?; try std.testing.expect(src.kindOf(sv2) != .delta); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // After pack, v2 is a delta whose base is the same-path predecessor v1 — the // §6.2 mlink analog reached a base the chronological window could not. const dv2 = packRidByName(&dst, &v2_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dv2)); var dw = (try dst.wire(gpa, dv2)).?; defer dw.deinit(gpa); var v1bin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&v1bin, &v1_name); try std.testing.expectEqualSlices(u8, &v1bin, dw.delta.baseHash()); // The store shrank (a tiny v1-based delta replaced the full v2 frame), every // artifact still reconstructs byte-for-byte, and no frame grew. try std.testing.expect(stats.bytes_after < stats.bytes_before); try packExpectContentEqual(gpa, &src, &dst); try packExpectNoFrameLarger(gpa, &src, &dst); } /// Seed the §6.2 same-path REVERT scenario: one tracked file whose content goes /// A→B→A across three check-ins — the third restores a byte-identical earlier /// blob. Because the store is content-addressed, the reverted version resolves to /// the SAME blob rid as v1, so the pack manifest walk records that rid twice, /// non-consecutively, for one path. Exercises `deriveCandidateHints`'s /// distinct-rid compaction: a blob must never be offered as its own same-path /// predecessor. Fills v1/v2's 64-hex names. fn packSeedSamePathRevert( gpa: std.mem.Allocator, log: *logstore.LogStore, v1_name: *[64]u8, v2_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); // v1 of tracked.txt (full) + its root check-in. v1_name.* = try packSeedBlob(log, v1); var c1: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &c1); // v2 (a one-line edit) + its check-in. v2_name.* = try packSeedBlob(log, v2); var c2: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &c1, "tracked.txt", v2_name, &c2); // Revert: tracked.txt returns to v1's byte-identical content. The blob is // already stored (content-addressed), so this check-in re-references v1's rid // — the non-consecutive duplicate that recordPathBlob's consecutive-only dedup // lets through. var c3: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &c2, "tracked.txt", v1_name, &c3); try log.commit(); } test "pack §6.2: a same-path revert (A→B→A) re-references one blob rid twice and packs cleanly (no self-predecessor panic)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var v1_name: [64]u8 = undefined; var v2_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSamePathRevert(gpa, &wlog, &v1_name, &v2_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); // Before the distinct-rid compaction in deriveCandidateHints this panicked on // a self-predecessor (v1 sorted adjacent to its own duplicate ⇒ equal offsets // fail the strict offset-below assert). Now the pack completes and every // artifact reconstructs byte-for-byte with no frame enlarged. const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); try packExpectContentEqual(gpa, &src, &dst); try packExpectNoFrameLarger(gpa, &src, &dst); try std.testing.expectEqual(dst.size(), stats.bytes_after); try std.testing.expectEqual(src.size(), stats.bytes_before); } test "pack §6.1 monotonic guard: a wire delta pack cannot beat is KEPT, not replaced by a larger frame (no regression)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var a_name: [64]u8 = undefined; var b_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedUnbeatableWireDelta(gpa, &wlog, &a_name, &b_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source holds B as a compact delta whose base is A. { const sb = packRidByName(&src, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb)); var sw = (try src.wire(gpa, sb)).?; defer sw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, sw.delta.baseHash()); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // (a) decode-all byte-identical pre/post. try packExpectContentEqual(gpa, &src, &dst); // (b) NO frame is larger post-pack than its pre-pack encoding. try packExpectNoFrameLarger(gpa, &src, &dst); // (c) the whole store did not regress on this wire-delta-heavy fixture. try std.testing.expect(stats.bytes_after <= stats.bytes_before); // Accounting exact: reported sizes are the real store sizes. try std.testing.expectEqual(src.size(), stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); // B stayed a delta (the compact A-based frame was kept, re-linked to A's new // rid) — the guard refused to replace it with the larger full zlib pack's // window-only re-encoding would have produced. const db = packRidByName(&dst, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db)); var dw = (try dst.wire(gpa, db)).?; defer dw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash()); // The kept frame is no larger than the source's (equal — copied verbatim). try std.testing.expect(dst.frameSpanOf(db) <= src.frameSpanOf(packRidByName(&src, &b_name).?)); } /// Seed a store exercising EVERY frame class pack must handle in one place: a /// raw frame, a zlib frame, a locally-created delta, a verbatim wire delta, an /// interned phantom (no frame on disk), and the config singleton. Fills the /// content-bearing artifacts' 64-hex names plus the phantom's binary digest. fn packSeedMixedStore( gpa: std.mem.Allocator, log: *logstore.LogStore, raw_name: *[64]u8, zlib_name: *[64]u8, delta_name: *[64]u8, wire_name: *[64]u8, phantom_digest: *[digest_len]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // raw: a unique blob large enough to be a real (if unrelated) base. var rbuf: std.ArrayList(u8) = .empty; defer rbuf.deinit(gpa); var i: usize = 0; while (i < 120) : (i += 1) try rbuf.print(gpa, "raw-only content row {d}\n", .{i}); const rbytes = rbuf.items; raw_name.* = try packSeedBlob(log, rbytes); // `put` => .raw // zlib: a large evolving blob stored as a full zlib container. const z = try packEvolvingBlob(gpa, 1); defer gpa.free(z); var z_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(z, &z_digest, .{}); const z_rid = try log.putZlib(&z_digest, z); _ = zit.artifact.artifactName(.sha3_256, z, zlib_name); // local delta: an edit of the zlib blob, delta-encoded against it. const d = try packEvolvingBlob(gpa, 2); defer gpa.free(d); var d_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(d, &d_digest, .{}); _ = try log.putArtifact(&d_digest, d, z_rid); _ = zit.artifact.artifactName(.sha3_256, d, delta_name); // wire delta: a hand-crafted §4.1 delta container against the raw blob (a // poor base) — the verbatim-retained peer form. const w = try packEvolvingBlob(gpa, 3); defer gpa.free(w); var w_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(w, &w_digest, .{}); _ = zit.artifact.artifactName(.sha3_256, w, wire_name); const r_rid = packRidByName(log, raw_name).?; const raw_delta = try zit.delta.create(gpa, rbytes, w); defer gpa.free(raw_delta); const container = try zit.container.encode(gpa, raw_delta, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&w_digest, r_rid, container); // phantom: an interned forward reference that never reaches disk. std.crypto.hash.sha3.Sha3_256.hash("absent forward-referenced artifact", phantom_digest, .{}); _ = try log.intern(phantom_digest); try log.commit(); } test "pack §6.1/§3: a mixed raw/zlib/delta/wire-delta/phantom/config store round-trips, config+phantom preserved, accounting exact" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var raw_name: [64]u8 = undefined; var zlib_name: [64]u8 = undefined; var delta_name: [64]u8 = undefined; var wire_name: [64]u8 = undefined; var phantom_digest: [digest_len]u8 = undefined; // A phantom never reaches disk, so we pack the in-memory writable store // directly (a reopen would drop it) — the case pack must tolerate without // choking or emitting a garbage frame. var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedMixedStore(gpa, &wlog, &raw_name, &zlib_name, &delta_name, &wire_name, &phantom_digest); // The seeded store really holds one frame of each class plus the phantom. try std.testing.expectEqual(logstore.FrameKind.raw, wlog.kindOf(packRidByName(&wlog, &raw_name).?)); try std.testing.expectEqual(logstore.FrameKind.zlib, wlog.kindOf(packRidByName(&wlog, &zlib_name).?)); try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &delta_name).?)); try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &wire_name).?)); try std.testing.expect(wlog.isPhantom(wlog.lookup(&phantom_digest).?)); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), wlog.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &wlog, &dst, Filter.keep_all, &.{}); // Decode-all byte-identical pre/post, and the artifact SETS match (the // phantom is excluded from both, so counts still agree). try packExpectContentEqual(gpa, &wlog, &dst); // §6.1 per-frame monotonicity: no frame grew in the repack. try packExpectNoFrameLarger(gpa, &wlog, &dst); // Config preserved verbatim in fields. var soc = (try wlog.readConfig(gpa)).?; defer soc.deinit(gpa); var doc = (try dst.readConfig(gpa)).?; defer doc.deinit(gpa); try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy); try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch); try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id); try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code); // Phantom preserved as a contentless forward reference: it carries no bytes, // so pack writes no frame for it (a phantom never reaches disk) and it is not // resurrected as a garbage artifact in the packed store. try std.testing.expect(dst.lookup(&phantom_digest) == null); // Accounting exact: the reported sizes are the actual store sizes and the // artifact tally is the source's real (non-phantom) frame count. try std.testing.expectEqual(wlog.size(), stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); const src_rids = try wlog.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); try std.testing.expectEqual(src_rids.len, stats.artifacts); } test "pack §7.7: packLog aborts on a source frame whose content does not name it (no packed frame written)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A frame keyed by a name that is NOT sha3-256 of its bytes: `put` stores // bytes verbatim under any key, so this models a corrupt/lying source frame // that survives `openReadOnly` (its CRC + embedded-hash label are intact). const lying_name = [_]u8{0xab} ** digest_len; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); _ = try wlog.put(&lying_name, "content these bytes do not name"); try wlog.commit(); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); try std.testing.expectError(error.Corrupt, packLog(gpa, &src, &dst, Filter.keep_all, &.{})); try std.testing.expect(dst.lookup(&lying_name) == null); // no frame landed on the abort } test "pack is idempotent: pack∘pack is byte-identical (0 additional re-deltas)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } // First pack: source (full) → L1. { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var f1 = try tmp.dir.createFile(io, "pack1", .{ .read = true, .truncate = true }); defer f1.close(io); var f1fs = storage.FileStorage.init(io, f1, 0); var dst = try logstore.LogStore.open(gpa, f1fs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); } // Second pack: L1 → L2, byte-for-byte equal, and no further re-deltaing. var l1fs: storage.FileStorage = undefined; var l1 = try packOpenReadOnly2(gpa, io, tmp.dir, "pack1", &l1fs); defer l1fs.file.close(io); defer l1.deinit(gpa); var f2 = try tmp.dir.createFile(io, "pack2", .{ .read = true, .truncate = true }); defer f2.close(io); var f2fs = storage.FileStorage.init(io, f2, 0); var l2 = try logstore.LogStore.open(gpa, f2fs.storageFile(), l1.hash_kind); defer l2.deinit(gpa); const stats2 = try packLog(gpa, &l1, &l2, Filter.keep_all, &.{}); try std.testing.expectEqual(l1.size(), l2.size()); const a = try tmp.dir.readFileAlloc(io, "pack1", gpa, .limited(1 << 24)); defer gpa.free(a); const b = try tmp.dir.readFileAlloc(io, "pack2", gpa, .limited(1 << 24)); defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); // byte-identical: pack∘pack == pack // An already-packed store still reports its deltas (they were re-created // identically), and packing it does not shrink further. try std.testing.expectEqual(l1.size(), stats2.bytes_after); } /// Reopen a named log file in `dir` read-only (a sibling to `packOpenReadOnly` /// that takes an arbitrary name, for the multi-generation idempotence test). fn packOpenReadOnly2(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, name: []const u8, fs_out: *storage.FileStorage) !logstore.LogStore { const f = try dir.openFile(io, name, .{}); const st = try f.stat(io); fs_out.* = storage.FileStorage.init(io, f, st.size); return logstore.LogStore.openReadOnly(gpa, fs_out.storageFile()); } test "pack: packLog never writes the source (a crash before the rename leaves the original byte-identical)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } const before = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(before); // Pack into a separate temp (the new generation). packLog only ever reads // the source; the atomic swap in `runPack` is the sole mutation of `.zit`. { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var f = try tmp.dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = true }); defer f.close(io); var ffs = storage.FileStorage.init(io, f, 0); var dst = try logstore.LogStore.open(gpa, ffs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); } // No rename happened (the "crash"): the original `.zit` is byte-for-byte // unchanged and still opens. const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(after); try std.testing.expectEqualSlices(u8, before, after); } /// The allocation-failure probe: pack a prebuilt source `.zit` into a fresh temp /// destination, using the failing allocator throughout. Every OOM point must /// unwind cleanly (no leak) and a fully-funded run must succeed. fn packOomProbe(alloc: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(alloc, io, dir, &sfs); defer sfs.file.close(io); defer src.deinit(alloc); var f = try dir.createFile(io, "oomdst", .{ .read = true, .truncate = true }); defer f.close(io); var ffs = storage.FileStorage.init(io, f, 0); var dst = try logstore.LogStore.open(alloc, ffs.storageFile(), src.hash_kind); defer dst.deinit(alloc); _ = try packLog(alloc, &src, &dst, Filter.keep_all, &.{}); } test "pack: checkAllAllocationFailures on packLog (every OOM path unwinds cleanly)" { const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(std.testing.allocator, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(std.testing.allocator); try packSeedFullStore(std.testing.allocator, &wlog, &tip); } try std.testing.checkAllAllocationFailures(std.testing.allocator, packOomProbe, .{ io, tmp.dir }); } test "pack∘derive == from-scratch derive: the packed store derives the identical repo view" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var src_view: zit.repoview.RepoView = undefined; try src_view.load(gpa, &src); defer src_view.deinit(gpa); var pfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer pfile.close(io); var pfs = storage.FileStorage.init(io, pfile, 0); var dst = try logstore.LogStore.open(gpa, pfs.storageFile(), src.hash_kind); _ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); dst.deinit(gpa); var rfs: storage.FileStorage = undefined; var packed_log = try packOpenReadOnly2(gpa, io, tmp.dir, "packed", &rfs); defer rfs.file.close(io); defer packed_log.deinit(gpa); var packed_view: zit.repoview.RepoView = undefined; try packed_view.load(gpa, &packed_log); defer packed_view.deinit(gpa); // Same derived view: main branch, artifact count, and the leaf set + order // (§7.5) are identical — pack changed only the storage layout. try std.testing.expectEqualStrings(src_view.main_branch, packed_view.main_branch); try std.testing.expectEqual(src_view.store.store().count(), packed_view.store.store().count()); var la = src_view.state.leaves(); var lb = packed_view.state.leaves(); while (la.next()) |ra| { const rb = lb.next() orelse return error.LeafCountMismatch; try std.testing.expectEqualStrings(src_view.hashOf(ra), packed_view.hashOf(rb)); } try std.testing.expect(lb.next() == null); } test "renderPackStats: 'packed' when the store shrank, 'already packed' otherwise" { var buf: [128]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 1695, .deltas = 1373, .dropped = 0, .shuns_recorded = 0, .bytes_before = 9_680_000, .bytes_after = 920_000 }); try std.testing.expectEqualStrings( "packed: 1695 artifacts walked, 1373 re-deltaed, 9680000 -> 920000 bytes\n", w.buffered(), ); } { var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 3, .deltas = 2, .dropped = 0, .shuns_recorded = 0, .bytes_before = 4096, .bytes_after = 4096 }); try std.testing.expectEqualStrings( "already packed: 3 artifacts walked, 2 re-deltaed, 4096 -> 4096 bytes\n", w.buffered(), ); } } test "renderPackStats: a reclaiming run names the dropped count; a run that drops nothing stays silent" { // Reclamation is the one pack outcome an operator must not have to infer // from a shrinking byte count: bytes shrink on an ordinary re-deltify too. // So the clause appears exactly when `dropped > 0`, and nowhere else. var buf: [128]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 3, .shuns_recorded = 0, .bytes_before = 8192, .bytes_after = 4096 }); try std.testing.expectEqualStrings( "packed: 12 artifacts walked, 4 re-deltaed, 3 shunned reclaimed, 8192 -> 4096 bytes\n", w.buffered(), ); } { // `artifacts` counts the source frames WALKED, so a run that dropped // every one of them still reports 12 walked and 12 reclaimed. var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 12, .deltas = 0, .dropped = 12, .shuns_recorded = 0, .bytes_before = 8192, .bytes_after = 64 }); try std.testing.expectEqualStrings( "packed: 12 artifacts walked, 0 re-deltaed, 12 shunned reclaimed, 8192 -> 64 bytes\n", w.buffered(), ); } { // A reclaim that GREW the log is still "packed". Reclaiming an artifact // other artifacts were deltified against moves its content into one of // them, so the log can legitimately end up larger — and "already packed" // on a run that just irreversibly deleted content would be the most // misleading line this verb could print. var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 7, .deltas = 5, .dropped = 1, .shuns_recorded = 0, .bytes_before = 5760, .bytes_after = 6248 }); try std.testing.expectEqualStrings( "packed: 7 artifacts walked, 5 re-deltaed, 1 shunned reclaimed, 5760 -> 6248 bytes\n", w.buffered(), ); } } test "rewriteAndSwap: a run the concurrency guard aborts leaves .zit untouched and records NO tombstone" { // THE "ONE DURABLE TRANSITION" INVARIANT, made observable. `zit shun --force` // must never leave a tombstone behind without also having removed the content // it names; the mechanism is that the rows are written into the TEMP and only // the rename makes anything durable. This drives the abort seam that is a live // guard rather than an injected fault — the no-repo-lock concurrency guard: a // concurrent writer extends `.zit` between our scan and the rename, so `validateTail` // fails after `packLog` has already written every row into the temp. If the // rows had been appended to the live log first, they would still be here. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const doomed_body = "content the operator named in this invocation " ** 4; var doomed: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(doomed_body, &doomed, .{}); { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wlog.deinit(gpa); _ = try packSeedBlob(&wlog, doomed_body); _ = try packSeedBlob(&wlog, "content with nothing against it " ** 4); try wlog.commit(); } // The scan the rewrite will be built from, and the tail key that pins it. var rfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &rfs); defer src.deinit(gpa); const src_key = try src.logKey(); // The concurrent writer. One appended frame is exactly what the guard exists // to detect; the snapshot in `src` is now stale. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); const st = try f.stat(io); var wfs = storage.FileStorage.init(io, f, st.size); var wlog = try logstore.LogStore.open(gpa, wfs.storageFile(), 0); defer wlog.deinit(gpa); _ = try packSeedBlob(&wlog, "a frame a concurrent writer appended " ** 3); try wlog.commit(); } const extended = try storeSnapshot(gpa, io, tmp.dir); defer gpa.free(extended); var set: NameSet = .{}; defer set.deinit(gpa); try set.add(gpa, &doomed); const rows = [_]logstore.ShunRecord{ .{ .hash = &doomed, .epoch = 1753000077, .reason = "takedown notice 4417" }, }; try std.testing.expectError(error.TailChanged, rewriteAndSwap( gpa, io, tmp.dir, &src, src_key, set.filter(), &rows, )); // THE PINS, tombstone FIRST because it is the one this test exists for: the // aborted run left no row, `.zit` is the concurrent writer's version byte for // byte (our stale rewrite never landed), the content the run intended to // remove is intact, and the temp is gone. var vfs: storage.FileStorage = undefined; var after_log = try packOpenReadOnly(gpa, io, tmp.dir, &vfs); defer after_log.deinit(gpa); try std.testing.expectEqual(@as(usize, 0), after_log.shunList().len); try std.testing.expect(!after_log.isShunned(&doomed)); try std.testing.expect(after_log.lookup(&doomed) != null); { const after = try storeSnapshot(gpa, io, tmp.dir); defer gpa.free(after); try std.testing.expectEqualSlices(u8, extended, after); } if (tmp.dir.openFile(io, pack_tmp_filename, .{})) |f| { var mf = f; mf.close(io); return error.TempSurvivedTheAbort; } else |_| {} } test "rewriteAndSwap: a run the concurrency guard aborts leaves .zit untouched and records NO tombstone (competing RENAME)" { // THE SIBLING SHAPE, and the one the caller's own file handle cannot see. The // test above simulates a concurrent writer that APPENDS to `.zit`; here a // competing rewrite (`zit pack`, or another `zit shun --force`) renames a whole // new generation over the name while our snapshot's handle stays open. That // handle then names an ORPHAN inode whose size and tail can never move again, // so a guard that re-stats it agrees with its own key forever and this run // renames a pre-competitor snapshot over the winner — silently restoring // content the winner shunned, with both runs exiting 0. Reopening `.zit` by // NAME is what turns it into `error.TailChanged`. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const doomed_body = "content the operator named in this invocation " ** 4; var doomed: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(doomed_body, &doomed, .{}); { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wlog.deinit(gpa); _ = try packSeedBlob(&wlog, doomed_body); _ = try packSeedBlob(&wlog, "content with nothing against it " ** 4); try wlog.commit(); } // The scan the rewrite will be built from, and the tail key that pins it. This // handle is the one the competing rename is about to orphan. var rfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &rfs); defer src.deinit(gpa); const src_key = try src.logKey(); // The competing rewrite: a NEW generation built beside `.zit` and renamed over // it. Nothing touches the inode `src` is reading, which is the whole point. { var f = try tmp.dir.createFile(io, "gen2", .{ .read = true, .truncate = true }); defer f.close(io); var gfs = storage.FileStorage.init(io, f, 0); var glog = try logstore.LogStore.open(gpa, gfs.storageFile(), src.hash_kind); defer glog.deinit(gpa); _ = try packSeedBlob(&glog, "the generation the competing rewrite produced " ** 3); try glog.commit(); } try tmp.dir.rename("gen2", tmp.dir, repo_filename, io); const winner = try storeSnapshot(gpa, io, tmp.dir); defer gpa.free(winner); var set: NameSet = .{}; defer set.deinit(gpa); try set.add(gpa, &doomed); const rows = [_]logstore.ShunRecord{ .{ .hash = &doomed, .epoch = 1753000077, .reason = "takedown notice 4417" }, }; try std.testing.expectError(error.TailChanged, rewriteAndSwap( gpa, io, tmp.dir, &src, src_key, set.filter(), &rows, )); // The winner's generation survives byte for byte, and the aborted run recorded // nothing — the loser did not resurrect its own snapshot's contents. { const after = try storeSnapshot(gpa, io, tmp.dir); defer gpa.free(after); try std.testing.expectEqualSlices(u8, winner, after); } var vfs: storage.FileStorage = undefined; var after_log = try packOpenReadOnly(gpa, io, tmp.dir, &vfs); defer after_log.deinit(gpa); try std.testing.expectEqual(@as(usize, 0), after_log.shunList().len); try std.testing.expect(after_log.lookup(&doomed) == null); if (tmp.dir.openFile(io, pack_tmp_filename, .{})) |f| { var mf = f; mf.close(io); return error.TempSurvivedTheAbort; } else |_| {} } /// Read `dir/.zit` whole (caller frees) — the byte-identical oracle for a run /// that must have changed nothing. fn storeSnapshot(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) ![]u8 { var file = try dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); const buf = try gpa.alloc(u8, @intCast(st.size)); errdefer gpa.free(buf); const n = try file.readPositionalAll(io, buf, 0); try std.testing.expectEqual(buf.len, n); return buf; } // --- `zit shun`'s pure renderers ------------------------------------------- // // Every operator-visible byte this verb emits is produced by a function of data // alone, for the same reason `renderPackStats` is: the exact wording of a // refusal that precedes irreversible deletion is the part most worth pinning, // and pinning it through a subprocess would make the e2e tests the only place // it is stated. const shun_name_a = "1c04b7" ++ "0" ** 58; const shun_name_b = "7bd1e0" ++ "1" ** 58; const shun_name_c = "9f2a1c" ++ "2" ** 58; test "renderShunRefusal: the counts and the word refusing share the first line, then one line per hash" { // The register is `clean`'s (`cleanRefuse`): an operator must not have to read // six lines of inventory to discover that nothing happened. var buf: [1024]u8 = undefined; var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = .manifest, .disposition = .remove }, .{ .name = shun_name_b, .kind = null, .disposition = .remove }, .{ .name = shun_name_c, .disposition = .absent }, }, 0, false); try std.testing.expectEqualStrings( "zit shun: 3 hashes would be shunned in this repository; 2 artifacts would be permanently removed; refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " checkin a check-in; its descendants become unresolvable\n" ++ " " ++ shun_name_b ++ " file file content; every check-in listing it loses that file\n" ++ " " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --force <HASH>...' to proceed\n", w.buffered(), ); } test "renderShunRefusal: singular agreement, an already-shunned line, and the nothing-held wording" { var buf: [1024]u8 = undefined; { // One target, held here: "1 hash", "1 artifact". var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = .wiki, .disposition = .remove }, .{ .name = shun_name_b, .disposition = .already }, }, 0, false); try std.testing.expectEqualStrings( "zit shun: 1 hash would be shunned in this repository; 1 artifact would be permanently removed; refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " wiki structural content; what it renders stops rendering\n" ++ " " ++ shun_name_b ++ " already shunned; content already gone\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --force <HASH>...' to proceed\n", w.buffered(), ); } { // A purely pre-emptive run: the tombstones are still irreversible, so the // refusal stands, but nothing would be removed and the line says so // instead of claiming a removal that is not going to happen. var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{.{ .name = shun_name_c, .disposition = .absent }}, 0, false); try std.testing.expectEqualStrings( "zit shun: 1 hash would be shunned in this repository; nothing you named holds content to remove; refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --force <HASH>...' to proceed\n", w.buffered(), ); } } test "renderShunRefusal: a reclaim-only run, and the collateral an earlier --defer left behind" { // THE CONSENT GATE MUST NAME EVERYTHING THE REWRITE DESTROYS, including what the // operator did not name. `renderShunStats` is honest after the fact that // `dropped` can exceed `recorded`; a refusal that only counted this // invocation's own targets would have the operator consent to one deletion and // get several. var buf: [1024]u8 = undefined; var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = null, .disposition = .reclaim }, }, 2, false); try std.testing.expectEqualStrings( "zit shun: no new hash would be recorded; 1 artifact would be permanently removed; " ++ "the same rewrite also removes 2 artifacts shunned earlier and not yet reclaimed; " ++ "refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " file already shunned; content still here and would be removed\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --force <HASH>...' to proceed\n", w.buffered(), ); } test "renderShunRefusal: a --defer run is described as a --defer run, and suggests --defer --force" { // THE CONSENT GATE MUST DESCRIBE THE OPERATION THE OPERATOR ASKED FOR. A // `--defer` run removes nothing now, so it must not say "would be permanently // removed"; it must not attribute rewrite collateral to a run that performs no // rewrite; and — the reason this is pinned byte for byte — the command it // suggests must be the deferred one. An operator who copies `zit shun --force` // out of a `--defer` refusal escalates a tombstone into a full-store reclaim of // the whole union set, including collateral this run never named. var buf: [1024]u8 = undefined; { // `deferred_held` is 2 here and the clause must NOT appear: nothing on this // path reclaims those rows. var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = .manifest, .disposition = .remove }, .{ .name = shun_name_b, .kind = null, .disposition = .remove }, .{ .name = shun_name_c, .disposition = .absent }, }, 2, true); try std.testing.expectEqualStrings( "zit shun: 3 hashes would be shunned in this repository; " ++ "the bytes of 2 artifacts stay on disk and stay reconstructible until 'zit pack'; " ++ "refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " checkin a check-in; its descendants become unresolvable\n" ++ " " ++ shun_name_b ++ " file file content; every check-in listing it loses that file\n" ++ " " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --defer --force <HASH>...' to proceed\n", w.buffered(), ); } { // Singular agreement on the deferred clause, and the `.reclaim` line: under // `--defer` the content stays rather than being removed. var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = .wiki, .disposition = .reclaim }, }, 0, true); try std.testing.expectEqualStrings( "zit shun: no new hash would be recorded; " ++ "the bytes of 1 artifact stay on disk and stay reconstructible until 'zit pack'; " ++ "refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " wiki already shunned; content still here and stays\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --defer --force <HASH>...' to proceed\n", w.buffered(), ); // The escalating suggestion must be ABSENT, not merely reworded: this is the // one substring an operator is most likely to paste. try std.testing.expect(std.mem.indexOf(u8, w.buffered(), "run 'zit shun --force") == null); } { // Nothing named holds content: the wording is the same on both paths, and // only the suggested command differs. var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{.{ .name = shun_name_c, .disposition = .absent }}, 0, true); try std.testing.expectEqualStrings( "zit shun: 1 hash would be shunned in this repository; " ++ "nothing you named holds content to remove; refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --defer --force <HASH>...' to proceed\n", w.buffered(), ); } } test "shunConsequence/shunTypeLabel: one arm per consequence, total over ArtifactType" { // The per-type diagnostic, pinned type by type. The switch is total, so this // test and the compiler together guarantee a type added later cannot ship // with no warning attached to its removal. try std.testing.expectEqualStrings("file", shunTypeLabel(null)); try std.testing.expectEqualStrings( "file content; every check-in listing it loses that file", shunConsequence(null), ); try std.testing.expectEqualStrings("checkin", shunTypeLabel(.manifest)); try std.testing.expectEqualStrings( "a check-in; its descendants become unresolvable", shunConsequence(.manifest), ); try std.testing.expectEqualStrings("cluster", shunTypeLabel(.cluster)); try std.testing.expectEqualStrings("sync housekeeping only; regenerable", shunConsequence(.cluster)); // The six that all stop rendering share one arm; their LABELS still differ, // so the operator can tell a ticket from a forum post. const rendering = "structural content; what it renders stops rendering"; inline for (.{ .control, .wiki, .ticket, .attachment, .event, .forum }) |k| { try std.testing.expectEqualStrings(rendering, shunConsequence(k)); } try std.testing.expectEqualStrings("tag", shunTypeLabel(.control)); try std.testing.expectEqualStrings("technote", shunTypeLabel(.event)); try std.testing.expectEqualStrings("forumpost", shunTypeLabel(.forum)); } test "renderShunBlockers: the unconditional stops, one line each, with no --force offered" { // The current-checkout stop is the only refusal in this verb with no // override, so its text must NOT mention --force — an operator who reads a // flag out of it will try the flag and be refused twice. var buf: [1024]u8 = undefined; var w: std.Io.Writer = .fixed(&buf); renderShunBlockers(&w, &.{ .{ .name = shun_name_a, .kind = .manifest, .disposition = .current_checkout }, .{ .name = "nothex", .disposition = .not_a_name }, .{ .name = "1c04", .disposition = .unresolved }, .{ .name = "7bd1", .disposition = .ambiguous }, // A permissive target contributes nothing to this block. .{ .name = shun_name_c, .disposition = .remove }, }); try std.testing.expectEqualStrings( "zit shun: " ++ shun_name_a ++ " is the check-in this checkout is on; refusing\n" ++ "run 'zit update' to a different check-in first\n" ++ "zit shun: 'nothex' is not an artifact name\n" ++ "zit shun: no artifact matches '1c04'; give the complete name to shun a hash this repository does not hold\n" ++ "zit shun: hash '7bd1' is ambiguous\n", w.buffered(), ); try std.testing.expect(std.mem.indexOf(u8, w.buffered(), "--force") == null); } test "renderShunStats/renderShunDeferred: the two success lines" { var buf: [256]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderShunStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 3, .shuns_recorded = 3, .bytes_before = 481232, .bytes_after = 402118, }); try std.testing.expectEqualStrings( "shunned: 3 hashes recorded, 3 artifacts removed, 12 -> 9 remaining, 481232 -> 402118 bytes\n", w.buffered(), ); } { // `dropped` legitimately EXCEEDS `recorded`: the same rewrite reclaims // shuns deferred by earlier runs, and an operator must be told that // rather than left to infer it from the byte count. var w: std.Io.Writer = .fixed(&buf); renderShunStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 4, .shuns_recorded = 1, .bytes_before = 8192, .bytes_after = 4096, }); try std.testing.expectEqualStrings( "shunned: 1 hash recorded, 4 artifacts removed, 12 -> 8 remaining, 8192 -> 4096 bytes\n", w.buffered(), ); } { // THE RECORDED COUNT IS THE PACK'S, NOT THE CALLER'S. `shuns_recorded` is // measured off the new generation's own shun table, so this line cannot // over-report a takedown even if a caller's intent and the pack's result ever // diverged. Two tombstones, and this rewrite reclaimed nothing. var w: std.Io.Writer = .fixed(&buf); renderShunStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 0, .shuns_recorded = 2, .bytes_before = 8192, .bytes_after = 8320, }); try std.testing.expectEqualStrings( "shunned: 2 hashes recorded, 0 artifacts removed, 12 -> 12 remaining, 8192 -> 8320 bytes\n", w.buffered(), ); } { // --defer promises no byte count: nothing on that path measured one. var w: std.Io.Writer = .fixed(&buf); renderShunDeferred(&w, 2, 2); try std.testing.expectEqualStrings( "shunned: 2 hashes recorded; run 'zit pack' to reclaim the space\n", w.buffered(), ); } { var w: std.Io.Writer = .fixed(&buf); renderShunDeferred(&w, 1, 1); try std.testing.expectEqualStrings( "shunned: 1 hash recorded; run 'zit pack' to reclaim the space\n", w.buffered(), ); } { // Re-deferring a hash already deferred: no row to add, and yet content still // to lose. "0 hashes recorded" would read as a no-op over live bytes. var w: std.Io.Writer = .fixed(&buf); renderShunDeferred(&w, 0, 1); try std.testing.expectEqualStrings( "shunned: no new hash to record; 1 artifact still holds content; run 'zit pack' to reclaim the space\n", w.buffered(), ); } { // The purely pre-emptive success line: no rewrite ran, so it claims no byte // count AND does not claim a removal. var w: std.Io.Writer = .fixed(&buf); renderShunRecordedOnly(&w, 1); try std.testing.expectEqualStrings( "shunned: 1 hash recorded; none is held here, so nothing was removed\n", w.buffered(), ); } } test "renderShunList: persisted order, the recorded date, and the reason only when given" { var buf: [512]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderShunList(&w, &.{}); try std.testing.expectEqualStrings("the shun table is empty\n", w.buffered()); } { // 1753660800 = 2025-07-28T00:00:00Z; 1752192000 = 2025-07-11T00:00:00Z. // A row with no reason prints no trailing separator at all. var w: std.Io.Writer = .fixed(&buf); renderShunList(&w, &.{ .{ .hash = "\x9f\x2a\x1c" ++ "\x00" ** 29, .epoch = 1753660800, .reason = "legal takedown ref #4417" }, .{ .hash = "\x1c\x04\xb7" ++ "\x00" ** 17, .epoch = 1753660800, .reason = "" }, .{ .hash = "\x7b\xd1\xe0" ++ "\x00" ** 29, .epoch = 1752192000, .reason = "committed credential" }, }); try std.testing.expectEqualStrings( "9f2a1c" ++ "00" ** 29 ++ " 2025-07-28 legal takedown ref #4417\n" ++ "1c04b7" ++ "00" ** 17 ++ " 2025-07-28\n" ++ "7bd1e0" ++ "00" ** 29 ++ " 2025-07-11 committed credential\n", w.buffered(), ); } } /// How many rows the store in `dir` holds in its shun table. fn shunRowsIn(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !usize { var fs: storage.FileStorage = undefined; var log = try packOpenReadOnly(gpa, io, dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); return log.shunList().len; } test "shunAppendRows: the append aborts on a changed tail and on a held temp lock, recording nothing" { // THE APPEND PATH'S TWO CONCURRENCY GUARDS, driven directly because neither is // reachable from the CLI in one process: both windows open between the moment // `runShun` classified its targets and the moment this function commits. // // The TAIL guard is what makes the append path's success lines honest. They are // claims about STORE STATE — "none is held here, so nothing was removed" — and a // `zit commit` landing a named hash in that window falsifies them: the tombstone // would be recorded, exit 0, and an operator under a takedown order told nothing // was held while the bytes were live and unreclaimed. // // The LOCK guard is what stops the row being written to an ORPHANED INODE. A // `zit pack` renaming its new generation over `.zit` after our `openFile` takes // the append with it, and the run would still report success — a deferred shun // that silently does not exist, having just told the operator to run `zit pack` // to finish it. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } const digest_a = [_]u8{0xab} ** digest_len; const digest_b = [_]u8{0xcd} ** digest_len; const rows_a = [_]logstore.ShunRecord{.{ .hash = &digest_a, .epoch = 1_753_660_800, .reason = "" }}; const rows_b = [_]logstore.ShunRecord{.{ .hash = &digest_b, .epoch = 1_753_660_800, .reason = "" }}; // The key the runner would have captured off its read-only snapshot. var key_1: logstore.LogStore.LogKey = undefined; { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); key_1 = try src.logKey(); } // Control: with a current key the append lands, so the guards below are failing // for the reason under test rather than because this path never works. try shunAppendRows(gpa, io, tmp.dir, key_1, &rows_a); try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir)); // No temp left behind: the lock file is removed on the success path too. try std.testing.expectError(error.FileNotFound, tmp.dir.openFile(io, pack_tmp_filename, .{})); // That append is itself the "concurrent writer": `key_1` is now stale. const before = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(before); try std.testing.expectError(error.TailChanged, shunAppendRows(gpa, io, tmp.dir, key_1, &rows_b)); // NOTHING WRITTEN: the check precedes the first `putShun`. try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir)); { const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(after); try std.testing.expectEqualSlices(u8, before, after); } var key_2: logstore.LogStore.LogKey = undefined; { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); key_2 = try src.logKey(); } { // Stand in for the concurrent rewrite by holding the temp lock, which makes // this deterministic rather than a race. var held = try tmp.dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = false, .lock = .exclusive, .lock_nonblocking = true, }); defer held.close(io); try std.testing.expectError(error.RewriteBusy, shunAppendRows(gpa, io, tmp.dir, key_2, &rows_b)); try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir)); const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(after); try std.testing.expectEqualSlices(u8, before, after); } tmp.dir.deleteFile(io, pack_tmp_filename) catch {}; // Released and current: the same call now lands. try shunAppendRows(gpa, io, tmp.dir, key_2, &rows_b); try std.testing.expectEqual(@as(usize, 2), try shunRowsIn(gpa, io, tmp.dir)); } /// Render one `ApplyError` through `openApplyFail` into `buf`. fn openFailText(buf: []u8, err: update.ApplyError) []const u8 { var w = Io.Writer.fixed(buf); var sink: [1]u8 = undefined; var wnull = Io.Writer.fixed(&sink); const s: cli.Streams = .{ .out = &wnull, .err = &w }; _ = openApplyFail(s, err); return w.buffered(); } // Same guard as the `zit update`, `zit merge` and `zit bisect` mappers. // `Corrupt` alleges the stored bytes do not match the name they are stored // under; `StateRecordInvalid` says only that an artifact was unusable as a // state record. A clone of an intact fossil repository reaches the second // without anything being damaged, so it must never be answered with a // corruption claim. test "open: the two read-path failures do not share a diagnostic" { var a: [256]u8 = undefined; var b: [256]u8 = undefined; const corrupt = openFailText(&a, error.Corrupt); const state = openFailText(&b, error.StateRecordInvalid); try std.testing.expect(!std.mem.eql(u8, corrupt, state)); try std.testing.expect(std.mem.indexOf(u8, corrupt, "corrupt") != null); try std.testing.expect(std.mem.indexOf(u8, state, "corrupt") == null); } // --------------------------------------------------------------------------- // `zit serve` must answer one sync round in time bounded by the // REQUEST, not by the served store. Two per-request costs made a clone // O(rounds x store): counting held artifacts by forcing a full inflate of every // artifact, and rebuilding the entire in-memory snapshot from the log on every // request. The guards below pin both as SCALING properties measured with work // counters (plaintext bytes materialised, snapshot builds performed) — never // wall-clock thresholds, which flake and prove nothing about the mechanism. // --------------------------------------------------------------------------- /// Seed `ms` with `n` distinct wire-backed artifacts exactly as /// `loadContentStore` does for a served repository: one §4.1 full container per /// artifact, with NO plaintext materialised. `ms.plaintextBytesHeld()` is 0 on /// return, so any later non-zero reading is inflation this store did not ask for. fn serveSeedWireStore(gpa: std.mem.Allocator, ms: *repostate.MemStore, n: usize) !void { var i: usize = 0; while (i < n) : (i += 1) { var body_buf: [64]u8 = undefined; const body = try std.fmt.bufPrint(&body_buf, "issue nine artifact {d}\n", .{i}); var name_buf: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, body, &name_buf); const container = try zit.container.encode(gpa, body, .default); defer gpa.free(container); _ = try ms.putWire(name, container); } std.debug.assert(ms.plaintextBytesHeld() == 0); } test "serve: the held-artifact count serve uses never materialises plaintext (constant in store size)" { const gpa = std.testing.allocator; // Two store sizes: the work this count performs must not grow with n. The // count VALUE alone would pass before and after the fix — it is the // materialised-plaintext reading that observes the defect, so that is what // is asserted, at both sizes, as the scaling statement. for ([_]usize{ 8, 64 }) |n| { var ms = repostate.MemStore.init(gpa); defer ms.deinit(); try serveSeedWireStore(gpa, &ms, n); try std.testing.expectEqual(n, ms.store().heldCount()); try std.testing.expectEqual(@as(usize, 0), ms.plaintextBytesHeld()); } } test "serve: a held artifact whose container does not inflate is still counted as servable" { const gpa = std.testing.allocator; var ms = repostate.MemStore.init(gpa); defer ms.deinit(); // §4.1 shape (4-byte length prefix) with a payload that is not a valid zlib // stream: `held()==true`, `content()==null` — the case `held()`'s contract at // repostate.zig names. `respond` retransmits it verbatim and `persistStore` // counts it, so serve's own count must agree with them. const bogus = "\x00\x00\x00\x0anot-zlib-bytes"; _ = try ms.putWire("f" ** 64, bogus); try std.testing.expectEqual(@as(usize, 1), ms.store().heldCount()); try std.testing.expect(ms.store().held(1)); try std.testing.expect(ms.store().content(1) == null); } /// Append one content frame (raw `digest` naming `bytes`) to the `.zit` in /// `dir`, creating the repository on first call — a separate writer's committed /// append, exactly what a `zit commit` alongside a live `zit serve` performs. fn serveAppendFrame(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, digest: []const u8, bytes: []const u8) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch try dir.createFile(io, repo_filename, .{ .read = true }); defer f.close(io); const st = try f.stat(io); var fs = storage.FileStorage.init(io, f, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer log.deinit(gpa); _ = try log.put(digest, bytes); try log.commit(); } /// A §4.1-framed read-only request naming `hash` in a `gimme` card. fn serveGimmeRequest(gpa: std.mem.Allocator, card: []const u8, hash: []const u8) ![]u8 { const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\n{s} {s}\n", .{ card, hash }); defer gpa.free(plain); return zit.container.encode(gpa, plain, .default); } test "serve: repeated rounds over an unchanged log build the snapshot once (constant in round count)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const digest_a = "a" ** digest_len; try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n"); var hex_buf: [2 * digest_len]u8 = undefined; const hex_a = digestToWireName(digest_a, &hex_buf); const request = try serveGimmeRequest(gpa, "gimme", hex_a); defer gpa.free(request); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); // Eight rounds, log unchanged: the snapshot is a function of the log, not of // the request, so the number of builds must not grow with the round count. for (0..8) |_| { const reply = try serveAnswer(&ctx, gpa, request, null, null); gpa.free(reply.body); } try std.testing.expectEqual(@as(u64, 1), ctx.loads); } test "serve: a responder's phantom interning cannot leak into the next round" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const digest_a = "a" ** digest_len; try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n"); var hex_buf: [2 * digest_len]u8 = undefined; const hex_a = digestToWireName(digest_a, &hex_buf); // §3.3 — an `igot` for a hash the server lacks makes `respond` intern a // phantom into whatever store it answers over. That mutation must not // survive into the reused snapshot. const unknown_hex = "c" ** (2 * digest_len); const igot_req = try serveGimmeRequest(gpa, "igot", unknown_hex); defer gpa.free(igot_req); const gimme_req = try serveGimmeRequest(gpa, "gimme", hex_a); defer gpa.free(gimme_req); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); // NOTE ON THIS GUARD'S ROLE: it is green against the pre-fix code (which // rebuilds per request and so cannot leak) — the round-count guard above is // the red one. This guard exists because the obvious WRONG way to satisfy // that guard — answer directly over one shared mutable snapshot — passes it // and truncates real clones, since the phantoms and memoisations `respond` // writes change what later rounds serve. Without this assertion that // implementation ships. var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const r1 = try serveAnswer(&ctx, gpa, igot_req, null, null); gpa.free(r1.body); const snap = ctx.snapshot.?.store(); try std.testing.expectEqual(@as(usize, 1), snap.count()); // the one on-disk artifact, nothing more try std.testing.expect(snap.lookup(unknown_hex) == null); // the phantom did not land here const r2 = try serveAnswer(&ctx, gpa, gimme_req, null, null); defer gpa.free(r2.body); try std.testing.expectEqual(@as(u64, 1), ctx.loads); // still one build: the log never changed // The reply is byte-identical to the one a server with no prior round produces. var fresh_file = try tmp.dir.openFile(io, repo_filename, .{}); defer fresh_file.close(io); var fresh_ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &fresh_file, .server = &server }; defer fresh_ctx.dropSnapshot(); const expected = try serveAnswer(&fresh_ctx, gpa, gimme_req, null, null); defer gpa.free(expected.body); try std.testing.expectEqualSlices(u8, expected.body, r2.body); } test "serve §3.8: a reused snapshot still sees a concurrent commit's append" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const digest_a = "a" ** digest_len; const digest_b = "b" ** digest_len; try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n"); var hex_a_buf: [2 * digest_len]u8 = undefined; var hex_b_buf: [2 * digest_len]u8 = undefined; const hex_a = digestToWireName(digest_a, &hex_a_buf); const hex_b = digestToWireName(digest_b, &hex_b_buf); const req_a = try serveGimmeRequest(gpa, "gimme", hex_a); defer gpa.free(req_a); const req_b = try serveGimmeRequest(gpa, "gimme", hex_b); defer gpa.free(req_b); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const r1 = try serveAnswer(&ctx, gpa, req_a, null, null); gpa.free(r1.body); try std.testing.expectEqual(@as(usize, 1), ctx.snapshot.?.store().count()); // A SEPARATE writer appends while the serve handle stays open. try serveAppendFrame(gpa, io, tmp.dir, digest_b, "content b\n"); const r2 = try serveAnswer(&ctx, gpa, req_b, null, null); gpa.free(r2.body); // The log grew, so the snapshot was rebuilt and the new artifact is servable: // a cache that never invalidates would still report one artifact and one load. try std.testing.expectEqual(@as(u64, 2), ctx.loads); try std.testing.expectEqual(@as(usize, 2), ctx.snapshot.?.store().count()); try std.testing.expect(ctx.snapshot.?.store().lookup(hex_b) != null); } /// Seed `dir`'s `.zit` with `n` distinct compressible artifacts, returning the /// wire (hex) name of the first. Compressible so the log retains a §4.1 /// container per frame — the shape a served repository actually has, and the /// only shape in which "answering a round does not inflate the store" has /// anything to say. fn serveSeedRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, n: usize, first_hex: *[2 * digest_len]u8) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch try dir.createFile(io, repo_filename, .{ .read = true }); defer f.close(io); const st = try f.stat(io); var fs = storage.FileStorage.init(io, f, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer log.deinit(gpa); var i: usize = 0; while (i < n) : (i += 1) { var body: [512]u8 = undefined; @memset(&body, 'a' + @as(u8, @intCast(i % 26))); var digest: [digest_len]u8 = undefined; @memset(&digest, 0); std.mem.writeInt(u64, digest[0..8], i + 1, .big); _ = try log.putZlib(&digest, &body); if (i == 0) _ = digestToWireName(&digest, first_hex); } try log.commit(); } test "serve: answering a round inflates only what the round emits (constant in store size)" { const gpa = std.testing.allocator; const io = std.testing.io; // The same read round against two store sizes. The reply is identical, so // the plaintext the server inflates to produce it must be identical too: any // dependence on n is the server paying for artifacts the round never names. // This is the assertion the whole change exists to make true — the pre-fix // held-count walked `content()` over every rid, and so did inflate the store. var inflated: [2]usize = undefined; for ([_]usize{ 4, 40 }, 0..) |n, k| { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var hex_buf: [2 * digest_len]u8 = undefined; try serveSeedRepo(gpa, io, tmp.dir, n, &hex_buf); const request = try serveGimmeRequest(gpa, "gimme", &hex_buf); defer gpa.free(request); const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); for (0..3) |_| { const reply = try serveAnswer(&ctx, gpa, request, null, null); gpa.free(reply.body); // Nothing the round materialised is still resident: the cached index // reads containers on demand and frees them between rounds, never // holding the repository's uncompressed history (or its containers). try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.residentBytes()); } const emitted = ctx.snapshot.?.materialized_bytes; // A CLONE round is the worst case for residency: it emits every artifact // and proves each one's declared size against the plaintext, so it // legitimately inflates the whole store. What must not survive it is the // plaintext — a snapshot that outlives the round would otherwise end up // holding the entire repository uncompressed, for the process's life. const clone_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 1\n", .{}); defer gpa.free(clone_plain); const clone_req = try zit.container.encode(gpa, clone_plain, .default); defer gpa.free(clone_req); const clone_reply = try serveAnswer(&ctx, gpa, clone_req, null, null); gpa.free(clone_reply.body); try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.residentBytes()); try std.testing.expect(ctx.snapshot.?.materialized_bytes >= emitted); try std.testing.expectEqual(@as(usize, n), ctx.snapshot.?.store().count()); inflated[k] = emitted; } try std.testing.expectEqual(inflated[0], inflated[1]); // ... and the constant is the round's own emission, not the store's: a // 40-artifact store holds 20480 bytes of plaintext, so an equality that // happened to hold at "everything" would still be caught here. try std.testing.expect(inflated[1] < 4 * 512); } test "serve: the pushing round does not rebuild inside its deadline; maintain rebuilds between connections" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.commit(); } const push_body = "pushed artifact bytes\n"; var name_buf: [64]u8 = undefined; const push_hash = zit.artifact.artifactName(.sha3_256, push_body, &name_buf); const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ push_hash, push_body.len, push_body }); defer gpa.free(plain); const push_req = try zit.container.encode(gpa, plain, .default); defer gpa.free(push_req); const read_req = try serveGimmeRequest(gpa, "gimme", push_hash); defer gpa.free(read_req); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const r0 = try serveAnswer(&ctx, gpa, read_req, null, null); gpa.free(r0.body); try std.testing.expectEqual(@as(u64, 1), ctx.loads); // The push makes the snapshot stale, but answering it must NOT pay the // O(store) rebuild: the reply is written inside a connection deadline, and // The failure is exactly the whole-repository rebuild landing inside that // budget. The rebuild is deferred to `maintain`, which the accept loop runs // after the connection closes — outside every deadline. So the pushing round // leaves `loads` untouched. const r1 = try serveAnswer(&ctx, gpa, push_req, null, null); gpa.free(r1.body); try std.testing.expectEqual(@as(u64, 1), ctx.loads); // no rebuild inside the round // The snapshot is momentarily stale — it predates the push — yet the push is // already durable on disk: `persistStore` ran inside the round. try std.testing.expect(ctx.snapshot != null); try std.testing.expect(ctx.snapshot.?.store().lookup(push_hash) == null); // The accept loop's post-connection maintenance hook does the O(store) // rebuild, between connections, with no connection open to charge it to. ServeCtx.maintain(&ctx, gpa); try std.testing.expectEqual(@as(u64, 2), ctx.loads); try std.testing.expect(ctx.snapshot.?.store().lookup(push_hash) != null); // The rebuilt snapshot describes the grown log exactly (the append-only stat // gate that `baseSnapshot` already carries). const st = try file.stat(io); try std.testing.expectEqual(st.size, ctx.snapshot_log_size); // A later read round is stat-gated to no rebuild, and still serves the push. const r2 = try serveAnswer(&ctx, gpa, read_req, null, null); defer gpa.free(r2.body); try std.testing.expectEqual(@as(u64, 2), ctx.loads); } // --------------------------------------------------------------------------- // sync §3.3 L527-530 / §8.8 L1777-1779 — the durable side of the phantom-scan // shun filter: the repository shun table, snapshotted off the log and answered // as a wire-name question. `RepoShunSet` lives above with `LogIndexStore`; these // tests pin the key domain and the end-to-end suppression. // --------------------------------------------------------------------------- /// The shunned artifact's digest and its complete 64-hex wire name. 's' is 0x73, /// so the name is the byte pattern spelled out — no runtime encode in the vector. const shun_test_digest = "s" ** digest_len; const shun_test_name = "73" ** digest_len; /// A second artifact the repo actually holds ('h' is 0x68), to prove the filter /// suppresses only the shunned name. const shun_test_held_digest = "h" ** digest_len; const shun_test_held_name = "68" ** digest_len; /// A third shunned digest whose hex spelling contains LETTER nibbles, so the /// uppercase-spelling question below is a real one. const shun_test_letter_digest = "\xab" ** digest_len; const shun_test_letter_name = "ab" ** digest_len; comptime { std.debug.assert(shun_test_name.len == 2 * digest_len); std.debug.assert(logstore.isCompleteDigest(shun_test_digest.len)); } /// Seed `dir`'s repository with one held artifact and one shun row naming an /// artifact it does NOT hold — exactly the state a `zit shun` on a hash the peer /// still offers leaves behind. fn seedShunnedRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); _ = try log.put(shun_test_held_digest, "held content\n"); try log.putShun(shun_test_digest, "operator request", 1); try log.putShun(shun_test_letter_digest, "operator request", 1); try log.commit(); } test "sync §3.3 L527-530: a repository shun row is a durable wire-name hit, keyed exactly" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedShunnedRepo(gpa, io, tmp.dir); var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const shun = &holdings.index.shun; const set = holdings.admission().shun; // The complete name is the key: a hit. try std.testing.expect(set.contains(shun_test_name)); // The held artifact is not shunned. try std.testing.expect(!set.contains(shun_test_held_name)); // A 12-hex PREFIX of the shunned name is a DIFFERENT key, not a hit. The // lookup never truncates, pads, or prefix-matches — a prefix must not be // able to masquerade as a shun-table hit. Nor may an incomplete name reach // `LogStore.isShunned`, whose caller contract (src/logstore.zig L1746-1757) // binds this boundary: the width check is a CHECKED rejection, not an // assert, because an assert is nothing in ReleaseFast and these names come // off the wire. // // These OUT-OF-DOMAIN widths are probed against `containsWireName` directly, // not through `zit.sync.ShunSet.contains`: the release-checked rule lives // here, in the implementation that owns it, while the wrapper asserts the // §3.2 width its own key domain promises. Going through the wrapper would // pin the assert instead of the rejection these lines mean to pin. try std.testing.expect(!RepoShunSet.containsWireName(shun, shun_test_name[0..12])); try std.testing.expect(!RepoShunSet.containsWireName(shun, shun_test_name[0 .. 2 * digest_len - 1])); // odd length try std.testing.expect(!RepoShunSet.containsWireName(shun, "")); // empty try std.testing.expect(!RepoShunSet.containsWireName(shun, "ab" ** 48)); // 96 hex — not one of the two forms // Correct WIDTH, not hex: in the wrapper's key domain, so probed through it. try std.testing.expect(!set.contains("z" ** (2 * digest_len))); // not hex // An UPPERCASE spelling of the shunned name decodes to the same digest and // so answers TRUE. artifact-format §3.3 L370-377 makes the token invalid but // lets an implementation accept it at name validation and fail it at // verification; over-suppressing a name that can never verify is the safe // direction. Pinned so the fail-closed direction is a decision, not an // accident of `hexToBytes` accepting case. try std.testing.expect(set.contains(shun_test_letter_name)); var upper: [2 * digest_len]u8 = undefined; for (shun_test_letter_name, 0..) |c, i| upper[i] = std.ascii.toUpper(c); try std.testing.expect(!std.mem.eql(u8, &upper, shun_test_letter_name)); // the case really differs try std.testing.expect(set.contains(&upper)); } test "sync §3.3 L527-530: an OOM inside the shun-set load leaves no borrowed key in the owned-key map" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedShunnedRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var fs = storage.FileStorage.init(io, file, st.size); var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile()); defer log.deinit(gpa); try std.testing.expect(log.shunList().len >= 2); // `load` dupes each row's digest because `shunList` borrows the STORE's own // bytes, and the dupe must PRECEDE the insert: insert-then-replace leaves the // borrowed slice live as a key on the one error path, after which `deinit` // frees the store's digest and `log.deinit` frees it again. A manual sweep // (not `checkAllAllocationFailures`) because the obligation is about which // POINTER survives the unwind, not just about leaking: the store is // allocated from the UNWRAPPED allocator, so a key that is not `load`'s own // dupe is freed through the wrap and the double free lands on // `testing.allocator`, which catches it. `fail_index` indexes `load`'s // allocations alone for the same reason. var i: usize = 0; while (i < 64) : (i += 1) { var fa = std.testing.FailingAllocator.init(gpa, .{ .fail_index = i }); var shun = RepoShunSet.init(fa.allocator()); defer shun.deinit(); if (shun.load(&log)) |_| { // The fail point lay at or past the last allocation: no failure was // forced, so every allocation site is now covered and the set is // whole. try std.testing.expect(shun.digests.count() >= 2); break; } else |e| { try std.testing.expectEqual(error.OutOfMemory, e); } } else return error.SweepNeverCompleted; } test "sync §3.3 L527-530: a pull client carrying the repository shun set creates the phantom but never gimme's it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedShunnedRepo(gpa, io, tmp.dir); var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); // The pull client, constructed exactly as `pullRun` does. var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = sync_servercode, .project_code = sync_projectcode, .version = .{ .version = sync_client_version }, .shun = holdings.admission().shun, }; defer client.deinit(gpa); // The peer announces BOTH artifacts. §3.3 r24(1) is unconditional on the // shun table, so the shunned one still becomes a phantom. const reply = "pragma server-version 1\nigot " ++ shun_test_name ++ "\n"; const obs = try client.consumeReply(gpa, holdings.store(), reply); try std.testing.expect(obs.phantom_created); try std.testing.expectEqual(@as(usize, 1), client.phantoms.count()); // …and the request never asks for it, in any round. var buf: std.ArrayListUnmanaged(u8) = .empty; defer buf.deinit(gpa); var round: usize = 0; while (round < 3) : (round += 1) { const request = try client.buildRequest(gpa, holdings.store(), &buf); try std.testing.expect(std.mem.indexOf(u8, request, "gimme " ++ shun_test_name) == null); // The held artifact is still announced — nothing else changed. try std.testing.expect(std.mem.indexOf(u8, request, "igot " ++ shun_test_held_name) != null); } } /// Does the decoded reply body carry `gimme NAME` as a whole card line? fn replyHasGimme(gpa: std.mem.Allocator, reply_container: []const u8, name: []const u8) !bool { const body = try zit.container.decode(gpa, reply_container, @intCast(logstore.max_payload)); defer gpa.free(body); var it = std.mem.splitScalar(u8, body, '\n'); while (it.next()) |line| { if (line.len == 6 + name.len and std.mem.startsWith(u8, line, "gimme ") and std.mem.eql(u8, line[6..], name)) return true; } return false; } test "sync §3.3 L527-530 / §8.8 L1777-1779: a served reply never gimme's a shunned phantom, and a shun row written between rounds is honoured by the next" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A repo holding one artifact and NO shun row yet. const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer seed.deinit(gpa); _ = try seed.put(shun_test_held_digest, "held content\n"); try seed.commit(); } // A client `igot` for a hash this repo lacks: §3.3 r24(1) L511-514 makes the // server a receiver, so `respond` interns the phantom and would beg for it. const igot_req = try serveGimmeRequest(gpa, "igot", shun_test_name); defer gpa.free(igot_req); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); // Round 1, no shun row: the reply asks for it. This is the behaviour the // filter must change, so the test is red on an unfiltered server. { const r = try serveAnswer(&ctx, gpa, igot_req, null, null); defer gpa.free(r.body); try std.testing.expect(try replyHasGimme(gpa, r.body, shun_test_name)); } // The operator shuns it BETWEEN rounds — an append to the same log. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); const st = try f.stat(io); var fs = storage.FileStorage.init(io, f, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putShun(shun_test_digest, "operator request", 1); try log.commit(); } // Round 2 honours it with no restart: the append grew the log, so // `baseSnapshot`'s stat gate rebuilds the index — and the rebuild re-loads // the shun table (`LogIndexStore.shun`). That is the freshness mechanism this // set rides; nothing here is memoised past a build. { const r = try serveAnswer(&ctx, gpa, igot_req, null, null); defer gpa.free(r.body); try std.testing.expect(!try replyHasGimme(gpa, r.body, shun_test_name)); // The unshunned holding is still advertised: only the shunned name went quiet. const body = try zit.container.decode(gpa, r.body, @intCast(logstore.max_payload)); defer gpa.free(body); try std.testing.expect(std.mem.indexOf(u8, body, "igot " ++ shun_test_held_name) != null); } try std.testing.expectEqual(@as(u64, 2), ctx.loads); // one build per log state, not per round } // --------------------------------------------------------------------------- // sync §9.5 L1988-1992 — the SEND side of the shun filter. // // "A subsequent read-authorized transfer path requests igot for every // eligible holding after the Section 3.3 shun, phantom, private, and // peer-has filters" // // `LogIndexStore`'s enumeration is that "every eligible holding", so the shun // filter belongs in the enumeration itself: a shunned holding is masked, which // makes the server neither `igot`-announce it (the §3.2 walk's `held()` gate // answers false) nor serve it in a clone batch (no wire form, no content), and // makes an inbound `gimme` for it land on §3.3 L532-535's *"If absent, silently // skip"* with no code on that path — the name is simply not in `by_name`. // // Which of those three the MASK is load-bearing for, stated honestly: the igot // announcement and the `gimme` miss are new here — nothing else suppressed them. // The clone batch was already covered by `LogStore.wire`/`content`'s own rid-keyed // read mask, which the pager gates on; the mask's contribution there is that the // pager's cursor stays contiguous over the survivors (see the clone-pager test). // // The mask is a MASK, not a removal, and §8.2 L1504-1506 is why: // // "SEQNO is a server blob-table RID. Iterate from supplied RID through // current maximum, silently skipping gaps; clients MUST NOT infer count // or size from range." // // Dropping the shunned frame from the enumeration would renumber every survivor // above it. The server is stateless (§8.9 L1783-1784) — the client's SEQNO *is* the // clone cursor — and a shun row appended between two rounds of one clone grows // the log, which is exactly what trips `ServeCtx.baseSnapshot`'s append-only stat // gate and forces a rebuild. Under renumbering the resuming client's cursor would // then step over a survivor and still be told the sequence was complete: a // silently short clone, the failure class `read_fault` exists to prevent. Keeping // the slot and answering it as a §8.2 GAP makes adapter-rid identity a pure // function of the log, independent of the shun table. // --------------------------------------------------------------------------- /// Five artifact digests in persisted order. Distinct byte fills, so each wire /// name is the pattern spelled out and no vector needs a runtime encode. const send_shun_digests = [5][]const u8{ "\x11" ** digest_len, "\x22" ** digest_len, "\x33" ** digest_len, "\x44" ** digest_len, "\x55" ** digest_len, }; const send_shun_names = [5][]const u8{ "11" ** digest_len, "22" ** digest_len, "33" ** digest_len, "44" ** digest_len, "55" ** digest_len, }; /// The FIRST, a MIDDLE and the LAST of the five are shunned, so an off-by-one or /// a misindexed mask cannot pass: the survivors are interleaved among them. fn sendShunIsShunned(i: usize) bool { return i % 2 == 0; } /// Seed `dir`'s repository holding all five `send_shun_digests` in order, with a /// live config frame BEFORE the first and another BETWEEN the second and third, /// and shun rows naming the first, middle and last. /// /// The interleaving is what an off-by-one cannot survive: shunning the FIRST, a /// MIDDLE and the LAST slot means a mask built by any walk other than the /// enumeration's own loop names one artifact and suppresses another, and the /// alignment test below reads that straight off the log. The config frames are /// here for a WEAKER reason than they look: they never enter /// `ridsInPersistedOrder` at all (a config frame is not interned — `LogStore.scan` /// only records its offset), so they exercise the `.config` guards' *presence*, not /// a live skip; they do not by themselves make a second walk misalign. fn seedSendShunRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); _ = try log.put(send_shun_digests[0], "content zero\n"); _ = try log.put(send_shun_digests[1], "content one\n"); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "cccc" }); _ = try log.put(send_shun_digests[2], "content two\n"); _ = try log.put(send_shun_digests[3], "content three\n"); _ = try log.put(send_shun_digests[4], "content four\n"); for (send_shun_digests, 0..) |d, i| { if (sendShunIsShunned(i)) try log.putShun(d, "operator request", 1); } try log.commit(); } test "sync §9.5 L1988-1992: a shunned holding is masked out of the adapter enumeration and its name is absent from by_name" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // §8.2 L1504-1506 — the rid space is the LOG's, not the survivors': all five // artifact frames still hold their slot (config frames were never in it). try std.testing.expectEqual(@as(usize, 5), index.rids.len); try std.testing.expectEqual(@as(usize, 5), s.count()); for (send_shun_names, 0..) |name, i| { const rid: repostate.Rid = @intCast(i + 1); if (sendShunIsShunned(i)) { // Not resolvable by name: the §3.2 gimme arm keys on `lookup`, so // this IS §3.3 L532-535's "If absent, silently skip". try std.testing.expect(index.by_name.get(name) == null); try std.testing.expect(s.lookup(name) == null); // Not announced (§3.2's igot walk gates on `held`), and nothing to // send in either form. try std.testing.expect(!s.held(rid)); try std.testing.expect(s.content(rid) == null); try std.testing.expect(s.wire(rid) == null); } else { try std.testing.expectEqual(@as(?repostate.Rid, rid), s.lookup(name)); try std.testing.expect(s.held(rid)); try std.testing.expect(s.content(rid) != null); } } // A masked frame is a policy gap, never a read fault: the round must not be // converted to `error.ServerStorage` by `serveAnswer`. try std.testing.expect(!index.read_fault); } /// Does the decoded reply body carry `VERB NAME` as a whole card line? fn replyHasCardFor(body: []const u8, verb: []const u8, name: []const u8) bool { var it = std.mem.splitScalar(u8, body, '\n'); while (it.next()) |line| { if (line.len != verb.len + 1 + name.len) continue; if (!std.mem.startsWith(u8, line, verb)) continue; if (line[verb.len] != ' ') continue; if (std.mem.eql(u8, line[verb.len + 1 ..], name)) return true; } return false; } test "sync §9.5 L1988-1992: a served round igot-announces the survivors and never a masked holding" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); // A round that makes the server run its §3.2 inventory walk over every rid. // The announced hash is one this repo does not hold, so nothing else about // the request steers which holdings the walk reaches. const req = try serveGimmeRequest(gpa, "igot", shun_test_name); defer gpa.free(req); const reply = try serveAnswer(&ctx, gpa, req, null, null); defer gpa.free(reply.body); const body = try zit.container.decode(gpa, reply.body, @intCast(logstore.max_payload)); defer gpa.free(body); for (send_shun_names, 0..) |name, i| { if (sendShunIsShunned(i)) { // §9.5 L1988-1992 — not among "every eligible holding": the walk's // `held()` gate answers false, so no igot names it. try std.testing.expect(!replyHasCardFor(body, "igot", name)); // Nor does it fall out of the walk as a PHANTOM to beg for. The // enumeration now hands the walk a non-held rid, and it is `Server.shun` // — the §3.3 L527-530 arm this repository's own table feeds through // `serveRound` — that keeps the reply quiet. That check is not dead // after this slice; the mask is what makes it load-bearing. try std.testing.expect(!replyHasCardFor(body, "gimme", name)); } else { try std.testing.expect(replyHasCardFor(body, "igot", name)); } } // A masked holding is not an error condition on any card. try std.testing.expect(std.mem.indexOf(u8, body, "error") == null); } test "sync §9.5 L1988-1992: the masked enumeration stays aligned — adapter rid r names rids[r-1] with shunned first, middle and last" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // `rids`, `names`, `masked`, `content_memo` and `wire_memo` are PARALLEL // arrays and `by_name` maps a wire name to that same index. Filtering or // masking one out of step with the others makes adapter rid r name a // DIFFERENT artifact's bytes — silent corruption on the wire, not an error. // Pin the whole alignment against the log itself, entry by entry. var shunned_seen: usize = 0; for (index.rids, 0..) |lr, a| { var buf: [2 * digest_len]u8 = undefined; const expect_name = digestToWireName(index.log.hashOf(lr), &buf); const rid: repostate.Rid = @intCast(a + 1); try std.testing.expectEqualStrings(expect_name, index.names[a]); try std.testing.expectEqualStrings(expect_name, s.hashOf(rid)); try std.testing.expectEqualStrings(send_shun_names[a], expect_name); // The mask indexes the SAME slot as `rids[a]` — the invariant a mask // built over any second walk over the log breaks (verified: a mask // written one slot out of step turns this expectation red). try std.testing.expectEqual(index.shun.digests.contains(index.log.hashOf(lr)), index.masked[a]); if (index.masked[a]) { shunned_seen += 1; try std.testing.expect(index.by_name.get(expect_name) == null); } else { try std.testing.expectEqual(@as(?repostate.Rid, rid), index.by_name.get(expect_name)); } } // The fixture really does interleave: three shunned, two survivors. try std.testing.expectEqual(@as(usize, 3), shunned_seen); try std.testing.expectEqual(@as(usize, 2), index.by_name.count()); } test "sync §9.5 L1988-1992 / §7.7 L1437-1440: a write against a masked wire name does not unmask it — no second identity, still not held, still not servable" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // The masked name is absent from `by_name`, so every write door below misses the // dedup key the survivors hit and would otherwise fall through to the appendix — // minting a SECOND, HELD entry for a name §9.5 says this server does not send, // and putting it back on the §3.2 walk. The mask outranks the write instead. const masked_name = send_shun_names[0]; try std.testing.expect(index.masked[0]); const before = s.count(); try std.testing.expectEqual(@as(repostate.Rid, 1), try s.intern(masked_name)); try std.testing.expectEqual(@as(repostate.Rid, 1), try s.put(masked_name, "content zero\n")); // Even bytes that DISAGREE with the frame: the mask is not a content check, and // this path must not read (nor assert on) a masked frame's plaintext. try std.testing.expectEqual(@as(repostate.Rid, 1), try s.put(masked_name, "different bytes\n")); try s.retain(1, .{ .full = "not a real container" }); // Nothing was created and nothing was unmasked. try std.testing.expectEqual(before, s.count()); try std.testing.expectEqual(@as(usize, 2), index.by_name.count()); try std.testing.expect(!s.held(1)); try std.testing.expect(s.content(1) == null); try std.testing.expect(s.wire(1) == null); // §3.3 L532-535 still applies to an inbound `gimme` for it: absent, silently // skipped, no second rid for the walk to trip over. try std.testing.expect(s.lookup(masked_name) == null); try std.testing.expect(!index.read_fault); // NEGATIVE SPACE: the appendix arm is intact for a name the log genuinely does // not carry — the guard above narrows the masked case only. const absent_name = "99" ** digest_len; const appendix_rid = try s.intern(absent_name); try std.testing.expectEqual(@as(repostate.Rid, 6), appendix_rid); try std.testing.expectEqual(before + 1, s.count()); try std.testing.expect(!s.held(appendix_rid)); } test "sync §7.7 L1444-1450 / §9.5 L1988-1992: the adapter owns its content mask — the frame's plaintext is still on disk and still readable through the maintenance door" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // §9.5's filter is a POLICY answer about what this server SENDS, not a deletion: // the frame, its bytes and its checksums are untouched on disk. §7.7 L1444-1450 // depends on exactly that — "For an admitted target whose DELTASRC name is in // the repository shun table, the receiver does not apply admission independently // to that source: content still held locally MAY serve as the delta basis" — so // "still held locally" is a property this slice must not have quietly destroyed. // `LogStore.contentUnmasked` is the maintenance door that reads under no mask // (`pack.zig`'s reader), and it is the only way to distinguish "the bytes are // gone" from "the send side refuses to hand them out". try std.testing.expect(index.masked[0]); const on_disk = try index.log.contentUnmasked(gpa, index.rids[0], null); defer if (on_disk) |b| gpa.free(b); try std.testing.expectEqualStrings("content zero\n", on_disk.?); // ...and the store still refuses, in both forms, without calling it a fault. try std.testing.expect(s.content(1) == null); try std.testing.expect(s.wire(1) == null); try std.testing.expect(!index.read_fault); // Nothing was materialised on the way: a refusal is answered from the mask, so // a masked frame is never inflated into the round's memo (which is also what // keeps a shunned artifact's plaintext out of a server's memory). try std.testing.expectEqual(@as(usize, 0), index.materialized_bytes); try std.testing.expectEqual(@as(usize, 0), index.residentBytes()); // NEGATIVE SPACE: the maintenance door is not a blanket unmask of the store — // a SURVIVOR reads identically through both, so the asymmetry above is the // mask's doing and not an artefact of which reader was called. const survivor = try index.log.contentUnmasked(gpa, index.rids[1], null); defer if (survivor) |b| gpa.free(b); try std.testing.expectEqualStrings(survivor.?, s.content(2).?); } test "sync §9.5 L1988-1992 / §7.7 L1437-1440: a served round's writes land in the round-local overlay — the base enumeration's write doors are never reached" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // `seedSendShunRepo`'s digests are synthetic, so no peer could legitimately push // one back — the responder hashes the payload and would reject the name long // before reaching any store door. This fixture names its two artifacts by their // REAL content address, so the push below is one a real peer could send. const shunned_body = "shunned artifact bytes\n"; const survivor_body = "surviving artifact bytes\n"; var shunned_buf: [64]u8 = undefined; var survivor_buf: [64]u8 = undefined; const shunned_name = zit.artifact.artifactName(.sha3_256, shunned_body, &shunned_buf); const survivor_name = zit.artifact.artifactName(.sha3_256, survivor_body, &survivor_buf); var shunned_dbuf: [digest_len]u8 = undefined; var survivor_dbuf: [digest_len]u8 = undefined; const shunned_digest = hexToDigest(shunned_name, &shunned_dbuf) orelse return error.BadTestName; const survivor_digest = hexToDigest(survivor_name, &survivor_dbuf) orelse return error.BadTestName; const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer seed.deinit(gpa); try seed.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); _ = try seed.put(shunned_digest, shunned_body); _ = try seed.put(survivor_digest, survivor_body); try seed.putShun(shunned_digest, "operator request", 1); try seed.commit(); } const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); // A peer pushes the very name this repository has shunned. const push_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ shunned_name, shunned_body.len, shunned_body }); defer gpa.free(push_plain); const push_req = try zit.container.encode(gpa, push_plain, .default); defer gpa.free(push_req); { const reply = try serveAnswer(&ctx, gpa, push_req, null, null); defer gpa.free(reply.body); // THE PROPERTY. `putImpl`/`internImpl` deliberately NARROW the store contract // for a masked name — they answer a rid whose `content` is null and whose // `held` is false — and the vtable's `error{OutOfMemory}` cannot signal that. // What keeps the narrowing safe is that a served round never reaches those // doors at all: `respond` writes into a per-request `repostate.OverlayStore` // that is discarded at round end. `appendix` is where any write that DID // reach this base would land, so its emptiness is that claim, measured. const base = ctx.snapshot.?; try std.testing.expectEqual(@as(usize, 0), base.appendix.store().count()); try std.testing.expectEqual(@as(usize, 2), base.store().count()); // ...and the push did not unmask the name it carried. try std.testing.expect(base.masked[0]); try std.testing.expect(base.store().lookup(shunned_name) == null); try std.testing.expect(!base.store().held(1)); try std.testing.expect(base.store().content(1) == null); try std.testing.expect(!base.read_fault); // The survivor is untouched by any of it. try std.testing.expectEqual(@as(?repostate.Rid, 2), base.store().lookup(survivor_name)); } // NEGATIVE SPACE: it is not the MASK that keeps the base's doors shut, it is the // overlay. A push of a name this repository has never seen — the case that has a // real appendix arm waiting for it — must land in the round-local store just the // same. (This round's `baseSnapshot` rebuilds over the grown log first, so the // snapshot read below is the one this round actually served from.) const fresh_body = "a name this repository has never held\n"; var fresh_buf: [64]u8 = undefined; const fresh_name = zit.artifact.artifactName(.sha3_256, fresh_body, &fresh_buf); const fresh_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ fresh_name, fresh_body.len, fresh_body }); defer gpa.free(fresh_plain); const fresh_req = try zit.container.encode(gpa, fresh_plain, .default); defer gpa.free(fresh_req); const reply2 = try serveAnswer(&ctx, gpa, fresh_req, null, null); defer gpa.free(reply2.body); try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.appendix.store().count()); } /// The `clone_seqno NEXT` cursor a clone reply carries (§8.2 L1517-1520: "An /// accepting clone reply emits zero or more clone cfile cards in RID order … then /// clone_seqno NEXT, push SERVERCODE PROJECTCODE, and final comment. … NEXT zero /// signals exhaustion."), read off the decoded body. fn cloneSeqnoOf(body: []const u8) !u64 { var it = std.mem.splitScalar(u8, body, '\n'); while (it.next()) |line| { if (!std.mem.startsWith(u8, line, "clone_seqno ")) continue; return std.fmt.parseInt(u64, line["clone_seqno ".len..], 10) catch return error.BadCloneSeqno; } return error.NoCloneSeqno; } /// Does the decoded reply body carry a `cfile` card for `name`? Keyed on the card /// prefix plus the separating space, so a name that is a prefix of another cannot /// answer for it. fn replyHasCfile(gpa: std.mem.Allocator, body: []const u8, name: []const u8) !bool { const needle = try std.fmt.allocPrint(gpa, "cfile {s} ", .{name}); defer gpa.free(needle); return std.mem.indexOf(u8, body, needle) != null; } // WHAT THIS TEST PINS, and what it does NOT. The masked frames' absence from the // clone stream is NOT this slice's doing: the pager gates on `store.wire()` then // `store.content()`, and `LogStore.wire`/`content` already answered null for a // shunned entry through their own rid-keyed read mask before the enumeration was // masked at all — so this test passes with `LogIndexStore`'s exclusion removed // (confirmed by mutation). What it does pin, and what nothing else pins, is that // the mask keeps the pager's persisted-order cursor CONTIGUOUS over the survivors: // each survivor arrives exactly once across paged batches and the sequence // terminates at NEXT 0. A mask written out of step with `rids` turns it red. test "sync §8.2 L1504-1506 / §9.5 L1988-1992: the clone pager walks the survivors contiguously across paged batches and terminates at NEXT 0" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); // A one-artifact batch cap forces the pager to page, so the persisted-order // cursor is exercised at every masked frame rather than swallowed by a single // full-store batch. const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, .clone_batch_max = 1, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); var delivered = [_]usize{0} ** 5; var seqno: u64 = 1; var rounds: usize = 0; while (rounds < 16) : (rounds += 1) { const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 {d}\n", .{seqno}); defer gpa.free(plain); const req = try zit.container.encode(gpa, plain, .default); defer gpa.free(req); const reply = try serveAnswer(&ctx, gpa, req, null, null); defer gpa.free(reply.body); // §8.1 L1489-1491 / §4.1 L885-887 — a clone reply is the RAW card/cfile stream under // `application/x-fossil-uncompressed`, never a zlib container. try std.testing.expectEqualStrings("application/x-fossil-uncompressed", reply.content_type); const body = reply.body; for (send_shun_names, 0..) |name, i| { if (try replyHasCfile(gpa, body, name)) delivered[i] += 1; } const next = try cloneSeqnoOf(body); if (next == 0) break; // A LOOP GUARD, not a protocol requirement: §8.2 L1521-1523 says the // opposite — "The client records a digit-only NEXT and does not validate // that a positive value advances from the supplied SEQNO". A pager that // returned a non-advancing cursor would spin this test to its round cap // instead of failing on the thing it is here to measure, so the guard // states the pager's own contract, with no normative claim behind it. try std.testing.expect(next > seqno); seqno = next; } else return error.CloneNeverExhausted; // THE PROOF: every survivor arrived EXACTLY ONCE across the paged batches, and // the sequence reached NEXT 0. The zero counts for the masked slots are carried // by the log's own read mask (see the note above the test), not by this slice; // the "exactly once" and the termination are the mask's contribution — a slot // masked out of step with `rids` drops or duplicates a survivor here. for (delivered, 0..) |n, i| { const want: usize = if (sendShunIsShunned(i)) 0 else 1; try std.testing.expectEqual(want, n); } } test "sync §9.5 L1988-1992: a shun row written between clone rounds is honoured by the next round WITHOUT renumbering the survivors" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A repo holding all five artifacts and NO shun row yet. const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer seed.deinit(gpa); try seed.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); for (send_shun_digests, 0..) |d, i| { var body_buf: [32]u8 = undefined; _ = try seed.put(d, try std.fmt.bufPrint(&body_buf, "distinct content {d}\n", .{i})); } try seed.commit(); } const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const clone_plain = "pragma client-version 20000\nclone 3 1\n"; const clone_req = try zit.container.encode(gpa, clone_plain, .default); defer gpa.free(clone_req); // Round 1, no shun row: every artifact is served, and artifact 3 sits at // adapter rid 3. This is the behaviour the mask must change, so the test is // red on an unfiltered server. { const reply = try serveAnswer(&ctx, gpa, clone_req, null, null); defer gpa.free(reply.body); const body = reply.body; // §8.1 L1489-1491 / §4.1 L885-887 — uncompressed clone stream for (send_shun_names) |name| try std.testing.expect(try replyHasCfile(gpa, body, name)); try std.testing.expectEqual(@as(?repostate.Rid, 3), ctx.snapshot.?.store().lookup(send_shun_names[2])); try std.testing.expectEqual(@as(?repostate.Rid, 5), ctx.snapshot.?.store().lookup(send_shun_names[4])); } // The operator shuns the MIDDLE artifact between rounds — an append, which is // exactly what trips `baseSnapshot`'s append-only stat gate and rebuilds. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); const st = try f.stat(io); var fs = storage.FileStorage.init(io, f, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putShun(send_shun_digests[1], "operator request", 1); try log.commit(); } // Round 2 honours it with no restart — AND the survivors above it keep their // adapter rids. A dense filter would shift artifact 3 to rid 2 and artifact 5 // to rid 4, so a client resuming this clone at cursor 3 would step over // artifact 3 entirely and still be told NEXT 0 (§8.2 L1504-1506 licenses the // gap; it does not license the shift). { const reply = try serveAnswer(&ctx, gpa, clone_req, null, null); defer gpa.free(reply.body); const body = reply.body; // §8.1 L1489-1491 / §4.1 L885-887 — uncompressed clone stream try std.testing.expect(!try replyHasCfile(gpa, body, send_shun_names[1])); for (send_shun_names, 0..) |name, i| { if (i == 1) continue; try std.testing.expect(try replyHasCfile(gpa, body, name)); } try std.testing.expectEqual(@as(usize, 5), ctx.snapshot.?.store().count()); try std.testing.expectEqual(@as(?repostate.Rid, 3), ctx.snapshot.?.store().lookup(send_shun_names[2])); try std.testing.expectEqual(@as(?repostate.Rid, 5), ctx.snapshot.?.store().lookup(send_shun_names[4])); // THE MASK ITSELF, at THIS layer. The absent `cfile` above does not prove it: // `LogStore.wire`/`content` answer null for a shunned entry through their own // read mask, so the clone stream would omit the frame even with the // enumeration unmasked. The rebuilt index is where a shun table read once at // startup instead of per generation shows up, so assert on it directly. try std.testing.expect(ctx.snapshot.?.masked[1]); try std.testing.expect(ctx.snapshot.?.store().lookup(send_shun_names[1]) == null); try std.testing.expect(!ctx.snapshot.?.store().held(2)); } try std.testing.expectEqual(@as(u64, 2), ctx.loads); // one build per log state } test "serve §3.8 / sync §9.5 L1988-1992: the snapshot cache key is not the size alone — a same-length write to the log invalidates it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const req = try serveGimmeRequest(gpa, "igot", shun_test_name); defer gpa.free(req); // Two read rounds over an untouched log: built once, then stat-gated. This is // the property the mtime term must NOT cost — a read round still does not // rebuild, so the cache is still a cache. for (0..2) |_| { const reply = try serveAnswer(&ctx, gpa, req, null, null); gpa.free(reply.body); } try std.testing.expectEqual(@as(u64, 1), ctx.loads); const before = try file.stat(io); // A write that leaves the log BYTE-IDENTICAL and exactly as long: the last byte // is lifted off and put straight back. Nothing a `zit` writer does today looks // like this — they all append — but "every writer only appends" is an assumption // about all present and future writers, not a checked property, and since this // snapshot began carrying the §3.3 shun table a missed rebuild no longer costs // stale CONTENT (self-correcting) but a stale SUPPRESSION TABLE: an artifact the // operator shunned served on, with nothing failing and nothing logged. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); var fs = storage.FileStorage.init(io, f, before.size); var last: [1]u8 = undefined; try std.testing.expectEqual(@as(usize, 1), try fs.storageFile().pread(before.size - 1, &last)); try fs.storageFile().truncate(before.size - 1); var reopened = storage.FileStorage.init(io, f, before.size - 1); try reopened.storageFile().append(&last); try reopened.storageFile().sync(); } // THE PREMISE, asserted rather than assumed: the log really is the same length // and the filesystem really did move the modification time. A filesystem that // could not express this scenario fails here, loudly, instead of letting the // conclusion below pass vacuously. const after = try file.stat(io); try std.testing.expectEqual(before.size, after.size); try std.testing.expect(after.mtime.nanoseconds != before.mtime.nanoseconds); // THE CONCLUSION: the size-only gate would have returned the stale snapshot and // its stale shun table. The size+mtime key rebuilds. { const reply = try serveAnswer(&ctx, gpa, req, null, null); gpa.free(reply.body); } try std.testing.expectEqual(@as(u64, 2), ctx.loads); try std.testing.expectEqual(after.mtime.nanoseconds, ctx.snapshot_log_mtime_ns); // ...and having rebuilt, it settles again: the key is stable, not a rebuild on // every round. { const reply = try serveAnswer(&ctx, gpa, req, null, null); gpa.free(reply.body); } try std.testing.expectEqual(@as(u64, 2), ctx.loads); } test "sync §7.2 L1217-1218 / §9.5 L1988-1992: a surviving delta whose basis is masked is served in FULL form, never as a dangling DELTASRC" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // `zit shun --defer` records the row without rewriting the generation, so a // SURVIVING frame in the current generation can still be a delta rooted on // the shunned one. Handing that delta's wire form out would emit a `cfile` // whose DELTASRC is never sent: the peer stores a permanently dangling delta // and gimme's a basis it will be silently skipped on — CORRUPT, not absent. const base_plain = "the quick brown fox jumps over the lazy dog, and then again once more"; const child_plain = "the quick brown fox jumps over the lazy dog, and then twice more, at last"; const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); const base_rid = try log.put(send_shun_digests[0], base_plain); const d = try zit.delta.create(gpa, base_plain, child_plain); defer gpa.free(d); _ = try log.putDelta(send_shun_digests[1], base_rid, d); try log.putShun(send_shun_digests[0], "operator request", 1); try log.commit(); } var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); { var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // The premise: the child really is a delta frame rooted on the masked one. try std.testing.expectEqual(logstore.FrameKind.delta, index.log.kindOf(index.rids[1])); // The masked basis is a §8.2 gap... try std.testing.expect(!s.held(1)); try std.testing.expect(s.wire(1) == null); // ...so the child's DELTA wire form is withheld — the responder falls back // to compressing `content()` on the fly (§7.2), which reconstructs against // the raw log and is unaffected by the name-level mask. try std.testing.expect(s.held(2)); try std.testing.expect(s.wire(2) == null); try std.testing.expectEqualStrings(child_plain, s.content(2).?); // Withholding a wire form for policy is not a read fault. try std.testing.expect(!index.read_fault); } // End to end through a clone round: the child arrives, the basis does not, and // the shunned NAME appears nowhere in the reply — no DELTASRC to dangle on. const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const clone_req = try zit.container.encode(gpa, "pragma client-version 20000\nclone 3 1\n", .default); defer gpa.free(clone_req); const reply = try serveAnswer(&ctx, gpa, clone_req, null, null); defer gpa.free(reply.body); const body = reply.body; // §8.1 L1489-1491 / §4.1 L885-887 — uncompressed clone stream try std.testing.expect(try replyHasCfile(gpa, body, send_shun_names[1])); try std.testing.expect(std.mem.indexOf(u8, body, send_shun_names[0]) == null); } test "sync §3.3 L527-530: the CLIENT's holdings snapshot carries the same masked-basis guard — a surviving delta on a masked base is held as full content" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // The same `zit shun --defer` shape as the serve-side test above, asked of // the CLIENT's builder: `ClientHoldings`, which every `zit pull`/`push`/ // `sync` client announces from. Serving the delta wire form would give the // snapshot an entry that `held()` calls true but whose DELTASRC names a // masked basis — so §3.2's igot walk would announce a name the gimme send // gate can never answer, and a push would spin the round with nothing to // send while reporting success. The artifact itself is NOT shunned; only // its basis is. const base_plain = "the quick brown fox jumps over the lazy dog, and then again once more"; const child_plain = "the quick brown fox jumps over the lazy dog, and then twice more, at last"; const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); const base_rid = try log.put(send_shun_digests[0], base_plain); const d = try zit.delta.create(gpa, base_plain, child_plain); defer gpa.free(d); _ = try log.putDelta(send_shun_digests[1], base_rid, d); try log.putShun(send_shun_digests[0], "operator request", 1); try log.commit(); } var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const s = holdings.store(); // The masked basis is ABSENT from the name map — `lookup` answers null, // which IS §3.3 L532-535's "If absent, silently skip" on the gimme path — // and a peer-announced intern lands on the masked slot rather than minting // a second identity: held by nothing, readable as nothing. try std.testing.expect(s.lookup(send_shun_names[0]) == null); const base_rid = try s.intern(send_shun_names[0]); try std.testing.expect(!s.held(base_rid)); try std.testing.expect(s.content(base_rid) == null); // The survivor is held AND readable — held-ness and readability agree, which // is the property a recorded dangling delta would have broken. const child_rid = s.lookup(send_shun_names[1]).?; try std.testing.expect(s.held(child_rid)); try std.testing.expectEqualStrings(child_plain, s.content(child_rid).?); try std.testing.expect(s.wire(child_rid) == null); // full content, no DELTASRC to dangle on } // --------------------------------------------------------------------------- // sync §7.7 L1428-1432 — the hash-policy half of the receive-side admission // predicate, from the REPOSITORY's stored config to the receiver that asks it. // // "shun-sha1 applies exactly when the receiving repository's effective // hash-policy value at the start of processing that card is the shun-sha1 // value defined by [FossilArtifact], Section 7." // // The tests below exist because every §7.7 unit test in `sync_test.zig` sets // `Client.hash_policy` / `Server.hash_policy` on a struct literal. That pins the // RULE and proves nothing about the WIRING, so a regression that dropped // `effectiveHashPolicy` — or read the wrong config field — would ship green. // These two close that: one over the stored byte, one over the served round. // --------------------------------------------------------------------------- /// Seed `dir` with a repository whose config frame names `policy` and which /// holds nothing. The empty holdings matter: the push below must be the only /// thing that could ever land. fn s77SeedPolicyRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, policy: artifact.HashPolicy) !void { const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = @intFromEnum(policy), .default_branch = "trunk", .project_id = "", .project_code = "", }); try log.commit(); } /// True when `dir`'s repository resolves `name` — read back off disk, after the /// served round has persisted, so this answers "did it LAND", not "was it /// staged". fn s77RepoHolds(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, name: []const u8) !bool { var file = try dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); return index.store().lookup(name) != null; } test "sync §7.7 L1428-1432: the SERVED round's admission predicate is read from the repository's stored hash-policy, not defaulted" { const gpa = std.testing.allocator; const io = std.testing.io; const content = "a SHA1-named artifact a peer pushes at us\n"; var nbuf: [64]u8 = undefined; const sha1_name = zit.artifact.artifactName(.sha1, content, &nbuf); try std.testing.expectEqual(@as(usize, 40), sha1_name.len); const plain = try std.fmt.allocPrint( gpa, "pragma client-version 20000\npush {s} {s}\nfile {s} {d}\n{s}", .{ serve_mod.server_servercode, serve_mod.server_projectcode, sha1_name, content.len, content }, ); defer gpa.free(plain); const request = try zit.container.encode(gpa, plain, .default); defer gpa.free(request); const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; // CONTROL: the same repository under `sha3` stores the pushed SHA1 name. If // this half ever stops holding, the shun-sha1 half below proves nothing. { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp.dir, .sha3); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const reply = try serveAnswer(&ctx, gpa, request, null, null); gpa.free(reply.body); try std.testing.expect(try s77RepoHolds(gpa, io, tmp.dir, sha1_name)); } // THE PROOF: the identical round against a repository whose stored config // names shun-sha1 discards it — with no error card (L1437-1440) — and the // ONLY difference between the two runs is that stored byte. A `serveRound` // that defaulted the policy instead of reading the snapshot would store here. { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const reply = try serveAnswer(&ctx, gpa, request, null, null); defer gpa.free(reply.body); const plain_reply = try zit.container.decode(gpa, reply.body, 1 << 20); defer gpa.free(plain_reply); try std.testing.expect(std.mem.indexOf(u8, plain_reply, "error ") == null); try std.testing.expect(!try s77RepoHolds(gpa, io, tmp.dir, sha1_name)); } } test "sync §7.7 L1428-1432: ClientHoldings carries the stored hash-policy into the one place a Client is built from repository state" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1); var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const admission = holdings.admission(); try std.testing.expectEqual(artifact.HashPolicy.shun_sha1, admission.hash_policy); // `syncClient` is the sole construction `pullRun`/`pushRun`/`syncRun` share, // so this one assertion covers all three verbs: a client that dropped either // half of `RepoAdmission` would apply half a filter with nothing failing. // A placeholder identity: this test is about `RepoAdmission`, and the // sub-manifest half of the same client is covered by the §7.2 tests. const identity: RepoSyncIdentity = .{ .project_code = "", .seen_submanifest = false, .enable_submanifests = false, .server_code = zit.sync.deriveServerCode(&.{"test-fixture-repo"}) }; for ([_]zit.sync.Operation{ .pull, .push, .sync }) |op| { const client = syncClient(op, &identity, null, admission); try std.testing.expectEqual(op, client.operation); try std.testing.expectEqual(artifact.HashPolicy.shun_sha1, client.hash_policy); // A 40-hex name is rejected by the very predicate this policy feeds. try std.testing.expectEqual( zit.sync.Admission.reject, zit.sync.admitInbound(client.hash_policy, client.shun, "a" ** 40), ); } // And the `.auto` repository next door admits the same name, so the // assertion above is reading the stored byte and not a constant. var tmp2 = std.testing.tmpDir(.{}); defer tmp2.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp2.dir, .auto); var holdings2 = try ClientHoldings.open(gpa, io, tmp2.dir); defer holdings2.deinit(io); const admission2 = holdings2.admission(); try std.testing.expectEqual(artifact.HashPolicy.auto, admission2.hash_policy); const c2 = syncClient(.pull, &identity, null, admission2); try std.testing.expectEqual( zit.sync.Admission.admit, zit.sync.admitInbound(c2.hash_policy, c2.shun, "a" ** 40), ); } test "sync §7.7 L1428-1432: a DAMAGED repository never yields a hash-policy — the admission predicate fails the load rather than defaulting" { // The property: no damaged repository may hand the receiver a policy value. // `readConfig` answers `Corrupt`/`Storage` for a config frame that will not // read or will not parse, and that is UNKNOWN, not "no defined policy": // answering `.auto` there would silently disable shun-sha1 and admit // SHA1-named content this repository is configured to refuse, with nothing // failing. §3.3's shun half already refuses to fail open (`RepoShunSet.load` // propagates, and `serveRound` returns `error.ServerStorage` rather than // serving tableless); the two halves of one predicate must not differ. // // HONESTY ABOUT WHAT THIS PINS: with the injury below, the fault is caught // UPSTREAM, at `LogStore.openReadOnly`, so this test does not by itself kill // a mutant that restores `effectiveHashPolicy`'s old blanket // `catch return .auto` — verified by applying that mutant, which leaves the // suite green. It is here as the end-to-end guard: the day the log open grows // tolerant of a damaged frame, the read below must still not answer `.auto`. // Pinning `effectiveHashPolicy`'s own catch would need a storage layer that // faults a single frame read, which this tree does not have. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1); // Damage the config frame in place: its default_branch bytes are on disk // verbatim, so overwriting them is a one-frame injury that leaves every other // frame — and the log's own scan — intact. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); const st = try f.stat(io); const buf = try gpa.alloc(u8, st.size); defer gpa.free(buf); var r = f.reader(io, buf); try r.interface.readSliceAll(buf); const at = std.mem.indexOf(u8, buf, "trunk") orelse return error.TestUnexpectedResult; var w = f.writer(io, &.{}); try w.seekTo(at); try w.interface.writeAll("\xff\xff\xff\xff\xff"); try w.interface.flush(); } try std.testing.expectError(error.Storage, ClientHoldings.open(gpa, io, tmp.dir)); } test "submanifests §8.1/§7.2: a repository that HOLDS a leaf arms its serve-side gate against a pre-S peer" { // The PRODUCTION wiring of the §7.2 gate, end to end through the durable // store: `persistStore` records §8.1's seen-submanifest indicator for a // stored SUBMANIFEST (which carries no S card at all, §4.4), the serve // snapshot re-reads it, and `serveRound` arms the gate with it. Every link // is load-bearing: // // * drop the `transferGatedArtifact` line in `phantomizeSubManifestRefs` // and the indicator stays false — a leaf is not "S-bearing" literally; // * drop `round_server.seen_submanifest = …` in `serveRound` and the gate // is unarmed, because §8.1's emission setting is false on every zit; // * drop `or self.seen_submanifest` from `Server.submanifestGateArmed` // and it is unarmed for the same reason. // // Each of the three leaves the leaf in the reply to a fossil-2.28 peer, // which §8 says can only hold it as "unreadable ... opaque content". const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var hex_buf: [2 * digest_len]u8 = undefined; try serveSeedRepo(gpa, io, tmp.dir, 1, &hex_buf); // §4.3: an F run with no D card is type SUBMANIFEST. The flat check-in // manifest beside it is everything that predates this draft. const leaf = "F a.txt " ++ ("3" ** 64) ++ "\n"; const flat = "C m\nD 2026-07-27T12:00:00.000\nF a.txt " ++ ("2" ** 64) ++ "\nU u\n"; var leaf_name: [2 * digest_len]u8 = undefined; var flat_name: [2 * digest_len]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, leaf, &leaf_name); _ = zit.artifact.artifactName(.sha3_256, flat, &flat_name); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(&leaf_name, leaf); _ = try ms.store().put(&flat_name, flat); _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); // §8.1 — the indicator is DURABLE and covers §7.2's type shape, not only // its S-card shape. Read back through the same function the CLI uses. var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = try repoSyncIdentity(gpa, io, tmp.dir, "test-fixture-repo", &pc_buf); try std.testing.expect(identity.seen_submanifest); // …and the emission setting is the OTHER, independent half of the arming // disjunction: this repository holds a gated artifact it received, and has // never been told to emit one. A single flag could not tell the two apart. try std.testing.expect(!identity.enable_submanifests); // …and the served reply refuses the leaf to a peer announcing fossil 2.28, // while still handing it the artifact it can read. const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; try std.testing.expect(!server.submanifests_enabled); // §8.1's default, stated var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const plain = try std.fmt.allocPrint( gpa, "pragma client-version 22800\ngimme {s}\ngimme {s}\n", .{ &leaf_name, &flat_name }, ); defer gpa.free(plain); const request = try zit.container.encode(gpa, plain, .default); defer gpa.free(request); const reply = try serveAnswer(&ctx, gpa, request, null, null); defer gpa.free(reply.body); const body = try zit.container.decode(gpa, reply.body, zit.sync.reply_body_cap); defer gpa.free(body); // Built through the same writer the server uses (§11.1's escape encoding // applies to the text), so the expectation cannot drift from the emission. var card_buf: [2 * zit.sync.submanifest_peer_version_error_text.len + 16]u8 = undefined; const refusal = zit.sync.writeSignalCard(.err, zit.sync.submanifest_peer_version_error_text, &card_buf); try std.testing.expect(std.mem.indexOf(u8, body, refusal) != null); try std.testing.expect(std.mem.indexOf(u8, body, &leaf_name) == null); try std.testing.expect(std.mem.indexOf(u8, body, &flat_name) != null); } test "submanifests §7.2/§10.1: the CLI announces a DATED client-version so the peer's version can reach the gate" { // §10.1 r11/r12 — the server replies `pragma server-version` exactly when // the received client-version carried DATE and TIME, and §7.2's gate reads // "the remote peer's announced version". An undated pragma therefore leaves // `Client.peer.version` at 0 for the whole session and an armed client // refuses its own trees to an opted-in server: pullable, never pushable. // (`sync_driver_test` drives that consequence end to end; this pins the one // CLI constant the whole chain hangs off.) const gpa = std.testing.allocator; try std.testing.expect(sync_client_version_info.datetime != null); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = sync_servercode, .project_code = sync_projectcode, .version = sync_client_version_info, }; defer client.deinit(gpa); var buf: std.ArrayListUnmanaged(u8) = .empty; defer buf.deinit(gpa); const request = try client.buildRequest(gpa, ms.store(), &buf); // The emitted pragma carries at least §10.1 r11's five tokens. const eol = std.mem.indexOfScalar(u8, request, '\n').?; var tokens: usize = 0; var it = std.mem.tokenizeScalar(u8, request[0..eol], ' '); while (it.next()) |_| tokens += 1; try std.testing.expect(std.mem.startsWith(u8, request, "pragma client-version ")); try std.testing.expect(tokens >= zit.sync.min_datetime_tokens); try std.testing.expect(zit.sync.serverEmitsServerVersion(.{ .version = sync_client_version, .datetime = sync_client_version_info.datetime, })); } test "configValue reads every name from one frame, and empty means no opinion" { var arena_state: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); // An untouched frame has no opinion about anything, so the versioned and // global tiers stay reachable. const bare: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p" }; try std.testing.expect((try configValue(arena, &bare, .ignore_glob)) == null); try std.testing.expect((try configValue(arena, &bare, .autosync)) == null); try std.testing.expect((try configValue(arena, &bare, .clearsign)) == null); try std.testing.expect((try configValue(arena, &bare, .dotfiles)) == null); try std.testing.expect((try configValue(arena, &bare, .crnl_glob)) == null); const set: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p", .ignore_glob = "*.o", .crnl_glob = "*.bat", .clean_glob = "*.tmp", .empty_dirs = "logs", .encoding_glob = "*.utf16", .dotfiles = "yes", .mimetypes = "md text/markdown", .clearsign_set = true, .clearsign = false, .autosync_set = true, .autosync_off = true, }; try std.testing.expectEqualStrings("*.o", (try configValue(arena, &set, .ignore_glob)).?); try std.testing.expectEqualStrings("*.bat", (try configValue(arena, &set, .crnl_glob)).?); try std.testing.expectEqualStrings("*.tmp", (try configValue(arena, &set, .clean_glob)).?); try std.testing.expectEqualStrings("logs", (try configValue(arena, &set, .empty_dirs)).?); try std.testing.expectEqualStrings("*.utf16", (try configValue(arena, &set, .encoding_glob)).?); // Verbatim: the operator wrote `yes`, so the listing says `yes`. try std.testing.expectEqualStrings("yes", (try configValue(arena, &set, .dotfiles)).?); try std.testing.expectEqualStrings("md text/markdown", (try configValue(arena, &set, .mimetypes)).?); // The booleans answer through their explicit-set bit, as they always have. try std.testing.expectEqualStrings("off", (try configValue(arena, &set, .clearsign)).?); try std.testing.expectEqualStrings("off", (try configValue(arena, &set, .autosync)).?); } test "resolveCrlfGlob falls back to crnl-glob from one frame" { // The commit gate holds the store's write lock and reads its config once, // so both spellings are answered from that frame -- no second store open, // and no way for the two reads to see different bytes. const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); const amb: cli.Ambient = .{ .now_epoch = 0, .user_env = null }; // The older spelling answers while the newer one is absent. const older: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p", .crnl_glob = "*.txt", }; const from_crnl = try resolveCrlfGlob(io, arena, tmp.dir, amb, &older); try std.testing.expectEqualStrings("*.txt", from_crnl.value); try std.testing.expectEqual(zit.settings.Source.repository, from_crnl.source); // `crlf-glob` present REPLACES it, exactly as fossil's // `db_get("crlf-glob", db_get("crnl-glob",""))` does. const newer: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p", .crlf_glob = "*.c", .crnl_glob = "*.txt", }; const from_crlf = try resolveCrlfGlob(io, arena, tmp.dir, amb, &newer); try std.testing.expectEqualStrings("*.c", from_crlf.value); // No frame and no versioned file: nothing is declared, so the class is // not checked at all. const none = try resolveCrlfGlob(io, arena, tmp.dir, amb, null); try std.testing.expectEqual(zit.settings.Source.default, none.source); try std.testing.expectEqualStrings("", none.value); } test "clone observer forwards phase edges without owning measurement state" { const Probe = struct { count: usize = 0, phase: ClonePhase = .transport, event: ClonePhaseEvent = .begin, fn callback(context: *anyopaque, phase: ClonePhase, event: ClonePhaseEvent) void { const self: *@This() = @ptrCast(@alignCast(context)); self.count += 1; self.phase = phase; self.event = event; } }; var probe = Probe{}; const observer = CloneObserver{ .context = &probe, .callback = Probe.callback }; observeClonePhase(&observer, .persist, .end); try std.testing.expectEqual(@as(usize, 1), probe.count); try std.testing.expectEqual(ClonePhase.persist, probe.phase); try std.testing.expectEqual(ClonePhaseEvent.end, probe.event); } // --------------------------------------------------------------------------- // `zit config pull AREA`: the explicit configuration request side // --------------------------------------------------------------------------- /// `zit config pull shun [URL]` — fossil's `configuration pull AREA ?URL?`. /// /// Configuration carriage is otherwise CLONE-TIME ONLY, and deliberately so: a /// shun table that refreshed on every sync would let any peer suppress content /// in this repository on its own say-so. This verb is how an operator asks for /// it anyway — explicitly, naming the area, against a peer they chose. /// /// `shun` is the one area zit models. The rest of fossil's areas describe a web /// interface and a user table zit has no equivalent for; naming one of those is /// a clean refusal rather than a silent no-op, so an operator learns which /// areas exist rather than watching nothing happen. /// Apply the peer's user rows to this repository's table. Shares /// `runConfigFromFile`'s two modes and its one-marker rule: `overwrite` /// replaces, and the marker lands before the first row and only when there /// is a row to land, so a pull that returned nothing never wipes the table it /// could not replace. /// /// An EMPTY result is reported rather than treated as success-with-nothing: /// against fossil, "no rows" is what an under-privileged client sees, so an /// operator who forgot `--user` must not read it as "the peer has no users". fn applyPeerUsers( gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, rows: []const zit.sync.OwnedUserRow, overwrite: bool, streams: cli.Streams, ) cli.Exit { if (rows.len == 0) { streams.err.writeAll( "zit config: the peer offered no user rows\n" ++ "zit config: fossil serves the user area to an administrator only -- " ++ "pass --user=NAME (and --password=, or $ZIT_PASSWORD)\n", ) catch {}; return .failure; } var file = cli.openStoreAppend(io, root_dir) catch return repoReadFail(streams, "config"); defer file.close(io); const st = file.stat(io) catch return repoReadFail(streams, "config"); var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch return repoReadFail(streams, "config"); defer log.deinit(gpa); if (overwrite) log.putUserReset(0) catch return repoReadFail(streams, "config"); var skipped: usize = 0; for (rows) |row| { log.putUser(.{ .login = row.login, .pw = row.pw, .cap = row.cap, .info = row.info, .mtime = row.mtime, }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), error.LoginTooLong, error.FieldTooLong => { skipped += 1; continue; }, }; } log.commit() catch return repoReadFail(streams, "config"); const table = log.userTable(gpa) catch return stateOom(streams, "config"); defer gpa.free(table); streams.out.print("user: {d} row{s} {s} from the peer; {d} in the table", .{ rows.len, plural(rows.len), if (overwrite) "adopted" else "merged", table.len, }) catch {}; if (skipped != 0) streams.out.print("; {d} unusable and skipped", .{skipped}) catch {}; streams.out.writeAll("\n") catch {}; if (skipped != 0) return .failure; return .ok; } /// `zit config`'s three shapes, spelled as fossil spells them /// (`configuration pull|import|merge`). const ConfigSub = enum { pull, import, merge }; const config_usage_line: []const u8 = "zit config: usage: zit config pull shun [URL] | zit config import|merge FILE\n"; /// The largest configuration export this verb will read whole. A user table /// is a handful of rows per operator; a file past this bound is not a /// configuration export and is refused rather than allocated for. const max_config_file_bytes: usize = 8 * 1024 * 1024; /// Apply a fossil configuration export file: the output of /// `fossil configuration export AREA FILE`, which is byte-identical to what /// the sync protocol carries for the same area (`config /AREA SIZE\n` then /// SIZE payload bytes then a newline), plus `#` comment lines at the top. /// /// That identity is the whole reason this path exists: a repository's users /// reach zit with no SQLite reader, no network, and no fossil server — one /// `fossil configuration export` on the migration host, and the SAME parser /// the wire path will use. /// /// `replace` is fossil's `CONFIGSET_OVERWRITE`. Applied ONCE, before the /// first row lands, exactly as `configure_receive`'s `configHasBeenReset` /// guard does — a per-row reset would leave only the last row of the file. fn runConfigFromFile( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, path: []const u8, replace: bool, repo_override: ?[]const u8, streams: cli.Streams, ) cli.Exit { // The user/shun areas are store tables; `-R` redirects which // store they land in. The FILE is still read from the cwd (fossil's // `configuration import FILENAME`), so only the store is redirected. const root = (cli.resolveRoot(arena, io, repo_override) catch return stateOom(streams, "config")) orelse { main.reportNoRoot(streams, io, "config", repo_override); return .failure; }; var root_dir = root.dir; defer root_dir.close(io); const text = std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(max_config_file_bytes)) catch { streams.err.print("zit config: cannot read '{s}'\n", .{path}) catch {}; return .failure; }; defer gpa.free(text); var users: std.ArrayList(zit.sync.UserRow) = .empty; defer users.deinit(gpa); var value_arena = std.heap.ArenaAllocator.init(gpa); defer value_arena.deinit(); const va = value_arena.allocator(); var skipped: usize = 0; var other_areas: usize = 0; var pos: usize = 0; while (pos < text.len) { const line_end = std.mem.indexOfScalarPos(u8, text, pos, '\n') orelse text.len; const line = text[pos..line_end]; // `configuration export` writes a `#` banner naming the repository and // the export time; blank lines separate nothing but are harmless. if (line.len == 0 or line[0] == '#') { pos = line_end + 1; continue; } const card = parseConfigCardLine(line) orelse { streams.err.print("zit config: '{s}' is not a configuration export\n", .{path}) catch {}; return .failure; }; const payload = zit.sync.consumeConfigPayload(text, line_end + 1, card.size) catch { streams.err.print("zit config: '{s}' is truncated\n", .{path}) catch {}; return .failure; }; pos = payload.next; if (!std.mem.eql(u8, card.area, zit.sync.config_user_group)) { other_areas += 1; continue; } const row = zit.sync.parseUserConfigPayload(payload.bytes) catch { skipped += 1; continue; }; const parsed = unquoteUserRow(va, row) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Malformed => { skipped += 1; continue; }, }; users.append(gpa, parsed) catch return stateOom(streams, "config"); } if (other_areas != 0 and users.items.len == 0) { streams.err.print( "zit config: '{s}' carries no /user cards ({d} card{s} in areas zit does not model)\n", .{ path, other_areas, plural(other_areas) }, ) catch {}; return .failure; } var file = cli.openStoreAppend(io, root_dir) catch return repoReadFail(streams, "config"); defer file.close(io); const st = file.stat(io) catch return repoReadFail(streams, "config"); var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch return repoReadFail(streams, "config"); defer log.deinit(gpa); // The marker lands before the first row and only when there is a row to // land: an import whose file held nothing applicable must not wipe the // table it could not replace. if (replace and users.items.len != 0) { log.putUserReset(0) catch return repoReadFail(streams, "config"); } for (users.items) |row| { log.putUser(.{ .login = row.login, .pw = row.pw, .cap = row.cap, .info = row.info, .mtime = row.mtime, }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), // A row this store cannot key or hold is reported, never silently // dropped: an operator migrating an access-control table needs to // know a login did not come across. error.LoginTooLong, error.FieldTooLong => { skipped += 1; continue; }, }; } log.commit() catch return repoReadFail(streams, "config"); const table = log.userTable(gpa) catch return stateOom(streams, "config"); defer gpa.free(table); streams.out.print("user: {d} row{s} {s}; {d} in the table", .{ users.items.len, plural(users.items.len), if (replace) "imported" else "merged", table.len, }) catch {}; if (skipped != 0) streams.out.print("; {d} unreadable and skipped", .{skipped}) catch {}; if (other_areas != 0) streams.out.print("; {d} card{s} in other areas ignored", .{ other_areas, plural(other_areas) }) catch {}; streams.out.writeAll("\n") catch {}; if (skipped != 0) return .failure; return .ok; } /// One `config /AREA SIZE` card header line. const ConfigCardLine = struct { area: []const u8, size: u32 }; fn parseConfigCardLine(line: []const u8) ?ConfigCardLine { const prefix = "config "; if (!std.mem.startsWith(u8, line, prefix)) return null; const rest = line[prefix.len..]; const sp = std.mem.indexOfScalar(u8, rest, ' ') orelse return null; const area = rest[0..sp]; if (area.len == 0 or area[0] != '/') return null; const size = std.fmt.parseInt(u32, rest[sp + 1 ..], 10) catch return null; return .{ .area = area, .size = size }; } /// Unquote a wire user row's fields onto `arena`. A field fossil omitted is /// the empty string here — zit models "no password" as empty rather than as /// a distinct NULL, which is also what it emits back on the wire. fn unquoteUserRow(arena: std.mem.Allocator, row: zit.sync.QuotedUserRow) error{ OutOfMemory, Malformed }!zit.sync.UserRow { const mtime = std.fmt.parseInt(i64, row.mtime_text, 10) catch return error.Malformed; return .{ .login = try unquoteField(arena, row.quoted_login), .pw = try unquoteField(arena, row.quoted_pw), .cap = try unquoteField(arena, row.quoted_cap), .info = try unquoteField(arena, row.quoted_info), .mtime = mtime, }; } fn unquoteField(arena: std.mem.Allocator, quoted: ?[]const u8) error{ OutOfMemory, Malformed }![]const u8 { const q = quoted orelse return ""; const buf = try arena.alloc(u8, q.len); const out = zit.sync.fossilUnquote(q, buf) catch return error.Malformed; return out; } pub fn runConfig( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { var area: ?[]const u8 = null; var url_arg: []const u8 = ""; var sub: ?ConfigSub = null; var overwrite = false; var login_user: ?[]const u8 = null; var login_password: ?[]const u8 = null; for (verb_args) |tok| { if (std.mem.eql(u8, tok, "-h") or std.mem.eql(u8, tok, "--help")) { streams.out.writeAll(main.config_help) catch {}; return .ok; } else if (std.mem.eql(u8, tok, "--overwrite")) { overwrite = true; } else if (std.mem.startsWith(u8, tok, "--user=")) { login_user = tok["--user=".len..]; } else if (std.mem.startsWith(u8, tok, "--password=")) { login_password = tok["--password=".len..]; } else if (tok.len > 1 and tok[0] == '-') { return cli.usageError(streams, "config", .unknown_option, tok); } else if (sub == null) { sub = std.meta.stringToEnum(ConfigSub, tok) orelse return cli.usageError(streams, "config", .unknown_option, tok); } else if (area == null) { area = tok; } else if (url_arg.len == 0) { url_arg = tok; } else { return cli.usageError(streams, "config", .extra_argument, tok); } } if (sub == null or area == null) { streams.err.writeAll(config_usage_line) catch {}; return .usage; } // `import`/`merge` take a FILE, not an area and a URL: fossil's own // `configuration import|merge FILENAME`. The two differ in exactly one // thing, which is why they share a body: `import` REPLACES the local table // (fossil's `CONFIGSET_OVERWRITE` — `DELETE FROM user` then `REPLACE // INTO`), `merge` folds rows in newest-mtime-wins. switch (sub.?) { .import, .merge => { if (url_arg.len != 0) return cli.usageError(streams, "config", .extra_argument, url_arg); // The file forms take their mode from the SUBCOMMAND, as fossil's // do; accepting a contradicting --overwrite would give one // invocation two ways to say the same thing and one way to lie. if (overwrite) return cli.usageError(streams, "config", .unknown_option, "--overwrite"); if (login_user != null or login_password != null) { streams.err.writeAll("zit config: --user/--password apply to 'config pull', not to a file\n") catch {}; return .usage; } return runConfigFromFile(gpa, io, arena, area.?, sub.? == .import, amb.repo_override, streams); }, .pull => {}, } // Fossil matches area names by PREFIX (`configure_name_to_mask`), and so // does the server side of this exchange, so the CLI accepts the same. const want = area.?; // Fossil matches by PREFIX, and orders `shun` before `user` in // `aGroupName`, so an ambiguous prefix resolves the way fossil resolves it. const group: []const u8 = if (want.len != 0 and std.mem.startsWith(u8, "shun", want)) zit.sync.config_shun_group else if (want.len != 0 and std.mem.startsWith(u8, "user", want)) zit.sync.config_user_group else { streams.err.print("zit config: unknown configuration area '{s}' (known: shun, user)\n", .{want}) catch {}; return .failure; }; const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "config")) orelse { main.reportNoRoot(streams, io, "config", amb.repo_override); return .failure; }; var root_dir = root.dir; defer root_dir.close(io); var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, url_arg, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), error.NoRemote => return noRemoteError(streams, "config"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "config", url, e); defer tr.close(); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); // §9.1 — the pull card carries this repository's adopted project code, and a // real peer refuses a mismatch, so the identity is loaded even though this // exchange moves no artifacts. The buffer outlives the run (the client // borrows it). var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), }; // A pull-shaped session that asks for one group. The in-memory store stays // EMPTY on purpose: this verb moves CONFIGURATION, never artifacts, so an // operator asking for a shun table cannot be handed a repository's worth of // content as a side effect. // fossil serves the user area to an ADMINISTRATOR only // (`configure.c`: `m &= ~(CONFIGSET_USER|CONFIGSET_SCRIBER)` for a client // without Admin), so a `/user` pull is credentialed or it comes back // empty. The §6.3 derivation is `resolvePushCredential`'s, reused rather // than restated: one place turns a password into a signing secret. var pw_buf: [max_password_len]u8 = undefined; defer std.crypto.secureZero(u8, &pw_buf); var user_buf: [logstore.max_config_str]u8 = undefined; const resolved = resolvePushCredential( gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = login_user, .password = login_password }, amb, &pw_buf, &user_buf, ) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), error.NoPassword => { streams.err.writeAll("zit config: no password supplied for --user\n") catch {}; return .failure; }, }; var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = &identity.server_code, .project_code = identity.project_code, .version = sync_client_version_info, .request_config = true, .config_group = group, .credential = if (resolved) |r| r.credential else null, }; defer client.deinit(gpa); _ = sync_driver.runSync(gpa, &client, ms.store(), &tr, max_stalled_sync_rounds) catch |e| return syncRunFail(streams, "config", url, e, tr.last_status); if (std.mem.eql(u8, group, zit.sync.config_user_group)) { return applyPeerUsers(gpa, io, root_dir, client.peerUserRows(), overwrite, streams); } const rows = client.peerShunRows(); if (rows.len == 0) { streams.out.writeAll("no shun rows offered by the peer\n") catch {}; return .ok; } var file = cli.openStoreAppend(io, root_dir) catch return repoReadFail(streams, "config"); defer file.close(io); const st = file.stat(io) catch return repoReadFail(streams, "config"); var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch return repoReadFail(streams, "config"); defer log.deinit(gpa); // Count by the TABLE's own size, not by successful calls: `putShun` is // idempotent on a name already present, so counting calls would report rows // the operator already had as newly adopted. const before = log.shunList().len; var skipped: usize = 0; for (rows) |row| { var digest_buf: [32]u8 = undefined; if (row.hash.len != 40 and row.hash.len != 64) { skipped += 1; continue; } const digest = digest_buf[0 .. row.hash.len / 2]; _ = std.fmt.hexToBytes(digest, row.hash) catch { skipped += 1; continue; }; // `putShun` is idempotent on a name already present, so re-running this // verb converges rather than accumulating. log.putShun(digest, row.reason, row.epoch) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), else => { skipped += 1; continue; }, }; } const added = log.shunList().len - before; log.commit() catch return repoReadFail(streams, "config"); streams.out.print("shun: {d} new row{s} adopted", .{ added, plural(added) }) catch {}; if (skipped != 0) streams.out.print("; {d} unreadable and skipped", .{skipped}) catch {}; streams.out.writeAll("\n") catch {}; return .ok; } test "the cluster sidecar is created by persist, adopted without a walk, rebuilt by one walk when lost, extended by a later persist" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Two blobs and a cluster naming one of them, persisted into the fresh log. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var names: [2][64]u8 = undefined; inline for (.{ "clustered blob", "loose blob" }, 0..) |body, i| { _ = zit.artifact.artifactName(.sha3_256, body, &names[i]); const rid = try ms.store().put(&names[i], body); const c = try zit.container.encode(gpa, body, .default); defer gpa.free(c); try ms.store().retain(rid, .{ .full = c }); } const cluster = try std.fmt.allocPrint(gpa, "M {s}\nZ 00000000000000000000000000000000\n", .{&names[0]}); defer gpa.free(cluster); var cn: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, cluster, &cn); { const rid = try ms.store().put(&cn, cluster); const c = try zit.container.encode(gpa, cluster, .default); defer gpa.free(c); try ms.store().retain(rid, .{ .full = c }); } _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); // 1. Adopted: persist created the index, so a load walks nothing. { var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const before = clusterindex.walk_rids; var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store()); defer r.deinit(); try std.testing.expectEqual(before, clusterindex.walk_rids); try std.testing.expectEqual(@as(usize, 1), r.cluster_names.len); try std.testing.expectEqualStrings(&cn, r.cluster_names[0]); try std.testing.expect(r.clustered.isSet(holdings.store().lookup(&names[0]).?)); try std.testing.expect(!r.clustered.isSet(holdings.store().lookup(&names[1]).?)); try std.testing.expect(!r.clustered.isSet(holdings.store().lookup(&cn).?)); } // 2. Lost: one full walk rebuilds it; the load after that adopts again. try tmp.dir.deleteFile(io, zit.repocache.clusters_filename); { var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const before = clusterindex.walk_rids; var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store()); defer r.deinit(); try std.testing.expectEqual(before + 3, clusterindex.walk_rids); try std.testing.expectEqual(@as(usize, 1), r.cluster_names.len); const again = clusterindex.walk_rids; var r2 = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store()); defer r2.deinit(); try std.testing.expectEqual(again, clusterindex.walk_rids); } // 3. Extended: a later persist records the cluster it writes; no walk. const cluster2 = try std.fmt.allocPrint(gpa, "M {s}\nZ 00000000000000000000000000000000\n", .{&names[1]}); defer gpa.free(cluster2); var cn2: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, cluster2, &cn2); { const rid = try ms.store().put(&cn2, cluster2); const c = try zit.container.encode(gpa, cluster2, .default); defer gpa.free(c); try ms.store().retain(rid, .{ .full = c }); } _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 3); { var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const before = clusterindex.walk_rids; var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store()); defer r.deinit(); try std.testing.expectEqual(before, clusterindex.walk_rids); try std.testing.expectEqual(@as(usize, 2), r.cluster_names.len); try std.testing.expect(r.clustered.isSet(holdings.store().lookup(&names[1]).?)); } }