Artifact e8dcdec181166e63a5d11e34a4ddecc5a276fbfa7a024fc92d1f63c9a68f3036:
- File
src/cmd/sync.zig
— part of check-in
[5f541eccc7]
at
2026-09-18 11:31:24
on branch trunk
— transport: answer an HTTP Basic challenge
A forge that gates a read answers a clone's first request with 401. The login card cannot answer that round, since its secret needs the project code the first reply teaches, and the status was fatal, so the credential never reached the wire. When a verb holds a plaintext password the transport keeps a Basic credential ready, sends nothing until a 401 names the Basic scheme, then repeats the round with the Authorization header and carries it on every later round, as fossil's --httpauth does. A 401 now says what the peer wanted. Proven by an e2e with an authenticating gate in front of zit serve. (user: dmestas, size: 692581) [annotate] [blame] [check-ins using]
//! zit sync / network / store-transfer verbs (serve, clone, open, pack, pull, //! push, sync, remote) and the helpers and tests private to them. //! //! Extracted verbatim from src/main.zig (release step S2). Every declaration //! below is byte-identical to its main.zig source; only this import prologue is //! new. Helpers still shared with other verbs stay in main.zig and are aliased //! back here so the moved bodies call them unqualified. main.zig's verb-dispatch //! switch calls into this file (`cmd_sync.runServe` ...); its test block imports //! it so these inline tests keep running under `zig build test`. const std = @import("std"); const zit = @import("zit"); const SparseClassification = zit.sparse.SparseClassification; const classifyKeepRids = zit.sparse.classifyKeepRids; const cli = @import("zit").cli; const extras = @import("../extras.zig"); const logstore = @import("zit").logstore; const storage = @import("zit").storage; const clusterindex = @import("zit").clusterindex; const update = @import("zit").update; const checkout = @import("zit").checkout; const Repository = @import("zit").Repository; const serve_mod = @import("../serve.zig"); const transport = @import("../transport.zig"); const sync_driver = @import("../sync_driver.zig"); const Io = std.Io; const File = Io.File; const repostate = zit.repostate; const managed = zit.managed; const artifact = zit.artifact; // Shared CLI helpers and test fixtures that stay in main.zig (other verbs use // them too); aliased back so the moved code and tests reference them by their // original unqualified names. const main = @import("../main.zig"); const runInit = main.runInit; const repo_filename = main.repo_filename; const renderStorage = main.renderStorage; const reasonOf = main.reasonOf; const stateOom = main.stateOom; pub const not_a_repo_msg = main.not_a_repo_msg; const repoReadFail = main.repoReadFail; const repoOpenFail = main.repoOpenFail; const openRepo = main.openRepo; const updateApplyFail = main.updateApplyFail; const digest_len = main.digest_len; const hexToDigest = main.hexToDigest; const digestToWireName = main.digestToWireName; const decodeWireName = main.decodeWireName; const open_help = main.open_help; const pack_help = main.pack_help; const shun_help = main.shun_help; const serve_help = main.serve_help; const clone_help = main.clone_help; const pull_help = main.pull_help; const push_help = main.push_help; const sync_help = main.sync_help; const remote_help = main.remote_help; const settings_help = main.settings_help; const packSeedBlob = main.packSeedBlob; const packSeedCheckin = main.packSeedCheckin; const packEvolvingBlob = main.packEvolvingBlob; const packOpenWritable = main.packOpenWritable; const packOpenReadOnly = main.packOpenReadOnly; /// Clone phase edges let benchmark code attribute the real command without /// coupling clocks or counters into production behavior. pub const ClonePhase = enum { transport, initialize, exchange, persist, config, warm_cache }; pub const ClonePhaseEvent = enum { begin, end }; pub const CloneObserver = struct { context: *anyopaque, callback: *const fn (context: *anyopaque, phase: ClonePhase, event: ClonePhaseEvent) void, }; fn observeClonePhase(observer: ?*const CloneObserver, phase: ClonePhase, event: ClonePhaseEvent) void { if (observer) |value| value.callback(value.context, phase, event); } /// `zit serve [--port N]` — put an HTTP face on the stateless sync responder. /// Opens `.zit` read-only and keeps it open for the server's lifetime; every /// request reads a fresh content view of it and is answered independently /// (sync §3.8). `std.http`/socket I/O lives entirely in `serve.zig`; this only /// wires the repository and the listen address. pub fn runServe( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, /// Carried solely so the served rounds can resolve this server's own /// content policy through the shared settings resolver. amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseServeArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "serve", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "serve", .extra_argument, tok), .bad_port => |tok| { streams.err.print("zit serve: invalid --port value '{s}'\n", .{tok}) catch {}; return .usage; }, }; if (parsed.help) { streams.out.writeAll(serve_help) catch {}; return .ok; } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "serve")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; // serve answers from the STORE: its open handle, its swap-detect // stat and its config reads all follow the resolved store root. if (cli.storeDirOf(io, root_dir) catch { root_dir.close(io); return repoReadFail(streams, "serve"); }) |resolved| { root_dir.close(io); root_dir = resolved; } var file = root_dir.openFile(io, repo_filename, .{}) catch { root_dir.close(io); return repoReadFail(streams, "serve"); }; defer file.close(io); const log_st = file.stat(io) catch { root_dir.close(io); return repoReadFail(streams, "serve"); }; // Bring-up order is load-bearing (rationale in `serve_mod.listen`'s // doc): open the listener FIRST — before ANY O(store) work, which here means // both the config read below and the prime, each a full-log scan. A client // that connects during either queues in the accept backlog instead of being // refused, so the readiness window starts at the bind rather than after the // scans. Announce "listening" only once the bind has actually succeeded (so // it never lies about a listener that failed to come up); the accept loop // then arms each connection's deadline on accept. The backlog property this // rests on is pinned by serve.zig's "listen opens the accept backlog before // serve" test. const address = std.Io.net.IpAddress.parse("127.0.0.1", parsed.port) catch { streams.err.print("zit serve: invalid --port value '{d}'\n", .{parsed.port}) catch {}; return .usage; }; var listener = serve_mod.listen(io, address) catch { streams.err.print("zit serve: cannot bind 127.0.0.1:{d}\n", .{parsed.port}) catch {}; return .failure; }; defer listener.deinit(io); streams.err.print("zit serve: listening on http://127.0.0.1:{d}/xfer (Ctrl-C to stop)\n", .{parsed.port}) catch {}; streams.err.flush() catch {}; var fs = storage.FileStorage.init(io, file, log_st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| { root_dir.close(io); switch (e) { error.OutOfMemory => streams.err.writeAll("zit serve: out of memory\n") catch {}, else => streams.err.print("zit serve: cannot read repository: {s}\n", .{reasonOf(e)}) catch {}, } return .failure; }; defer log.deinit(gpa); defer root_dir.close(io); // §9.1/§8.2 — emit the repository's stored project code in the clone // reply's `push` card, so a clone learns the shared project identity. A // repo predating the stored code, or an unreadable config, falls back to // the fixed placeholder (informational; servercode never gates a clone). var maybe_cfg: ?logstore.OwnedConfig = log.readConfig(gpa) catch null; defer if (maybe_cfg) |*oc| oc.deinit(gpa); const project_code: []const u8 = if (maybe_cfg) |oc| (if (oc.config.project_code.len > 0) oc.config.project_code else serve_mod.server_projectcode) else serve_mod.server_projectcode; // §9.1 — a real instance identity in the clone reply's `push` card, // derived from this checkout's absolute path exactly as init derives the // project code (stable, distinct per instance, no ambient RNG). The // placeholder constant remains only as the test fixtures' server identity. const serve_code = zit.sync.deriveServerCode(&.{root.abs}); const server = zit.sync.Server{ .version = serve_mod.server_version, .credential = null, .self_servercode = &serve_code, .project_code = project_code, }; var ctx = ServeCtx{ .io = io, .root_dir = root_dir, .file = &file, .diag = streams.err, .server = &server, .amb = amb, .fronted = parsed.fronted }; defer ctx.dropSnapshot(); const responder = serve_mod.Responder{ .ctx = &ctx, .answerFn = serveAnswer, .maintainFn = ServeCtx.maintain, .fronted = parsed.fronted }; if (ctx.baseSnapshot(gpa)) |_| {} else |_| return repoReadFail(streams, "serve"); serve_mod.serve(io, gpa, &listener, responder, .{}) catch |e| switch (e) { error.Accept => { streams.err.writeAll("zit serve: connection accept failed\n") catch {}; return .failure; }, }; return .ok; } /// The per-request context threaded into `serve.Responder`: the open log to read /// a fresh store from, the server identity to answer with, and the injected IO + /// repository directory used to make pushed content durable (a push request /// ingests new artifacts into the per-request store, which must be persisted into /// the served repository's on-disk log — sync §3.3). const ServeCtx = struct { io: std.Io, root_dir: std.Io.Dir, /// The ambient environment, carried only so `resolveSetting` (the one /// settings resolver) is reachable per round. Reading the config frame /// directly here instead would be a second reader, and the two drifting is /// the bug `signingSettings` documents having shipped once already. amb: ?cli.Ambient = null, /// Behind a trusted front. Turns on the per-round membership /// rule and keeps the repository image warm between connections so that /// rule reads O(image), not O(store), after a push. fronted: bool = false, /// The open read handle on the served `.zit`. `respond` re-reads the CURRENT /// on-disk log from it (§3.8): a fresh stat fixes the committed end-of-log, /// so a check-in appended by a separate `zit commit` while `serve` runs is /// advertised on the next request — no restart. /// /// The handle pins an INODE, not a path. `zit pack` and `zit shun --force` /// replace `.zit` by rename, which would leave a held-open handle reading /// the replaced inode forever (its fstat never moves again — no cache key /// over the handle can detect the swap). `baseSnapshot` therefore stats /// the PATH beside the handle each round and REOPENS on an inode change: /// the swap is noticed at the next round boundary, so a shun that /// rode a generation rewrite stops being served with no restart, while an /// in-flight round keeps its consistent pre-swap view. A pointer into /// `runServe`'s frame so the reopen and the shutdown close agree on one /// current handle. file: *File, /// Where `serveAnswer` names a failed request, or null to stay silent. The /// tests and `driveHttp` leave it unset; `runServe` points it at stderr. diag: ?*std.Io.Writer = null, server: *const zit.sync.Server, /// The most recent snapshot of the on-disk log, reused across requests. A /// log-backed INDEX (not a full-container copy): it reads the containers it /// serves on demand and holds only the index between rounds. Built from /// the long-lived allocator `serve` threads into every answer, so it outlives /// the request that built it, and owned here — heap-allocated because it owns /// the `FileStorage` its log borrows; `dropSnapshot` frees it. Never handed to /// `respond` directly — see `serveAnswer`. snapshot: ?*LogIndexStore = null, /// The snapshot's cluster index, rebuilt with it. clusters: ?clusterindex.Resolved = null, /// The log size `snapshot` was built from — one half of the cache key that /// `baseSnapshot` stats per request (see there for why size alone is not it). snapshot_log_size: u64 = 0, /// The log's last-modification time (ns) when `snapshot` was built. Together /// with the size it is the cache key: a write that does not change the length /// still moves `mtime`. A `std.Io.Timestamp`, unwrapped to its `nanoseconds` /// here so the key is two scalar comparisons. snapshot_log_mtime_ns: i96 = 0, /// How many snapshots this server has built. Not used by the server; it is /// the observable that makes "snapshot builds do not grow with the round /// count" a testable property rather than a wall-clock claim. loads: u64 = 0, /// Free the cached snapshot, leaving the context usable: the next /// `baseSnapshot` rebuilds. Both a caller finishing with the context and the /// rebuild path itself want exactly this. fn dropSnapshot(self: *ServeCtx) void { if (self.clusters) |*c| c.deinit(); self.clusters = null; if (self.snapshot) |s| s.destroy(); self.snapshot = null; self.snapshot_log_size = 0; self.snapshot_log_mtime_ns = 0; } /// Drop plaintext the cached snapshot memoised while answering, keeping the /// retained containers it serves from. Called once per round, after every /// borrow of the snapshot has ended. /// /// Inflating is how `content()` answers a wire-backed entry, and the /// responder's own §7.2 size proofs call `content()` on every artifact they /// emit. Against a throwaway per-request store that plaintext died with the /// request; against a store that outlives the request it would accumulate /// until the server held the whole repository's uncompressed history. The /// containers stay, so the next round answers identically. /// /// The memo is given up rather than capped because in an optimized build /// there is almost nothing to give up: the size proofs are `runtime_safety` /// only, so a clone round inflates nothing and the memo would stay empty /// regardless. Measured over four clones of a 3001-artifact repository, /// ReleaseFast: 12 MB resident and 0.09 s of server CPU with this release, /// against 26 MB and 0.15 s without it. A safety-checked build does re-prove /// every emitted artifact each round and so pays to re-inflate; that cost /// belongs to the proofs, not to the cache. fn releaseSnapshotPlaintext(self: *ServeCtx) void { if (self.snapshot) |s| s.releaseTransient(); } /// §3.8 — the store describing the CURRENT on-disk log, rebuilt only when the /// log has actually grown. The §3.8 guarantee is that a check-in appended by /// a separate `zit commit` is advertised on the NEXT request with no restart; /// that needs a fresh `stat` per request, not a fresh rebuild per request. /// Rebuilding regardless made every round cost the whole repository, which is /// what turned a clone of a large store into O(rounds x store) server work. /// /// A torn tail from an append caught mid-write is rejected by the read-only /// scanner exactly as before; the previous snapshot is dropped only once a /// replacement is fully built, so a failed rebuild leaves the cache coherent. /// /// The rebuild itself is O(store) and must therefore not fall inside a /// request whose connection carries a deadline. `runServe` primes the cache /// before it accepts anything. A round that appends to the log does NOT /// rebuild before it returns — it deliberately leaves the snapshot stale /// (describing the pre-append log) rather than pay the O(store) rebuild inside /// its own reply deadline. The append-only stat gate above makes that /// stale snapshot self-correcting: the size no longer matches, so the next /// `baseSnapshot` rebuilds. That next call is the accept loop's `maintain` /// hook (see `ServeCtx.maintain`), run between connections with no connection /// open — so the deferred rebuild still lands outside every connection /// deadline, never on an innocent later request. Nothing here can bound the /// startup build itself; that is bounded by the store, and always was. /// /// THE CACHE KEY IS NOT THE SIZE ALONE, and since this snapshot began carrying /// the §3.3 shun table it cannot be. A stale snapshot used to cost stale CONTENT /// — a peer misses a newly committed artifact and the next round supplies it — /// which is self-correcting. It now also decides whether the SUPPRESSION table is /// re-read, and a missed re-read serves an artifact the operator shunned, with /// nothing failing and nothing logged. "Every writer only appends, so the length /// always grows" is true of `zit shun --defer` and `zit commit` today, but it is /// an assumption about all present and future writers rather than a checked /// property: a generation rewrite landing on the same byte length, or any /// out-of-band mutation, would keep the old mask forever. So the key is size AND /// mtime — an in-place rewrite that does not change the length still moves the /// mtime. Two scalar comparisons against one `stat` this function already /// performs, and they can only over-invalidate (which costs a rebuild), never /// under-invalidate on a write that reaches THIS INODE. /// /// WHAT THE CACHE KEY CANNOT COVER, THE PATH STAT DOES. `st` comes /// from an `fstat` on the held handle, so it detects every write reaching /// that inode and no rename ABOVE it: a `zit shun --force` (or `zit pack`) /// generation swap leaves the old inode's size and mtime frozen forever, /// with the shun row in a file this handle would never see. So the path is /// stat'ed beside the handle each round, and an inode mismatch reopens the /// handle (dropping the snapshot) before anything is answered — the round /// then serves the live generation. Failure on the path stat or the reopen /// keeps the old handle and answers from it: availability first, and the /// next round retries the swap detection. fn baseSnapshot(self: *ServeCtx, gpa: std.mem.Allocator) serve_mod.AnswerError!repostate.ArtifactStore { var st = self.file.stat(self.io) catch return error.ServerStorage; if (self.root_dir.statFile(self.io, repo_filename, .{})) |path_st| { if (path_st.inode != st.inode) { if (self.root_dir.openFile(self.io, repo_filename, .{})) |fresh_handle| { self.file.close(self.io); self.file.* = fresh_handle; self.dropSnapshot(); st = self.file.stat(self.io) catch return error.ServerStorage; } else |_| {} } } else |_| {} if (self.snapshot) |s| { if (st.size == self.snapshot_log_size and st.mtime.nanoseconds == self.snapshot_log_mtime_ns) return s.store(); } // Build the fresh index BEFORE dropping the old one: a failed rebuild // leaves the previous snapshot (and its size) intact, so the stat gate // keeps mismatching until a rebuild succeeds and stale content is never // served in the meantime. `openFromFile` fully frees itself on any error. const fresh = LogIndexStore.openFromFile(gpa, self.io, self.file.*, st.size) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; self.dropSnapshot(); self.snapshot = fresh; // Resolve the cluster index over the new snapshot, then drop // the cluster plaintext it memoised: the index keeps only names and bits. self.clusters = loadClusterIndex(gpa, self.io, self.root_dir, &fresh.log, fresh.store()) catch null; fresh.releaseTransient(); self.snapshot_log_size = st.size; self.snapshot_log_mtime_ns = st.mtime.nanoseconds; self.loads += 1; return fresh.store(); } /// The accept loop's between-connections maintenance hook (see /// `serve.Responder.maintainFn`). Runs after a connection has closed and /// before the next accept, so no connection is open to charge its O(store) /// work to. It re-primes the cache via the same stat-gated `baseSnapshot`: /// after a pure read round the stat is unchanged and this is one `stat`; after /// a round that appended (a push) the log grew, so this rebuilds the snapshot /// the pushing round deliberately left stale rather than pay inside its /// deadline. A failed rebuild is swallowed for the same reason /// `baseSnapshot` leaves the cache coherent on failure: `baseSnapshot` drops /// the old snapshot only after a fresh one is fully built and updates /// `snapshot_log_size` only on success, so a failed rebuild RETAINS the /// previous (stale) snapshot with its old size — the stat gate keeps /// mismatching until a rebuild succeeds, and stale content is never served in /// the meantime. The next round (or the next `maintain`) retries, still /// outside any earlier round's budget. Type-erased to match the `maintainFn` /// seam. fn maintain(ctx_ptr: *anyopaque, gpa: std.mem.Allocator) void { const self: *ServeCtx = @ptrCast(@alignCast(ctx_ptr)); const before = self.snapshot_log_size; if (self.baseSnapshot(gpa)) |_| {} else |_| {} // A push left the zero-parse image stale, and the next // protected push would re-derive it inside its own deadline. Derive it // here instead, between connections, where O(store) is allowed — and // only when the log actually grew, so a pull-heavy fronted server does // not pay an image load per connection. Only a fronted server has a // resolver to warm; a failure is retried by the round itself, which // derives on demand and fails closed on a fault. if (self.fronted and self.snapshot_log_size != before) { var repo: Repository = undefined; if (Repository.open(&repo, gpa, self.io, self.root_dir, .{ .mode = .fast_read })) |_| { repo.deinit(gpa); } else |_| {} } } }; /// The `serve.Responder` seam: snapshot the repository into an in-memory store, /// answer the framed request over it, and — when the request pushed new content — /// make that content durable in the served repository's on-disk log (sync §3.3). /// /// `respond` stays sans-IO: it only `put`s pushed artifacts into the snapshot /// store it is given. The durable write is this serve-layer's job and reuses the /// idempotent `persistStore` (a re-push of an already-stored hash appends nothing /// — content is addressed by hash), the very path clone/pull persist through. A /// pure gimme/pull request makes nothing new servable, so the overlay stays /// empty and the log is never reopened; only a genuine push touches disk. /// /// `respond` is answered over a per-request WRITE OVERLAY of the cached snapshot /// (`repostate.OverlayStore`), never over the snapshot itself: the responder /// interns phantoms and upgrades entries as it works, and letting those survive /// into the next round changes what that round serves. Reads fall through to the /// snapshot, so reuse costs nothing; writes are confined to the overlay, and the /// persist pass — which is idempotent over everything already in the log — can /// only append what the overlay holds. /// /// The pushed bytes were size-validated by `respond`'s request scan before they /// entered the store, so `persistStore` only ever writes well-formed content, and /// a mid-write fault surfaces before `commit`, never as a partial log append. fn serveAnswer(ctx_ptr: *anyopaque, gpa: std.mem.Allocator, request_container: []const u8, cookie: ?[]const u8, protect: ?[]const u8, request_compressed: bool) serve_mod.AnswerError!serve_mod.XferReply { const ctx: *ServeCtx = @ptrCast(@alignCast(ctx_ptr)); return serveAnswerInner(ctx, gpa, request_container, cookie, protect, request_compressed) catch |e| { // NAME THE FAILURE. `serve.zig` collapses every AnswerError into one // bare status -- `ServerStorage`, `OutOfMemory` and `ResponseTooLarge` // all become 500 with no body and no log line -- so an operator (or a // bench harness) sees "HTTP 500" and has no way to learn which of three // unrelated faults produced it. This is the server-side half of naming // the status the client now reports; without both ends a 500 is a dead // end for anyone diagnosing from the outside. // // Single funnel on purpose: every `AnswerError` this responder can // produce passes through here, so a future cause is reported without a // second edit. Diagnostics only -- the error is re-raised unchanged and // the status the peer sees does not move. if (ctx.diag) |w| { w.print("zit serve: request failed: {s}\n", .{@errorName(e)}) catch {}; w.flush() catch {}; } return e; }; } fn serveAnswerInner(ctx: *ServeCtx, gpa: std.mem.Allocator, request_container: []const u8, cookie: ?[]const u8, protect: ?[]const u8, request_compressed: bool) serve_mod.AnswerError!serve_mod.XferReply { const reply = try serveRound(ctx, gpa, request_container, cookie, protect, request_compressed); // Past this point nothing borrows the snapshot: the overlay is gone with // `serveRound`'s frame and the reply owns its own bytes. Dropping the // plaintext memo is bounded (it frees only what this round inflated), so it // is safe inside the deadline. // // A round that appended (a push) has left the snapshot stale — it still // describes the pre-push log. It is NOT rebuilt here: an O(store) rebuild // inside the reply's connection deadline is exactly the failure this avoids. The // rebuild is the accept loop's job, via `ServeCtx.maintain`, run after this // connection closes and before the next accept — outside every deadline. The // append-only stat gate in `baseSnapshot` makes the stale snapshot self- // correcting: the next `baseSnapshot` (in `maintain` or the next round) sees // the grown size and rebuilds, so no `appended` signal needs to be threaded // out of the round. // // A post-scan read fault this round mapped a held artifact to null. The §8.2 // clone pager skips such a rid and still reports the sequence complete // (NEXT 0), silently finishing a clone MISSING an artifact — a failure the // copy-based prime could not produce (a fault surfaced at prime, and a // retained-but-uninflatable container was still served verbatim). Read the // flag BEFORE releasing the round's memos (which clears it), then surface it // as a storage error so the client retries; the next round re-reads. const faulted = if (ctx.snapshot) |s| s.read_fault else false; ctx.releaseSnapshotPlaintext(); if (faulted) { gpa.free(reply.body); return error.ServerStorage; } return reply; } /// One round over a per-request write overlay. Split out of `serveAnswer` so the /// overlay — which borrows the cached snapshot — is provably destroyed before /// `serveAnswer` touches that snapshot's storage. A round that pushes new content /// persists it durably here; the snapshot it read from is left describing the /// pre-push log, and `baseSnapshot`'s append-only stat gate rebuilds it on the /// next call (in the accept loop's `maintain`, never inside this round's /// deadline), so this round signals nothing back to its caller. fn serveRound( ctx: *ServeCtx, gpa: std.mem.Allocator, request_container: []const u8, cookie: ?[]const u8, protect: ?[]const u8, request_compressed: bool, ) serve_mod.AnswerError!serve_mod.XferReply { var overlay = repostate.OverlayStore.init(gpa, try ctx.baseSnapshot(gpa)); defer overlay.deinit(); // sync §3.3 L527-530 / §8.8 L1777-1779 — answer this round with THIS // snapshot's shun table. `ctx.server` is the caller's long-lived config and // stays const; the per-round copy carries the fields that are properties of // the snapshot rather than of the server, so the reply's phantom scan never // begs for an artifact this repository has shunned (a client `igot` for a // shunned hash still interns its phantom — §3.3 r24(1) L511-514 — it is just // never requested). // `ctx.baseSnapshot` above published it — the rebuild path assigns // `self.snapshot`, the cached path requires it non-null — so the null arm is // unreachable today. It is a CHECKED error rather than a fail-open `.none` // because "the filter is not skipped" must hold in a release build: a future // ctx that primes a store without publishing the snapshot, or an early-out // added above, must fail the round rather than silently serve with no shun // table and re-beg every shunned hash with nothing failing. // // submanifests §8.1/§7.2 — the same per-round freshness arms the send gate. // The long-lived `ctx.server` is a build identity (version, servercode, // project code) and outlives every snapshot; the seen-submanifest indicator, // the §8.1 emission setting, and the two settings this server only REPORTS // are facts about the CURRENT log and are re-read whenever the snapshot is. // Copying per round is what keeps them from drifting: a push that stores this // repository's first tree arms the gate for the next round rather than for // the next process, and a `zit submanifest enable`/`retune` against a served // repository is obeyed by the next round rather than the next process. These // read the snapshot unconditionally for the same reason the shun table does — // serving a stale default is the failure, not a fallback. var round_server = ctx.server.*; const snapshot = ctx.snapshot orelse return error.ServerStorage; round_server.shun = snapshot.shun.shunSet(); // §9.3 r28 — and the row view, for a `reqconfig /shun`. round_server.shun_rows = snapshot.shun_rows; // Sparse-cone RFC section 6.2 -- cone-scoped requests answer filtered. round_server.cone_classifier = snapshot.coneClassifier(); // Bounded announcement: the clusters carry what they name. if (ctx.clusters) |*c| { round_server.clustered = &c.clustered; round_server.cluster_names = c.cluster_names; } // This server's own content policy, re-read per round like every // other setting here, so `zit settings binary-glob ...` against a served // repository is obeyed by the next round rather than the next process. // // Resolved from the SERVER's settings, never from the artifacts arriving in // the push: a push carrying its own policy would otherwise arrive bearing // permission to land. The versioned tier is still consulted, and that is // deliberate -- it reads the server's own working tree, which a push does // not touch, so adopting a project's policy stays an act the operator takes // by updating, exactly as `zit trust` is for before-commit consent. var policy_arena = std.heap.ArenaAllocator.init(gpa); defer policy_arena.deinit(); if (ctx.amb) |amb| { // The repository tier comes from the snapshot's per-build // config read, NOT from `resolveSetting`, whose repository read opens // the store and re-scans every frame header. That scan per setting // per round is what made a 1.17M-artifact clone take half an hour. // The snapshot's (size, mtime) key preserves the per-round freshness // contract: a settings write grows the log and rebuilds the copy. const pa = policy_arena.allocator(); const repo_bin: ?[]const u8 = if (snapshot.policy_binary_glob.len != 0) snapshot.policy_binary_glob else null; const repo_crlf: ?[]const u8 = if (snapshot.policy_crlf_glob.len != 0) snapshot.policy_crlf_glob else null; const bin = resolveSettingWith(ctx.io, pa, ctx.root_dir, amb, .binary_glob, repo_bin) catch zit.settings.Resolved{ .value = "", .source = .default }; const crlf = resolveSettingWith(ctx.io, pa, ctx.root_dir, amb, .crlf_glob, repo_crlf) catch zit.settings.Resolved{ .value = "", .source = .default }; round_server.push_policy = .{ .binary_glob = bin.value, .crlf_glob = crlf.value }; } // Sparse-cone RFC section 5.3 -- a sparse origin refuses clones it does // not cover. Read per round like every other snapshot-freshness fact: an // `update --cone` against the served tree is obeyed by the next round. var origin_cone = zit.repocache.readStoreCone(gpa, ctx.io, ctx.root_dir); defer if (origin_cone) |*c| c.deinit(gpa); round_server.origin_cone = if (origin_cone) |*c| c else null; // sync §7.7 L1428-1432 — the other half of the same admission predicate, from // the same snapshot, so `ingest` never pairs this round's shun table with a // stale policy. round_server.hash_policy = snapshot.hash_policy; round_server.seen_submanifest = snapshot.seen_submanifest; round_server.submanifests_enabled = snapshot.submanifests_enabled; round_server.submanifest_k = snapshot.submanifest_k; round_server.submanifest_auto_retune = snapshot.submanifest_auto_retune; // The front's membership rule for THIS push, and the resolver it // needs. Both set together: a rule with no resolver fails closed in // `pushProtectFault`, which is the right answer to a misconfiguration but // not a state this code should be able to produce. The resolver opens the // repository lazily -- a round that pushes nothing protected never pays -- // and the view it opens reads the on-disk log, which every earlier round // already persisted, so a parent pushed last round resolves; a parent in // this round is resolved in-round by the rule itself. var resolver = RoundBranchResolver{ .ctx = ctx, .gpa = gpa }; defer resolver.deinit(); if (protect) |globs| if (globs.len != 0) { round_server.push_policy.protect_globs = globs; round_server.branch_of = resolver.resolver(); }; const reply = try serve_mod.handleXfer(gpa, &round_server, overlay.store(), request_container, cookie, request_compressed); errdefer gpa.free(reply.body); // sync §9.3 r33 — an AUTHORIZED pushed config card's settings, applied to // the served repository here because this is where the config lives; the // responder is sans-IO and only reports the decision. Unreachable today by // construction: `runServe` configures no credential, so `Server.hasAdmin` is // false for every requester and every push is rejected (the e2e pins that). // It is wired anyway because the alternative — a server that can AUTHORIZE a // push and then silently drop it — is the failure mode worth ruling out. if (reply.config_apply.any()) { updateConfig(gpa, ctx.io, ctx.root_dir, .{ .enable_submanifests = reply.config_apply.enable, .submanifest_k = reply.config_apply.k, .submanifest_auto_retune = reply.config_apply.auto_retune, }) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; } if (overlay.writtenCount() > 0) { // submanifests §7.1 — the durable half of the push-side duty; the // reply-visible half is `sync.ingest`, which phantomizes into the very // store the §3.2 walk reads. The gap COUNT is discarded on purpose: a // stateless server (§3.8/§8.9) has no operator to report to, and §7.1 // asks for a phantom, not a diagnostic. _ = persistStore(gpa, ctx.io, ctx.root_dir, overlay.store(), null, 0) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; } return reply; } /// `sync.BranchResolver` over the served repository's own view, /// opened on first use and only for the round. `.fast_read` takes the /// zero-parse image when it is fresh (`ServeCtx.maintain` keeps it so) and /// derives when it is not, so the answer is crosslink's own in either case -- /// which is the point: "which branch does this check-in resolve to" has one /// authority, and this is it, not a re-implementation over raw tags. const RoundBranchResolver = struct { ctx: *ServeCtx, gpa: std.mem.Allocator, repo: Repository = undefined, opened: bool = false, failed: bool = false, fn resolver(self: *RoundBranchResolver) zit.sync.BranchResolver { return .{ .ctx = self, .branchOfFn = branchOf, .mainBranchFn = mainBranch }; } fn deinit(self: *RoundBranchResolver) void { if (self.opened) self.repo.deinit(self.gpa); self.* = undefined; } /// Open once; a failure is sticky for the round so the rule fails closed /// on every question rather than retrying a broken read per artifact. fn open(self: *RoundBranchResolver) error{Unavailable}!*Repository { if (self.failed) return error.Unavailable; if (!self.opened) { Repository.open(&self.repo, self.gpa, self.ctx.io, self.ctx.root_dir, .{ .mode = .fast_read }) catch { self.failed = true; return error.Unavailable; }; self.opened = true; } return &self.repo; } fn branchOf(ctx_ptr: *anyopaque, hash: []const u8) error{Unavailable}!?[]const u8 { const self: *RoundBranchResolver = @ptrCast(@alignCast(ctx_ptr)); const repo = try self.open(); const rid = repo.lookup(hash) orelse return null; // `branchOf` is defined on check-ins; a P card naming a blob (or a // phantom) resolves to nothing rather than tripping its precondition. if (!repo.view.state.hasEvent(rid)) return null; return repo.branchOf(rid); } fn mainBranch(ctx_ptr: *anyopaque) error{Unavailable}![]const u8 { const self: *RoundBranchResolver = @ptrCast(@alignCast(ctx_ptr)); const repo = try self.open(); return repo.view.main_branch; } }; /// A serve snapshot that reads container bytes from the log ON DEMAND instead of /// copying the whole repository into RAM at prime. It owns a read-only /// `LogStore` — whose open scan already whole-frame-CRC'd every byte, so the /// single read stays; the second read `loadContentStore` performed does not — and /// a resident INDEX: the adapter-rid→log-rid map plus the lowercase-hex wire names /// the responder walks. `held()` answers from that index with zero I/O; `wire()` /// and `content()` read the one frame they are asked for and memoise it for the /// round; `releaseTransient` drops those memos between rounds, so a round's /// resident bytes are bounded by what it emitted, not by the store. /// /// Heap-allocated because it owns a `storage.FileStorage` whose `storageFile()` /// vtable points back into the struct — the box must not move once the log borrows /// it. `openFromFile` takes that ownership (the serve path); `open` borrows a /// caller-owned `StorageFile` (a test instrumenting read traffic). /// /// Adapter rids `1..=rids.len` name the log's non-config artifact frames in /// `ridsInPersistedOrder` — the SAME enumeration `loadContentStore` assigns, so /// adapter rid r is byte-for-byte the rid the old MemStore snapshot gave, which is /// what keeps the clone pager's persisted-order cursor (sync §8.2) stable. A frame /// named in the repository shun table keeps its slot in that enumeration and is /// MASKED rather than removed (see `masked`), so the shun table never renumbers /// a rid and §8.2 L1504-1506's "silently skipping gaps" carries the exclusion. Rids /// above `rids.len` are an appendix `MemStore` holding any hash a caller `intern`s /// or `put`s that the log does not carry. The serve path never writes the base /// (every write lands in the per-request `OverlayStore`), so the appendix stays /// empty in production; it exists so this store honours the whole `ArtifactStore` /// interface — composed over the existing `MemStore` rather than hand-rolled — /// instead of trapping on a write it will never see. const LogIndexStore = struct { gpa: std.mem.Allocator, /// Owned storage binding on the `openFromFile` path (the log borrows /// `&self.fs`); `undefined` on the `open` path, where the caller owns the /// storage and keeps it alive for this store's life. fs: storage.FileStorage, log: logstore.LogStore, /// Adapter rid a (1-based) → `rids[a-1]` = the log rid. Persisted order, config /// frames removed. rids: []logstore.Rid, /// Adapter rid a → `names[a-1]` = its lowercase-hex wire name (sync §3.3), /// backed by `name_arena`. The single site the binary-log digest becomes the /// wire name — the boundary `loadContentStore` held, kept to one place. names: [][]const u8, name_arena: std.heap.ArenaAllocator, by_name: std.StringHashMapUnmanaged(repostate.Rid) = .empty, /// Sparse-cone RFC section 6.2 -- single-entry classification memo for /// cone-scoped requests: the server is stateless per round, so without /// this every round of a cone-scoped clone would re-parse the store. /// Keyed by the cone digest; dropped with the snapshot, so a rebuilt /// snapshot can never serve a stale classification. cone_memo_digest: ?[32]u8 = null, cone_memo: ?SparseClassification = null, /// sync §9.5 L1988-1992 — adapter rid a → `masked[a-1]`: this frame's name /// is in the repository shun table, so the enumeration MASKS it. §9.5 asks for /// "igot for every eligible holding after the Section 3.3 shun … filters", and /// this array is where that filter lives: a masked entry answers `held` false, /// `content`/`wire` null, and its name is left out of `by_name`, so the server /// does not announce it (§3.2's walk gates on `held`), and an inbound `gimme` /// for it — which keys on `lookup` — lands on §3.3 L532-535's "If absent, /// silently skip" with no code on that path. /// /// THE CLONE HALF IS AN ERRATUM CANDIDATE, not a citation. Because the mask /// lives in the enumeration, a masked holding is also absent from the clone /// batch — but no draft clause says a clone server withholds one. §9.5's /// sentence is scoped to the `send-catalog` pragma's igot inventory; §3.4 /// L545-547 hands clone to §8.1-§8.4, and §8.2 L1504-1506 licenses only the /// MECHANISM ("silently skipping gaps"), never the policy. Applying §3.3's /// shun set to the clone reply is this implementation's extension, taken /// because the alternative — a repository under a takedown order handing the /// content to every new clone — is not a behaviour an operator would read the /// drafts' silence as licensing. Erratum candidate against sync §8.1/§8.2: /// state whether a clone server's batch is filtered by the §3.3 shun set. /// /// SCOPE, stated exactly: this masks what THIS BASE SNAPSHOT enumerates. A /// served round answers over a per-request `repostate.OverlayStore` wrapping the /// base, and its `putImpl` keys on `base.lookup` — which the mask answers null /// for — so a peer that PUSHES `file <masked name>` in the same round would /// create an overlay-local rid that genuinely holds those bytes and is /// announced by that round's §3.2 walk. That route is closed, but NOT here: it /// is closed by receive-side admission (§7.7 L1437-1440: "A non-clone file /// whose target is shunned is consumed and discarded after its payload and /// pending private marker; it emits no error … and makes no repository /// mutation."), which `scanCard` applies before the card can enter `scan.files` /// and `ingest` re-applies before the `put`, both reading `Server.shun` — the /// set `serveRound` builds from THIS SAME snapshot's table, so the two can /// never disagree about a name. Recorded here because the send-side filter /// genuinely cannot reach that path: what makes the guarantee hold is the pair, /// and removing either half reopens it. /// /// A MASK, not a removal, and §8.2 L1504-1506 is why: "SEQNO is a server /// blob-table RID. Iterate from supplied RID through current maximum, /// silently skipping gaps; clients MUST NOT infer count or size from range." /// Dropping the frame from the enumeration would renumber every survivor above /// it. The server is stateless (§8.9 L1783-1784: "Section 3.7 is authoritative; /// the stateless server signals no completion") — the client's SEQNO *is* the clone /// cursor — and a shun row appended between two rounds of one clone grows the /// log, which is exactly what trips `ServeCtx.baseSnapshot`'s append-only stat /// gate and rebuilds this index. Renumbering there would make a resuming /// client's cursor step over a survivor and still be told the sequence was /// complete: a silently short clone, the failure class `read_fault` exists to /// prevent. Keeping the slot and answering it as a §8.2 gap makes adapter-rid /// identity a pure function of the log, independent of the shun table. /// /// Filled in the SAME loop as `rids`/`names`/`content_memo`/`wire_memo`, off /// the same index — the FILL alignment is structural (one loop, one counter), /// not asserted. A mask derived from ANY second walk is the one way adapter rid /// r could come to name one artifact and mask another, and nothing about the log /// makes such a walk look wrong on inspection: it would agree on length and on /// set membership and differ only in which slot carries which answer. The /// LENGTHS are a separate question and the fill loop cannot speak to them; they /// are gated by a checked error at the end of `build` (see there). /// /// AND A PURE FUNCTION OF `shun.digests` — the other half of the same /// invariant, and the one a later filter is most likely to break. `wireImpl`'s /// delta-basis check asks the SET (`shun.digests.contains(d.baseHash())`) while /// `held`/`content`/`wire` by rid ask THIS ARRAY; the two agree only because the /// fill loop below defines `masked[a]` as exactly that membership. §9.5 /// L1988-1992 names FOUR filters — "igot for every eligible holding after the /// Section 3.3 shun, phantom, private, and peer-has filters" — and this array is /// the natural landing site for the other three. The day one of them widens /// `masked` beyond shun membership, the basis check silently stops following it /// and the server re-emits a DELTASRC it will not send. Widening this array /// therefore obliges the same change to resolve the basis THROUGH it (basis /// digest → adapter rid → `masked[rid - 1]`) rather than through the set. masked: []bool, /// Per-round transient memos over the persisted rids: the owned plaintext / /// wire container that `content()`/`wire()` read on demand, released each round /// by `releaseTransient`. content_memo: []?[]u8, wire_memo: []?WireHold, /// Hashes not in the log, interned/put by a caller. Empty on the serve path. appendix: repostate.MemStore, /// sync §3.3 L527-530 / §8.8 L1777-1779 — the served repository's shun table, /// re-loaded on every build. That is the freshness mechanism the rest of this /// index already rides: `ServeCtx.baseSnapshot` rebuilds whenever the log has /// grown, and a `zit shun` grows it, so a shun row written between two served /// rounds is honoured by the next round with no restart (§3.8's guarantee /// applied to the shun table rather than to check-ins). shun: RepoShunSet, /// sync §9.3 r28 — the same shun table as ROWS, for the `/shun` config group /// a peer requests. `shun` above answers "is this artifact suppressed" for /// the send filter; this carries the operator metadata (when, and why) that /// a peer needs to reconstruct the tombstone rather than merely obey it. /// /// The reason text borrows `self.log`'s own row storage (which this /// snapshot owns and outlives every round served from it), but the NAMES do /// not: the log keys shun rows by BINARY digest and the wire carries /// lowercase hex, so the hex is materialised once here rather than per /// round. `shun_hex` is its backing. shun_rows: []zit.sync.ShunRow, shun_hex: []u8, /// sync §7.7 L1428-1432 — the served repository's effective hash-policy, read /// off the same build as `shun` because it is the other half of the same /// receive-side admission predicate and must never be a build behind it. hash_policy: artifact.HashPolicy, /// Total plaintext this store has read on demand from the log, monotonic /// across `releaseTransient` — the work counter that makes "answering a round /// materialises only what it emits" testable (the log-backed twin of /// `MemStore.inflated_bytes`). Container reads are excluded, as they were /// there: this measures inflation, not verbatim retransmission. materialized_bytes: usize = 0, /// Set true when an on-demand `content()`/`wire()` read this round mapped a /// real error to null (a post-scan I/O fault or an OOM building the wire /// form) — NOT when a frame legitimately has no plaintext or no wire form. /// The vtable cannot signal an error, so the §8.2 clone pager would skip the /// null rid and still report the sequence complete, shipping a clone missing /// an artifact; `serveAnswer` reads this after the round and converts a set /// flag to `error.ServerStorage` so the client retries instead. Cleared with /// the round's memos by `releaseTransient`, so every round starts clean. read_fault: bool = false, /// draft-fossil-submanifests-00 §8.1 — this log's durable seen-submanifest /// indicator, read from the config frame of the SAME log generation this /// index describes. It arms the §7.2 send gate (`serveRound`), and it lives /// here rather than beside the long-lived `sync.Server` so it is re-read /// whenever the snapshot is: a push that lands the repository's first tree /// grows the log, the stat gate rebuilds the index, and the very next round /// serves that tree through an ARMED gate. Cached on a stale snapshot it /// would stay false for the life of the process. seen_submanifest: bool = false, /// draft-fossil-submanifests-00 §8.1 — this log's `enable-submanifests` /// setting, read from the same config frame and for the same reason: §7.2 /// makes the reply's `avoid-submanifests` pragma a function of it, so a /// stale copy would advertise the wrong policy for the life of the process. submanifests_enabled: bool = zit.sync.submanifest_emission_default, /// draft-fossil-submanifests-00 §6.3 / §6.4 — this log's pinned /// partition parameter and its auto-retune switch, read off the same config /// frame as the two above and for the same freshness reason: sync §9.3 /// serves them to a cloning peer, so an operator who re-pins k must be /// answered by the next round, not by the next process. submanifest_k: u8 = 0, submanifest_auto_retune: bool = false, /// The repository tier of the two push-policy settings, /// copied (owned) off the SAME per-build config read as the four facts /// above. Before this rode the snapshot, `serveAnswer` resolved the pair /// through a fresh `LogStore.openReadOnly` per setting per round -- a /// full frame-header scan of the log each time, which on a 1.17M-artifact /// store turned every round into ~11 seconds of dead re-scanning and a /// clone into half an hour. Freshness is unchanged: a settings /// write appends a config frame, the log grows, and the (size, mtime) /// snapshot key rebuilds this copy for the next round -- exactly the /// contract `seen_submanifest` above already relies on. An empty string /// is the repository having no opinion, as in `repositorySetting`. policy_binary_glob: []const u8 = "", policy_crlf_glob: []const u8 = "", /// A memoised wire form for one persisted rid: the owned §4.1 container and, /// for the delta form, the basis' owned hex name (DELTASRC) and the /// reconstructed target size (delta §3.2). Freed by `releaseTransient`. const WireHold = struct { container: []u8, delta: ?Delta = null, const Delta = struct { src_hex: []u8, target_size: u64 }; }; /// Build over a caller-owned `StorageFile` (the log borrows it; `fs` unused). fn open(gpa: std.mem.Allocator, storage_file: storage.StorageFile) logstore.OpenError!*LogIndexStore { const self = try gpa.create(LogIndexStore); errdefer gpa.destroy(self); self.gpa = gpa; self.fs = undefined; try self.build(storage_file); return self; } /// Build over an owned `FileStorage` bound to `file` (the serve path). `size` /// must equal `file`'s current length (`FileStorage.init`'s precondition). fn openFromFile(gpa: std.mem.Allocator, io: std.Io, file: File, size: u64) logstore.OpenError!*LogIndexStore { const self = try gpa.create(LogIndexStore); errdefer gpa.destroy(self); self.gpa = gpa; self.fs = storage.FileStorage.init(io, file, size); try self.build(self.fs.storageFile()); return self; } fn build(self: *LogIndexStore, storage_file: storage.StorageFile) logstore.OpenError!void { self.materialized_bytes = 0; // `self` came from `gpa.create` (undefined memory) and is filled field by // field, so the struct-declaration default `read_fault = false` never runs. // `serveAnswer` reads this flag on the FIRST round after a prime/rebuild, // before any `releaseTransient` clears it — leave it uninitialised and a // garbage-true value spuriously fails a healthy round. Zero it here. self.read_fault = false; // Same undefined-memory rule for the cone memo: garbage here reads as // a live classification and deinit frees wild pointers. self.cone_memo = null; self.cone_memo_digest = null; self.name_arena = std.heap.ArenaAllocator.init(self.gpa); errdefer self.name_arena.deinit(); self.by_name = .empty; errdefer self.by_name.deinit(self.gpa); self.appendix = repostate.MemStore.init(self.gpa); errdefer self.appendix.deinit(); self.log = try logstore.LogStore.openReadOnly(self.gpa, storage_file); errdefer self.log.deinit(self.gpa); // submanifests §8.1 — one config read per index build, never per round; // both settings come off the same frame. self.seen_submanifest = false; self.submanifests_enabled = zit.sync.submanifest_emission_default; self.submanifest_k = 0; self.submanifest_auto_retune = false; self.policy_binary_glob = ""; self.policy_crlf_glob = ""; if (try self.log.readConfig(self.gpa)) |*oc_const| { var oc = oc_const.*; defer oc.deinit(self.gpa); self.seen_submanifest = oc.config.seen_submanifest; self.submanifests_enabled = oc.config.enable_submanifests; self.submanifest_k = oc.config.submanifest_k; self.submanifest_auto_retune = oc.config.submanifest_auto_retune; self.policy_binary_glob = try self.gpa.dupe(u8, oc.config.binary_glob); errdefer self.gpa.free(self.policy_binary_glob); self.policy_crlf_glob = try self.gpa.dupe(u8, oc.config.crlf_glob); } // Function-scope: a failure in ANY later build step must free these // (a zero-length free of the "" default is a no-op). errdefer self.gpa.free(self.policy_binary_glob); errdefer self.gpa.free(self.policy_crlf_glob); // sync §3.3 L527-530 — the shun table, off this same build's open. Copied // rather than borrowed so the set has one shape on both the serve and the // client path (one hex→digest boundary, one lookup). self.shun = RepoShunSet.init(self.gpa); errdefer self.shun.deinit(); try self.shun.load(&self.log); // §9.3 r28 — the row view of that same table, off the same open, so the // group a peer requests can never be a build behind the filter. const rows = self.log.shunList(); var hex_bytes: usize = 0; for (rows) |r| hex_bytes += 2 * r.hash.len; self.shun_hex = try self.gpa.alloc(u8, hex_bytes); errdefer self.gpa.free(self.shun_hex); self.shun_rows = try self.gpa.alloc(zit.sync.ShunRow, rows.len); errdefer self.gpa.free(self.shun_rows); var hex_at: usize = 0; for (rows, self.shun_rows) |src_row, *dst| { const span = self.shun_hex[hex_at..][0 .. 2 * src_row.hash.len]; _ = std.fmt.bufPrint(span, "{x}", .{src_row.hash}) catch unreachable; hex_at += span.len; dst.* = .{ .hash = span, .epoch = src_row.epoch, .reason = src_row.reason }; } // sync §7.7 L1428-1432 — and the effective hash-policy off the same open. self.hash_policy = try effectiveHashPolicy(self.gpa, &self.log); const persisted = try self.log.ridsInPersistedOrder(self.gpa); defer self.gpa.free(persisted); // The `.config` test is a GUARD, not a live skip: a config frame is never // interned (`LogStore.scan` records only its offset, and `putConfig` appends // without touching `entries`), so `ridsInPersistedOrder` — which walks // `entries` — cannot yield one today. Kept because this counting loop and // the filling loop below must agree on `n` exactly, and because // `loadContentStore` states the same guard: the day a config frame becomes // interned, both stay correct instead of both silently naming it as an // artifact. var n: usize = 0; for (persisted) |lr| { if (self.log.kindOf(lr) != .config) n += 1; } self.rids = try self.gpa.alloc(logstore.Rid, n); errdefer self.gpa.free(self.rids); self.names = try self.gpa.alloc([]const u8, n); errdefer self.gpa.free(self.names); self.masked = try self.gpa.alloc(bool, n); errdefer self.gpa.free(self.masked); self.content_memo = try self.gpa.alloc(?[]u8, n); errdefer self.gpa.free(self.content_memo); self.wire_memo = try self.gpa.alloc(?WireHold, n); errdefer self.gpa.free(self.wire_memo); try self.by_name.ensureTotalCapacity(self.gpa, @intCast(n)); var a: usize = 0; for (persisted) |lr| { if (self.log.kindOf(lr) == .config) continue; // The one site the binary-log digest becomes the wire name (sync §3.3), // exactly as `loadContentStore` did — held to one place. var name_buf: [2 * digest_len]u8 = undefined; const wire_name = digestToWireName(self.log.hashOf(lr), &name_buf); const owned = try self.name_arena.allocator().dupe(u8, wire_name); self.rids[a] = lr; self.names[a] = owned; self.content_memo[a] = null; self.wire_memo[a] = null; // sync §9.5 L1988-1992 — the shun filter, applied to the enumeration // itself and off THIS iteration's index, so every parallel array plus // the mask derive from one loop and one counter (see `masked`). const is_masked = self.shun.digests.contains(self.log.hashOf(lr)); self.masked[a] = is_masked; // Hashes are unique per log entry (content-addressed intern), so no key // collides; adapter rid is a+1. A masked name is deliberately absent: // `lookup` answering null is what makes an inbound `gimme` for it hit // §3.3 L532-535's "If absent, silently skip" with no code of its own. if (!is_masked) self.by_name.putAssumeCapacity(owned, @intCast(a + 1)); a += 1; } // THE PARALLEL-ARRAY LENGTH GATE. `heldImpl`, `contentImpl` and `wireImpl` // bounds-test an incoming rid against `rids.len` ALONE and then index // `masked`/`content_memo`/`wire_memo` with it. Their alignment is what makes // adapter rid r name `rids[r-1]`; a length that disagreed would be an // out-of-bounds read, and in ReleaseFast/ReleaseSmall the slice bounds check // is gone too, so it would be a silent one: a garbage `masked` true withholds // a survivor (the silently short clone `read_fault` exists to prevent), a // garbage false serves an artifact the operator shunned (the §9.5 L1988-1992 // policy this whole filter exists to enforce). Both must be impossible in a // RELEASE build, where `std.debug.assert` compiles to nothing — so this is a // CHECKED error, paid once per build (five comparisons against an O(store) // walk), not an assertion. It also carries the fill loop's own `a == n`, which // is the same question asked of the counter rather than of the allocations. if (a != n or self.rids.len != n or self.names.len != n or self.masked.len != n or self.content_memo.len != n or self.wire_memo.len != n) return error.Corrupt; } /// Free every per-round transient memo, restoring the store to "index only, /// no materialised container bytes" — the shape between rounds. Every borrowed /// `content()`/`wire()` slice handed out this round is invalidated here, so it /// runs only once every borrow of the snapshot has ended (see /// `ServeCtx.releaseSnapshotPlaintext`). fn releaseTransient(self: *LogIndexStore) void { for (self.content_memo) |*slot| { if (slot.*) |b| self.gpa.free(b); slot.* = null; } for (self.wire_memo) |*slot| { if (slot.*) |wh| { self.gpa.free(wh.container); if (wh.delta) |d| self.gpa.free(d.src_hex); } slot.* = null; } // A round's fault is a property of that round's reads; clearing it here // (once every borrow of the round's memos has ended) lets a retry start // clean over the same snapshot. self.read_fault = false; } /// Total container/plaintext bytes currently memoised — a test/accounting /// probe. ~0 right after a prime (the whole point: no whole-store copy) and /// after `releaseTransient`; rises only as a round materialises what it emits. fn residentBytes(self: *const LogIndexStore) usize { var total: usize = 0; for (self.content_memo) |slot| { if (slot) |b| total += b.len; } for (self.wire_memo) |slot| { if (slot) |wh| total += wh.container.len; } return total; } fn deinit(self: *LogIndexStore) void { if (self.cone_memo) |*c| c.deinit(self.gpa); self.releaseTransient(); self.gpa.free(self.content_memo); self.gpa.free(self.wire_memo); self.gpa.free(self.rids); self.gpa.free(self.names); self.gpa.free(self.masked); self.by_name.deinit(self.gpa); self.name_arena.deinit(); self.appendix.deinit(); self.shun.deinit(); self.gpa.free(self.shun_rows); self.gpa.free(self.shun_hex); self.gpa.free(self.policy_binary_glob); self.gpa.free(self.policy_crlf_glob); self.log.deinit(self.gpa); } /// Free the store's internals and the heap box. `self` is invalid after. fn destroy(self: *LogIndexStore) void { const gpa = self.gpa; self.deinit(); gpa.destroy(self); } /// The ConeClassifier vtable target (sparse-cone RFC section 6.2). Any /// failure -- undecodable member, invalid cone, memo OOM -- answers null: /// the reply degrades to dense, never to an error (section 6.3). fn classifyCone( context: *anyopaque, members: []const []const u8, not_members: []const []const u8, ) ?*const std.DynamicBitSetUnmanaged { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (members.len == 0) return null; if (members.len + not_members.len > zit.cone.max_members) return null; var arena_state = std.heap.ArenaAllocator.init(self.gpa); defer arena_state.deinit(); const aa = arena_state.allocator(); var decoded: std.ArrayListUnmanaged([]const u8) = .empty; for (members) |member| { const buf = aa.alloc(u8, member.len) catch return null; const plain = artifact.decodeEscapes(member, buf); decoded.append(aa, plain) catch return null; } var decoded_not: std.ArrayListUnmanaged([]const u8) = .empty; for (not_members) |member| { const buf = aa.alloc(u8, member.len) catch return null; const plain = artifact.decodeEscapes(member, buf); decoded_not.append(aa, plain) catch return null; } var cone = zit.cone.Cone.init(self.gpa, decoded.items, decoded_not.items) catch return null; defer cone.deinit(self.gpa); const digest = cone.digest(self.gpa) catch return null; if (self.cone_memo_digest) |have| { if (std.mem.eql(u8, &have, &digest)) return &self.cone_memo.?.keep; } const classification = classifyKeepRids(self.gpa, self.store(), &cone) catch return null; if (self.cone_memo) |*old_memo| old_memo.deinit(self.gpa); self.cone_memo = classification; self.cone_memo_digest = digest; return &self.cone_memo.?.keep; } fn coneClassifier(self: *LogIndexStore) zit.sync.ConeClassifier { return .{ .context = @ptrCast(self), .vtable = &.{ .classify = classifyCone } }; } fn store(self: *LogIndexStore) repostate.ArtifactStore { return .{ .context = self, .vtable = &vtable }; } const vtable = repostate.ArtifactStore.VTable{ .intern = internImpl, .put = putImpl, .content = contentImpl, .hashOf = hashOfImpl, .lookup = lookupImpl, .count = countImpl, .wire = wireImpl, .retain = retainImpl, .held = heldImpl, }; fn wireFrameOf(wh: *const WireHold) repostate.WireFrame { if (wh.delta) |d| return .{ .container = wh.container, .delta = .{ .src_hash = d.src_hex, .target_size = d.target_size } }; return .{ .container = wh.container }; } fn countImpl(context: *anyopaque) usize { const self: *LogIndexStore = @ptrCast(@alignCast(context)); return self.rids.len + self.appendix.store().count(); } /// The adapter rid of a MASKED entry named `hash`, or null when this /// enumeration does not carry it under a mask. The mask deliberately keeps the /// name out of `by_name` (that absence IS §3.3 L532-535's "If absent, silently /// skip" on the gimme path), so the write doors below need this second, explicit /// question to avoid MINTING A SECOND IDENTITY for a name the enumeration /// already holds — see `putImpl`. A scan rather than a second name map, so /// "masked" has exactly one representation in this struct. /// /// THE TIME BOUND, in code rather than in prose. The walk is O(`rids.len`) per /// call and its callers are the vtable's write doors, whose call count belongs /// to a caller this struct does not bound: interning k names the log does not /// carry would otherwise pay k full walks plus a `std.mem.eql` per masked slot. /// An empty shun table is the case in which no slot can match, and it is also /// the overwhelmingly common one — a repository that has never shunned anything /// — so answering it in O(1) leaves the linear walk reachable only where a shun /// table exists to bound it. (Not reached on the serve path either way: every /// served write lands in the per-request `OverlayStore`.) fn maskedRidFor(self: *const LogIndexStore, hash: []const u8) ?repostate.Rid { if (self.shun.digests.count() == 0) return null; for (self.masked, 0..) |m, i| { if (!m) continue; if (std.mem.eql(u8, self.names[i], hash)) return @intCast(i + 1); } return null; } fn internImpl(context: *anyopaque, hash: []const u8) error{OutOfMemory}!repostate.Rid { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (self.by_name.get(hash)) |rid| return rid; // A MASKED name resolves to ITS OWN slot, not to a fresh appendix phantom: // the enumeration carries that frame at a fixed adapter rid (§8.2 L1504-1506 // gap), and it is already exactly what §3.3 L511-514 would create — a record // with the name and no content ("an igot HASH creates a phantom only when // HASH is absent"; `held` is false and `content`/`wire` are null here). A // second identity for one name would be the only way this store could hand // the walk two rids naming the same artifact. // // Not the mask's safety argument, which lives one layer up: this store never // sees a served `intern` at all — `respond` runs over an `OverlayStore` whose // `internImpl` probes `base.lookup` and, on the mask's null, interns into its // OWN round-local `added` store, which is discarded at round end. What keeps // that round's reply from begging for the name is `Server.shun` (§3.3 // L527-530), which `serveRound` sets from this same snapshot's table. if (self.maskedRidFor(hash)) |rid| return rid; return self.rids.len + try self.appendix.store().intern(hash); } fn lookupImpl(context: *anyopaque, hash: []const u8) ?repostate.Rid { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (self.by_name.get(hash)) |rid| return rid; if (self.appendix.store().lookup(hash)) |l| return self.rids.len + l; return null; } fn putImpl(context: *anyopaque, hash: []const u8, content_bytes: []const u8) error{OutOfMemory}!repostate.Rid { const self: *LogIndexStore = @ptrCast(@alignCast(context)); // A persisted hash is content-addressed and already carries these bytes, // so `put` is idempotent and `content(rid)` still reads them from the log // — the serve path never reaches this arm (writes go to the overlay). if (self.by_name.get(hash)) |rid| { // Parity with `MemStore.putImpl`: a repeat put must carry identical // bytes (a differing answer is a hash collision). Verified only under // runtime safety, and only when the log frame actually inflates — a // read fault leaves `content` null and the assert is skipped rather // than firing spuriously. Inflate through a THROWAWAY read (not // `contentImpl`), so this assertion path never memoises a container or // bumps `materialized_bytes`/`residentBytes` — the probes the round // tests assert on stay untouched by an assert that never runs on serve. if (std.debug.runtime_safety) { if (self.log.content(self.gpa, self.rids[@intCast(rid - 1)]) catch null) |existing| { defer self.gpa.free(existing); std.debug.assert(std.mem.eql(u8, existing, content_bytes)); } } return rid; } // A MASKED name is not in `by_name`, so without this arm a `put` of one // would fall through to the appendix and create a SECOND, HELD entry for it: // `held` true, `content` the pushed bytes, and `lookup` resolving the masked // name to it — the §3.2 walk would then announce the shunned name and the // clone pager would emit its `cfile`, undoing the whole §9.5 L1988-1992 // filter through a write. The mask outranks the write: the frame keeps its // one adapter rid and stays a §8.2 L1504-1506 gap. // // A DELIBERATE narrowing of the store contract's "after `put`, `content` is // readable" — `content(rid)` stays null here. The alternative is a name this // server has been told never to send becoming servable because a peer pushed // it back, and admitting a shunned target is a receive-side decision (§7.7 // L1437-1440: "consumed and discarded … makes no repository mutation"), not // something a store adapter may take by accident. // // The narrowing cannot be SIGNALLED — the vtable's error set is // `error{OutOfMemory}` — so what keeps it safe is that no caller is in a // position to be surprised by it. On serve that holds because `respond` runs // over a per-request `repostate.OverlayStore` and every served write lands // THERE, never on this base. That is a property of the serve path rather than // of this function, so it is PINNED BY A TEST at this layer ("a served round's // writes land in the round-local overlay — the base enumeration's write doors // are never reached") instead of resting on a claim this comment makes about // code two modules away. A caller that wires this store in as a WRITABLE store // is outside that proof and must read this arm before relying on `put`. if (self.maskedRidFor(hash)) |rid| return rid; return self.rids.len + try self.appendix.store().put(hash, content_bytes); } fn hashOfImpl(context: *anyopaque, rid: repostate.Rid) []const u8 { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (rid <= self.rids.len) return self.names[@intCast(rid - 1)]; return self.appendix.store().hashOf(rid - self.rids.len); } fn heldImpl(context: *anyopaque, rid: repostate.Rid) bool { const self: *LogIndexStore = @ptrCast(@alignCast(context)); // Every persisted (non-phantom, non-config) frame carries a form to serve // verbatim, so it is held — answered from the index with no frame read. // A MASKED frame (sync §9.5 L1988-1992) is the exception: it is a §8.2 // L1504-1506 gap, so it is not held and §3.2's walk never announces it. // This line is the one place the send side genuinely disagreed with the // log: `LogStore.content`/`wire` already read a shunned frame as ABSENT, // while this answered true from the index with no frame read — so the // inventory walk advertised an artifact no round could then send, and (with // the name still in `by_name`) `emitCloneReply`'s D3 basis proof could reach // a resolvable rid whose two forms were both null. `LogStore.shunned`'s doc // names exactly this disagreement as a SYNC decision to settle; §9.5's // "every eligible holding after the Section 3.3 shun … filters" settles it. if (rid <= self.rids.len) return !self.masked[@intCast(rid - 1)]; return self.appendix.store().held(rid - self.rids.len); } fn retainImpl(context: *anyopaque, rid: repostate.Rid, ingest: repostate.WireIngest) error{OutOfMemory}!void { const self: *LogIndexStore = @ptrCast(@alignCast(context)); // Advisory (sync §7.2): a persisted rid already carries a stored form, so // first-form-wins makes this a no-op — the log is the retained form. if (rid <= self.rids.len) return; return self.appendix.store().retain(rid - self.rids.len, ingest); } fn contentImpl(context: *anyopaque, rid: repostate.Rid) ?[]const u8 { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (rid > self.rids.len) return self.appendix.store().content(rid - self.rids.len); const i: usize = @intCast(rid - 1); // sync §9.5 L1988-1992 — a masked frame has nothing to hand out. Checked // before the memo so the mask cannot be bypassed by a slot some earlier // round filled, and it is a plain null: a policy gap, never a read fault. // // `LogStore.content` ALREADY answers null for a shunned entry (its own // rid-keyed read mask), so today this line changes no answer: it is a // structural guard, not covered behaviour. Measured, not assumed — deleting // this line leaves the whole suite green, because the read one line below // returns the same null. What IS covered, by "the adapter owns its content // mask", is the surrounding fact the guard exists to protect: the frame's // plaintext is still on disk and still readable through // `LogStore.contentUnmasked`, so the null is a policy refusal over present // bytes rather than an absence. That test is what would turn red the day the // two layers were consolidated into one. It is here // because the adapter must OWN its mask rather than inherit it. §7.7 // L1444-1449 wants a held-but-shunned artifact to remain usable as a delta // BASIS — "For an admitted target whose DELTASRC name is in the repository // shun table, the receiver does not apply admission independently to that // source: content still held locally MAY serve as the delta basis" — and the // only reason that works today is that basis resolution walks byte offsets // and never re-enters `content`. The moment that basis path is given an // explicitly unmasked door — the repair `LogStore.shunned`'s own doc // contemplates — the log's mask stops covering this one, and the send-side // exclusion must not evaporate with it. if (self.masked[i]) return null; if (self.content_memo[i]) |b| return b; // Read the frame on demand and memoise for the round. The vtable cannot // signal an error, so a read fault is recorded on `read_fault` and // returned as null; `serveAnswer` turns a set flag into // `error.ServerStorage` so the client retries rather than accepting a // silently short clone. A frame that legitimately has no plaintext is a // plain null and does NOT set the flag. // // THIS IS THE PRIMARY CORRUPTION CHANNEL IN SERVE, and it did not used // to be. `openReadOnly` once CRC-verified every payload while building // the snapshot, which meant a damaged store failed to open at all and // this catch saw only genuine post-scan faults -- transient EIO, or rot // arriving after a clean scan. Both open paths now scan header-only and // `readFrameAt` CRC-checks every frame it reads instead, so ordinary // stored corruption reaches serve HERE, on the read, rather than at // snapshot build. // // That is a better shape, not a worse one: one bad frame used to make // the server refuse to serve anything, and now it fails only the round // that touches the damage while every healthy artifact still moves. But // it means this arm is live traffic rather than a rare event, and // anything reasoning about how often it fires should start from that. const maybe = self.log.content(self.gpa, self.rids[i]) catch { self.read_fault = true; return null; }; const bytes = maybe orelse return null; self.content_memo[i] = bytes; self.materialized_bytes += bytes.len; return bytes; } fn wireImpl(context: *anyopaque, rid: repostate.Rid) ?repostate.WireFrame { const self: *LogIndexStore = @ptrCast(@alignCast(context)); if (rid > self.rids.len) return self.appendix.store().wire(rid - self.rids.len); const i: usize = @intCast(rid - 1); // sync §9.5 L1988-1992 — same gap as `contentImpl`, checked ahead of the // memo, and owned here for the same reason (see there). if (self.masked[i]) return null; if (self.wire_memo[i]) |*wh| return wireFrameOf(wh); // A frame read fault is a real error recorded on `read_fault` so // `serveAnswer` can fail the round; a `.raw`/phantom frame legitimately // has no wire form (the responder compresses `content()` instead) and // yields a plain null that does NOT set the flag. As at `contentImpl`, // this arm now sees ordinary stored corruption rather than only // post-scan faults -- no open path CRC-verifies payloads any more. var w = (self.log.wire(self.gpa, self.rids[i]) catch { self.read_fault = true; return null; }) orelse return null; switch (w) { .full => |c| { self.wire_memo[i] = .{ .container = c }; }, .delta => |d| { // sync §9.5 L1988-1992 — the delta form is servable only while its // basis is. `zit shun --defer` records the row without rewriting // the generation, so a SURVIVING frame here can be a delta rooted // on a MASKED one; emitting its `cfile` would ship a DELTASRC this // server never sends, leaving the peer a permanently dangling delta // whose basis it gimmes and is silently skipped on (§3.3 L532-535) // — corrupt rather than absent, and `emitCloneReply`'s D3 basis // lookup would trap under `runtime_safety` on the way. Withhold the // wire form instead; the responder's §7.2 compress-on-the-fly arm // then emits the FULL form, and `log.content` reconstructs through // the frame chain by BYTE OFFSET, which the name-level mask does // not reach. A plain null, not a read fault: policy, not I/O. // // Asked as the SAME predicate the fill loop applied — the raw digest // against the shun table — so "masked" has one spelling in this // struct. (`by_name`-absence would answer identically for a log // delta, whose basis is always a non-config frame in the // enumeration, but it would make the exclusion mean two things and // pay a hex encode plus a map lookup to say the second one.) if (self.shun.digests.contains(d.baseHash())) { w.deinit(self.gpa); return null; } // The basis precedes this frame in the log, so its wire name is the // hex of the base digest the wire read carried (delta §3.2 target // size alongside). var src_buf: [2 * digest_len]u8 = undefined; const src_hex = digestToWireName(d.baseHash(), &src_buf); const owned_src = self.gpa.dupe(u8, src_hex) catch { // Failing to produce the wire form here would silently drop a // held delta from the clone; record it so the round errors. w.deinit(self.gpa); self.read_fault = true; return null; }; self.wire_memo[i] = .{ .container = d.container, .delta = .{ .src_hex = owned_src, .target_size = d.target_size } }; }, } return wireFrameOf(&self.wire_memo[i].?); } }; /// Snapshot every stored artifact from the persisted log into `ms`, skipping /// config frames (not artifacts). A content-bearing frame becomes a held entry; /// a content-absent frame becomes a phantom — exactly the held/phantom split /// `respond` walks (sync §3.2/§3.3). /// /// The log keys frames by the RAW binary digest, but the sync wire names an /// artifact by that digest's lowercase hex (sync §3.3 — a `file`/`igot`/`gimme` /// HASH is 40/64 hex characters). The store the responder walks must therefore /// hold the hex name, or `respond` would emit non-hex bytes that break the /// peer's card tokenizer. Every hash crossing this boundary is hex-encoded once, /// here — the single place binary-log identity becomes wire identity. fn loadContentStore( gpa: std.mem.Allocator, log: *logstore.LogStore, ms: *repostate.MemStore, ) serve_mod.AnswerError!void { const rids = log.ridsInPersistedOrder(gpa) catch return error.OutOfMemory; defer gpa.free(rids); for (rids) |rid| { if (log.kindOf(rid) == .config) continue; var name_buf: [2 * digest_len]u8 = undefined; const name = digestToWireName(log.hashOf(rid), &name_buf); // sync §4.1/§7.2 — copy the stored container into the snapshot verbatim // so the clone serves it without inflating here and re-compressing at // serve. `wire()` is null only for a `.raw` frame or a phantom; those // fall back to the plaintext path below. A `.zlib` frame carries the // full form; a `.delta` frame carries the stored delta container plus // its basis name and reconstructed size (the §7.2 USIZE, delta §3.2). var maybe_wire = log.wire(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; if (maybe_wire) |*w| { defer w.deinit(gpa); const recorded: bool = switch (w.*) { .full => |container| blk: { _ = ms.putWire(name, container) catch return error.OutOfMemory; break :blk true; }, .delta => |d| blk: { // sync §3.3 L527-530 — the delta form is usable only while its // basis is. `zit shun --defer` records the row without // rewriting the generation, so a SURVIVING frame here can be a // delta rooted on a shunned one; that basis reads null through // this loop's own masked doors and interns as a PHANTOM, so // recording the delta would leave an entry the store calls held // (`wire != null`) whose `content()` can never resolve. §3.2's // igot walk would then announce a name no `gimme` can ever be // answered with, and the push would spin the round with nothing // to send. Fall through to the plaintext path instead: the // artifact itself is NOT shunned, so `log.content` reads it // through the frame chain by BYTE OFFSET — which the name-level // mask does not reach — and it is snapshotted as a held full // form. Exactly `LogIndexStore.wireImpl`'s masked-basis guard, // stated for the client's own holdings snapshot. const base = d.baseHash(); if (logstore.isCompleteDigest(base.len) and log.isShunned(base)) break :blk false; // The basis frame precedes this one in the log (delta_of < // offset), so it was already inserted at an earlier rid; its // wire name is the hex of the base's binary digest. var src_name_buf: [2 * digest_len]u8 = undefined; const src_name = digestToWireName(base, &src_name_buf); _ = ms.putWireDelta(name, d.container, src_name, d.target_size) catch return error.OutOfMemory; break :blk true; }, }; if (recorded) continue; } const maybe = log.content(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; if (maybe) |bytes| { defer gpa.free(bytes); _ = ms.store().put(name, bytes) catch return error.OutOfMemory; } else { _ = ms.store().intern(name) catch return error.OutOfMemory; } } } // --------------------------------------------------------------------------- // `zit clone URL DIR` — the interop milestone: a real client converging against // a real `zit serve` over HTTP. Create a fresh repo at DIR, open an HTTP // transport to URL, pump the sans-IO clone client (`sync.Client`) to §3.7 // convergence through `sync_driver.runSync`, then persist the received // artifacts into DIR's on-disk log so the clone is durable. Anonymous only. // --------------------------------------------------------------------------- /// §9.1 — a fixed placeholder repository identity emitted in the client's /// pull/push operation cards. Neither a fresh clone nor a pulled repository /// carries a persistent SERVERCODE/PROJECTCODE of its own yet (real per-repo /// sync identity is future work); the stateless server echoes the peer's codes /// and never gates on a configured project code (sync §9.1), so any well-formed /// 16-hex pair carries a content-only exchange. Shared by clone and pull. const sync_servercode = "0000000000000000"; const sync_projectcode = "0000000000000000"; /// §10.1 — the release integer the sync client announces as `pragma /// client-version`. A fixed build identity, matching `serve`'s own version /// constant style. (`announcedVersion` raises what actually goes on the wire — /// submanifests §7.2.) const sync_client_version: u32 = 22700; /// §10.1 r11/r12 — the DATED client-version pragma every zit run announces. /// /// The DATE and TIME are not decoration: §10.1 r11/r12 make the server's /// `server-version` reply conditional on them ("at least five tokens total"), /// and submanifests §7.2 gates every upload on "the remote peer's announced /// version". An undated pragma therefore leaves `Client.peer.version` at 0 for /// the whole session — below every threshold — so an armed client would refuse /// its own trees to a server that would have announced support if only it had /// been asked. Soliciting the reply is what makes the gate a peer-version gate /// rather than a permanent refusal. Fossil sends the dated form too (§A.4). const sync_client_version_info: zit.sync.VersionInfo = .{ .version = sync_client_version, .datetime = .{ .date = 20260715, .time = 0 }, }; /// An explicit bound on CONSECUTIVE ROUNDS THAT MOVE NO BYTES (TigerStyle: /// every loop is bounded). See `sync_driver.runSync` for why the bound counts /// stalls rather than rounds: a total-round cap multiplies out against the /// per-round batch size into an accidental ceiling on repository size, which a /// large healthy repository trips while converging perfectly. /// /// This value bounds only unproductive looping, so it is a statement about how /// long a peer may accomplish nothing — not about how much there is to /// transfer. §3.7's longest by-design stall is the unversioned three-round /// grace window ((d)(2)); config exchange (f) and private-content discovery (c) /// are each shorter. 16 clears every one of them several times over while still /// failing a stuck peer in well under a second of round trips. const max_stalled_sync_rounds: usize = 16; /// The canonical form of a clone URL for persistence: rebuilt from parsed /// components — scheme://[user@]host[:port]/path[?query] — so a userinfo /// password is excluded BY CONSTRUCTION: a component the rebuild never names /// cannot survive into the stored value, even if a future parser change adds /// fields (fossil url.c's canonical form works the same way). The username is /// kept, matching fossil. The REQUESTED URL is what canonicalizes, never the /// redirect-resolved one: an alias in front of a moving host is load-bearing, /// and the redirect chain re-resolves on every sync anyway. pub fn canonicalRemoteUrl(gpa: std.mem.Allocator, url: []const u8) error{ OutOfMemory, InvalidUrl }![]u8 { const uri = std.Uri.parse(url) catch return error.InvalidUrl; const bytesOf = struct { fn f(c: std.Uri.Component) []const u8 { return switch (c) { .raw => |r| r, .percent_encoded => |p| p, }; } }.f; var out: std.ArrayList(u8) = .empty; errdefer out.deinit(gpa); try out.appendSlice(gpa, uri.scheme); try out.appendSlice(gpa, "://"); if (uri.user) |u| { try out.appendSlice(gpa, bytesOf(u)); try out.append(gpa, '@'); } const host = uri.host orelse return error.InvalidUrl; try out.appendSlice(gpa, bytesOf(host)); if (uri.port) |p| try out.print(gpa, ":{d}", .{p}); const path = bytesOf(uri.path); if (path.len == 0) try out.append(gpa, '/') else try out.appendSlice(gpa, path); if (uri.query) |q| { try out.append(gpa, '?'); try out.appendSlice(gpa, bytesOf(q)); } return out.toOwnedSlice(gpa); } test "canonicalRemoteUrl: password excluded by construction, username kept" { const gpa = std.testing.allocator; const cases = [_]struct { in: []const u8, want: []const u8 }{ .{ .in = "http://user:secret@host/repo", .want = "http://user@host/repo" }, .{ .in = "http://user@host:8080/repo", .want = "http://user@host:8080/repo" }, .{ .in = "https://host", .want = "https://host/" }, .{ .in = "https://host/a/b?x=1#frag", .want = "https://host/a/b?x=1" }, }; for (cases) |c| { const got = try canonicalRemoteUrl(gpa, c.in); defer gpa.free(got); try std.testing.expectEqualStrings(c.want, got); try std.testing.expect(std.mem.indexOf(u8, got, "secret") == null); } try std.testing.expectError(error.InvalidUrl, canonicalRemoteUrl(gpa, "not a url")); } /// `zit clone URL DIR` — thin wiring over three deep modules: `transport` /// (HTTP/IO), `sync.Client` + `sync_driver.runSync` (sans-IO §3.7 loop), and /// `LogStore` (persistence). The URL is used verbatim (sync §4 r1 — the client /// never appends /xfer). All-or-nothing failures surface an honest exit-1 /// diagnostic; a bad URL scheme is rejected before any repo is created. pub fn runClone( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { return runCloneObserved(gpa, io, arena, verb_args, streams, amb, null); } /// `runClone` with benchmark-owned phase attribution. Sync §8.2's project-code /// adoption remains in the observed config phase; a null observer is the /// ordinary CLI path, which reads no clock and owns no counters. pub fn runCloneObserved( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, observer: ?*const CloneObserver, ) cli.Exit { const exit = cloneObserved(gpa, io, arena, verb_args, streams, amb, observer); if (exit != .ok) return exit; // A clone is followed by the `open` the operator would type next // (Fossil 2.x), run inside DIR by this same binary so the two verbs stay // one implementation; --no-open keeps the bare store. It runs only here, // after cloneObserved returned and released its exclusive store lock — // the child takes that same lock. The child opens the same cone the clone // elided to: members from the flags and from --cone-file (read here, // relative to this cwd) are passed explicitly. const parsed = switch (cli.parseCloneArgs(verb_args)) { .ok => |p| p, else => return exit, }; if (parsed.help or !parsed.open) return exit; var members = parsed.cone_members; var len = parsed.cone_len; var not_members = parsed.cone_not_members; var not_len = parsed.cone_not_len; if (parsed.cone_file) |cf| { if (cli.appendConeFile(arena, io, cf, &members, len, ¬_members, not_len)) |lens| { len = lens.members; not_len = lens.not_members; } } return autoOpen(gpa, io, arena, streams, parsed.dir, members[0..len], not_members[0..not_len]); } /// Spawn this same binary as `zit open` inside the freshly cloned DIR. fn autoOpen( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, dir: []const u8, cone: []const []const u8, cone_not: []const []const u8, ) cli.Exit { streams.out.flush() catch {}; streams.err.flush() catch {}; const exe = std.process.executablePathAlloc(io, gpa) catch { streams.err.writeAll("zit clone: cloned, but cannot locate zit to open the working tree; run 'zit open' inside the directory\n") catch {}; return .failure; }; defer gpa.free(exe); var argv = std.ArrayList([]const u8).empty; argv.append(arena, exe) catch return .failure; argv.append(arena, "open") catch return .failure; for (cone) |m| argv.append(arena, std.fmt.allocPrint(arena, "--cone={s}", .{m}) catch return .failure) catch return .failure; for (cone_not) |m| argv.append(arena, std.fmt.allocPrint(arena, "--cone-not={s}", .{m}) catch return .failure) catch return .failure; var child = std.process.spawn(io, .{ .argv = argv.items, .cwd = .{ .path = dir }, }) catch { streams.err.print("zit clone: cloned, but 'zit open' could not be started; run it inside '{s}'\n", .{dir}) catch {}; return .failure; }; const term = child.wait(io) catch return .failure; return switch (term) { .exited => |code| if (code == 0) .ok else .failure, else => .failure, }; } fn cloneObserved( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, observer: ?*const CloneObserver, ) cli.Exit { const parsed = switch (cli.parseCloneArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "clone", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "clone", .extra_argument, tok), .no_url => return cli.usageError(streams, "clone", .no_url, ""), .no_dir => return cli.usageError(streams, "clone", .no_dir, ""), .empty_value => |tok| return cli.usageError(streams, "clone", .empty_value, tok), .too_many_cones => |tok| return cli.usageError(streams, "clone", .too_many_cones, tok), }; if (parsed.help) { streams.out.writeAll(clone_help) catch {}; return .ok; } // Open the transport first: a malformed URL fails before any repo lands on // disk (nothing half-made to roll back). `open` performs no network I/O. observeClonePhase(observer, .transport, .begin); // sync §6 — a private repository is unreadable anonymously. The login is // --user or the URL's user@; the secret cannot be derived until the round-0 // reply names the project code, so the client holds name and password and // signs from round 1, exactly as fossil does. The plaintext lives only in // pw_buf/argv/env and is scrubbed on every exit. var pw_buf: [max_password_len]u8 = undefined; defer std.crypto.secureZero(u8, &pw_buf); const login_user: ?[]const u8 = blk: { const u = parsed.user orelse urlUser(parsed.url) orelse break :blk null; if (u.len == 0) break :blk null; if (zit.sync.containsFossilWhitespace(u)) { streams.err.writeAll("zit clone: --user must not contain whitespace\n") catch {}; return .usage; } break :blk u; }; const login: ?zit.sync.PendingLogin = if (login_user) |u| blk: { const password = parsed.password orelse amb.push_password orelse (readPasswordNoEcho(streams, u, &pw_buf) orelse { if (parsed.user != null) { streams.err.print("zit clone: no password for '{s}' (give --password, set ZIT_PASSWORD, or run on a terminal)\n", .{u}) catch {}; return .failure; } // A user named only by the URL is a hint: clone anonymously, and say so. streams.err.print("zit clone: no password for '{s}' from the URL (set ZIT_PASSWORD or pass --user/--password); cloning anonymously\n", .{u}) catch {}; break :blk null; }); break :blk .{ .user = u, .password = password }; } else null; var tr = transport.Transport.open(io, gpa, parsed.url) catch |e| { observeClonePhase(observer, .transport, .end); return syncOpenFail(streams, "clone", parsed.url, e); }; observeClonePhase(observer, .transport, .end); defer tr.close(); if (login) |l| tr.setBasicAuth(l.user, l.password) catch return stateOom(streams, "clone"); // Create a fresh `.zit` at DIR by reusing the init sequence, quietly (its // own success line is suppressed; clone prints its own). A pre-existing repo // there is init's "already exists" error, propagated verbatim. observeClonePhase(observer, .initialize, .begin); const init_rc = runInit(gpa, io, arena, &.{ "-q", "--", parsed.dir }, streams); observeClonePhase(observer, .initialize, .end); if (init_rc != .ok) return init_rc; // Converge an in-memory store against the peer, then persist it into DIR. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); // §9.1 — the clone client's own instance identity, derived from the // destination it is creating (informational; a fresh instance is exactly // what a clone is). const clone_code = blk: { const cwd = (cli.currentPathAlloc(arena, io) catch return stateOom(streams, "clone")) orelse break :blk zit.sync.deriveServerCode(&.{parsed.dir}); const abs = std.fs.path.resolve(arena, &.{ cwd, parsed.dir }) catch return stateOom(streams, "clone"); break :blk zit.sync.deriveServerCode(&.{abs}); }; // Live per-round progress, on stderr and ONLY when stderr is a // terminal: an operator watching a slow clone sees each round land instead // of silence, while piped stderr (scripts, tests) stays byte-identical to // before. The line is \r-overwritten in place — never fossil's // percent-flood — and finished with one accounting line when the exchange // ends, either way it ends. const stderr_tty = std.Io.File.stderr().isTty(io) catch false; var progress: CloneProgress = .{ .err = streams.err }; const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null; observeClonePhase(observer, .exchange, .begin); var clone_members = parsed.cone_members; var clone_len = parsed.cone_len; var clone_not_members = parsed.cone_not_members; var clone_not_len = parsed.cone_not_len; if (parsed.cone_file) |cf| { const lens = cli.appendConeFile(arena, io, cf, &clone_members, clone_len, &clone_not_members, clone_not_len) orelse { streams.err.print("zit clone: cannot read --cone-file '{s}' (or it overflows the {d}-member cap)\n", .{ cf, zit.cone.max_members }) catch {}; return .failure; }; clone_len = lens.members; clone_not_len = lens.not_members; } // The destination log, open BEFORE the exchange: a dense clone // persists each round as it converges instead of holding the repository // resident, so the log has to exist while rounds are still arriving. A // failed exchange leaves DIR holding a partial, uncommitted `.zit` the // operator deletes with the directory — the same recovery as before, with // different debris. var target = std.Io.Dir.cwd().openDir(io, parsed.dir, .{}) catch return renderStorage(streams, parsed.dir, "cannot reopen the clone target directory"); defer target.close(io); var file = cli.openStoreAppend(io, target) catch return renderStorage(streams, parsed.dir, "cannot write the cloned repository"); defer file.close(io); const st = file.stat(io) catch return renderStorage(streams, parsed.dir, "cannot write the cloned repository"); var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"), }; defer log.deinit(gpa); // Stream the persist unless a cone is requested. A cone-scoped // clone must classify its keep set over the CONVERGED store (sparse-cone // RFC section 6.3), so it keeps the batched path; every dense clone drains // per round and releases what it wrote, bounding peak memory by the round // batch plus the deltas whose basis has not arrived (git index-pack's // shape, applied to fossil's reverse-delta wire). const cone_scoped = clone_len + clone_not_len != 0; // Every cluster this clone writes, for the index it creates. var clone_clusters: std.ArrayListUnmanaged([]const u8) = .empty; defer { for (clone_clusters.items) |n| gpa.free(n); clone_clusters.deinit(gpa); } var stream_pass: ?PersistPass = if (cone_scoped) null else PersistPass.init(gpa, &log, ms.store(), null, &ms, 0, &clone_clusters) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"), }; defer if (stream_pass) |*pass| pass.deinit(); const drain: ?sync_driver.RoundDrain = if (stream_pass) |*pass| .{ .ctx = pass, .run = PersistPass.drainErased } else null; var peer_error: PeerErrorCopy = .{}; var outcome = cloneRun(gpa, &tr, &ms, &clone_code, login, sink, drain, clone_members[0..clone_len], clone_not_members[0..clone_not_len], &peer_error) catch |e| { progress.finishLine(); observeClonePhase(observer, .exchange, .end); // A peer that stated its refusal is quoted; anything else keeps the // generic mapping (sparse-cone RFC section 5.3 diagnostic path). if (e == error.FatalProtocolError) { if (peer_error.text()) |text| { streams.err.print("zit clone: the peer refused: {s}\n", .{text}) catch {}; return .failure; } } return syncRunFail(streams, "clone", parsed.url, e, tr.last_status, basicStateOf(&tr)); }; // §9.3 r28 — the latched shun rows are the one thing this outcome owns. defer outcome.deinit(gpa); progress.finishLine(); if (stderr_tty) { streams.err.print( "clone exchange: {d} rounds, {d} protocol bytes received, {d} sent\n", .{ outcome.rounds, outcome.received_bytes, outcome.sent_bytes }, ) catch {}; } observeClonePhase(observer, .exchange, .end); // Sparse-cone RFC section 6.3: a cone-scoped clone filters LOCALLY -- // correct against every peer, including cone-ignorant ones. The keep set // is classified over the converged in-memory store before any frame is // written; nothing is discarded silently (the count prints below). var clone_cone: ?zit.cone.Cone = null; defer if (clone_cone) |*c| c.deinit(gpa); var keep_bits: ?SparseClassification = null; defer if (keep_bits) |*k| k.deinit(gpa); if (cone_scoped) { clone_cone = zit.cone.Cone.init(gpa, clone_members[0..clone_len], clone_not_members[0..clone_not_len]) catch |e| return cloneConeFail(streams, e); keep_bits = classifyKeepRids(gpa, ms.store(), &clone_cone.?) catch return stateOom(streams, "clone"); } const clone_keep: ?*const std.DynamicBitSetUnmanaged = if (keep_bits) |*k| &k.keep else null; observeClonePhase(observer, .persist, .begin); const persisted = blk: { if (stream_pass) |*pass| { // Streaming: the rounds already drained; settle any straggler the // last round interned, then the leftovers and the bookkeeping. const summary = res: { pass.drainNew() catch |e| break :res @as(PersistError!PersistSummary, e); break :res pass.finish(); }; break :blk summary; } break :blk persistStoreOpen(gpa, &log, ms.store(), clone_keep, 0, &clone_clusters); } catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"), }; const written = persisted.written; observeClonePhase(observer, .persist, .end); // §8.2 and submanifests §6.3/§8.1 — adopt every origin-owned setting in // one field-preserving overlay. Missing fields leave `init`'s values // untouched, and one combined frame avoids intermediate clone config state. observeClonePhase(observer, .config, .begin); const project_code = outcome.projectCode(); // Save the clone URL as the default remote (fossil's behavior), unless // --once. The canonical rebuild strips a userinfo password by construction; // `zit remote` remains the explicit override and the audit view. The // InvalidUrl arm is unreachable in practice — `transport.open` parsed this // same URL — and degrades to not-saving rather than failing a completed // clone. const saved_remote: ?[]const u8 = if (parsed.once) null else canonicalRemoteUrl(arena, parsed.url) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.InvalidUrl => null, }; const cloned_patch: ConfigPatch = .{ .project_code = if (project_code.len > 0) project_code else null, // The login the peer accepted from round 1 on, stored the way a push // stores its own, so the next pull or push authenticates unasked. .login = outcome.login, .remote_url = saved_remote, .enable_submanifests = outcome.submanifest.enable, .submanifest_k = outcome.submanifest.k, .submanifest_auto_retune = outcome.submanifest.auto_retune, }; if (cloned_patch.any()) { updateConfigOpen(gpa, &log, cloned_patch) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"), }; } // §9.3 r28 — adopt the origin's shun table. Written BEFORE the commit // below, so a clone is durable with its tombstones or not at all: a // repository that came up holding the content but not the record of its // suppression is the exact state this carriage exists to prevent. // // The wire carries hex; the log keys by binary digest. A name that will not // decode is skipped rather than fatal — the artifacts are already down, and // failing a finished clone over one unparseable row helps nobody. The count // is reported so a skip is never silent. var shun_adopted: usize = 0; var shun_skipped: usize = 0; for (outcome.shun_rows) |row| { var digest_buf: [32]u8 = undefined; if (row.hash.len != 40 and row.hash.len != 64) { shun_skipped += 1; continue; } const digest = digest_buf[0 .. row.hash.len / 2]; _ = std.fmt.hexToBytes(digest, row.hash) catch { shun_skipped += 1; continue; }; log.putShun(digest, row.reason, row.epoch) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), else => { shun_skipped += 1; continue; }, }; shun_adopted += 1; } if (shun_adopted != 0 or shun_skipped != 0) { streams.out.print("adopted {d} shun row{s} from the origin", .{ shun_adopted, plural(shun_adopted) }) catch {}; if (shun_skipped != 0) streams.out.print("; {d} unreadable and skipped", .{shun_skipped}) catch {}; streams.out.writeAll("\n") catch {}; } // Every persisted artifact and the optional combined config frame must be // durable before the cache observes this log or clone reports success. log.commit() catch return renderStorage(streams, parsed.dir, "cannot write the cloned repository"); // A store carrying a credential is owner-only (best-effort, as for push). if (outcome.login != null) file.setPermissions(io, File.Permissions.fromMode(0o600)) catch {}; refreshClusterSidecar(gpa, io, target, &log, clone_clusters.items, true) catch return stateOom(streams, "clone"); observeClonePhase(observer, .config, .end); // The committed writable log is still open, so reuse its derivation path // directly instead of reopening `.zit` solely to prime the disposable cache. // Say so first: on a large store this phase is minutes of derive, and an // operator watching a silent clone cannot tell transfer from derivation. streams.out.print("clone from {s}: transferred {d} artifact{s} in {d} round{s}; deriving caches\n", .{ parsed.url, written, plural(written), outcome.rounds, plural(outcome.rounds) }) catch {}; observeClonePhase(observer, .warm_cache, .begin); Repository.warmCacheFromLog(gpa, io, target, &log); observeClonePhase(observer, .warm_cache, .end); if (clone_cone) |*c| { // Policy lands with the disposable tier (RFC section 5.2 loss rule). zit.repocache.writeStoreCone(gpa, io, target, c); streams.out.print( "cloned {d} artifacts into {s} in {d} rounds ({d} elided by the cone)\n", .{ written, parsed.dir, outcome.rounds, persisted.elided }, ) catch {}; } else { streams.out.print("cloned {d} artifact{s} into {s} in {d} round{s}\n", .{ written, plural(written), parsed.dir, outcome.rounds, plural(outcome.rounds) }) catch {}; } if (project_code.len > 0) streams.out.print("project-code: {s}\n", .{project_code}) catch {}; reportUvSkipped(streams, "clone", outcome.uv_skipped); // submanifests §7.1/§7.3 — a clone converges on the cfile cursor, so it // copies the peer's holdings faithfully, INCLUDING an incomplete tree // closure the peer itself was carrying. The phantoms the persist hook minted // say so; report them, because a check-in whose file set cannot be produced // (§5.2) is not a converged clone, and "exit 0 with nothing said" is exactly // how a repository stays quietly broken. Reported AFTER the success line and // AFTER the repository is fully written: a partial clone is still progress, // and this is the same shape `pull` already uses for artifacts it did not // obtain. if (persisted.submanifest_gaps > 0) { streams.err.print( "zit clone: {d} sub-manifest references are missing; the cloned repository cannot enumerate every check-in\n", .{persisted.submanifest_gaps}, ) catch {}; return .failure; } return .ok; } /// Render an `open` diagnostic to stderr and fail. Best-effort on the write: a /// diagnostic that cannot be printed must not change the exit code. fn openDiag(s: cli.Streams, msg: []const u8) cli.Exit { s.err.print("zit open: {s}\n", .{msg}) catch {}; return .failure; } /// Map a materialization fault to its `open` diagnostic, one arm per /// `update.ApplyError` variant and NO `else`: a new variant must fail this /// compile and force a decision here, exactly as it does in `updateApplyFail`. /// /// Only `InvalidCheckinId` and `CheckinNotFound` are genuinely unreachable — /// `open` hands the engine a hash it just resolved out of this repository's own /// derived state, never a caller-supplied name — and even those get honest /// prose rather than a corruption claim. Everything else is reachable on an /// intact repository: the content/manifest arms describe a repository that /// really is damaged, while `SymlinkUnsupported` / `InvalidPath` / /// `ReservedPath` are raised while planning the TARGET manifest, so a clone of /// any repository containing a symlink or a path this platform cannot represent /// reaches them with nothing corrupt at all. Reporting those as corruption /// would be a false accusation against a healthy repository; the prose mirrors /// `updateApplyFail`'s. /// Decode a lowercase-hex artifact name to its raw digest for a presence probe. /// Null for anything that cannot be a stored name, which the caller reads as /// "not a name this store could hold" and therefore not a missing-content claim. fn openDecodeName(name: []const u8, out: *[logstore.max_hash_len]u8) ?[]const u8 { if (name.len == 0 or name.len % 2 != 0) return null; const bytes = name.len / 2; if (bytes > logstore.max_hash_len) return null; _ = std.fmt.hexToBytes(out[0..bytes], name) catch return null; return out[0..bytes]; } /// The tip resolved to a check-in whose file content this repository does not /// hold. Named as a CLASS rather than as a missing file, because the missing /// file is a symptom: the usual cause is that the store holds artifacts which /// parse as check-ins of a different repository, and no amount of looking at the /// named path explains that. /// /// This refuses. It never falls back to another check-in — picking a different /// tip because the real one is unusable would be inventing history, and the /// operator would have no way to know it happened. fn openForeignCheckin( s: cli.Streams, target: []const u8, branch: []const u8, missing: usize, first: []const u8, ) cli.Exit { s.err.print( "zit open: the tip of '{s}' is check-in {s}, whose file content this repository does not hold\n", .{ branch, target }, ) catch {}; s.err.print( "zit open: {d} file(s) are missing, starting with '{s}'\n", .{ missing, first }, ) catch {}; s.err.writeAll( "zit open: this happens when a repository stores artifacts that parse as check-ins of\n" ++ "zit open: ANOTHER repository -- imported mirror metadata is the common source -- so they\n" ++ "zit open: join the branch as check-ins whose blobs were never here. The working tree was\n" ++ "zit open: not modified. 'zit leaves' shows the branch's check-ins; 'zit update <check-in>'\n" ++ "zit open: materializes a specific one.\n", ) catch {}; return .failure; } fn openApplyFail(s: cli.Streams, err: update.ApplyError) cli.Exit { return switch (err) { // applyCheckin never emits this: only applyMerge's cross-cone guard // does, and this switch maps a checkin materialization. The unreachable // IS the assertion; a future emitter trips it in tests immediately. error.SparseMergeUnsupported => unreachable, error.FileContentMissing => blk: { var b: [512]u8 = undefined; break :blk openDiag(s, update.stageFaultReason(&b)); }, error.OutOfMemory => openDiag(s, "out of memory"), error.InvalidCheckinId => openDiag(s, "cannot read repository: the resolved check-in ID is not a valid artifact name"), error.CheckinNotFound => openDiag(s, "cannot read repository: the resolved check-in is not present"), error.CheckinContentMissing => openDiag(s, "cannot read repository: check-in content is missing"), error.AggregateChecksumMismatch => openDiag(s, "the check-in's R checksum disagrees with its content"), error.CheckinNameMismatch => openDiag(s, "cannot read repository: check-in content does not match its name"), error.TargetNotManifest => openDiag(s, "cannot read repository: the check-in is not a valid manifest"), error.BaselineNotFound => openDiag(s, "cannot read repository: the check-in baseline is not found"), error.BaselineContentMissing => openDiag(s, "cannot read repository: the check-in baseline content is missing"), error.BaselineNameMismatch => openDiag(s, "cannot read repository: the check-in baseline content does not match its name"), error.BaselineNotManifest => openDiag(s, "cannot read repository: the check-in baseline is not a valid manifest"), error.BaselineIsDelta => openDiag(s, "cannot read repository: the check-in baseline is not fully resolvable"), error.StateRecordInvalid => openDiag(s, "cannot read repository: " ++ update.state_record_invalid_reason), error.Corrupt => openDiag(s, "cannot read repository: repository is corrupt"), error.SymlinkUnsupported => openDiag(s, "the check-in contains an unsupported symlink entry"), error.InvalidPath => openDiag(s, "the check-in names an invalid file path"), error.ReservedPath => openDiag(s, "the check-in names a reserved file path"), error.WorktreeNotClean => openDiag(s, "working tree is not clean"), error.UnmanagedCollision => openDiag(s, "an untracked file or directory is in the way"), error.CurrentStateInvalid => openDiag(s, "current working-tree state is invalid"), error.Storage => openDiag(s, "cannot update working tree: input/output error"), error.TransactionUnavailable => openDiag(s, "cannot create a transaction directory"), error.RollbackFailed => openDiag(s, "open failed and could not be rolled back; the previous working-tree state is retained for recovery"), error.CleanupFailed => openDiag(s, "open committed, but the transaction directory could not be removed"), }; } /// `zit open` — the second half of `zit clone URL DIR`: turn a repository whose /// checkout is still UNBORN into a materialized working tree. Fossil's own UX is /// the same two steps (`fossil clone` then `fossil open`), and splitting them /// keeps `runClone` strictly repo-only. /// /// Thin wiring over the shipped machinery, deliberately: probe the checkout /// record, resolve the tip of the main branch out of the derived repo state, and /// hand materialization to the SAME `update.applyCheckin` engine every other /// working-tree move rides (journal-armed, transactional, rollback-safe). Only /// the tip resolution is open's own work, and it is load-bearing: `update` /// accepts nothing but a literal 40/64-hex check-in ID, so a freshly cloned tree /// has no way to name its own tip. /// /// It writes NO checkout record and NO managed set of its own. `init` — which /// `clone` delegates repo creation to — already wrote the unborn record; an /// absent managed set already reads as the empty set; and the update engine is /// the single writer of the born record and the managed set. Adding a second /// writer of that bootstrap state would only create two things to keep in sync. pub fn runOpen( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { // One demand-fetch retry, statically bounded (TigerStyle: no unbounded // recursion -- the second attempt runs with fetching disabled). // `zit open URL` is `zit clone URL` — cloned into a directory // named from the URL and opened there. for (verb_args) |tok| { if (std.mem.startsWith(u8, tok, "http://") or std.mem.startsWith(u8, tok, "https://")) { return runClone(gpa, io, arena, verb_args, streams, amb); } if (tok.len > 0 and tok[0] != '-') break; } return openAttempt(gpa, io, arena, verb_args, streams, amb, true); } /// The `--store` bootstrap: validate both ends, then write the container. /// The current directory must not already be a checkout (either control /// file), and PATH must hold a readable `.zit` -- checked here so the /// operator hears "no store at PATH" instead of a discovery miss. The /// container write is `checkout.write`'s atomic splice plus one /// `store_location` section carrying the store root's ABSOLUTE path: a /// worktree is addressed from anywhere, so a relative pointer would go /// stale with every `cd`. const Bootstrap = union(enum) { created, repaired, failed: cli.Exit }; fn bootstrapDetached( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, store_arg: []const u8, ) Bootstrap { var cwd = std.Io.Dir.cwd(); if (cwd.statFile(io, repo_filename, .{})) |_| { streams.err.writeAll("zit open: this directory holds the repository itself; --store points a DETACHED tree elsewhere\n") catch {}; return .{ .failed = .failure }; } else |_| {} const repairing = if (cwd.statFile(io, checkout.filename, .{})) |_| true else |_| false; // PATH may name the store root or the `.zit` file itself; either way the // pointer stored is the DIRECTORY, absolute. const named_file = std.mem.endsWith(u8, store_arg, "/.zit") or std.mem.eql(u8, store_arg, ".zit"); const store_root_rel = if (named_file) (std.fs.path.dirname(store_arg) orelse ".") else store_arg; var store_root = cwd.openDir(io, store_root_rel, .{}) catch { streams.err.print("zit open: no directory at '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; defer store_root.close(io); _ = store_root.statFile(io, repo_filename, .{}) catch { streams.err.print("zit open: no repository ('.zit') under '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; const abs = store_root.realPathFileAlloc(io, ".", arena) catch { streams.err.print("zit open: cannot resolve '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; if (repairing) { // Re-open: the checkout exists and its pointer (or the store it // pointed at) went stale — a moved store strands its satellites, and // a stranded tree can hold uncommitted work, so the pointer must be // repairable in place. The one gate that matters: the NEW store must // contain this tree's current check-in, or the recorded tree state // would be a lie against the store it now names. "Same store at a // new address" passes; "some other repository" is refused. var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined; const current = checkout.read(io, cwd, &ck_buf) catch { streams.err.writeAll("zit open: cannot read the checkout state\n") catch {}; return .{ .failed = .failure }; }; if (current) |state| if (state.version.len != 0) { var digest_buf: [digest_len]u8 = undefined; const digest = hexToDigest(state.version, &digest_buf) orelse { streams.err.writeAll("zit open: the checkout state is corrupt\n") catch {}; return .{ .failed = .failure }; }; var store_file = store_root.openFile(io, repo_filename, .{}) catch { streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; defer store_file.close(io); const store_st = store_file.stat(io) catch { streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; var store_fs = storage.FileStorage.init(io, store_file, store_st.size); var store_log = logstore.LogStore.openReadOnly(gpa, store_fs.storageFile()) catch { streams.err.print("zit open: cannot read the repository under '{s}'\n", .{store_root_rel}) catch {}; return .{ .failed = .failure }; }; defer store_log.deinit(gpa); if (store_log.lookup(digest) == null) { streams.err.print( "zit open: the repository under '{s}' does not contain this tree's check-in '{s}'; refusing to re-point at a different repository\n", .{ store_root_rel, state.version }, ) catch {}; return .{ .failed = .failure }; } }; checkout.updateSections(io, cwd, &.{.{ .tag = .store_location, .payload = abs }}) catch { streams.err.writeAll("zit open: cannot write the checkout state\n") catch {}; return .{ .failed = .failure }; }; streams.out.print("checkout now points at '{s}'\n", .{abs}) catch {}; return .repaired; } checkout.write(io, cwd, .{ .version = "", .branch = "trunk", .merge_parent = "", .special_op = "", .manifest_flags = "" }) catch { streams.err.writeAll("zit open: cannot write the checkout state\n") catch {}; return .{ .failed = .failure }; }; checkout.updateSections(io, cwd, &.{.{ .tag = .store_location, .payload = abs }}) catch { cwd.deleteFile(io, checkout.filename) catch {}; streams.err.writeAll("zit open: cannot write the checkout state\n") catch {}; return .{ .failed = .failure }; }; return .created; } fn openAttempt( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, allow_fetch: bool, ) cli.Exit { const parsed = switch (cli.parseOpenArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "open", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "open", .extra_argument, tok), .repeated_option => |tok| return cli.usageError(streams, "open", .repeated_option, tok), .empty_value => |tok| return cli.usageError(streams, "open", .empty_value, tok), .bad_manifest => |tok| return cli.usageError(streams, "open", .bad_manifest, tok), .too_many_cones => |tok| return cli.usageError(streams, "open", .too_many_cones, tok), }; if (parsed.help) { streams.out.writeAll(open_help) catch {}; return .ok; } // `--store PATH` bootstraps a DETACHED checkout in the current // directory: an unborn checkout record (exactly what `init` writes) plus // the store pointer, and nothing else. Everything after this block is the // ordinary open — discovery finds the fresh container, `Repository.open` // follows the pointer, and the materialization below neither knows nor // cares that the store lives elsewhere. if (parsed.store) |store_arg| { switch (bootstrapDetached(gpa, io, arena, streams, store_arg)) { .created => {}, // A repair is complete in itself: the tree is already open at its // check-in, so the materialization below has nothing to do. .repaired => return .ok, .failed => |rc| return rc, } } // The full derive: resolving the main branch reads the tag tables, which the // `.fast_read` image path does not carry. var repo: Repository = undefined; var root_dir: std.Io.Dir = undefined; const rc = openRepo(gpa, io, arena, streams, "open", null, false, false, &repo, &root_dir); if (rc != .ok) return rc; defer root_dir.close(io); defer repo.deinit(gpa); // BORN-ness, not presence, is the "already open" signal. Every fresh // repository carries a checkout record — `init` writes an unborn one // (`version == ""`, branch `trunk`) as its step 8, and `clone` delegates to // `init` — so a presence-based gate would reject `zit clone && zit open` on // every clone, the primary path this verb exists to serve. Only a non-empty // version means a tree has actually been materialized against a check-in. var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined; const current = checkout.read(io, root_dir, &ck_buf) catch |e| switch (e) { error.Storage => return openDiag(streams, "cannot read the checkout state: input/output error"), error.Corrupt => return openDiag(streams, "the checkout state is corrupt"), }; if (current) |state| if (state.version.len != 0) { streams.err.print( "zit open: this working tree is already open at check-in '{s}'; use 'zit update' to change check-ins\n", .{state.version}, ) catch {}; return .failure; }; // The target: an explicit CHECKIN resolved like `zit info` (§8.3), or the // tip of the main branch (repo-state §9 current-trunk). Everything below — // preflight, materialization, the manifest opt-in — is target-generic. const main_branch = repo.mainBranch(); const tip = if (parsed.checkin) |name| repo.resolve(name, .checkin) catch |e| { switch (e) { error.NotFound => streams.err.print("zit open: check-in '{s}' not found\n", .{name}) catch {}, error.Ambiguous => streams.err.print("zit open: name '{s}' is ambiguous\n", .{name}) catch {}, error.OutOfMemory => return openDiag(streams, "out of memory"), } return .failure; } else // Switched, not an unbound `catch`: the success message below is an // exit-0 claim that the tree is already correct, and it is only honest // for `NotFound`. Naming the arm means a later Storage/OOM arm on // `currentTrunk` fails this compile instead of silently reporting a // genuine failure as success on a materialization verb. repo.state().currentTrunk(main_branch) catch |e| switch (e) { // A repository with no check-in on its main branch is ALREADY in its // correct end state — the unborn record and an empty tree. There is // nothing to materialize and nothing wrong, so this is reported, not // failed, and the record is left exactly as it was found. An // explicit CHECKIN never lands here: in an empty repository it // resolves to NotFound above, a failure, because the operator asked // for something specific and did not get it. error.NotFound => { streams.out.print( "open: no check-in on '{s}' yet; the working tree is already in its correct (empty) state\n", .{main_branch}, ) catch {}; return .ok; }, // Settling an equal-mtime tie by ancestry allocates (repo-state // `MtimeMaxPick`). A failure here is a genuine failure — reporting // it as the exit-0 "already correct" message would be the lie the // switch above exists to prevent. error.OutOfMemory => return openDiag(streams, "out of memory"), }; // Both borrow the derived view, which outlives the update below. const target = repo.hashOf(tip); const branch = repo.branchOf(tip); // An assertion, not a check: unlike `update`, whose target is user-supplied // and therefore validated, open's target comes out of this repository's own // derived state. A malformed name here is a defect in the derivation, not // bad input, so it belongs in the negative space rather than in a // diagnostic path no correct run can reach. std.debug.assert(artifact.validateName(target)); // Distinct from the open above, which shares `repoOpenFail`'s wording: this // one has already derived the view successfully and is opening the log a // SECOND time, read-only, for materialization. Reporting both identically // made a failure here indistinguishable from a failure to derive — and they // are different repositories' worth of problem. const log = repo.logStore(gpa) catch |e| { switch (e) { error.Corrupt => streams.err.print( "zit open: the repository derived cleanly but its log will not reopen for reading: corrupt\n", .{}, ) catch {}, else => return repoOpenFail(streams, "open", e), } return .failure; }; // Sparse-cone RFC section 4.1: `open --cone` materializes the first // checkout already scoped. Absent flag = dense, exactly the shipped // behavior; `open` has no --dense because a fresh checkout IS dense. var open_members = parsed.cone_members; var open_len = parsed.cone_len; var open_not_members = parsed.cone_not_members; var open_not_len = parsed.cone_not_len; if (parsed.cone_file) |cf| { const lens = cli.appendConeFile(arena, io, cf, &open_members, open_len, &open_not_members, open_not_len) orelse { streams.err.print("zit open: cannot read --cone-file '{s}' (or it overflows the {d}-member cap)\n", .{ cf, zit.cone.max_members }) catch {}; return .failure; }; open_len = lens.members; open_not_len = lens.not_members; } var open_cone: ?zit.cone.Cone = null; defer if (open_cone) |*c| c.deinit(gpa); const sparse: update.SparseOpt = if (open_len + open_not_len != 0) blk: { open_cone = zit.cone.Cone.init(gpa, open_members[0..open_len], open_not_members[0..open_not_len]) catch |e| return openConeFail(streams, e); break :blk .{ .set = &open_cone.? }; } else .inherit; // PREFLIGHT: does this repository actually hold the content of the check-in // it just resolved as the tip? // // The format is content-addressed and types artifacts by STRUCTURE, so a // stored blob whose bytes happen to be a valid check-in manifest IS a // check-in — repo-state §4.4 grants opacity only to artifacts a conformant // parser DECLINES to classify, and a foreign manifest classifies fine. A // repository that tracks a file containing one (a Fossil->Git mirror's // `manifest`, a test fixture, a documentation example) therefore holds // check-ins of a DIFFERENT repository, whose file content it never had. That // is conformant derivation, not corruption, and Fossil behaves the same way. // // What is not acceptable is spending several minutes materializing before // discovering it. `stagePlan` already refuses safely — nothing is written // until every byte is verified and staged — so this changes no outcome; it // moves the refusal earlier and lets it NAME the situation, which a bare // missing-content error cannot. // // Presence-only: `lookup` asks whether a name is held, reconstructing // nothing. Cost is one probe per file in the tip, against a derive that // already walked every artifact in the store. if (repo.files(gpa, tip)) |maybe_list| { if (maybe_list) |list_const| { var list = list_const; defer list.deinit(gpa); // Sparse-cone RFC section 5.2: absence has three classes, and // this preflight must not read a deliberately sparse store as the // foreign-check-in class. A missing blob whose path the STORE // cone does not cover is elided -- healthy. Elided files that the // requested materialization would need refuse by the elision // class (section 6.4; demand fetch is a later wave), never as // foreign, and never as corruption. var store_cone = zit.repocache.readStoreCone(gpa, io, root_dir); defer if (store_cone) |*c| c.deinit(gpa); var missing: usize = 0; var first_missing: ?[]const u8 = null; var elided: usize = 0; var first_elided: ?[]const u8 = null; var it = list.iterator(); while (it.next()) |f| { const h = f.hash orelse continue; var digest_buf: [logstore.max_hash_len]u8 = undefined; const raw = openDecodeName(h, &digest_buf) orelse continue; if (log.lookup(raw) == null) { if (store_cone) |*c| { if (!c.covers(f.filename)) { elided += 1; if (first_elided == null) first_elided = f.filename; continue; } } missing += 1; if (first_missing == null) first_missing = f.filename; } } if (missing != 0) return openForeignCheckin(streams, target, branch, missing, first_missing.?); if (elided != 0) { // The materialization cone decides whether elision matters: // an open scoped inside the held content proceeds; one that // needs elided bytes refuses, naming the class and the count. const wants_elided = blk: { switch (sparse) { .set => |c| { if (c) |request| { if (!request.covers(first_elided.?)) { var again = list.iterator(); var any = false; while (again.next()) |f| { const h2 = f.hash orelse continue; var db2: [logstore.max_hash_len]u8 = undefined; const raw2 = openDecodeName(h2, &db2) orelse continue; if (log.lookup(raw2) == null and request.covers(f.filename)) { any = true; break; } } break :blk any; } break :blk true; } break :blk true; // dense request needs everything }, .inherit => break :blk true, // no worktree cone yet on open } }; if (wants_elided) { // Sparse-cone RFC section 6.4, policy form: the requested // materialization IS the new coverage intent. Write it as // the store cone (null = dense), run an ordinary pull -- // whose suppression and receive filtering read the NEW // policy -- and reprobe. Anything the peer lacked stays a // transit phantom for later synchronization to heal. var url_buf: [logstore.max_config_str]u8 = undefined; const fetchable = allow_fetch and blk: { _ = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch break :blk false; break :blk true; }; if (!fetchable) { streams.err.print( "zit open: {d} file(s) of this check-in are elided by the store's sparse cone, starting with '{s}'\n" ++ "zit open: no saved remote to fetch them from; open inside the cone\n" ++ "zit open: ('zit open --cone=...'), set a remote ('zit remote URL'), or re-clone\n", .{ elided, first_elided.? }, ) catch {}; return .failure; } streams.err.print( "zit open: {d} file(s) are elided by the store's sparse cone; fetching from the saved remote\n", .{elided}, ) catch {}; const requested: ?*const zit.cone.Cone = switch (sparse) { .set => |c| c, .inherit => null, // a dense open drops the cone }; zit.repocache.writeStoreCone(gpa, io, root_dir, requested); const pull_exit = runPull(gpa, io, arena, &.{}, streams, .{ .now_epoch = 0, .user_env = null }); if (pull_exit != .ok) return pull_exit; // Everything resolved so far (tip, branch, this log // handle) predates the fetch; re-attempt from scratch with // fetching disabled, so a peer that also lacked the // content produces the elided refusal rather than a loop. return openAttempt(gpa, io, arena, verb_args, streams, amb, false); } } } } else |_| { // A files() fault is not this preflight's to report: the engine below // reaches the same data and has the diagnostics for it. Falling through // keeps this advisory, never a new way to fail. } // `open` is the ONE verb that SETS the manifest opt-in. `--manifest` // is the operator's explicit choice and always wins; `--manifest=off` is // remembered as the refusal. Absent, the PROJECT's // `manifest` setting supplies it, which is fossil's model: a clone gets // build stamps because the project asked for them, not because each // developer remembered to. The tree is not materialized yet, so the // versioned tier is read out of the check-in itself, as fossil's // `db_get_versioned` does through `historical_blob`. // The verb's own `log` is already open, so the project setting is // resolved from the config it holds rather than through a second open. var open_config: ?logstore.OwnedConfig = log.readConfig(gpa) catch null; defer if (open_config) |*c| c.deinit(gpa); const open_cfg: ?*const logstore.Config = if (open_config) |*c| &c.config else null; const manifest: update.ManifestOpt = if (parsed.manifest) |m| .{ .set = m } else .{ .inherit_or = blk: { const r = checkinSetting(gpa, io, arena, &repo, tip, amb, open_cfg, .manifest) catch break :blk ""; break :blk r.value; }, }; // The check-in is named on failure because `open` RESOLVES its own target // (the tip of the configured main branch) rather than being handed one. A // diagnostic that omits it cannot distinguish "materializing the expected // check-in went wrong" from "resolved a different check-in than the operator // believes is the tip" — and the operator has no other way to see which. update.applyCheckinWith(gpa, io, root_dir, log, target, manifest, sparse, &repo, null) catch |e| { streams.err.print("zit open: materializing check-in {s} on {s}\n", .{ target, branch }) catch {}; return openApplyFail(streams, e); }; // The `empty-dirs` shape, laid over the tree the check-in // just produced. Read AFTER the materialization because a versioned // `.fossil-settings/empty-dirs` does not exist on disk before it; fossil's // checkout reads the same value out of the check-in it is installing, so // the two see the same list. { const dirs: []const u8 = blk: { const r = localSetting(io, arena, root_dir, amb, open_cfg, .empty_dirs) catch break :blk ""; break :blk r.value; }; update.ensureEmptyDirs(io, root_dir, dirs, streams.err); } // A successful open stamps this tree into the store's worktree // registry (best-effort; discovery data never gates the open). if (std.process.currentPathAlloc(io, gpa)) |own_abs| { defer gpa.free(own_abs); cli.stampWorktreeRegistry(gpa, io, root_dir, own_abs); } else |_| {} streams.out.print("opened {s} on {s}\n", .{ target, branch }) catch {}; return .ok; } /// Map a Cone.init refusal for `open`. Same wording discipline as update's /// mapper: states the rule, never says "missing" or "corrupt". fn openConeFail(s: cli.Streams, err: zit.cone.Error) cli.Exit { switch (err) { error.OutOfMemory => s.err.writeAll("zit open: out of memory\n") catch {}, error.InvalidMember => s.err.writeAll( "zit open: invalid --cone member: members are repository-relative paths using '/', with no '.', '..', or empty components\n", ) catch {}, error.ReservedMember => s.err.writeAll("zit open: a --cone member names a reserved control path\n") catch {}, error.Bounds => s.err.writeAll("zit open: --cone needs between 1 and 255 members (exclusions included), at least one inclusion\n") catch {}, error.DeadExclusion => s.err.writeAll("zit open: a --cone-not member no --cone member covers is a dead rule\n") catch {}, error.Contradiction => s.err.writeAll("zit open: one path given as both --cone and --cone-not\n") catch {}, } return .failure; } /// The fields a `updateConfig` overlay may change; a null field is left exactly /// as the stored config has it. Naming a single change site keeps the "copy the /// whole config, overlay one field" discipline in one place — a caller that /// means to change only the project code can never silently drop the credential /// or the saved remote (the class of bug the pre-`updateConfig` re-puts had). const ConfigPatch = struct { project_code: ?[]const u8 = null, login: ?zit.sync.Credential = null, remote_url: ?[]const u8 = null, /// draft-fossil-submanifests-00 §6.3/§8.1 and §6.4 — the three /// project-scope settings sync §9.3's scalar channel carries. Optional for /// the same reason every field here is: a peer that named only one of them /// must leave the other two exactly as stored. enable_submanifests: ?bool = null, submanifest_k: ?u8 = null, submanifest_auto_retune: ?bool = null, /// The autosync setting, stored inverted (true = off). Local, like /// `remote_url`; only the settings verb writes it. autosync_off: ?bool = null, ignore_glob: ?[]const u8 = null, /// The two signing settings, both LOCAL and both deliberately /// outside every config group: fossil classifies `pgp-command` sensitive /// (it is a command line a peer would otherwise get to choose) and puts /// `clearsign` in no group at all. clearsign: ?bool = null, pgp_command: ?[]const u8 = null, /// The rest of fossil's checkout settings at the repository tier. crnl_glob: ?[]const u8 = null, clean_glob: ?[]const u8 = null, empty_dirs: ?[]const u8 = null, encoding_glob: ?[]const u8 = null, dotfiles: ?[]const u8 = null, mimetypes: ?[]const u8 = null, attachment_size_limit: ?[]const u8 = null, /// The explicit-set bits, which are what make "this repository has no /// opinion" representable and therefore let the global tier be reached. autosync_set: ?bool = null, clearsign_set: ?bool = null, keep_glob: ?[]const u8 = null, manifest: ?[]const u8 = null, before_commit: ?[]const u8 = null, /// The two content-policy globs. Data settings, so a tracked /// `.zit-settings/` file outranks whatever is stored here. binary_glob: ?[]const u8 = null, crlf_glob: ?[]const u8 = null, /// The consent record. Empty string is a real value here (revoke), /// so it is written whenever the field is present, not when non-empty. before_commit_trust: ?[]const u8 = null, editor: ?[]const u8 = null, autosync_mode: ?[]const u8 = null, fn any(self: ConfigPatch) bool { return self.project_code != null or self.login != null or self.remote_url != null or self.enable_submanifests != null or self.submanifest_k != null or self.submanifest_auto_retune != null or self.autosync_off != null or self.ignore_glob != null or self.clearsign != null or self.pgp_command != null or self.autosync_set != null or self.clearsign_set != null or self.keep_glob != null or self.manifest != null or self.before_commit != null or self.before_commit_trust != null or self.binary_glob != null or self.crlf_glob != null or self.editor != null or self.autosync_mode != null; } }; /// Read the live config from an already-open log, overlay `patch`, and append /// the result while preserving every unnamed field. The caller owns durability, /// so a clone can combine its origin settings in one config frame and commit /// its complete result once. fn updateConfigOpen( gpa: std.mem.Allocator, log: *logstore.LogStore, patch: ConfigPatch, ) PersistError!void { var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse return error.Corrupt; // init always writes a config frame defer oc.deinit(gpa); // Copy every field, then overlay only the named ones. `next`'s string slices // borrow `oc.backing` / `patch`; both outlive the `putConfig` memcpy below. var next = oc.config; if (patch.project_code) |c| next.project_code = c; if (patch.login) |cred| { next.login_user = cred.user; // Point at the function-scoped `patch` storage, not the block-local // capture `cred`: `next` is read by `putConfig` after this block closes. next.login_secret_hex = &patch.login.?.secret_hex; } if (patch.remote_url) |u| next.remote_url = u; if (patch.enable_submanifests) |v| next.enable_submanifests = v; if (patch.submanifest_k) |v| next.submanifest_k = v; if (patch.submanifest_auto_retune) |v| next.submanifest_auto_retune = v; if (patch.autosync_off) |v| next.autosync_off = v; if (patch.ignore_glob) |v| next.ignore_glob = v; if (patch.clearsign) |v| next.clearsign = v; if (patch.pgp_command) |v| next.pgp_command = v; if (patch.autosync_set) |v| next.autosync_set = v; if (patch.clearsign_set) |v| next.clearsign_set = v; if (patch.keep_glob) |v| next.keep_glob = v; if (patch.manifest) |v| next.manifest = v; if (patch.before_commit) |v| next.before_commit = v; if (patch.before_commit_trust) |v| next.before_commit_trust = v; if (patch.binary_glob) |v| next.binary_glob = v; if (patch.crlf_glob) |v| next.crlf_glob = v; if (patch.editor) |v| next.editor = v; if (patch.autosync_mode) |v| next.autosync_mode = v; if (patch.crnl_glob) |v| next.crnl_glob = v; if (patch.clean_glob) |v| next.clean_glob = v; if (patch.empty_dirs) |v| next.empty_dirs = v; if (patch.encoding_glob) |v| next.encoding_glob = v; if (patch.dotfiles) |v| next.dotfiles = v; if (patch.mimetypes) |v| next.mimetypes = v; if (patch.attachment_size_limit) |v| next.attachment_size_limit = v; log.putConfig(next) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; } /// Open `dir/.zit` for a field-preserving config update, retaining the legacy /// wrapper for pull, sync, serve, and remote callers that do not already own a /// writable log. pub fn updateConfig( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, patch: ConfigPatch, ) PersistError!void { var file = cli.openStoreAppend(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; defer log.deinit(gpa); try updateConfigOpen(gpa, &log, patch); log.commit() catch return error.Storage; } /// §8.2 — persist the origin's project code into the freshly cloned repo at /// `dir/.zit`, preserving every other field `init` wrote. A no-op when the peer /// sent no code. Thin wrapper over `updateConfig` (the field-preserving RMW). fn storeClonedProjectCode( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, project_code: []const u8, ) PersistError!void { if (project_code.len == 0) return; // peer sent none; keep the local init code return updateConfig(gpa, io, dir, .{ .project_code = project_code }); } /// draft-fossil-submanifests-00 §6.3/§8.1 (and §6.4) over sync §9.3 /// — persist the origin's three sub-manifest settings into the freshly cloned /// repo at `dir/.zit`, preserving every other field `init` wrote. A no-op when /// the peer named none. Thin wrapper over `updateConfig`, the same shape /// `storeClonedProjectCode` has and for the same reason. /// /// One-way and clone-only, deliberately. A fresh clone has nothing of its own /// to conflict with, so adoption needs no merge; an EXISTING repository would, /// and §9.3's channel resolves conflicts by MTIME, which zit's config frame does /// not store (see `sync.submanifest_config_mtime`). Refreshing settings on a /// later pull is therefore a real conflict-resolution feature with a config /// schema change behind it, not plumbing this wrapper can be widened into — /// `runPull`/`runPush` deliberately have no seam here. fn storeClonedSubmanifestSettings( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, cfg: zit.sync.SubmanifestConfig, ) PersistError!void { if (!cfg.any()) return; // peer sent none; keep the local init defaults return updateConfig(gpa, io, dir, .{ .enable_submanifests = cfg.enable, .submanifest_k = cfg.k, .submanifest_auto_retune = cfg.auto_retune, }); } /// Drives a modern-clone client (§3.8 CLONE_ROUND0 → EXCHANGE) to convergence /// over `tr`, filling `ms` with the peer's artifacts; returns the round count. /// Sans-IO on this side of `tr.exchange`: the state machine only moves /// `[]const u8` bytes. Its closed error set is the driver's own causes unioned /// with the transport's, so `runClone` switches the whole surface exhaustively. /// The result of a clone run: the round count plus the origin's project code /// (§8.2), copied out of the sans-IO client so the caller can persist the /// shared identity after the client is torn down. const CloneOutcome = struct { rounds: usize, /// The login the peer accepted from round 1 on: the derived secret, copied /// out so the CLI can store it in the new repository once it exists. The /// user borrows argv. login: ?zit.sync.Credential = null, uv_skipped: u64 = 0, /// Uncompressed protocol bytes moved (`RunStats` totals), for the /// end-of-clone accounting line. sent_bytes: usize, received_bytes: usize, project_code_buf: [zit.sync.max_projectcode_len]u8, project_code_len: usize, /// submanifests §6.3/§8.1 over sync §9.3 — the origin's three project-scope /// settings, learned from the cleanup round's config card. Copied out of the /// sans-IO client for the same reason the project code is: the client is /// torn down before the caller persists anything. Owns no memory (three /// optional scalars), so no buffer twin is needed. submanifest: zit.sync.SubmanifestConfig, /// §9.3 r28 — the origin's shun table, learned from the same cleanup round. /// Unlike the settings above this OWNS memory (a row count the wire /// decides), so `deinit` is not optional: the client that parsed these rows /// is torn down before the caller writes anything durable. /// /// Carrying the shun table is not a nicety. fossil's shun table is its /// takedown mechanism; a clone that drops it will, on its next sync with /// any third party, re-accept exactly the content an operator was ordered /// to remove. shun_rows: []zit.sync.OwnedShunRow = &.{}, fn projectCode(self: *const CloneOutcome) []const u8 { return self.project_code_buf[0..self.project_code_len]; } fn deinit(self: *CloneOutcome, gpa: std.mem.Allocator) void { for (self.shun_rows) |row| { gpa.free(row.hash); gpa.free(row.reason); } gpa.free(self.shun_rows); self.shun_rows = &.{}; } }; /// The stderr live-progress sink a TTY clone runs with. Each completed /// round overwrites one line in place (\r, no newline); `finishLine` closes it /// with a newline exactly once, so whatever stderr carries next starts clean. /// Flushed per round — a progress line that sits in a buffer until the clone /// ends reports nothing. const CloneProgress = struct { err: *std.Io.Writer, emitted: bool = false, fn emit(ctx: *anyopaque, info: sync_driver.RoundInfo) void { const self: *CloneProgress = @ptrCast(@alignCast(ctx)); self.emitted = true; self.err.print( "\rround {d}: {d} bytes received", .{ info.round, info.total_received }, ) catch return; self.err.flush() catch {}; } fn finishLine(self: *CloneProgress) void { if (!self.emitted) return; self.emitted = false; self.err.writeAll("\n") catch return; self.err.flush() catch {}; } }; /// Map a Cone.init refusal for clone. Same wording discipline as the other /// verbs' mappers. fn cloneConeFail(s: cli.Streams, err: zit.cone.Error) cli.Exit { switch (err) { error.OutOfMemory => s.err.writeAll("zit clone: out of memory\n") catch {}, error.InvalidMember => s.err.writeAll( "zit clone: invalid --cone member: members are repository-relative paths using '/', with no '.', '..', or empty components\n", ) catch {}, error.ReservedMember => s.err.writeAll("zit clone: a --cone member names a reserved control path\n") catch {}, error.Bounds => s.err.writeAll("zit clone: --cone needs between 1 and 255 members (exclusions included), at least one inclusion\n") catch {}, error.DeadExclusion => s.err.writeAll("zit clone: a --cone-not member no --cone member covers is a dead rule\n") catch {}, error.Contradiction => s.err.writeAll("zit clone: one path given as both --cone and --cone-not\n") catch {}, } return .failure; } /// A bounded copy of a peer error-card text, taken before the aborted sync /// client goes out of scope so the CLI can quote the refusal. const PeerErrorCopy = struct { buf: [zit.sync.peer_error_cap]u8 = undefined, len: usize = 0, fn set(self: *PeerErrorCopy, source: ?[]const u8) void { const text_in = source orelse return; std.debug.assert(text_in.len <= self.buf.len); @memcpy(self.buf[0..text_in.len], text_in); self.len = text_in.len; } fn text(self: *const PeerErrorCopy) ?[]const u8 { if (self.len == 0) return null; return self.buf[0..self.len]; } }; fn cloneRun( gpa: std.mem.Allocator, tr: *transport.Transport, ms: *repostate.MemStore, self_servercode: []const u8, login: ?zit.sync.PendingLogin, sink: ?sync_driver.RoundSink, drain: ?sync_driver.RoundDrain, cone_members: []const []const u8, cone_not_members: []const []const u8, peer_error_out: *PeerErrorCopy, ) sync_driver.RunError(*transport.Transport)!CloneOutcome { var client = zit.sync.Client{ .state = zit.sync.initialState(.clone), .operation = .clone, .self_servercode = self_servercode, .project_code = sync_projectcode, .version = sync_client_version_info, .pending_login = login, // Sparse-cone RFC section 6.1: request server-side filtering; the // local persist filter stays as the section 6.3 belt regardless. .cone_members = cone_members, .cone_not_members = cone_not_members, }; defer client.deinit(gpa); const stats = sync_driver.runSyncProgress(gpa, &client, ms.store(), tr, max_stalled_sync_rounds, sink, drain) catch |e| { // A refusing peer's words outlive the aborted client (bounded copy) // so the caller renders them instead of a generic protocol error. peer_error_out.set(client.peerErrorText()); return e; }; // §8.2 — the peer's project code, learned from the clone reply's `push` // card, copied out before the client is deinitialized. var outcome = CloneOutcome{ .rounds = stats.rounds, .uv_skipped = client.uv_skipped, .sent_bytes = stats.sent_bytes, .received_bytes = stats.received_bytes, .project_code_buf = undefined, .project_code_len = 0, // §9.3 — the origin's sub-manifest settings, latched from the config // card the §8.4 cleanup round asked for. .submanifest = client.peerSubmanifestConfig(), }; const pc = client.peerProjectCode(); @memcpy(outcome.project_code_buf[0..pc.len], pc); outcome.project_code_len = pc.len; outcome.login = client.credential; // §9.3 r28 — copy the latched shun rows out of the client, which the // `defer` above tears down on return. const latched = client.peerShunRows(); if (latched.len != 0) { var rows = try gpa.alloc(zit.sync.OwnedShunRow, latched.len); var filled: usize = 0; errdefer { for (rows[0..filled]) |r| { gpa.free(r.hash); gpa.free(r.reason); } gpa.free(rows); } for (latched, rows) |src_row, *dst| { const h = try gpa.dupe(u8, src_row.hash); errdefer gpa.free(h); const r = try gpa.dupe(u8, src_row.reason); dst.* = .{ .hash = h, .epoch = src_row.epoch, .reason = r }; filled += 1; } outcome.shun_rows = rows; } return outcome; } /// The closed error set of `persistStore`: memory, a storage write fault, or a /// corrupt/oversized peer artifact name that will not decode to a log key. const PersistError = error{ OutOfMemory, Storage, Corrupt }; /// The disposition of one artifact in `persistStore`'s ordered pass. const PersistOutcome = union(enum) { /// A non-phantom frame for this artifact now exists in the log — written /// here, or already present from a prior cascade / a pull re-persist. persisted, /// A reverse-delta whose basis is a content-bearing store entry not YET /// written to the log. It is deferred on the basis wire-hash and re-driven /// once that basis lands, so its `delta_of` offset points strictly backward. deferred: []const u8, }; /// What one `persistStore` sweep did: the artifacts it made durable, and the /// sub-manifest references (draft-fossil-submanifests-00 §7.1) it phantomized /// and could not resolve from this repository's own holdings. const PersistSummary = struct { /// Content-bearing artifacts in the store — the count every caller reports. written: usize, /// §7.1 — distinct S references still standing as phantoms when the sweep /// ended: this repository holds an artifact whose tree closure is open. /// Zero for every repository that carries no S card, which is every /// repository predating this draft. submanifest_gaps: usize, /// Sparse-cone RFC section 6.3 -- artifacts deliberately not persisted /// under the clone's cone. Zero for every dense clone. elided: usize = 0, }; /// draft-fossil-submanifests-00 §7.1 — the S references one persist sweep saw. /// /// Rids, not names: `LogStore` entries are append-only, so an interned rid stays /// valid for the sweep, and whether a reference is still a GAP can only be /// decided once the sweep has ended — a sub-manifest may be persisted after the /// tree manifest that names it, and `putZlib` upgrades the phantom minted for it /// in place. Deciding per S card instead would report a gap for every reference /// whose artifact happens to sit later in rid order. const SubManifestRefs = struct { rids: std.AutoArrayHashMapUnmanaged(logstore.Rid, void) = .empty, /// Delta-dialect memory: this sweep stored a B-card manifest, so the /// `seen_delta_manifest` config indicator is written alongside `seen`'s /// on the same false->true edge discipline. seen_delta: bool = false, /// submanifests §8.1 — set the first time this sweep stores an artifact /// inside §7.2's gated set: "When the first S-bearing artifact is stored, an /// implementation SHOULD record a local seen-submanifest indicator". Both /// §7.2 shapes count, leaf sub-manifests included — see /// `phantomizeSubManifestRefs` for why the literal S-card-only reading would /// leave the gate blind to exactly the artifacts that carry a tree's files. /// It is also the durable gate on the §7.1 phantom REBUILD (see /// `persistStore`), which is why it tracks artifacts seen, not gaps left. seen: bool = false, fn deinit(self: *SubManifestRefs, gpa: std.mem.Allocator) void { self.rids.deinit(gpa); self.* = undefined; } /// Distinct references that never became a content-bearing frame. fn unresolved(self: *const SubManifestRefs, log: *const logstore.LogStore) usize { var n: usize = 0; for (self.rids.keys()) |rid| { if (log.isPhantom(rid)) n += 1; } std.debug.assert(n <= self.rids.count()); return n; } }; /// draft-fossil-submanifests-00 §7.1 — "Storing a structural artifact that /// carries S cards whose referenced sub-manifests are absent MUST create a /// phantom for each missing reference, with the referencing artifact's privacy /// classification, exactly as an absent delta source creates a source phantom." /// /// This is §7.1's SECOND storing boundary. The receive path /// (`sync.Client.absorbSubManifestRefs`) covers artifacts a pull absorbs; it /// cannot cover a clone, which stores every artifact wire-backed and never /// inflates one (so it has no plaintext to scan), nor a server-side push /// ingest, which absorbs nothing at all. Both of those reach the durable store /// only HERE — and here the plaintext is already in hand, reconstructed for the /// §7.7 verify, so the whole duty costs one `sBearingCandidate` scan of bytes /// this function was handed anyway. The clone's O(wire) receive bound is /// untouched: no container is inflated that §7.7 was not already inflating. /// /// A clone from a peer whose OWN closure is incomplete is exactly the case the /// receive path cannot see: the cursor delivers every artifact the peer holds, /// and the peer does not hold the sub-manifests. Without this, such a clone /// records its gap nowhere and never asks for the leaves. /// /// Privacy: zit's phantom is an interned entry with no content and carries no /// per-entry privacy at either boundary, so an S-reference phantom is minted /// exactly as an absent delta source's is — which is what §7.1 asks for. fn phantomizeSubManifestRefs( gpa: std.mem.Allocator, log: *logstore.LogStore, content: []const u8, seen: *SubManifestRefs, ) PersistError!void { // §8.1's indicator, over BOTH §7.2 shapes rather than over S cards alone. // // §8.1 says "when the first S-bearing artifact is stored"; a LEAF // sub-manifest carries F cards and no S card (§4.4), so the literal reading // records nothing for a repository holding leaves. That reading cannot be // right: this indicator is what arms the §7.2 transfer gate, §7.2 gates // leaves by TYPE, and §8's whole reason for the transfer layer is that a // pre-S reader can only hold a leaf as "unreadable ... opaque content". §8.2 // reads the same way — "a repository containing an S-bearing artifact" is a // repository holding any part of a tree, and a repository that holds a tree's // leaves is exactly that. So the indicator answers `transferGatedArtifact`. // // Cheap where it runs: the predicate rejects any byte string whose first two // bytes are not a card letter followed by SP or LF — every file blob, in two // comparisons. A structural artifact pays one allocation-free walk of its // card lines, beside the full-artifact rehash §7.7 already charges here. if (!seen.seen and repostate.transferGatedArtifact(content)) seen.seen = true; // The delta-dialect twin: remember that this repository stores delta // manifests, so `zit commit` keeps the dialect (config `seen_delta`). if (!seen.seen_delta and repostate.deltaManifestArtifact(content)) seen.seen_delta = true; var refs: std.ArrayListUnmanaged([]const u8) = .empty; defer refs.deinit(gpa); // Validate-then-apply lives in the shared extractor: a malformed card run // mints nothing, so a half-read artifact never leaves a half-minted phantom // set. `content` outlives `refs`, which borrows its hash tokens. if (!try repostate.collectSubManifestRefs(gpa, content, &refs)) return; for (refs.items) |name| { var buf: [digest_len]u8 = undefined; // §3.5 admits a 40- or 64-hex name and `decodeWireName` decodes both; a // name too wide for this repository's digest cannot key a frame here, so // it is a corrupt reference rather than a gap. const digest = decodeWireName(name, &buf) orelse return error.Corrupt; const rid = log.intern(digest) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, }; try seen.rids.put(gpa, rid, {}); } } /// Persist one artifact `rid` into `log`, choosing its stored form, and verify /// §7.7 integrity (the reconstructed content MUST hash to its wire name) BEFORE /// any durable write. A retained delta whose basis is already a held log frame /// under the read depth bound is stored VERBATIM as a `.delta`; a retained delta /// whose basis is a not-yet-written content-bearing store entry is DEFERRED /// (when `allow_defer`) so the basis frame is written first (reverse-delta /// ordering); every other case — no wire form, a full container, or an /// absent / phantom / over-deep basis — lands as the full form, never a /// phantom-based or unreadable delta. Idempotent: an artifact already holding a /// non-phantom frame returns `.persisted` without a re-write or re-verification /// (§7.7 was proven when it was first persisted). /// Record a cluster's wire name as it is written (see /// `persistArtifact`'s `clusters_out`). The dupe is owned by the list. fn noteCluster(gpa: std.mem.Allocator, clusters_out: ?*std.ArrayListUnmanaged([]const u8), name: []const u8, content: []const u8) error{OutOfMemory}!void { const list = clusters_out orelse return; if (!clusterindex.isCluster(content)) return; const owned = try gpa.dupe(u8, name); errdefer gpa.free(owned); try list.append(gpa, owned); } fn persistArtifact( gpa: std.mem.Allocator, store: repostate.ArtifactStore, log: *logstore.LogStore, rid: repostate.Rid, allow_defer: bool, /// Streaming persist: defer even on a basis the store has not /// SEEN yet — mid-exchange, an unseen basis is usually still in flight, so /// falling back to a full frame here would write different bytes than the /// converged pass. The final leftover drain runs with `allow_defer` off, /// which restores the full-form fallback for a basis that truly never /// arrived. Always false on the batched paths, whose store IS converged. defer_unseen: bool, /// A bounded plaintext memo shared across `persistStore`'s whole sweep. Each /// artifact reconstructs its own plaintext exactly once here — for the §7.7 /// verify — and primes it under its just-written frame's offset, so a later /// reverse-delta naming this frame as basis reuses that plaintext instead of /// re-walking (re-inflating, re-applying) the chain. Bounded (LRU, fixed byte /// + entry caps): a cache miss just costs a fresh reconstruction, never a /// wrong answer, so the stored bytes are identical with or without it. cache: *logstore.PlaintextCache, /// submanifests §7.1 — the sweep-wide S-reference record. Every plaintext /// this function reconstructs for the §7.7 verify is offered to it, so the /// phantom duty attaches to STORING and not to any one arrival path. srefs: *SubManifestRefs, /// submanifests §7.1/§8.1 — REBUILD the S-reference phantom set for an /// artifact that is ALREADY a durable frame. A phantom is an interned entry /// with no frame (`LogStore.intern`: "Phantoms never reach disk"), so the /// §7.1 phantoms this repository minted on a previous run are gone the /// moment the process exits; the gap has to be re-derived from the durable /// artifacts that name it. Without this an open tree closes itself on the /// next run: the tree manifest is already persisted, the early return skips /// its scan, and the repository reports converged while it still cannot /// enumerate that check-in. /// /// Passed as a parameter rather than read here because the decision is the /// repository's, not this artifact's: §8.1's durable seen-submanifest /// indicator is false for every repository predating this draft, so the /// rescan — and its one inflate per already-persisted artifact — is /// unreachable for all of them and the persist sweep is what it always was. rescan_persisted: bool, /// Where to record the wire name of every cluster this call /// writes, so the persist keeps the cluster index warm without a rescan. clusters_out: ?*std.ArrayListUnmanaged([]const u8), ) PersistError!PersistOutcome { const name = store.hashOf(rid); var digest_buf: [digest_len]u8 = undefined; const digest = decodeWireName(name, &digest_buf) orelse return error.Corrupt; if (log.lookup(digest)) |ex| { if (!log.isPhantom(ex)) { if (rescan_persisted) { // Read through the sweep's own memo, so an artifact this sweep // already reconstructed costs nothing to rescan. const plain = (log.contentCached(gpa, ex, cache) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse return .persisted; defer gpa.free(plain); try phantomizeSubManifestRefs(gpa, log, plain, srefs); } return .persisted; } } // A genuine phantom carries nothing durable — skip it (never a drop of held // content). `held` classifies without inflating a wire-backed entry. if (!store.held(rid)) return .persisted; // §7.7 L1465/L1469-1471 — the repository transaction-commit hook. Recompute // the artifact hash over the reconstructed bytes and abort BEFORE any frame is // written on a mismatch (a lying container inflates but does not name its // content). The verify plaintext is reconstructed against the LOG basis (a // read bounded by `max_delta_chain`) — NOT the in-memory wire chain, which, // now a clone stores every artifact wire-backed, is the full reverse-delta // history and would exceed the read depth bound. `persistStore` writes basis // frames before the deltas that name them (reverse-delta order), so a delta's // basis is already a log frame here. if (store.wire(rid)) |wf| { if (wf.delta) |d| { var base_buf: [digest_len]u8 = undefined; const base_digest = decodeWireName(d.src_hash, &base_buf) orelse return error.Corrupt; const maybe_base = log.lookup(base_digest); if (maybe_base == null or log.isPhantom(maybe_base.?)) { // The basis is not yet a log frame. Defer on it (reverse-delta // order: the newer basis frame must be written first) when it is a // held store entry that WILL be persisted. if (allow_defer) { if (store.lookup(d.src_hash)) |bs_rid| { if (store.held(bs_rid)) return .{ .deferred = d.src_hash }; } if (defer_unseen) return .{ .deferred = d.src_hash }; } // Basis absent and not deferrable (§3.3): fall back to FULL form // from the entry's OWN plaintext — a put+retain entry carries it // directly (never a phantom-based delta). In a converged clone this // is unreachable (every delta's basis is a written frame, handled // below); a wire-delta entry whose basis truly never arrived cannot // reconstruct — a hard error, never a fabricated artifact. const content = store.content(rid) orelse return error.Corrupt; if (!zit.artifact.verify(name, content)) return error.Corrupt; // submanifests §7.1 — the plaintext §7.7 just verified is the same // bytes the phantom duty is stated over; scan it here rather than // forcing a second inflate anywhere else. try phantomizeSubManifestRefs(gpa, log, content, srefs); try noteCluster(gpa, clusters_out, name, content); const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } const base_rid = maybe_base.?; const base_off = log.offsetOf(base_rid); // Reuse the basis plaintext memoised when the basis frame was // persisted — reverse-delta order writes every basis before the // delta that names it, so the memo is normally warm. Only a miss // (first use, or an LRU eviction) reconstructs it by walking the // chain; either way `base_plain` is owned and freed below. const base_plain = if (cache.get(base_off)) |hit| blk: { // §7.7 positive space: a memo entry is the exact plaintext that // named its wire hash at the basis's own persist, so the reused // bytes still hash to the basis name (debug-only assertion; the // hash is never recomputed on the hot path). std.debug.assert(zit.artifact.verify(d.src_hash, hit)); break :blk gpa.dupe(u8, hit) catch return error.OutOfMemory; } else blk: { const p = (log.contentCached(gpa, base_rid, cache) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse return error.Corrupt; // `contentCached` does not memoise a top-level full-form basis; // prime it here so sibling reverse-deltas on the same basis reuse // this reconstruction instead of repeating it. try cache.put(gpa, base_off, p); break :blk p; }; defer gpa.free(base_plain); const raw = zit.container.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(raw); const content = zit.delta.apply(gpa, base_plain, raw) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(content); if (!zit.artifact.verify(name, content)) return error.Corrupt; // submanifests §7.1 — the plaintext §7.7 just verified is the same // bytes the phantom duty is stated over; scan it here rather than // forcing a second inflate anywhere else. try phantomizeSubManifestRefs(gpa, log, content, srefs); try noteCluster(gpa, clusters_out, name, content); // Store the delta VERBATIM iff its basis is a held frame under the READ // depth bound; an over-deep basis falls back to full form (never an // unreadable chain). `delta_of` is the basis's backward offset — // strictly lower than this new frame, the invariant `putDeltaContainer` // asserts. if (log.depthOf(base_rid) < logstore.max_delta_chain) { const new_rid = log.putDeltaContainer(digest, base_rid, wf.container) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the just-verified plaintext so an older reverse-delta // naming this frame as basis reuses it (no chain re-walk). try cache.put(gpa, log.offsetOf(new_rid), content); } else { const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; try cache.put(gpa, log.offsetOf(new_rid), content); } return .persisted; } // §7.2 full form — decode to verify, then store the container verbatim. const content = zit.container.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(content); if (!zit.artifact.verify(name, content)) return error.Corrupt; // submanifests §7.1 — the plaintext §7.7 just verified is the same // bytes the phantom duty is stated over; scan it here rather than // forcing a second inflate anywhere else. try phantomizeSubManifestRefs(gpa, log, content, srefs); try noteCluster(gpa, clusters_out, name, content); const new_rid = log.putContainer(digest, wf.container) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the just-verified full-form plaintext so a reverse-delta naming // this frame as basis reuses it instead of re-inflating the container. try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } // §4.1 — no retained wire (a raw pull `put`): the plaintext is memoised in the // store, no chain to walk. Verify and compress it; the hash names the // UNCOMPRESSED bytes, so compression is address-transparent. const content = store.content(rid) orelse return error.Corrupt; if (!zit.artifact.verify(name, content)) return error.Corrupt; // submanifests §7.1 — the plaintext §7.7 just verified is the same // bytes the phantom duty is stated over; scan it here rather than // forcing a second inflate anywhere else. try phantomizeSubManifestRefs(gpa, log, content, srefs); try noteCluster(gpa, clusters_out, name, content); const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the plaintext so a reverse-delta naming this frame as basis reuses it. try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } /// Evidence seam: the persist drain's deferred waiting set — the /// reverse-deltas resident because their basis frame has not been written yet — /// at its peak, as a frame count and as wire container bytes. This is the /// number the streaming-persist decision hinges on: it is exactly what a /// per-round drain would have to keep (or spill) instead of the whole store. /// Off unless `ZIT_PERSIST_WAITING` asks for it at exit, and it changes /// nothing else. pub var persist_waiting_peak: u64 = 0; /// Evidence seam: what the last sync run moved over the wire and /// what its cluster index cost — bounded announcement is a claim about bytes /// per round, and this is where it is measured. `ZIT_SYNC_STATS` at exit. pub var sync_rounds: u64 = 0; pub var sync_sent_bytes: u64 = 0; pub var sync_received_bytes: u64 = 0; pub var cluster_index_names: u64 = 0; fn recordSyncStats(stats: sync_driver.RunStats) void { sync_rounds = stats.rounds; sync_sent_bytes = stats.sent_bytes; sync_received_bytes = stats.received_bytes; } pub var persist_waiting_peak_bytes: u64 = 0; /// Persists every content-bearing artifact in `store` into an already-open /// writable log. The caller owns durability, allowing clone to append artifacts /// and its combined config overlay before one final commit. fn persistStoreOpen( gpa: std.mem.Allocator, log: *logstore.LogStore, store: repostate.ArtifactStore, keep: ?*const std.DynamicBitSetUnmanaged, start_after: repostate.Rid, clusters_out: ?*std.ArrayListUnmanaged([]const u8), ) PersistError!PersistSummary { var pass = try PersistPass.init(gpa, log, store, keep, null, start_after, clusters_out); defer pass.deinit(); try pass.drainNew(); return pass.finish(); } /// The persist drain as a RESUMABLE pass: `drainNew` persists every /// content-bearing artifact interned since the last call, and `finish` settles /// the leftovers and the sub-manifest bookkeeping. The batched shape /// (`persistStoreOpen`) is init + one drainNew + finish; a streaming clone /// calls drainNew after every round instead, with `mem` set so each persisted /// artifact's resident bytes are released the moment its frame is durable — /// git index-pack's shape (stream to disk, defer a delta whose base is unseen, /// resolve when it lands) applied to fossil's reverse-delta wire. const PersistPass = struct { gpa: std.mem.Allocator, log: *logstore.LogStore, store: repostate.ArtifactStore, keep: ?*const std.DynamicBitSetUnmanaged, /// Set on the streaming path: the concrete store behind `store`, so a /// persisted rid's bytes can be released. Null keeps every byte resident /// until the caller drops the store — the batched behaviour. mem: ?*repostate.MemStore, /// One bounded plaintext memo for the whole pass: a basis reconstructed for /// its own §7.7 verify is reused by every reverse-delta that names it, so a /// delta family reconstructs each basis ~once rather than once per /// referencing delta. Bounded (LRU byte + entry caps) — never the whole repo. cache: logstore.PlaintextCache, /// submanifests §7.1 — one record for the whole pass, so a reference is /// judged a gap only after every artifact this store carries has landed. srefs: SubManifestRefs = .{}, rescan: bool, waiting: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(repostate.Rid)) = .empty, ready: std.ArrayListUnmanaged(repostate.Rid) = .empty, written: usize = 0, elided: usize = 0, /// The first rid `drainNew` has not seeded yet — the resume cursor. next_seed: repostate.Rid = 1, /// See `persistArtifact`; null records nothing. clusters_out: ?*std.ArrayListUnmanaged([]const u8) = null, /// Streaming only: rids the cursor passed while they were /// phantoms. Fossil interns a delta's unseen DELTASRC as a phantom the /// moment the delta arrives; the content lands at that SAME rid in a later /// round, behind the cursor. Each drainNew re-checks these first and /// drives the ones that have since gained content; whatever is left at /// `finish` is a true phantom, skipped exactly as the batched pass skips /// it. Empty on the batched path, whose store is already converged. revisit: std.ArrayListUnmanaged(repostate.Rid) = .empty, // Evidence-seam running totals for the peak counters above. waiting_count: u64 = 0, waiting_bytes: u64 = 0, fn init( gpa: std.mem.Allocator, log: *logstore.LogStore, store: repostate.ArtifactStore, keep: ?*const std.DynamicBitSetUnmanaged, mem: ?*repostate.MemStore, // Begin the walk AFTER this rid, so a pull persists only the // artifacts it received (which the holdings store appends beyond its // log-backed prefix) instead of re-walking every already-durable // holding. Zero means start at rid 1 (clone/push/serve: the whole // store IS the new content). start_after: repostate.Rid, clusters_out: ?*std.ArrayListUnmanaged([]const u8), ) PersistError!PersistPass { // submanifests §8.1/§7.1 — THE REBUILD-ON-OPEN INVARIANT, stated once here. // // §7.1's phantoms are not durable and are not made durable: a phantom is an // interned entry with no frame, exactly as an absent delta source's is // ("exactly as an absent delta source creates a source phantom"), and // `LogStore` writes none of either kind ("Phantoms never reach disk"). The // `.zit` on-disk format version is frozen (`logstore.version`), so minting a // durable phantom record would reject every repository already written. // // What makes the in-memory phantoms CORRECT across a restart is this // invariant: the S-reference gap set is a pure function of the durable // content-bearing frames, and this sweep re-derives it in full on every run. // A repository whose §8.1 indicator says it has ever stored an S reference // rescans its already-persisted artifacts (`rescan_persisted`); one that has // not — every repository predating this draft — rescans nothing and pays // nothing. The indicator itself IS durable, which is the whole trick: one // config byte survives the restart instead of a phantom table. const rescan = blk: { var oc = log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; if (oc) |*c| { defer c.deinit(gpa); break :blk c.config.seen_submanifest; } break :blk false; }; return .{ .gpa = gpa, .log = log, .store = store, .keep = keep, .mem = mem, .cache = logstore.PlaintextCache.init(), .rescan = rescan, .next_seed = start_after + 1, .clusters_out = clusters_out, }; } /// The type-erased `RoundDrain` entry: persist everything the round that /// just folded in has interned. fn drainErased(ctx: *anyopaque) sync_driver.DrainError!void { const self: *PersistPass = @ptrCast(@alignCast(ctx)); return self.drainNew(); } fn deinit(self: *PersistPass) void { self.cache.deinit(self.gpa); self.srefs.deinit(self.gpa); var it = self.waiting.valueIterator(); while (it.next()) |list| list.deinit(self.gpa); self.waiting.deinit(self.gpa); self.ready.deinit(self.gpa); self.revisit.deinit(self.gpa); } /// Drive one rid through `persistArtifact`, drain the waiters its frame /// unblocks, and — on the streaming path — release everything persisted. fn drive(self: *PersistPass, first: repostate.Rid) PersistError!void { const gpa = self.gpa; try self.ready.append(gpa, first); while (self.ready.pop()) |rid| { switch (try persistArtifact(gpa, self.store, self.log, rid, true, self.mem != null, &self.cache, &self.srefs, self.rescan, self.clusters_out)) { .persisted => { // `rid`'s frame now exists: re-drive any reverse-deltas that // deferred on it as their (now-written) basis. if (self.waiting.fetchRemove(self.store.hashOf(rid))) |kv| { var list = kv.value; defer list.deinit(gpa); for (list.items) |w| { self.waiting_count -= 1; self.waiting_bytes -= if (self.store.wire(w)) |wf| wf.container.len else 0; } try self.ready.appendSlice(gpa, list.items); } if (self.mem) |m| m.releasePersisted(rid); }, .deferred => |base_hash| { const gop = try self.waiting.getOrPut(gpa, base_hash); if (!gop.found_existing) gop.value_ptr.* = .empty; try gop.value_ptr.append(gpa, rid); // Evidence seam: a deferred rid is exactly what // streaming persist keeps resident; meter it. self.waiting_count += 1; self.waiting_bytes += if (self.store.wire(rid)) |wf| wf.container.len else 0; if (self.waiting_count > persist_waiting_peak) persist_waiting_peak = self.waiting_count; if (self.waiting_bytes > persist_waiting_peak_bytes) persist_waiting_peak_bytes = self.waiting_bytes; }, } } } /// Persist every content-bearing artifact interned since the last call. /// /// Reverse-delta persist ordering (§7.2/§8.2). Fossil stores an OLDER /// artifact as a delta of a NEWER one, so a delta cfile can land in the store /// at a LOWER rid than the basis it points at, while the log's `delta_of` is a /// BACKWARD offset that REQUIRES the basis frame to precede the delta. Persist /// therefore drains a worklist: a delta whose basis is not yet a frame is /// deferred (keyed by the basis wire-hash) and re-driven the instant that /// basis lands. Each content-bearing artifact seeds the drain once in rid /// order; each deferred delta is re-driven exactly once, so the pass is O(n) /// and terminates — a chain bottoms out at fossil's full newest artifact, and /// content addressing forbids a cycle. fn drainNew(self: *PersistPass) PersistError!void { // Revived phantoms first: their frames may be the bases this round's // deltas are about to defer on. var i: usize = 0; while (i < self.revisit.items.len) { const rid = self.revisit.items[i]; if (self.store.held(rid)) { _ = self.revisit.swapRemove(i); self.written += 1; try self.drive(rid); } else i += 1; } const n = self.store.count(); while (self.next_seed <= n) : (self.next_seed += 1) { const seed = self.next_seed; if (!self.store.held(seed)) { // Residual phantom: not durable. Mid-exchange it may yet gain // content at this very rid — remember it for the next drain. if (self.mem != null) try self.revisit.append(self.gpa, seed); continue; } if (self.keep) |k| { // Sparse-cone RFC section 6.3: elision writes nothing -- the frame // simply never lands, and rebuild-on-open re-interns the dangling // references as phantoms the cone classifies. A kept delta whose // basis is elided falls to the leftover drain below and stores // FULL FORM, so elision can never orphan held content. if (!k.isSet(seed)) { self.elided += 1; continue; } } self.written += 1; // count each content-bearing artifact exactly once, at its rid try self.drive(seed); } } /// Settle the leftovers and the sub-manifest bookkeeping; the pass is done. fn finish(self: *PersistPass) PersistError!PersistSummary { const gpa = self.gpa; const log = self.log; // Defensive drain: a deferred basis is itself content-bearing and therefore // persisted above (draining its waiters), so `waiting` is normally empty // here. Should a basis nonetheless be missing, its waiters land as full form // rather than be dropped — every content-bearing artifact stays durable. var leftover = self.waiting.iterator(); while (leftover.next()) |e| { for (e.value_ptr.items) |rid| { switch (try persistArtifact(gpa, self.store, log, rid, false, false, &self.cache, &self.srefs, self.rescan, self.clusters_out)) { .persisted => if (self.mem) |m| m.releasePersisted(rid), .deferred => unreachable, // allow_defer=false never defers } } } // submanifests §7.1 — decided once, at the end: a reference is a gap only // if nothing in this whole store turned out to name it. const gaps = self.srefs.unresolved(log); // submanifests §8.1 — "When the first S-bearing artifact is stored, an // implementation SHOULD record a local seen-submanifest indicator so // operational surfaces can warn before the setting is enabled or a pre-S // client is granted access." Written once, on the false->true edge, so a // steady-state sync appends no config frame; it is also what arms the // rebuild above on every subsequent run of this repository. if ((self.srefs.seen or self.srefs.seen_delta) and !self.rescan) { var oc = log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; var cfg: logstore.Config = if (oc) |c| c.config else .{ .hash_policy = 0, .default_branch = "", .project_id = "" }; defer if (oc) |*c| c.deinit(gpa); const already = (cfg.seen_submanifest or !self.srefs.seen) and (cfg.seen_delta_manifest or !self.srefs.seen_delta); if (!already) { if (self.srefs.seen) cfg.seen_submanifest = true; if (self.srefs.seen_delta) cfg.seen_delta_manifest = true; log.putConfig(cfg) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; } } return .{ .written = self.written, .submanifest_gaps = gaps, .elided = self.elided }; } }; /// Opens `dir/.zit` for the legacy persistence callers, preserving their /// artifact result and commit boundary while routing the write through the /// already-open-log implementation. fn persistStore( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, store: repostate.ArtifactStore, keep: ?*const std.DynamicBitSetUnmanaged, start_after: repostate.Rid, ) PersistError!PersistSummary { var file = cli.openStoreAppend(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; defer log.deinit(gpa); // A log with no entries at open is being CREATED by this persist, // so the clusters it records are the whole index; otherwise they extend one. const fresh = log.entryCount() == 0; var clusters: std.ArrayListUnmanaged([]const u8) = .empty; defer { for (clusters.items) |n| gpa.free(n); clusters.deinit(gpa); } const summary = try persistStoreOpen(gpa, &log, store, keep, start_after, &clusters); log.commit() catch return error.Storage; try refreshClusterSidecar(gpa, io, dir, &log, clusters.items, fresh); return summary; } /// Keep the cluster index warm across a persist: adopt the stored /// index (its key still validates — this persist only appended), append the /// clusters just written, re-key to the committed log, store. With no valid /// index to extend, only a persist that created the log writes one; any /// other store waits for the next sync open, which builds it by one walk. /// Advisory on storage (a failed write leaves the sidecar as it was); honest /// on memory, so the persist's OOM contract holds through it. fn refreshClusterSidecar( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *const logstore.LogStore, written: []const []const u8, fresh: bool, ) error{OutOfMemory}!void { var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); const a = arena.allocator(); var names: std.ArrayListUnmanaged([]const u8) = .empty; var adopted = false; if (try zit.repocache.loadClusterIndex(gpa, io, dir)) |blob| { defer gpa.free(blob); if (try clusterindex.parse(a, blob, log)) |p| { for (p.names) |n| try names.append(a, try a.dupe(u8, n)); adopted = true; } } if (!adopted and !fresh) return; for (written) |w| try names.append(a, w); const blob = (try clusterindex.serialize(gpa, log, log.entryCount(), names.items)) orelse return; defer gpa.free(blob); zit.repocache.storeClusterIndex(io, dir, blob); } /// The cluster index for one sync open: adopt the sidecar when its /// key validates, walk only the tail it has not seen (or everything, once, on /// a miss), store the refreshed sidecar, and resolve it against `store`. /// Nothing here is an error beyond memory: any refusal is a rebuild, and a /// store the walk cannot read simply stays unindexed — unbounded, never wrong. fn loadClusterIndex( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *logstore.LogStore, store: repostate.ArtifactStore, ) error{OutOfMemory}!clusterindex.Resolved { var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); const a = arena.allocator(); var names: std.ArrayListUnmanaged([]const u8) = .empty; var seen: std.StringHashMapUnmanaged(void) = .empty; defer seen.deinit(gpa); var after: repostate.Rid = 0; var adopted = false; if (try zit.repocache.loadClusterIndex(gpa, io, dir)) |blob| { defer gpa.free(blob); if (try clusterindex.parse(a, blob, log)) |p| { for (p.names) |n| { const owned = try a.dupe(u8, n); if ((try seen.getOrPut(gpa, owned)).found_existing) continue; try names.append(a, owned); } after = @intCast(p.n_rids); adopted = true; } } // Phantoms never reach disk, so an index written after a persist may count // entries this scan does not see; clamping never skips a real frame the // writer had not already recorded. const n: repostate.Rid = @intCast(log.entryCount()); if (after > n) after = n; if (after < n) { var walked: std.ArrayListUnmanaged([]const u8) = .empty; try clusterindex.walkClusters(gpa, a, log, after, &walked); for (walked.items) |w| { if ((try seen.getOrPut(gpa, w)).found_existing) continue; try names.append(a, w); } } if (!adopted or after < n) { if (try clusterindex.serialize(gpa, log, n, names.items)) |blob| { defer gpa.free(blob); zit.repocache.storeClusterIndex(io, dir, blob); } } cluster_index_names = names.items.len; return clusterindex.resolve(gpa, store, names.items); } // --------------------------------------------------------------------------- // `zit pack` — repack the store into a compact new generation. // // On-disk storage is UNSPECIFIED by the RFCs (they fix the wire format and // content-addressed naming, §4.1 L889-892/L918-919 and artifact §3.1 L318-326), so pack is free to reorganize // the log as long as every artifact's exact content and name survive and the // §7.2 verbatim wire bytes are still servable. It re-ingests every artifact, // in the source's offset order, into a FRESH log through `putArtifact` with a // lineage base hint — a check-in against its parent manifest, a file blob // against the same path's blob in the parent check-in (the commit path's own // §4.1/§7.2 compaction heuristic, applied offline). Offset order plus the // strictly-backward `delta_of` invariant means every base is already written // before the delta that names it, so no worklist is needed (unlike // `persistStore`, whose input is rid-ordered). // // The new log is written to a temporary file beside `.zit`, fsynced, and then // atomically renamed over `.zit`: a crash anywhere before the rename leaves the // original byte-identical (the temp is discarded next run). The `.zit-cache` // invalidates itself on the tail change and rebuilds on the next read verb. // --------------------------------------------------------------------------- /// pack's core store-to-store transform lives in the `zit` module (pack.zig) /// so the DST/fuzz harnesses drive the REAL pass in-process; the CLI wiring /// below (temp file, atomic rename, stats rendering) stays here. These aliases /// keep every call site and test reading /// `packLog`/`PackError`/`PackStats`/`Filter`/`shunFilter`. const PackError = zit.pack.PackError; const PackStats = zit.pack.PackStats; const packLog = zit.pack.packLog; const Filter = zit.pack.Filter; const shunFilter = zit.pack.shunFilter; /// The name of the transient new-generation file the rewriting verbs share, /// beside `.zit`. A fixed name (never a random one) so a temp left by a crashed /// run is reused rather than accumulating, and so `extras.zig` can name it in /// the single never-tracked predicate. Because it IS fixed and TWO verbs now /// reach for it, `rewriteAndSwap` holds an exclusive advisory lock on it for the /// whole rewrite — see there. const pack_tmp_filename = extras.pack_tmp_filename; /// `zit pack` — repack the current repository's store into a compact new /// generation and atomically swap it in. Thin CLI wiring over `packLog`: parse /// (no positional), discover the checkout root, open `.zit` read-only and /// capture its tail key, stream the packed log into a temp file, fsync, re-check /// the source tail (the no-repo-lock concurrency guard), then rename the temp /// over `.zit`. On any failure the temp is deleted and the original store is /// intact. /// /// The integrity guarantee this rewrite depends on is UNCHANGED, but it is no /// longer collected at open. `openReadOnly` used to whole-frame-CRC-verify every /// frame while scanning, so a corrupt source aborted before a byte was written — /// an eager pre-flight. It scans header-only now. The check that matters happens /// on the reads instead: `packLog` pulls every artifact through `srcContent`, /// which propagates `error.Corrupt` rather than skipping the frame, and /// `readFrameAt` CRC-checks every frame it reads. So a rotted payload still /// aborts the pack with the temp deleted and `.zit` untouched — it is caught in /// the copy loop rather than at open, and the operator sees the same outcome. /// This ordering is load-bearing for a verb that RENAMES OVER the original: if /// `srcContent` were ever changed to answer null on a corrupt read, the rewrite /// would silently omit the damaged artifact and then delete the only copy of it. /// The packed-generation memo's byte shape: magic, then the four /// `LogKey` fields little-endian — 32 bytes exactly. No self-checksum: a /// damaged memo can only fail the equality below, and the cost of a miss is /// one real pack. const packed_key_magic = "zPK1"; const packed_key_len = 4 + 8 + 8 + 8 + 4; fn serializePackedKey(buf: *[packed_key_len]u8, key: logstore.LogStore.LogKey) []const u8 { @memcpy(buf[0..4], packed_key_magic); std.mem.writeInt(u64, buf[4..12], key.end, .little); std.mem.writeInt(u64, buf[12..20], key.frame_count, .little); std.mem.writeInt(u64, buf[20..28], key.last_frame_offset, .little); std.mem.writeInt(u32, buf[28..32], key.last_header_crc, .little); return buf[0..packed_key_len]; } fn parsePackedKey(blob: []const u8) ?logstore.LogStore.LogKey { if (blob.len != packed_key_len or !std.mem.eql(u8, blob[0..4], packed_key_magic)) return null; return .{ .end = std.mem.readInt(u64, blob[4..12], .little), .frame_count = std.mem.readInt(u64, blob[12..20], .little), .last_frame_offset = std.mem.readInt(u64, blob[20..28], .little), .last_header_crc = std.mem.readInt(u32, blob[28..32], .little), }; } /// Record the just-swapped generation's key so the next identity-filter /// pack over an unchanged log is a no-op, and refresh the scan-index sidecar /// the rename invalidated — the open below pays the scan either way, so the /// next read verb should not pay it again. Best-effort throughout: any failure /// leaves the memo absent, which costs a real pack, never an answer. fn recordPackedGeneration(gpa: std.mem.Allocator, io: std.Io, store_dir: std.Io.Dir) void { var file = store_dir.openFile(io, repo_filename, .{}) catch return; defer file.close(io); const st = file.stat(io) catch return; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch return; defer log.deinit(gpa); const key = log.logKey() catch return; var buf: [packed_key_len]u8 = undefined; zit.repocache.storePackedKey(io, store_dir, serializePackedKey(&buf, key)); refreshSettingLogIndex(gpa, io, store_dir, &log); } pub fn runPack( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { var drop_elided = false; var pack_args: std.ArrayListUnmanaged([]const u8) = .empty; defer pack_args.deinit(gpa); for (verb_args) |tok| { if (std.mem.eql(u8, tok, "--drop-elided")) { drop_elided = true; } else { pack_args.append(gpa, tok) catch return stateOom(streams, "pack"); } } const help = switch (cli.parseNoPositionalArgs(pack_args.items)) { .ok => |a| a.help, .unknown_option => |tok| return cli.usageError(streams, "pack", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "pack", .extra_argument, tok), }; if (help) { streams.out.writeAll(pack_help) catch {}; return .ok; } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "pack")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // pack is store-scoped end to end: the reads, the pack-tmp lock // and the final rename must all land beside the STORE, so a detached // worktree resolves its pointer once here and the whole flow inherits it. if (cli.storeDirOf(io, root_dir) catch return repoReadFail(streams, "pack")) |resolved| { root_dir.close(io); root_dir = resolved; } // Open the source read-only: `openReadOnly` scans and whole-frame-CRC-checks // every frame, so a corrupt source aborts here (no temp written). The tail // key pins the log we packed for the pre-rename concurrency re-check. // // LIMIT OF THE RECLAIM'S DAMAGE TOLERANCE, stated where it is imposed: this // pre-flight runs before `packLog` sees the filter, so it covers the shunned // frames too. The reclaim contract's filter-before-reconstruction ordering // rescues a shunned artifact whose CONTENT no longer hashes to its name (and // one that will not decode) — but NOT one whose frame CHECKSUM is damaged, // which fails here with the reclaim never attempted. `pack_help` says so in // those terms; narrowing the pre-flight to kept frames would trade a // whole-store integrity gate for a rarer recovery path and is not this // slice's call to make. var src_file = root_dir.openFile(io, repo_filename, .{}) catch return repoReadFail(streams, "pack"); defer src_file.close(io); const src_st = src_file.stat(io) catch return repoReadFail(streams, "pack"); var src_fs = storage.FileStorage.init(io, src_file, src_st.size); var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| { switch (e) { error.OutOfMemory => streams.err.writeAll("zit pack: out of memory\n") catch {}, else => streams.err.print("zit pack: cannot read repository: {s}\n", .{reasonOf(e)}) catch {}, } return .failure; }; defer src.deinit(gpa); const src_key = src.logKey() catch return packFail(streams, error.Corrupt); // The no-op gate: a log whose key equals the one the last pack // recorded has had NO append since (a commit, a shun row, and a pull all // grow the log), so there is nothing to compact and nothing to reclaim // that the last pack did not already handle. Gated to the identity // shape: a `--drop-elided` run composes a cone filter the memo knows // nothing about, so it always runs. if (!drop_elided) { if (zit.repocache.tryLoadPackedKey(gpa, io, root_dir)) |blob| { defer gpa.free(blob); if (parsePackedKey(blob)) |recorded| { if (std.meta.eql(recorded, src_key)) { streams.out.writeAll("already packed; nothing to do\n") catch {}; return .ok; } } } } // `zit pack` is where the repository's deferred shuns are RECLAIMED: every // artifact the source's shun table names is dropped from the new generation // instead of relocated into it (the pack filter). The tombstones themselves // carry forward, so the name keeps reading as absent and a later sync will // not re-fetch what the operator removed. pack names no NEW tombstone — that // is `zit shun`'s job — so `extra_shuns` is empty. // Sparse-cone RFC section 5.4 -- narrow-then-compact, the spec's ONLY // deletion: an explicit `pack --drop-elided` drops what the store cone // elides (blobs referenced exclusively out-of-cone), composed with the // ordinary shun reclaim. Refused without a store cone: with no policy on // record there is nothing "elided" to drop, and a bare pack must never // guess one. var elided_dropper: ?ElidedDropper = null; defer if (elided_dropper) |*d| d.deinit(gpa); if (drop_elided) { var cone = zit.repocache.readStoreCone(gpa, io, root_dir) orelse { streams.err.writeAll( "zit pack: --drop-elided needs a store cone and this store has none\n" ++ "zit pack: narrow first ('zit update --cone=...'), or drop the flag\n", ) catch {}; return .failure; }; elided_dropper = ElidedDropper.build(gpa, src_fs.storageFile(), &cone) orelse { cone.deinit(gpa); streams.err.writeAll("zit pack: cannot classify the store for --drop-elided\n") catch {}; return .failure; }; cone.deinit(gpa); } const pack_filter: Filter = if (elided_dropper) |*d| d.composed(shunFilter(&src)) else shunFilter(&src); const stats = rewriteAndSwap( gpa, io, root_dir, &src, src_key, pack_filter, &.{}, ) catch |e| switch (e) { error.TailChanged => return tailChangedFail(streams, "pack"), error.RewriteBusy => return rewriteBusyFail(streams, "pack"), else => |pe| return packFail(streams, pe), }; renderPackStats(streams.out, stats); // Only the identity-filter shape records the memo: a --drop-elided pack's // outcome depends on a cone the memo does not capture, so the next plain // pack must run for real. if (!drop_elided) recordPackedGeneration(gpa, io, root_dir); return .ok; } /// `rewriteAndSwap`'s causes: everything `packLog` can fail with, plus the two /// aborts that are not failures of the rewrite at all and must not be reported as /// if they were. `TailChanged` means the source log is intact and NEWER than the /// snapshot we packed; `RewriteBusy` means another rewrite already owns the temp. /// In both the original store is untouched and the operator's next move is /// obvious, whereas every `PackError` arm means the new generation could not be /// built. pub const RewriteError = PackError || error{ TailChanged, RewriteBusy }; /// The new-generation rewrite `pack` and `shun` share: create the fixed-name /// readable temp beside `.zit`, stream `packLog(src → temp)` through `filter` /// (recording `extra_shuns` in the new generation only), fsync, re-check the /// source tail against `src_key`, then atomically rename the temp over `.zit`. /// Returns the pack's own stats; on every failing path the temp is deleted and /// the original store is byte-identical. /// /// ONE durable transition — the rename — and that is why the two verbs share /// this body rather than each having its own: the property is a property of this /// sequence, so a second copy would be a second place for it to stop holding. /// `src` is already open (both callers must read it before they can decide what /// to filter); the pre-rename check deliberately reads NEITHER it nor the /// caller's file handle. /// /// THAT IS THE GUARD, and it takes a fresh HANDLE, not merely a fresh stat. /// `storage.FileStorage` caches the append watermark it was constructed with and /// `size()` returns that cached value without a stat (src/storage.zig /// `sizeImpl`), so re-checking the tail through the caller's existing view /// compares `key.end` against the very number the key was derived from — it /// cannot fail, and the concurrency guard is then a no-op that reads as a guard. /// The caller's HANDLE fails the same way against the other concurrent writer: /// once a competing rewrite has renamed its generation over `.zit`, that handle /// names an ORPHAN inode whose size and tail are frozen, so it agrees with the /// key forever — and this run would then rename a snapshot taken before the /// winner existed over the winner, silently un-shunning what it just shunned. /// Reopening `.zit` BY NAME under the lock is what makes both an append (`zit /// commit`) and a rename (a racing `pack`/`shun --force`) detectable, and — for /// `shun` — is what makes "an aborted run leaves NO tombstone" something more /// than an assertion about code that never runs. Pinned by the two "a run the /// concurrency guard aborts leaves .zit untouched and records NO tombstone" /// tests, one per concurrent-writer shape. Same argument, same shape, as /// `shunAppendRows`. /// /// MUTUAL EXCLUSION, and why it lives here rather than in either verb. The temp /// has a FIXED name, so two overlapping rewrites would each truncate the other's /// in-flight new generation and then fail the §7.7 reconstruct-and-rehash pass on /// frames that had become holes — reported as `error.Corrupt`, i.e. two healthy /// processes telling the operator their repository is corrupt. A cron'd `zit pack` /// overlapping an operator's `zit shun --force` is an entirely ordinary way to hit /// that, and the shun is the run that must not silently not happen. So the temp is /// opened under an EXCLUSIVE, NON-BLOCKING advisory lock: the second arrival gets /// `error.RewriteBusy` and says so, instead of a corruption claim. The lock is /// held by the open file description, so a crashed run releases it — which is why /// this is a lock rather than an `exclusive` create, whose leftover would block /// every future rewrite. Truncation is therefore ours to do, AFTER the lock is /// held (`truncate` in the open would race the lock acquisition). /// /// Parameterised by DATA only — a drop predicate and a tombstone list — never by /// a policy callback. Which names those are, what to tell the operator, and /// whether to refuse at all stay entirely in the callers; nothing here knows /// which verb it is running, which is why it needs no verb label. /// /// `pub` for one reason: `src/sim/shun_sweep.zig` drives THIS FUNCTION under a /// fault-injecting `std.Io`. The property the sweep exists to pin — one durable /// transition, so a crash leaves either the old store or the new one — is a /// property of this sequence and of nothing else, so the sweep has to reach the /// sequence. It deliberately does NOT drive `runPack`/`runShun`: both begin at /// `cli.discoverRoot`, which reads the process's current directory, and a sweep /// that depends on ambient cwd is a sweep whose failures do not replay. pub fn rewriteAndSwap( gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, src: *logstore.LogStore, src_key: logstore.LogStore.LogKey, filter: Filter, extra_shuns: []const logstore.ShunRecord, ) RewriteError!PackStats { // New-generation file: a readable temp beside `.zit` (createFileAtomic opens // write-only, but pack reads its own delta bases back, so we roll our own // readable temp and rename it). Opened under the exclusive non-blocking lock // this function's doc explains; `WouldBlock` is another rewrite in progress, // not a storage failure. Cleaned up on every path except the successful // rename — and on the busy path we must NOT delete it, since it belongs to the // run that holds the lock. var tmp_file = root_dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = false, .lock = .exclusive, .lock_nonblocking = true, }) catch |e| switch (e) { error.WouldBlock => return error.RewriteBusy, else => return error.Storage, }; var renamed = false; // UNLINK BEFORE CLOSE, not after. `close` releases the advisory lock, and the // lock is what makes this path ours; unlinking after it would let a second // rewrite create and lock a fresh temp at the same name in between, which this // run's `deleteFile` would then pull out from under it — leaving that run to // write a complete new generation into an unlinked inode and fail its own // rename with ENOENT, the "healthy process told the operator something is // wrong" outcome the lock exists to prevent. Matches `shunAppendRows`. defer { if (!renamed) root_dir.deleteFile(io, pack_tmp_filename) catch {}; tmp_file.close(io); } // Discard whatever a prior crashed run left, now that the lock makes us the // only writer. `LogStore.open` below is handed watermark 0 and writes from // offset 0, so trailing bytes from a longer stale temp would survive past the // new generation's end — a checked truncation, not an optimisation. tmp_file.setLength(io, 0) catch return error.Storage; var dst_fs = storage.FileStorage.init(io, tmp_file, 0); var dst = try logstore.LogStore.open(gpa, dst_fs.storageFile(), src.hash_kind); const stats = packLog(gpa, src, &dst, filter, extra_shuns) catch |e| { dst.deinit(gpa); return e; }; dst.deinit(gpa); // `packLog` already fsynced; deinit only frees memory // No repo lock exists: if the source log changed under us since we scanned // it, abort rather than clobber a concurrent write with our stale snapshot. // O(1) (one stat + one header read); the original stays intact. For `shun` // this is also what makes an aborted run leave NO tombstone behind: the rows // were written into the temp this path is about to delete. // // A FRESH HANDLE, a fresh stat and a fresh storage view, for the reason given // in the doc above: the caller's view answers `size()` from a cached // watermark, so asking it would always agree with the key — and the caller's // HANDLE would agree just as unconditionally once a competing rewrite has // renamed its generation over `.zit`, since that leaves the handle on an // orphan inode whose size and tail can never move again. Reopening by NAME is // what makes that case an abort; the caller's handle only ever catches an // append to the same inode. var check_file = root_dir.openFile(io, repo_filename, .{}) catch return error.Storage; defer check_file.close(io); const now_st = check_file.stat(io) catch return error.Storage; var check_fs = storage.FileStorage.init(io, check_file, now_st.size); if (!logstore.LogStore.validateTail(check_fs.storageFile(), src_key)) return error.TailChanged; // The single durable transition: atomically replace `.zit` with the packed // log. A crash before this leaves the original untouched; after it, the // complete new log. There is no window where `.zit` is partial. root_dir.rename(pack_tmp_filename, root_dir, repo_filename, io) catch return error.Storage; renamed = true; return stats; } /// The no-repo-lock concurrency guard's diagnostic, general over `verb` because /// both rewriting verbs abort on it identically. Not a crash and not corruption: /// the wording leads with what the operator most needs to know. fn tailChangedFail(s: cli.Streams, verb: []const u8) cli.Exit { s.err.print( "zit {s}: repository changed during {s}; aborted (original is intact)\n", .{ verb, verb }, ) catch {}; return .failure; } /// The temp-lock diagnostic, general over `verb` for the same reason: both /// rewriting verbs contend for one fixed-name temp. Says what is happening and /// what to do, and — critically — does NOT say "corrupt", which is what an /// unguarded collision used to produce. fn rewriteBusyFail(s: cli.Streams, verb: []const u8) cli.Exit { s.err.print( "zit {s}: another repository rewrite is in progress; try again when it finishes\n", .{verb}, ) catch {}; return .failure; } /// Render pack's failure `err` as `zit pack: <reason>` on stderr, exit 1. The /// original store is always intact when this runs (every write went to the /// temp, which the caller's `defer` deletes). fn packFail(s: cli.Streams, err: PackError) cli.Exit { s.err.print("zit pack: {s}\n", .{reasonOf(err)}) catch {}; return .failure; } /// One human line summarizing a repack. "already packed" only when the run /// changed NOTHING an operator would care about — the log did not shrink AND /// nothing was reclaimed (the store was already compact; pack still verified every /// frame, its integrity-scrub value). A reclaim is never described as "already /// packed" even when the log grew, which it legitimately can: dropping an artifact /// that others were deltified against moves its content into one of them, so a /// byte count is exactly the wrong signal for whether work happened. /// /// The reclaim clause appears only when the run actually dropped something: a /// repository with no deferred shuns is the common case and "0 shunned reclaimed" /// on every pack would be noise, while a run that DID remove bytes must say so — /// dropping content silently is the one outcome an operator must never have to /// infer from a byte count. `pack_help` states the clause the same conditional way. /// /// `artifacts` is labelled "walked" rather than "packed" because `PackStats` /// counts SOURCE frames: on a reclaiming run `artifacts - dropped` is what the new /// generation holds, and an unlabelled first number would read as the packed /// count. Split from `runPack` so the exact wording is unit tested without a /// filesystem. fn renderPackStats(out: *std.Io.Writer, stats: PackStats) void { const unchanged = stats.bytes_after >= stats.bytes_before and stats.dropped == 0; const verb = if (unchanged) "already packed" else "packed"; out.print("{s}: {d} artifacts walked, {d} re-deltaed, ", .{ verb, stats.artifacts, stats.deltas }) catch {}; if (stats.dropped > 0) out.print("{d} shunned reclaimed, ", .{stats.dropped}) catch {}; out.print("{d} -> {d} bytes\n", .{ stats.bytes_before, stats.bytes_after }) catch {}; } // --------------------------------------------------------------------------- // `zit shun` — record artifacts in this repository's shun table and, by default, // remove their bytes in the same operation. // // NO DRAFT SPECIFIES THIS VERB. sync §3.3 (docs/rfcs/ // draft-fossil-sync-protocol-00.txt L518-519) defines the repository shun table // as "a persistent, repository-local set keyed by complete artifact name", // L520-521 grants that "Local administration MAY add or remove names", and // L527-530 makes the set mask the phantom scan. That is the whole of it: no // draft names a command, an option, a report, a refusal, or the operation that // removes a shunned artifact's bytes. Everything below — the refuse-by-default // policy, the `--force` gate, the unconditional current-checkout stop, the // per-type diagnostics — is this implementation's decision, and the choices are // stated at the point that makes them rather than attributed to a section. // // The one advisory the plan of record cites for sub-manifests cannot be // implemented or quoted here: there is no `submanifest` variant in // `artifact.ArtifactType` on this branch (it runs manifest=1 .. forum=8) and no // sub-manifest draft in docs/rfcs/. `shunConsequence`'s switch is total over // that enum, so the day a ninth type lands the compiler demands its diagnostic // — including the wider-blast-radius warning a sub-manifest deserves. // --------------------------------------------------------------------------- const NameSet = zit.pack.NameSet; /// What a `zit shun` run will do about one hash the operator named. Split into /// three groups by consequence, because the runner acts on the group, not the /// member: the three `blocking` values stop the whole invocation before anything /// durable happens; `remove`/`absent` each need a new tombstone; `reclaim` needs /// no row but still has bytes to remove; `already` needs nothing at all. const ShunDisposition = enum { /// Held here: the rewrite drops its bytes and records its tombstone. remove, /// Already in the set from an earlier `--defer`, but the BYTES ARE STILL HERE: /// no new row to write, and yet the removal the operator just asked for has /// genuinely not happened. Grouping this with `already` is what would make /// `zit shun --force H` after `zit shun --defer --force H` print "nothing to /// do" and exit 0 with the content still readable — the worst sentence this /// verb could produce, on the one path where an operator is acting under an /// order. So it is its own disposition: no tombstone, but a rewrite. reclaim, /// Not held here: the tombstone is recorded and nothing is reclaimed. A /// pre-emptive shun is meaningful and free — sync §3.3 L527-530 filters the /// phantom scan on the set, so a name recorded before the artifact ever /// arrives is never requested. absent, /// Already in the set AND already gone: a true no-op. `putShun` is idempotent /// as set membership, so the durable row keeps the epoch and reason it was /// written with. already, /// The manifest of the current checkout. REFUSED, with no `--force` /// override — the one hard stop. current_checkout, /// Not a lowercase-hex artifact-name prefix at all. not_a_name, /// An abbreviated prefix matching no stored artifact. Refused rather than /// recorded: the shun table is keyed by COMPLETE name, so a prefix cannot be /// completed and a row keyed on one would name nothing. unresolved, /// A prefix matching more than one stored artifact. ambiguous, /// Whether this disposition stops the whole invocation before any durable /// write. A total switch so a value added later cannot silently join the /// permissive group. fn blocks(self: ShunDisposition) bool { return switch (self) { .current_checkout, .not_a_name, .unresolved, .ambiguous => true, .remove, .reclaim, .absent, .already => false, }; } /// Whether this disposition needs a new tombstone row. fn records(self: ShunDisposition) bool { return switch (self) { .remove, .absent => true, .reclaim, .already, .current_checkout, .not_a_name, .unresolved, .ambiguous => false, }; } /// Whether this disposition has BYTES for the rewrite to drop. Separate from /// `records` precisely because the two are not the same question: `absent` /// records without removing and `reclaim` removes without recording. fn reclaims(self: ShunDisposition) bool { return switch (self) { .remove, .reclaim => true, .absent, .already, .current_checkout, .not_a_name, .unresolved, .ambiguous => false, }; } }; /// One hash the operator named, resolved. `name` is the full lowercase-hex /// artifact name — or, for the three unresolvable dispositions, the operator's /// own token, which is all there is to name. `digest` is the raw binary key the /// shun table and the pack filter both use, empty when there is none. `kind` is /// null for an opaque blob (an artifact that does not parse as a structural one), /// which is a first-class answer here exactly as it is for `zit artifact`. const ShunTarget = struct { name: []const u8, digest: []const u8 = &.{}, kind: ?artifact.ArtifactType = null, disposition: ShunDisposition, }; /// The report label for a target's type. Structural artifacts use §5.1 Table 3's /// external name (`artifact.ArtifactType.externalName`), the same vocabulary /// `zit artifact` prints, so an operator sees one word per type across the CLI; /// a blob is "file", matching `zit artifact`'s "type: file". fn shunTypeLabel(kind: ?artifact.ArtifactType) []const u8 { const t = kind orelse return "file"; return t.externalName(); } /// The consequence of removing an artifact of this type, in one clause. /// /// A TOTAL switch on `artifact.ArtifactType`, deliberately: the alternative is a /// default arm, and a type added later would then get no diagnostic at all while /// the operator was told the removal was safe. The six types that all lose the /// same thing share one arm rather than repeating one sentence six times. /// /// NO COUNTS, and that is a deliberate reduction from the intended report shape. /// "N descendant check-ins become unresolvable" needs a public descendants /// accessor over `repostate`'s child map plus a BFS, and "src/secret.key, in 6 /// check-ins" needs a reverse F-card index that exists nowhere in this tree — /// producing either means decoding every manifest in the repository on the /// refusal path, which is supposed to be the cheap path. The clause without the /// count states the same consequence; the count is worth revisiting only /// alongside the index that would make it cheap. fn shunConsequence(kind: ?artifact.ArtifactType) []const u8 { const t = kind orelse return "file content; every check-in listing it loses that file"; return switch (t) { .manifest => "a check-in; its descendants become unresolvable", .cluster => "sync housekeeping only; regenerable", .submanifest => "part of a check-in's file list; every check-in whose tree names it stops enumerating", .control, .wiki, .ticket, .attachment, .event, .forum => "structural content; what it renders stops rendering", }; } /// English plural suffix for `n`. Used only in these reports. fn plural(n: usize) []const u8 { return if (n == 1) "" else "s"; } /// The `-es` plural suffix, for the one noun these reports pluralize that does /// not take a bare `-s`: "hash" / "hashes". A separate function rather than a /// caller-supplied string so no call site can spell it "hashs". fn pluralEs(n: usize) []const u8 { return if (n == 1) "" else "es"; } /// Render the hard stops: one line per blocking target, then nothing happens. /// Every arm of the switch prints, including the non-blocking ones — a renderer /// with an `unreachable` arm would be undefined behavior in `ReleaseFast`, which /// is a poor trade for a line of output nobody should ever see. fn renderShunBlockers(err: *std.Io.Writer, targets: []const ShunTarget) void { for (targets) |t| switch (t.disposition) { .current_checkout => { err.print("zit shun: {s} is the check-in this checkout is on; refusing\n", .{t.name}) catch {}; err.writeAll("run 'zit update' to a different check-in first\n") catch {}; }, .not_a_name => err.print("zit shun: '{s}' is not an artifact name\n", .{t.name}) catch {}, .unresolved => err.print( "zit shun: no artifact matches '{s}'; give the complete name to shun a hash this repository does not hold\n", .{t.name}, ) catch {}, .ambiguous => err.print("zit shun: hash '{s}' is ambiguous\n", .{t.name}) catch {}, .remove, .reclaim, .absent, .already => {}, }; } /// Whether `src` still holds reclaimable BYTES under `digest` — an interned, /// non-phantom frame. This is exactly `packLog`'s own `dropped` predicate (it /// counts non-phantom rids the filter drops), so "would be removed" here and "was /// removed" there cannot disagree. No decode, so the refusal path stays cheap. fn holdsBytes(src: *const logstore.LogStore, digest: []const u8) bool { const rid = src.lookup(digest) orelse return false; return !src.isPhantom(rid); } /// How many rows already in `src`'s shun table still hold bytes that this run's /// rewrite would drop, EXCLUDING the ones `targets` names (those are counted as /// `.reclaim` targets and listed by name, so counting them here too would double /// them). This is the clause the refusal — the consent gate — would otherwise be /// silent about: `renderShunStats` is honest that `dropped` can exceed `recorded`, /// but that honesty arrives after the deletion. fn deferredStillHeld(src: *const logstore.LogStore, targets: []const ShunTarget) usize { var n: usize = 0; rows: for (src.shunList()) |row| { if (!holdsBytes(src, row.hash)) continue; for (targets) |t| { if (std.mem.eql(u8, t.digest, row.hash)) continue :rows; } n += 1; } return n; } /// Render the default refusal: what this run would do, then stop. The register is /// `clean`'s (`cleanRefuse` in cmd/tracking.zig) — the counts and the word /// "refusing" in ONE line, so an operator learns nothing happened before reading /// six lines of inventory, then the flag that proceeds, and never a prompt. /// /// THREE counts, because they answer three different questions and only the first /// used to be stated. How many rows become permanent (`recording` — including the /// pre-emptive ones, which remove nothing). How much content this invocation's own /// targets destroy (`removing` — which is NOT a subset of `recording`: a hash /// shunned earlier with `--defer` records nothing and still has bytes to lose). /// And `deferred_held`, the content the same rewrite reclaims from rows an EARLIER /// run recorded — collateral the operator never named, which the success line /// already admits to and the consent gate must not hide. /// /// `defer_reclaim` IS A PARAMETER BECAUSE THE TWO RUNS ARE DIFFERENT OPERATIONS, /// and this block's whole job is to state the operation accurately before an /// irreversible act. A `--defer` run removes nothing NOW: describing it as "would /// be permanently removed" overstates it, the `deferred_held` collateral clause is /// about a rewrite that is not going to happen, and — worst — a suggestion of /// `zit shun --force` would have an operator who copies it escalate a deferred /// tombstone into a full-store reclaim of the whole union set, including collateral /// the run never named. The suggested command is therefore always the one the /// operator actually asked for, plus `--force`. fn renderShunRefusal( err: *std.Io.Writer, targets: []const ShunTarget, deferred_held: usize, defer_reclaim: bool, ) void { var recording: usize = 0; var removing: usize = 0; for (targets) |t| { if (t.disposition.records()) recording += 1; if (t.disposition.reclaims()) removing += 1; } err.writeAll("zit shun: ") catch {}; if (recording == 0) { err.writeAll("no new hash would be recorded") catch {}; } else { err.print("{d} hash{s} would be shunned in this repository", .{ recording, pluralEs(recording) }) catch {}; } if (removing == 0) { err.writeAll("; nothing you named holds content to remove") catch {}; } else if (defer_reclaim) { // Named, still held, and deliberately NOT reclaimed by this run. "stays // reconstructible" rather than merely "unreclaimed": the mask is on the // name, so the plaintext is still materialisable through any surviving // delta, which is precisely what an operator under a takedown order would // otherwise assume they had already prevented. err.print( "; the bytes of {d} artifact{s} stay on disk and stay reconstructible until 'zit pack'", .{ removing, plural(removing) }, ) catch {}; } else { err.print("; {d} artifact{s} would be permanently removed", .{ removing, plural(removing) }) catch {}; } // Collateral of the REWRITE, so it is silent on the deferred path: there is no // rewrite in a `--defer` run and those rows keep their bytes untouched. if (deferred_held != 0 and !defer_reclaim) { err.print( "; the same rewrite also removes {d} artifact{s} shunned earlier and not yet reclaimed", .{ deferred_held, plural(deferred_held) }, ) catch {}; } err.writeAll("; refusing without --force:\n\n") catch {}; for (targets) |t| switch (t.disposition) { .remove => err.print(" {s} {s} {s}\n", .{ t.name, shunTypeLabel(t.kind), shunConsequence(t.kind), }) catch {}, .reclaim => if (defer_reclaim) { err.print(" {s} {s} already shunned; content still here and stays\n", .{ t.name, shunTypeLabel(t.kind), }) catch {}; } else { err.print(" {s} {s} already shunned; content still here and would be removed\n", .{ t.name, shunTypeLabel(t.kind), }) catch {}; }, .absent => err.print(" {s} recorded only (not present in this repository)\n", .{t.name}) catch {}, .already => err.print(" {s} already shunned; content already gone\n", .{t.name}) catch {}, // Unreachable in practice (`renderShunBlockers` ran and the caller // exited), but printed rather than asserted: see `renderShunBlockers`. .current_checkout, .not_a_name, .unresolved, .ambiguous => err.print( " {s} cannot be shunned\n", .{t.name}, ) catch {}, }; err.writeAll( \\ \\This is irreversible: zit undo does not recover a shun, and the content \\cannot be re-fetched from a peer. \\ ) catch {}; err.print("run 'zit shun {s}--force <HASH>...' to proceed\n", .{ if (defer_reclaim) "--defer " else "", }) catch {}; } /// One human line summarizing a completed reclaiming shun, in `renderPackStats`'s /// register. `dropped` can EXCEED `shuns_recorded`, because the same rewrite also /// reclaims any shun that was deferred earlier — an operator must be told that /// rather than left to infer it from a byte count. Split out so the exact wording /// is unit tested without a filesystem. /// /// EVERY NUMBER HERE COMES FROM THE PACK, none from the caller's intent — which is /// the whole reason `PackStats.shuns_recorded` exists. The rows this run meant to /// write and the rows the new generation actually holds are two different facts, /// and the one an operator is owed after a takedown is the second. `packLog`'s /// row-count gate makes them equal or fails the pack; if that gate were ever /// weakened, this line would under-report rather than over-report. fn renderShunStats(out: *std.Io.Writer, stats: PackStats) void { // `dropped` is counted among the source frames walked, so this subtraction // cannot underflow. Asserted anyway, and precisely because it is a tautology: // it is the only unchecked arithmetic in this verb's reporting, and an // underflow would print an absurd remaining count rather than trap. std.debug.assert(stats.dropped <= stats.artifacts); out.print("shunned: {d} hash{s} recorded, {d} artifact{s} removed, {d} -> {d} remaining, {d} -> {d} bytes\n", .{ stats.shuns_recorded, pluralEs(stats.shuns_recorded), stats.dropped, plural(stats.dropped), stats.artifacts, stats.artifacts - stats.dropped, stats.bytes_before, stats.bytes_after, }) catch {}; } /// `--defer`'s success line. It does NOT promise a byte count: nothing on this /// path measured one, and the only honest number would require doing the rewrite /// the operator just declined. /// /// `recorded == 0` with `holding != 0` is the re-`--defer` of a hash already /// deferred: no row to add, and yet content still to lose. Saying "0 hashes /// recorded" there would read as a no-op on a repository that still holds the /// bytes, so that case gets its own clause naming what is left. fn renderShunDeferred(out: *std.Io.Writer, recorded: usize, holding: usize) void { if (recorded == 0) { out.print( "shunned: no new hash to record; {d} artifact{s} still hold{s} content; run 'zit pack' to reclaim the space\n", .{ holding, plural(holding), if (holding == 1) "s" else "" }, ) catch {}; return; } out.print( "shunned: {d} hash{s} recorded; run 'zit pack' to reclaim the space\n", .{ recorded, pluralEs(recorded) }, ) catch {}; } /// The success line for a run with tombstones to write and NOTHING to reclaim — /// the purely pre-emptive shun. It takes the append path rather than the rewrite /// (see `runShun`), so there is no byte count to report and, more to the point, /// nothing was removed: a line ending in a store that had GROWN would be the /// wrong thing to print under the word "shunned". fn renderShunRecordedOnly(out: *std.Io.Writer, recorded: usize) void { out.print( "shunned: {d} hash{s} recorded; none is held here, so nothing was removed\n", .{ recorded, pluralEs(recorded) }, ) catch {}; } /// `--list`: the shun table in persisted order, one row per line — full artifact /// name, the UTC date it was recorded, and the reason when one was given. The /// date comes from `cli.formatWriterTimestamp`'s first ten bytes, so this shares /// the CLI's one epoch formatter rather than growing a second. fn renderShunList(out: *std.Io.Writer, rows: []const logstore.ShunRecord) void { if (rows.len == 0) { out.writeAll("the shun table is empty\n") catch {}; return; } for (rows) |row| { var hex: [2 * digest_len]u8 = undefined; var ts: [19]u8 = undefined; const date = cli.formatWriterTimestamp(row.epoch, &ts)[0..10]; out.print("{s} {s}", .{ digestToWireName(row.hash, &hex), date }) catch {}; if (row.reason.len != 0) out.print(" {s}", .{row.reason}) catch {}; out.writeAll("\n") catch {}; } } /// Render shun's failure `err` as `zit shun: <reason>` on stderr, exit 1. On /// every path that reaches this the store is byte-identical: either nothing was /// written yet, or every write went to the temp the rewrite's `defer` deletes. fn shunFail(s: cli.Streams, err: anyerror) cli.Exit { s.err.print("zit shun: {s}\n", .{reasonOf(err)}) catch {}; return .failure; } /// The two shun-row rejections (`BadKey`, `ReasonTooLong`) that are unreachable by /// construction here — every digest passed `isCompleteDigest` and every reason /// passed the length gate — given a REAL phrase rather than left to `reasonOf`, /// whose `else` arm renders both as "input/output error". Being unreachable is /// exactly why the phrase matters: if one ever fires it is a key-width or /// reason-length bug, and telling the operator their disk failed would send them /// looking at hardware. Nothing durable has happened on either path. fn shunRowBug(s: cli.Streams) cli.Exit { s.err.writeAll("zit shun: internal error: a shun row was rejected as malformed; nothing was changed\n") catch {}; return .failure; } /// `shunAppendRows`' failure, with its two unreachable-by-construction arms routed /// to `shunRowBug` instead of through `reasonOf`'s "input/output error" catch-all. /// A torn append is truncated by the log's own recovery, so a failure here leaves /// the shun simply absent, never half present. /// /// `TailChanged` and `RewriteBusy` join the set for the reason `RewriteError` /// carries them: neither is a failure of the append at all, both leave the store /// exactly as it was, and both have the same operator-facing meaning here as they /// do on the rewrite path — which is why they route to the SAME two diagnostics /// rather than growing a second pair of sentences for the append. const ShunAppendError = logstore.OpenError || logstore.ShunError || logstore.CommitError || error{ TailChanged, RewriteBusy }; fn shunAppendFail(s: cli.Streams, err: ShunAppendError) cli.Exit { return switch (err) { error.OutOfMemory => stateOom(s, "shun"), error.BadKey, error.ReasonTooLong => shunRowBug(s), error.TailChanged => tailChangedFail(s, "shun"), error.RewriteBusy => rewriteBusyFail(s, "shun"), else => shunFail(s, err), }; } /// Canonicalize a `zit shun` positional to a lowercase-hex artifact-name prefix, /// or null when it is not one. The same band `zit artifact` accepts (4..64 hex), /// deliberately: an operator pastes the same token into both verbs. fn canonicalizeShunHex(arg: []const u8, out: *[2 * logstore.max_hash_len]u8) ?[]const u8 { if (arg.len < 4 or arg.len > out.len) return null; for (arg, 0..) |c, i| { const lc = std.ascii.toLower(c); if (!((lc >= '0' and lc <= '9') or (lc >= 'a' and lc <= 'f'))) return null; out[i] = lc; } return out[0..arg.len]; } /// Decode a COMPLETE lowercase-hex artifact name (40 or 64 hex, the two forms /// artifact-format §3.2 defines) into `out`, returning the used prefix, or null /// when `hex` is not one of those two widths. The width, not `hash_kind`, decides /// — the same rule `logstore.isCompleteDigest` enforces on the write end, so /// nothing here interprets a digest. fn shunHexToDigest(hex: []const u8, out: *[logstore.max_hash_len]u8) ?[]const u8 { if (hex.len != 40 and hex.len != 64) return null; const n = hex.len / 2; std.debug.assert(n <= out.len); _ = std.fmt.hexToBytes(out[0..n], hex) catch return null; return out[0..n]; } /// Append `rows` to the live store and fsync — `--defer`'s whole effect, and the /// path a `--force` run with nothing to reclaim takes too. One frame per row plus /// one `commit`, so a crash mid-append leaves a torn tail the log's existing /// recovery truncates: the shun is then simply absent, never half present. /// Deliberately NOT the rewrite path: nothing here is atomic across rows, and /// nothing needs to be, because each row is independently meaningful. /// /// IT CARRIES BOTH OF `rewriteAndSwap`'S CONCURRENCY PROTECTIONS, for reasons that /// are this path's own and not a symmetry argument. /// /// The LOCK — the same exclusive non-blocking lock on the same fixed /// `pack_tmp_filename`, so an append and a rewrite exclude each other. Without it a /// `zit pack` (or another `zit shun --force`) that renames its new generation over /// `.zit` after our `openFile` leaves us appending to an ORPHANED INODE: the row /// lands nowhere anyone will ever read, and this run still reports success. On the /// `--defer` path that means the deferred shun silently does not exist while the /// help text has just told the operator to run `zit pack` to finish it. A cron'd /// pack overlapping an operator's shun is an entirely ordinary way to hit that. /// /// The TAIL RE-VALIDATION against `src_key` — because the caller classified its /// targets against a snapshot taken before the lock existed, and both success /// lines this path can print (`renderShunRecordedOnly`'s "none is held here, so /// nothing was removed", `renderShunDeferred`'s count of what still holds content) /// are claims about STORE STATE, not about the argument list. A `zit commit` /// landing a named hash in that window makes the first of them false: the /// tombstone would be recorded, exit 0, and an operator under a takedown order /// told nothing was held while the bytes were live. Re-checking the tail on the /// very handle the append will go through — under the lock, so no rename can have /// intervened — turns that into an abort. `error.TailChanged` here means nothing /// was written: the check precedes the first `putShun`. /// /// The check reads the REOPENED handle rather than the caller's, and that is the /// stronger of the two: a rewrite that completed just before we took the lock has /// left the caller's handle pointing at an orphan whose tail still matches its own /// key, while `.zit` by name is now a different generation. Validating the handle /// we are about to WRITE is what makes that case an abort instead of an append /// driven by a stale snapshot. fn shunAppendRows( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, src_key: logstore.LogStore.LogKey, rows: []const logstore.ShunRecord, ) ShunAppendError!void { var lock_file = dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = false, .lock = .exclusive, .lock_nonblocking = true, }) catch |e| switch (e) { error.WouldBlock => return error.RewriteBusy, else => return error.Storage, }; // Ours for the duration and removed on every path: an append leaves no new // generation behind, so a zero-length temp outliving the run would be litter. // The lock lives in the open file description, so a crash releases it. defer { dir.deleteFile(io, pack_tmp_filename) catch {}; lock_file.close(io); } var file = cli.openStoreAppend(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); // A FRESH handle and a FRESH stat, for the same reason `rewriteAndSwap` takes // them: the caller's view answers `size()` from the cached watermark its key // was derived from, so asking it could never disagree. if (!logstore.LogStore.validateTail(fs.storageFile(), src_key)) return error.TailChanged; // The header of the existing repo carries the hash kind; `open` reads it from // the non-empty file, so the passed value is a don't-care (0). var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer log.deinit(gpa); for (rows) |row| try log.putShun(row.hash, row.reason, row.epoch); try log.commit(); } /// `zit shun` — record one or more artifacts in this repository's shun table and, /// unless `--defer`, remove their bytes in the same operation. /// /// POLICY, decided here and not by any draft: refuse by default and name the /// consequence; require `--force` for BOTH paths, because `--defer` also appends /// an irreversible tombstone and the read mask hides the content the instant it /// lands; never prompt; refuse the current checkout's own check-in with no /// override, because that is the one target whose removal breaks the tree the /// operator is standing in and `zit update` is a cheap way out. /// /// CRASH SAFETY. `--force` performs exactly ONE durable transition. When there is /// content to remove that is the rename inside `rewriteAndSwap`, and the new /// tombstones are written into the NEW generation (`packLog`'s `extra_shuns`), /// never appended to `.zit` first — two durable transitions would expose a state /// where the tombstone exists and the content still does. It also means an aborted /// run leaves no tombstone at all: the rows were in the temp that the abort /// deletes. Pinned by "runShun: a --force run the temp lock turns away records NO /// tombstone". /// /// When NOTHING is held — every named hash is pre-emptive — the single durable /// transition is instead the append plus commit, and the rewrite is skipped /// altogether. It is not load-bearing there: the rule exists to stop a tombstone /// outliving its content, and with no content to remove the append is already /// crash-safe under the log's torn-tail recovery. Running a whole-store rewrite to /// reclaim nothing would cost O(store) plus transient room for both generations and /// end with a report whose byte count had GROWN. /// /// The rename is visible to a running `zit serve` DESPITE its held handle: /// serve's `baseSnapshot` stats the path beside the handle each round and /// reopens on an inode change, so the round after this swap serves the /// rewritten generation — shunned content gone — with no restart. pub fn runShun( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseShunArgs(arena, verb_args) catch return stateOom(streams, "shun")) { .ok => |a| a, .unknown_option => |tok| return cli.usageError(streams, "shun", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "shun", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(shun_help) catch {}; return .ok; } if (!parsed.list and parsed.hashes.len == 0) { return cli.usageError(streams, "shun", .no_hash, ""); } // A reason the durable row cannot hold is refused BEFORE anything is opened, // and refused rather than truncated: the note on a shun is often the legal // reference the operator will be asked for, and silently keeping a prefix of // it is worse than making them shorten it themselves. // // A CONTROL BYTE IS REFUSED THE SAME WAY, and this is the reason `--reason` is // validated at all rather than merely bounded. `--list` is the only view of a // durable, irreversible, un-removable record, and `renderShunList` prints one // row per line: a newline inside a reason forges a row there that is // indistinguishable from a real one. Reasons arrive from scripts and from pasted // takedown text as often as they are typed, so this is not a hand-typed-only // hazard. Rejecting at input rather than escaping at render keeps the fix to one // place and keeps the stored bytes exactly what the operator supplied. NO DRAFT // SPEAKS TO THIS and `logstore.ShunError` has no arm for it — a local decision, // taken the same way the length gate was: refuse rather than mangle. if (parsed.reason) |r| { if (r.len > logstore.max_shun_reason) { streams.err.print( "zit shun: --reason is {d} bytes; the recorded note is at most {d}\n", .{ r.len, logstore.max_shun_reason }, ) catch {}; return .failure; } for (r) |c| { if (c >= 0x20 and c != 0x7f) continue; streams.err.writeAll( "zit shun: --reason contains a control byte; the recorded note must be a single line of text\n", ) catch {}; return .failure; } } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "shun")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // shun is store-scoped end to end: the reads, the pack-tmp lock // and the final rename must all land beside the STORE, so a detached // worktree resolves its pointer once here and the whole flow inherits it. if (cli.storeDirOf(io, root_dir) catch return repoReadFail(streams, "shun")) |resolved| { root_dir.close(io); root_dir = resolved; } // Open the source read-only, exactly as `pack` does, and for the same // reasons — including the same caveat: this open no longer CRC-verifies // payloads (it scans header-only), so a corrupt source aborts during the // rewrite's own reads rather than here. Nothing is written either way. The // tail key pins the log for the pre-rename re-check. See `runPack`'s doc // comment for why that ordering matters to a verb that renames over `.zit`. var src_file = root_dir.openFile(io, repo_filename, .{}) catch return repoReadFail(streams, "shun"); defer src_file.close(io); const src_st = src_file.stat(io) catch return repoReadFail(streams, "shun"); var src_fs = storage.FileStorage.init(io, src_file, src_st.size); var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "shun"), else => |re| return shunFail(streams, re), }; defer src.deinit(gpa); if (parsed.list) { renderShunList(streams.out, src.shunList()); return .ok; } const src_key = src.logKey() catch return shunFail(streams, error.Corrupt); // The current checkout's own check-in, for the one refusal with no override. // A `.zit-checkout` we cannot parse is a hard failure, not an empty answer: // the guard would silently become vacuous and the operator would be allowed // to delete the check-in they are standing on. var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined; const ck_state = checkout.read(io, root_dir, &ck_buf) catch |e| return main.checkoutFail(streams, "shun", e); const ck_version: []const u8 = if (ck_state) |s| s.version else ""; // Classified, then DEDUPLICATED BY RESOLVED DIGEST — which is why it cannot be // done in the parser. `zit shun --force H <prefix-of-H>` is one artifact named // twice, and an operator pasting a takedown list beside an abbreviation from // `zit log` cannot see that it is. Left folded in, the duplicate builds two // identical rows, `putShun` collapses them, and `packLog`'s row-count gate — // whose documented precondition is that `extra_shuns` rows are DISTINCT — fires // `error.Corrupt`: the verb would tell the operator their repository is corrupt, // on the one occasion they can least afford to doubt it. Deduping here rather // than in `packLog` keeps that gate a gate. O(n^2) over an argument list. const target_buf = arena.alloc(ShunTarget, parsed.hashes.len) catch return stateOom(streams, "shun"); var target_count: usize = 0; tokens: for (parsed.hashes) |tok| { const t = classifyShunTarget(gpa, arena, &src, ck_version, tok) catch |e| return shunFail(streams, e); // Only a resolved target can be a duplicate: the unresolvable dispositions // carry the operator's own token and no digest, and two different bad tokens // both deserve their own diagnostic line. if (t.digest.len != 0) { for (target_buf[0..target_count]) |prev| { if (std.mem.eql(u8, prev.digest, t.digest)) continue :tokens; } } target_buf[target_count] = t; target_count += 1; } const targets = target_buf[0..target_count]; // THE GUARD RUNS BEFORE ANY DURABLE WRITE — the ordering is the point, not an // implementation detail. A refused run must leave the store byte-identical, // which it cannot do if a tombstone was recorded while deciding. for (targets) |t| { if (t.disposition.blocks()) { renderShunBlockers(streams.err, targets); return .failure; } } // Two counts, and they are independent: `absent` records without removing, // `reclaim` removes without recording. Either one alone is work to do. var recorded: usize = 0; var reclaiming: usize = 0; for (targets) |t| { if (t.disposition.records()) recorded += 1; if (t.disposition.reclaims()) reclaiming += 1; } if (recorded == 0 and reclaiming == 0) { // Every named hash was already in the set AND its content already gone: a // genuine no-op, so it needs no `--force` and it is not a failure. Reaching // this on a hash whose bytes SURVIVED is the defect `.reclaim` exists to // prevent — it would report "nothing to do" over content the operator had // just been ordered to remove. streams.out.print( "shunned: nothing to do; {d} hash{s} already shunned\n", .{ targets.len, pluralEs(targets.len) }, ) catch {}; return .ok; } if (!parsed.force) { renderShunRefusal(streams.err, targets, deferredStillHeld(&src, targets), parsed.defer_reclaim); return .failure; } // ONE owned name set drives both the drop predicate and the tombstone list, // so a name cannot be recorded without also being dropped. It is the UNION of // the source's already-deferred shuns and the hashes named now: `shunFilter` // covers only the first half, and half a drop set would relocate into the new // generation exactly the bytes this run reports as removed. var set: NameSet = .{}; defer set.deinit(gpa); for (src.shunList()) |row| set.add(gpa, row.hash) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "shun"), // A persisted row whose key is not a complete digest is a corrupt store, // and that is the one place `BadKey` here can come from. error.BadKey => return shunFail(streams, error.Corrupt), }; const row_buf = arena.alloc(logstore.ShunRecord, recorded) catch return stateOom(streams, "shun"); var n: usize = 0; for (targets) |t| { if (!t.disposition.records()) continue; set.add(gpa, t.digest) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "shun"), error.BadKey => return shunRowBug(streams), }; row_buf[n] = .{ .hash = t.digest, .epoch = amb.now_epoch, .reason = parsed.reason orelse "" }; n += 1; } // SLICED, not asserted-and-passed-whole. `recorded` and `n` count the same pure // predicate over the same unmutated `targets`, so they cannot differ today — but // `row_buf` is arena memory and the FULL buffer used to be what went to // `packLog` and `shunAppendRows`. Had they ever differed, `row_buf[n..]` would // be undefined `ShunRecord`s whose `hash` is a garbage slice, handed to // `putShun` on the way to a durable irreversible write — in `ReleaseFast`, where // the assert below does not exist. Taking the slice makes the mismatch // unrepresentable and leaves the assert as documentation of the tautology. const rows = row_buf[0..n]; std.debug.assert(rows.len == recorded); // Nothing to reclaim: take the append path even without `--defer`. See the // crash-safety note in the doc above — the rewrite is not what makes this safe, // and running it would cost a whole-store copy to remove nothing. if (reclaiming == 0) { shunAppendRows(gpa, io, root_dir, src_key, rows) catch |e| return shunAppendFail(streams, e); renderShunRecordedOnly(streams.out, rows.len); return .ok; } if (parsed.defer_reclaim) { // `rows` is empty when every target was already tombstoned and only its // bytes remain; skip the append rather than fsync for nothing. if (rows.len != 0) { shunAppendRows(gpa, io, root_dir, src_key, rows) catch |e| return shunAppendFail(streams, e); } renderShunDeferred(streams.out, rows.len, reclaiming); return .ok; } // THE CURRENT-CHECKOUT STOP, RE-ASKED OF THE DROP SET ITSELF — one hashmap // lookup, at the point of consequence. `classifyShunTarget` enforces it over the // NAMED TOKENS, but `set` is the union of those with every row already in the // shun table, and the rewrite deletes the union. A table that already contained // this check-in's manifest would therefore have `--force` delete the check-in the // operator is standing in while the per-token guard never fired — the refusal // line would count it only as an anonymous integer of earlier deferred // collateral. That state is not constructible through zit today (`--defer` goes // through the same per-token guard, and S1's mask makes `zit update` to a shunned // check-in fail), which is exactly why the invariant belongs here as well: the // property `shun_help` promises is about what gets deleted, not about the // argument list. if (ck_version.len != 0) { var ck_raw: [logstore.max_hash_len]u8 = undefined; if (shunHexToDigest(ck_version, &ck_raw)) |ck_digest| { if (set.contains(ck_digest)) { renderShunBlockers(streams.err, &.{.{ .name = ck_version, .digest = ck_digest, .disposition = .current_checkout, }}); return .failure; } } } const stats = rewriteAndSwap( gpa, io, root_dir, &src, src_key, set.filter(), rows, ) catch |e| switch (e) { error.TailChanged => return tailChangedFail(streams, "shun"), error.RewriteBusy => return rewriteBusyFail(streams, "shun"), else => |pe| return shunFail(streams, pe), }; // Reported from the PACK's observation of the new generation, not from // `rows.len`: see `renderShunStats`. They agree or `packLog`'s row-count gate // failed the pack, which is why this is an assert rather than a branch. std.debug.assert(stats.shuns_recorded == rows.len); renderShunStats(streams.out, stats); return .ok; } /// Resolve one operator token against `src` into a `ShunTarget`. Impure only in /// that it reads the store: no policy, no output, no durable write, so the /// runner's whole decision is a fold over the results. /// /// RESOLUTION RULE. Every token is offered to prefix resolution FIRST, at every /// width in `canonicalizeShunHex`'s band — the same band and the same resolver /// `zit artifact` uses (cmd/history.zig), which is what makes "paste the same /// token into both verbs" true rather than merely intended. Only when nothing /// matches does a COMPLETE name (40 or 64 hex — artifact-format §3.2's two forms) /// fall back to being used literally, because a hash this repository does not hold /// is precisely the pre-emptive case worth recording; an ABBREVIATED prefix that /// matches nothing is refused, since the shun table is keyed by complete name /// (sync §3.3 L518-519), a prefix cannot be completed, and a row keyed on one would /// read as a removal while naming nothing. /// /// TRYING RESOLUTION FIRST IS THE LOAD-BEARING PART, not an optimisation. Treating /// a complete-WIDTH token as a complete NAME without asking the store means a /// 40-hex abbreviation of a 64-hex artifact (and a 64-hex abbreviation in a sha1 /// repository) records a tombstone naming nothing, reports success, and leaves the /// content the operator was ordered to remove still readable — the `unresolved` /// refusal exists to stop exactly that, and a width-first rule walks past it. /// It also un-hides the current-checkout guard, which cannot fire on a token that /// never resolved to the check-in's name. fn classifyShunTarget( gpa: std.mem.Allocator, arena: std.mem.Allocator, src: *logstore.LogStore, ck_version: []const u8, tok: []const u8, ) (logstore.ContentError || error{OutOfMemory})!ShunTarget { var canon_buf: [2 * logstore.max_hash_len]u8 = undefined; const canon = canonicalizeShunHex(tok, &canon_buf) orelse return .{ .name = tok, .disposition = .not_a_name }; var raw_buf: [logstore.max_hash_len]u8 = undefined; var digest: []const u8 = undefined; if (src.resolveHexPrefix(canon) catch |e| switch (e) { error.Ambiguous => return .{ .name = tok, .disposition = .ambiguous }, }) |rid| { digest = src.hashOf(rid); } else if (shunHexToDigest(canon, &raw_buf)) |d| { digest = d; } else { return .{ .name = tok, .disposition = .unresolved }; } std.debug.assert(logstore.isCompleteDigest(digest.len)); const owned_digest = try arena.dupe(u8, digest); var hex_buf: [2 * digest_len]u8 = undefined; const name = try arena.dupe(u8, digestToWireName(owned_digest, &hex_buf)); // THE CURRENT-CHECKOUT STOP IS CHECKED FIRST, ahead of the shun table. An // already-tombstoned check-in whose bytes survive is otherwise `.reclaim`, and // reclaiming it would delete the check-in the operator is standing on — the one // thing this verb refuses outright. `--defer` goes through this same guard, so // the state cannot arise through zit today; the ordering is what keeps that // true if it ever does. if (std.mem.eql(u8, name, ck_version)) return .{ .name = name, .digest = owned_digest, .disposition = .current_checkout, }; const already = src.isShunned(owned_digest); const rid = src.lookup(owned_digest) orelse return .{ .name = name, .digest = owned_digest, .disposition = if (already) .already else .absent, }; // THROUGH THE MAINTENANCE READER, deliberately. `content` applies the shun // mask, so an already-shunned artifact whose bytes are still on disk would read // as absent and be reported as nothing left to do — which is the whole defect // `.reclaim` exists to prevent. Shunning is maintenance, not an artifact read. // // A phantom is an announced name with no frame: there are no bytes to reclaim, // so it is the `absent` case with a rid attached. const content = (try src.contentUnmasked(gpa, rid, null)) orelse return .{ .name = name, .digest = owned_digest, .disposition = if (already) .already else .absent, }; defer gpa.free(content); // The type verdict, through the same door `zit artifact` uses: a structural // artifact classifies (`parseWindow` runs §5's `classify` internally), and // anything that does not parse is an opaque blob — a first-class answer, not // an error. A blob is by far the common shun target. const held: ShunDisposition = if (already) .reclaim else .remove; var m = artifact.parseWindow(gpa, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return .{ .name = name, .digest = owned_digest, .kind = null, .disposition = held }, }; defer m.deinit(); return .{ .name = name, .digest = owned_digest, .kind = m.kind, .disposition = held }; } /// Maps a transport-open failure (bad URL) to the `verb` diagnostic + exit 1. /// Shared by clone and pull — the only difference is the verb label. fn syncOpenFail(s: cli.Streams, verb: []const u8, url: []const u8, err: transport.TransportError) cli.Exit { switch (err) { error.InvalidUrl => s.err.print("zit {s}: '{s}' is not a valid URL\n", .{ verb, url }) catch {}, error.UnsupportedScheme => s.err.print("zit {s}: unsupported URL scheme in '{s}' (use http or https)\n", .{ verb, url }) catch {}, error.OutOfMemory => s.err.print("zit {s}: out of memory\n", .{verb}) catch {}, else => s.err.print("zit {s}: cannot open '{s}'\n", .{ verb, url }) catch {}, } return .failure; } /// Maps a sync run (transport + driver) failure to the `verb` diagnostic + exit /// 1. Exhaustive over the driver's own causes and the transport's, so the whole /// closed error set is accounted for (no `else`). Shared by clone and pull. /// `status` is the transport's `last_status` -- the HTTP status that produced an /// `error.ProtocolError`, or null for every other cause. Named in the message /// because "an unexpected HTTP status" sends an investigation nowhere: 500, 413, /// 400 and 417 are four different bugs and the operator could not tell which one /// they had hit. /// Tolerate-and-report — one stderr line naming what the session /// declined: unversioned files a fossil peer offered that zit has no store /// for. Zero is silent; the count makes the data non-transfer VISIBLE, which /// is the entire posture (the alternative was an abort nobody chose). fn reportUvSkipped(s: cli.Streams, verb: []const u8, count: u64) void { if (count == 0) return; s.err.print( "zit {s}: {d} unversioned file(s) not transferred (unsupported)\n", .{ verb, count }, ) catch {}; } /// What the transport did about an HTTP Basic challenge, for the 401 sentence. const BasicState = enum { none, challenged, sent }; fn basicStateOf(tr: *const transport.Transport) BasicState { if (tr.basic_armed) return .sent; if (tr.basic_challenged) return .challenged; return .none; } fn syncRunFail(s: cli.Streams, verb: []const u8, url: []const u8, err: sync_driver.RunError(*transport.Transport), status: ?std.http.Status, basic: BasicState) cli.Exit { const reason: []const u8 = switch (err) { error.OutOfMemory => "out of memory", // Not "did not converge": the guard counts rounds that moved no bytes, // so this fires only when the peer kept the exchange open while // delivering nothing. A slow or very large transfer never reaches it. error.RoundLimitExceeded => "the peer kept the exchange open without transferring anything", error.FatalProtocolError => "the peer sent a fatal protocol error", error.RequestTooLarge => "the sync request exceeded the transport limit", error.ReplyTruncated, error.ReplyMalformed => "the peer sent a malformed reply", // The streaming drain's own faults, surfaced through the // driver: writing the destination log failed, or a received artifact // did not verify against its name. error.Storage => "cannot write the cloned repository", error.Corrupt => "a received artifact failed verification", error.ReplyTooLarge => "the peer's reply exceeded the size limit", error.RedirectLimit => "too many HTTP redirects", error.InsecureRedirect => "the peer redirected HTTPS to plaintext HTTP", error.RedirectNotAllowed => "the peer redirected on a transport that forbids it", error.MissingLocation => "the peer sent a redirect with no location", error.InvalidUrl => "a redirect target was not a valid URL", error.UnsupportedScheme => "a redirect named an unsupported URL scheme", error.ProtocolError => blk: { if (status) |st| { if (st == .unauthorized) { // The one status that means "ask again, properly". Which // half is missing decides the sentence. if (basic == .sent) { s.err.print("zit {s}: the peer refused the login (HTTP 401)\n", .{verb}) catch {}; } else if (basic == .challenged) { s.err.print("zit {s}: the peer wants a login (HTTP 401): give --user and --password, or set ZIT_PASSWORD\n", .{verb}) catch {}; } else { s.err.print("zit {s}: the peer answered HTTP 401 (Unauthorized) without saying how to log in\n", .{verb}) catch {}; } return .failure; } s.err.print("zit {s}: the peer answered HTTP {d} ({s})\n", .{ verb, @intFromEnum(st), st.phrase() orelse "unrecognized" }) catch {}; return .failure; } break :blk "the peer answered with an unexpected HTTP status"; }, error.ConnectionFailed => { s.err.print("zit {s}: cannot connect to {s}\n", .{ verb, url }) catch {}; return .failure; }, }; s.err.print("zit {s}: {s}\n", .{ verb, reason }) catch {}; return .failure; } /// The causes `resolveRemoteUrl` can surface: a store read failure, OOM, or the /// repository having no URL to use (`NoRemote` — neither an explicit arg nor a /// saved remote). Closed set so the caller switches exhaustively. const RemoteResolveError = error{ OutOfMemory, Storage, NoRemote }; /// sync §4/§6 — resolve the peer URL for a sync-family verb: an explicit `arg` /// (non-empty) wins; otherwise the saved remote (`config.remote_url`) is used; /// a repository with neither is a usage error (`NoRemote`). The saved URL is /// copied into `buf` so it outlives the read-only config frame. `buf` need only /// live until `transport.Transport.open` (which dupes the URL). An out-of-spec /// stored URL longer than the config-field cap is treated as none. fn resolveRemoteUrl( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, arg: []const u8, buf: *[logstore.max_config_str]u8, ) RemoteResolveError![]const u8 { if (arg.len > 0) return arg; var file = cli.openStoreRead(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.NoRemote; defer oc.deinit(gpa); const saved = oc.config.remote_url; if (saved.len == 0 or saved.len > buf.len) return error.NoRemote; @memcpy(buf[0..saved.len], saved); return buf[0..saved.len]; } /// sync §4/§6 — the usage error (exit 2) when a sync-family `verb` is given no /// URL and the repository has no saved remote to fall back to. Names the fix. /// Shared by pull, push, and sync so the wording lives in one place. fn noRemoteError(s: cli.Streams, verb: []const u8) cli.Exit { s.err.print("zit {s}: no repository URL given and no saved remote\n", .{verb}) catch {}; s.err.writeAll("set one with 'zit remote URL', or pass the URL explicitly\n") catch {}; return .usage; } // --------------------------------------------------------------------------- // `zit pull URL` — incremental sync into an EXISTING repository. The sibling of // clone: it reuses the same transport + `runSync` + persist machinery, but opens // the cwd's `.zit` (never creating one), announces what it already holds so the // peer sends only the missing artifacts (sync §3.2/§3.3 igot/gimme), and drives // to §3.7 termination. Receive-only (§3.3 r24: pull obtains but never uploads). // --------------------------------------------------------------------------- /// `zit pull URL` — open the current repository, converge it toward the peer at /// URL, and persist the newly received artifacts. Thin wiring over the same deep /// modules clone uses: `transport` (HTTP/IO), `sync.Client` + `runSync` (sans-IO /// §3.7 loop), and `LogStore` (persistence). With URL omitted the saved remote /// (`zit remote`) is used; a repo with neither is a usage error. Anonymous only. pub fn runPull( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parsePullArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "pull", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "pull", .extra_argument, tok), // An omitted URL is not an error: it falls back to the saved remote below. .no_url => cli.PullArgs{ .url = "" }, }; if (parsed.help) { streams.out.writeAll(pull_help) catch {}; return .ok; } if (parsed.user) |u| { if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) { streams.err.writeAll("zit pull: --user must not contain whitespace\n") catch {}; return .usage; } } // Locate the current repository before any network I/O: a pull outside a repo // is a usage failure, not a wasted connection. const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "pull")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), error.NoRemote => return noRemoteError(streams, "pull"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "pull", url, e); defer tr.close(); // The client's holdings view: igot names, held/phantom, shun mask, and // hash-policy off the log's frame index, with content lazy (the // old whole-store `MemStore` snapshot held the repository's plaintext // resident for the run). var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), }; defer holdings.deinit(io); var admission = holdings.admission(); // Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once // and suppress spontaneous requests for elided phantoms. Both the // classification and the wrapper outlive the run below. var sync_cone: ?zit.cone.Cone = null; defer if (sync_cone) |*c| c.deinit(gpa); var sync_classification: ?SparseClassification = null; defer if (sync_classification) |*c| c.deinit(gpa); var sync_elided: RepoElidedSet = undefined; if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| { sync_cone = c; sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch return stateOom(streams, "pull"); sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() }; admission.elided = sync_elided.shunSet(); admission.cone_members = sync_cone.?.members; admission.cone_not_members = sync_cone.?.not_members; } // §9.1 — announce this repository's adopted project code in the pull card; // a real peer refuses a PROJECTCODE mismatch, so the placeholder cannot be // sent against fossil. The buffer outlives the run (the client borrows it). // The cluster index: what the held clusters already announce // (never announced again individually) and what they promised that this // repository never collected (seeded as phantoms, so the run asks for it). // The clusters' plaintext is memoised by the holdings adapter while it // resolves; drop it — the index keeps only names and bits. var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "pull"); defer clusters.deinit(); holdings.index.releaseTransient(); var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), }; // sync §6 — a read of a private repository is as gated as a write. The // login comes from --user, the URL's user@, or the login a previous // authenticated push or pull stored here; with none the pull is anonymous. var pw_buf: [max_password_len]u8 = undefined; defer std.crypto.secureZero(u8, &pw_buf); var user_buf: [logstore.max_config_str]u8 = undefined; const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf, &user_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), error.NoPassword => { streams.err.print("zit pull: no password for '{s}' (give --password, set ZIT_PASSWORD, or run on a terminal)\n", .{parsed.user orelse urlUser(url) orelse ""}) catch {}; return .failure; }, }; // Live per-round progress on a terminal, the same sink clone uses. const stderr_tty = std.Io.File.stderr().isTty(io) catch false; var progress: CloneProgress = .{ .err = streams.err }; const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null; defer progress.finishLine(); if (resolved) |r| armBasicAuth(&tr, r.credential.user, r.plaintext) catch return stateOom(streams, "pull"); const outcome = pullRun(gpa, &tr, holdings.store(), identity, if (resolved) |r| r.credential else null, admission, sink, &clusters) catch |e| return syncRunFail(streams, "pull", url, e, tr.last_status, basicStateOf(&tr)); // Persist a freshly derived secret only after the peer accepted the // authenticated session (mirrors push and sync). if (resolved) |r| { if (r.persist) { storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), else => return repoReadFail(streams, "pull"), }; } } // Sparse-cone RFC section 6.3 applies to EVERY receive: a dense peer // volunteers content the client never announced, so the filter runs at // persist. Reclassified over the post-run store, so a brand-new commit's // blobs classify by their own F cards -- new in-cone content persists, // new out-of-cone content elides with the rest. var post_classification: ?SparseClassification = null; defer if (post_classification) |*c| c.deinit(gpa); if (sync_cone) |*c| { post_classification = classifyKeepRids(gpa, holdings.store(), c) catch return stateOom(streams, "pull"); } // Persist only what this pull received. The holdings store appends // received artifacts beyond its log-backed prefix, so the walk begins at that // boundary and never re-visits the already-durable holdings. A pull that // received nothing persists nothing: a no-op pull was O(store) here (every // holding read once from disk to be early-returned), 196 s on a 905k store. const persisted = persistStore(gpa, io, root_dir, holdings.store(), if (post_classification) |*c| &c.keep else null, holdings.logBackedCount()) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), error.Corrupt => return renderStorage(streams, root.abs, "the peer sent a malformed artifact name"), }; // The walk started past the holdings, so `written` is exactly the received // count (each received artifact is persisted once). const received = persisted.written; // The peer named artifacts this run never obtained. Everything is persisted // — a partial pull is still progress — but the exit status says so, because // "exit 0 with nothing said" is indistinguishable from a converged sync and // leaves a repository quietly out of date. This holds whether `received` is // zero or not: the invariant is convergence, not motion. // A promise a held cluster made that this peer could not keep is // reported, not failed on: the run asked on its own initiative (no peer // igot set the expectation) and the repository IS converged against this // peer. Fossil asks for these every round, silently; zit says so once. if (outcome.cluster_gaps > 0) { streams.err.print( "zit pull: {d} artifact{s} named by held clusters could not be obtained from this peer\n", .{ outcome.cluster_gaps, plural(outcome.cluster_gaps) }, ) catch {}; } if (outcome.unresolved > 0) { streams.err.print( "zit pull: the peer holds {d} artifacts this pull did not obtain\n", .{outcome.unresolved}, ) catch {}; return .failure; } // submanifests §7.1/§7.3 — the same reasoning one level down. A pull can // converge against the peer (nothing left to gimme) while the tree closure // this repository now holds is still open, because the peer's own closure // was open: §7.1's "at most d+1 pull rounds" bounds a tree whose levels are // ARRIVING, not one whose leaves nobody has. The phantoms the persist hook // minted are that distinction, so report them rather than call it converged. // // §7.3/§9.4 — two counts, because they answer different questions and the // draft names both: `submanifest_gaps` is the durable store's missing-hash // count, `deferred_trees` is the session's DEPENDENT-keyed record ("enough // to rediscover that manifest"). A run reporting the first without the // second tells the operator something is missing but not what it breaks. if (persisted.submanifest_gaps > 0 or outcome.deferred_trees > 0) { // Two sentences, not one, because the two counts have different // LIFETIMES and a single sentence reads as one fact. `submanifest_gaps` // is a property of the durable store and survives a restart; // `deferred_trees` is this SESSION's §9.4 record and is zero for a run // that received no S card — including every re-run against an already // open closure. Printed together they said "N references are missing; 0 // check-ins cannot be enumerated", which is exactly backwards. if (persisted.submanifest_gaps > 0) { streams.err.print( "zit pull: {d} sub-manifest references are missing; the check-ins that reference them cannot be enumerated\n", .{persisted.submanifest_gaps}, ) catch {}; } if (outcome.deferred_trees > 0) { streams.err.print( "zit pull: {d} check-ins received this session are deferred until they arrive\n", .{outcome.deferred_trees}, ) catch {}; } return .failure; } if (received == 0) { streams.out.print("pull from {s}: already in sync\n", .{url}) catch {}; } else { streams.out.print("pull from {s}: received {d} artifact{s} in {d} round{s}\n", .{ url, received, plural(received), outcome.rounds, plural(outcome.rounds) }) catch {}; reportUvSkipped(streams, "pull", outcome.uv_skipped); } return .ok; } /// Drives a pull client (§3.2/§3.3) to convergence over `tr`, filling `ms` with /// the artifacts the peer holds and this side lacked; returns the round count. /// The client announces `ms`'s holdings via igot and requests the rest via /// gimme (receive-only — §3.3 r24: pull obtains content but never uploads). /// Sans-IO on this side of `tr.exchange`. Its closed error set is the driver's /// own causes unioned with the transport's, so `runPull` switches it exhaustively. fn pullRun( gpa: std.mem.Allocator, tr: *transport.Transport, holdings: repostate.ArtifactStore, identity: RepoSyncIdentity, credential: ?zit.sync.Credential, admission: RepoAdmission, sink: ?sync_driver.RoundSink, clusters: ?*const clusterindex.Resolved, ) sync_driver.RunError(*transport.Transport)!PullOutcome { var client = syncClient(.pull, &identity, credential, admission); defer client.deinit(gpa); if (clusters) |c| { client.clustered = &c.clustered; try client.seedClusterPhantoms(gpa, holdings, c.missing); } const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null); recordSyncStats(stats); return .{ .rounds = stats.rounds, .unresolved = client.unresolvedPhantoms(holdings), .cluster_gaps = client.clusterGaps(holdings), // submanifests §7.3/§9.4 — the DEPENDENT-keyed half of the verdict: how // many received check-ins still have a recorded S-reference gap. .deferred_trees = client.deferredTrees(), .uv_skipped = client.uv_skipped, }; } /// What one pull run reports: the round count, and how many artifacts the peer /// announced (§3.3 r24 — by igot, or as a member of a cluster it sent) that this /// side still lacks when §3.7 reached its clean terminal. A nonzero `unresolved` /// is a pull that stopped short of what it was told existed; the caller must not /// report success for it, however well-formed every card on the wire was. /// submanifests §7.3/§9.4 — `deferred_trees` is the §9.4 record read back: the /// count of DEPENDENTS (check-ins) whose processing this session deferred on a /// sub-manifest that never arrived. `unresolved` counts missing HASHES; a /// dependent-keyed count is what §9.4 requires be recorded and is what an /// operator can act on ("which check-in is broken", not "how many blobs"). const PullOutcome = struct { rounds: usize, unresolved: usize, deferred_trees: usize = 0, uv_skipped: u64 = 0, cluster_gaps: usize = 0 }; /// sync §3.3 L527-530 / §8.8 L1777-1779 — this repository's shun table as the /// `zit.sync.ShunSet` the phantom scan asks. Owns a copy of every shunned /// DIGEST, taken while the caller already has the log open, so the set outlives /// the handle it was read through. /// /// FRESHNESS, stated exactly. This is a SNAPSHOT, not a live view: `load` reads /// `shunList` once and the set answers from that copy thereafter. On the SERVE /// path that is per-round freshness, because `LogIndexStore` rebuilds — and so /// re-loads this — whenever the stat gate sees the log grow, which a `zit shun` /// append does; a shun written between two served rounds is honoured by the next /// one. On the CLIENT path it is a PROCESS snapshot: `ClientHoldings` builds /// its index once at run start and never rebuilds mid-run, so a shun applied /// mid-run lands on the next process, not the next round. That is a /// deliberate limit, recorded here rather than implied away — the protocol layer /// re-probes every scan (`sync.ShunSet`), so nothing else has to change if this /// ever grows a re-read. /// /// This is the ONE place a wire NAME becomes a log DIGEST for shun purposes — /// the boundary `LogIndexStore.names` already declares is held to one place. The /// width/hex validation is a CHECKED rejection rather than an assert, because an /// assert is nothing in ReleaseFast: `logstore.LogStore.isShunned`'s caller /// contract (see its doc) binds this boundary to reject a non-complete name /// BEFORE decoding it, so the width is gated on the NAME, ahead of the decode. /// The sync-facing elided-set: answers RFC section 5.2 rule 3 over a /// classification computed once at run start. Same total-function contract /// as `RepoShunSet`; borrows both the classification and the store, so both /// must outlive the run. /// `pack --drop-elided`'s filter half (sparse-cone RFC section 5.4): built /// once over the SOURCE log, drops exactly the classification's elide set. /// A digest that does not resolve in the index is kept -- unknown provenance /// is never deleted. const ElidedDropper = struct { index: *LogIndexStore, classification: SparseClassification, inner: zit.pack.Filter, fn build(gpa: std.mem.Allocator, src: storage.StorageFile, cone: *const zit.cone.Cone) ?ElidedDropper { const index = LogIndexStore.open(gpa, src) catch return null; const classification = classifyKeepRids(gpa, index.store(), cone) catch { index.deinit(); return null; }; return .{ .index = index, .classification = classification, .inner = zit.pack.Filter.keep_all }; } fn drop(ctx: *anyopaque, digest: []const u8) bool { const self: *ElidedDropper = @ptrCast(@alignCast(ctx)); if (self.inner.dropFn(self.inner.ctx, digest)) return true; // shun reclaim first var name_buf: [2 * logstore.max_hash_len]u8 = undefined; const name = digestToWireName(digest[0..digest.len], &name_buf); const rid = self.index.store().lookup(name) orelse return false; return self.classification.elided(rid); } fn composed(self: *ElidedDropper, inner: zit.pack.Filter) zit.pack.Filter { self.inner = inner; return .{ .ctx = @ptrCast(self), .dropFn = drop }; } fn deinit(self: *ElidedDropper, gpa: std.mem.Allocator) void { self.classification.deinit(gpa); self.index.destroy(); self.* = undefined; } }; const RepoElidedSet = struct { classification: *const SparseClassification, store: repostate.ArtifactStore, fn contains(context: *anyopaque, name: []const u8) bool { const self: *const RepoElidedSet = @ptrCast(@alignCast(context)); const rid = self.store.lookup(name) orelse return false; return self.classification.elided(rid); } fn shunSet(self: *RepoElidedSet) zit.sync.ShunSet { return .{ .context = @ptrCast(self), .vtable = &.{ .contains = contains } }; } }; const RepoShunSet = struct { gpa: std.mem.Allocator, /// Keys are owned binary digests; the value side carries nothing — this is a /// set. Empty is the common case (no repository has a shun row until an /// operator writes one), and an empty map allocates nothing. digests: std.StringHashMapUnmanaged(void) = .empty, fn init(gpa: std.mem.Allocator) RepoShunSet { return .{ .gpa = gpa }; } /// Copy every shun row's digest out of `log`. `shunList` borrows the store's /// own bytes, which die with the handle they were read through, so each key /// is duped. Re-loading over a populated set replaces it wholesale, which is /// what a `LogIndexStore` rebuild wants. fn load(self: *RepoShunSet, log: *const logstore.LogStore) error{OutOfMemory}!void { self.clear(); const rows = log.shunList(); // State the bound where it is relied on rather than narrowing usize into // the map's u32 capacity argument by an unchecked cast: a truncated // reservation would let `getOrPutAssumeCapacity` below write past the // capacity it assumes, which is silent corruption in ReleaseFast instead // of the `error.OutOfMemory` a shun table that large has already earned. // This is the one such count an operator grows by direct action. const row_count = std.math.cast(u32, rows.len) orelse return error.OutOfMemory; try self.digests.ensureTotalCapacity(self.gpa, row_count); for (rows) |row| { // Dupe BEFORE inserting. `row.hash` belongs to the store and dies // with the handle, so the key must be owned — and inserting the // borrowed slice first and replacing it after leaves a BORROWED key // in an owned-key map on the one error this function can return. // Every unwind path then runs `clear`, which frees the store's own // live digest buffer: a double free at `log.deinit`, and a // use-after-free of `shun_entries[i].hash` while the store is open. const owned = try self.gpa.dupe(u8, row.hash); // Capacity is already reserved for every row, so this cannot fail. const gop = self.digests.getOrPutAssumeCapacity(owned); // Two rows naming the same digest collapse to one entry, and the // second dupe is surplus — say so rather than leaking it. if (gop.found_existing) self.gpa.free(owned) else gop.key_ptr.* = owned; } } fn clear(self: *RepoShunSet) void { var it = self.digests.keyIterator(); while (it.next()) |k| self.gpa.free(k.*); self.digests.clearAndFree(self.gpa); } fn deinit(self: *RepoShunSet) void { self.clear(); } fn shunSet(self: *RepoShunSet) zit.sync.ShunSet { return .{ .context = self, .vtable = &vtable }; } const vtable: zit.sync.ShunSet.VTable = .{ .contains = containsWireName }; /// Exact-key lookup on the digest a COMPLETE wire name decodes to. A name /// whose WIDTH is not one of artifact-format §3.2 L347-350's two forms is /// rejected before the decode, which is the order `isShunned`'s caller /// contract fixes; a correct-width token that is not hex fails the decode. /// Neither can be a key any shun row carries (`putShun` rejects a /// non-complete key with `error.BadKey`, and the open-time scan fails /// `error.Corrupt` on one), so `false` is the exact answer. Never a prefix /// match, never a truncate, never a pad: a prefix must not be able to /// masquerade as a shun-table hit. /// /// An UPPERCASE-hex spelling of a shunned name decodes to the same digest /// and therefore answers TRUE. artifact-format §3.3 L370-377 makes such a /// token invalid but explicitly lets an implementation accept it at name /// validation and fail it at verification, and this direction is the safe /// one: it over-suppresses a name that can never verify against content /// anyway. No spelling of a shunned name can BYPASS the filter, which is the /// only direction §3.3 L527-530 cares about. fn containsWireName(context: *anyopaque, name: []const u8) bool { const self: *RepoShunSet = @ptrCast(@alignCast(context)); if (self.digests.count() == 0) return false; if (name.len != 2 * logstore.sha1_digest_len and name.len != 2 * logstore.sha3_256_digest_len) return false; var digest_buf: [digest_len]u8 = undefined; const digest = decodeWireName(name, &digest_buf) orelse return false; std.debug.assert(logstore.isCompleteDigest(digest.len)); return self.digests.contains(digest); } }; /// Opens `dir/.zit` read-only and snapshots its stored artifacts into `ms` — the /// held/phantom split the pull client announces via `igot` (sync §3.2/§3.3) — /// and its shun table into `shun` (sync §3.3 L527-530), both off the one open. /// Built by `loadContentStore`, which is the CLIENT's builder and not `serve`'s: /// `serve` builds a `LogIndexStore` instead, whose §8.2 slot-preserving mask, /// masked write doors and `read_fault` channel this snapshot has no use for. The /// two share the `digestToWireName` encoding and the held/phantom split, and /// nothing else — a filter added to one does NOT reach the other, which is why /// the §3.3 masked-basis guard is stated in both. The /// read-only handle is closed before `persistStore` reopens the log read-write, /// which is why `shun` owns its keys rather than borrowing the store's. /// sync §7.7 L1428-1432 — this repository's EFFECTIVE hash-policy value, as the /// §7.7 target-admission predicate needs it: read from the repository's own /// stored config, never from peer state or a card field. /// /// The stored byte is opaque to storage (`logstore.Config.hash_policy`), so two /// cases have no defined policy: a repository carrying no config frame at all, /// and a byte naming no artifact-format §7.1 Table 7 ordinal. Neither is /// shun-sha1 — the ONLY distinction §7.7 draws — so both answer `.auto` and the /// receive path admits, rather than failing a whole sync over a field this /// clause reads for one boolean. That is a read-side default, not a repair: the /// stored byte is left exactly as found. /// /// A READ FAULT IS NOT ONE OF THOSE TWO CASES. `readConfig` also returns /// `Storage` (the frame would not read) and `Corrupt` (the frame is not a config /// frame, or does not parse), and both mean the policy is UNKNOWN rather than /// undefined. Answering `.auto` there would silently disable the shun-sha1 half /// of the §7.7 predicate and admit SHA1-named content this repository is /// configured to refuse, with nothing failing — the same fail-open `serveRound` /// refuses for the shun-table half ("must fail the round rather than silently /// serve with no shun table"). So they propagate, exactly as `RepoShunSet.load` /// does three lines above the serve-side call site. fn effectiveHashPolicy(gpa: std.mem.Allocator, log: *logstore.LogStore) error{ OutOfMemory, Storage }!artifact.HashPolicy { var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, // A config frame that will not read or will not parse is a storage-level // failure of this repository, not a policy answer. error.Storage, error.Corrupt => return error.Storage, }) orelse return .auto; defer oc.deinit(gpa); return std.enums.fromInt(artifact.HashPolicy, oc.config.hash_policy) orelse .auto; } /// sync §3.3 L527-530 / §7.7 L1434-1437 — the two repository-local inputs to the /// receive-side filters, as ONE value. They are halves of one question (§7.7's /// combined target predicate reads both; §3.3's phantom scan reads the table), /// they are always read off the same open, and a client wired with one but not /// the other silently applies half a filter with nothing failing. Travelling as /// a pair is what makes that unrepresentable: `syncClient` takes this struct, so /// there is exactly one place either field is handed to a `Client`. /// /// `shun` borrows the caller's `RepoShunSet`, which must outlive the run. const RepoAdmission = struct { shun: zit.sync.ShunSet, hash_policy: artifact.HashPolicy, /// RFC section 5.2 rule 3 -- `.none` for every dense store. elided: zit.sync.ShunSet = .none, /// RFC section 6.1 -- the store cone's raw members, borrowed from the /// run-lifetime cone; empty for a dense store. The client encodes them /// as `pragma cone MEMBER` cards so a cone-aware peer filters replies. cone_members: []const []const u8 = &.{}, /// The store cone's exclusion members (`pragma cone-not MEMBER`). cone_not_members: []const []const u8 = &.{}, }; /// The ONE construction of a sync `Client` from repository state. `pullRun`, /// `pushRun` and `syncRun` differ only in the operation tag and whether a /// credential is named, so everything else — the servercode, the version, and /// both halves of `RepoAdmission` — is stated here once. Three copies of this /// literal is how a repository filter comes to be wired into two verbs and not /// the third. fn syncClient( op: zit.sync.Operation, identity: *const RepoSyncIdentity, credential: ?zit.sync.Credential, admission: RepoAdmission, ) zit.sync.Client { return .{ .state = zit.sync.initialState(op), .operation = op, .self_servercode = &identity.server_code, .project_code = identity.project_code, .version = sync_client_version_info, // sync §6 — a named credential makes `buildRequest` emit the §6.1 login // card; null leaves the session anonymous. .credential = credential, // sync §3.3 L527-530 — the per-round phantom scan's shun filter. .shun = admission.shun, .elided = admission.elided, .cone_members = admission.cone_members, .cone_not_members = admission.cone_not_members, // sync §7.7 L1428-1432 — this repository's effective hash-policy, the // SHA1-name half of the receive-side target-admission predicate. .hash_policy = admission.hash_policy, // submanifests §8.1/§7.2 — the durable indicator and the §8.1 emission // setting together arm the §7.2 upload gate. A pull uploads nothing, so // both are inert there; they are set for every operation so no caller has // to remember which ones upload, which is the same reason the credential // and the admission pair live here rather than at three call sites. .seen_submanifest = identity.seen_submanifest, .submanifests_enabled = identity.enable_submanifests, }; } /// The sync client's holdings view, log-backed and lazy. The clients /// used to snapshot every stored artifact into a `MemStore` (`loadRepoHoldings`) /// before the first wire byte, which resident-loaded the whole store's /// plaintext per pull/push/sync — 3.7 GB for a 147,920-artifact store, an OOM /// kill at tcl scale — just to announce igot names. This seats the client on /// the serve side's `LogIndexStore` instead: names, held/phantom, the /// shun mask, the hash-policy, and the config facts come off the frame index; /// content and wire forms materialize per-rid on demand; received artifacts /// land in the index's appendix `MemStore` and are the only new bytes /// resident. `persistStore` then walks the same store — every pre-held rid /// early-outs on the append log's own `lookup`, so persist stays flat too. /// /// The read handle stays open across the run and the persist; readers take no /// lock (`cli.openStoreRead`), so the append side's exclusive lock is not /// contested by it. const ClientHoldings = struct { file: File, index: *LogIndexStore, fn open(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) error{ OutOfMemory, Storage }!ClientHoldings { var file = cli.openStoreRead(io, dir) catch return error.Storage; errdefer file.close(io); const st = file.stat(io) catch return error.Storage; const index = LogIndexStore.openFromFile(gpa, io, file, st.size) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; return .{ .file = file, .index = index }; } fn store(self: *ClientHoldings) repostate.ArtifactStore { return self.index.store(); } /// The number of already-durable, log-backed artifacts. Received /// artifacts are appended beyond this, so persist begins here and never /// re-walks the holdings a pull did not change. fn logBackedCount(self: *ClientHoldings) repostate.Rid { return @intCast(self.index.rids.len); } /// The §3.3/§7.7 admission pair, borrowed from the index — which outlives /// the run exactly as the caller-owned `RepoShunSet` used to. fn admission(self: *ClientHoldings) RepoAdmission { return .{ .shun = self.index.shun.shunSet(), .hash_policy = self.index.hash_policy }; } fn deinit(self: *ClientHoldings, io: std.Io) void { self.index.destroy(); self.file.close(io); } }; /// sync §9.1 — the repository's stored project code, copied into `buf`, or the /// zero placeholder when the repo carries none (a repo predating the stored /// code) or an out-of-spec-length one. `init` derives a project code and a /// clone adopts the origin's (`storeClonedProjectCode`), so a real repo /// announces its adopted code here; the pull/push client emits it verbatim as /// the operation card's PROJECTCODE, and a real peer (fossil §9.1, ABNF /// `pull-card = "pull" SP servercode SP projectcode`) refuses a mismatch — so a /// zero placeholder would be rejected. Bounded by `max_projectcode_len`: a /// longer stored field is out of spec and never truncated into the wire. fn repoProjectCode( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, buf: *[zit.sync.max_projectcode_len]u8, ) error{ OutOfMemory, Storage }![]const u8 { return (try repoSyncIdentity(gpa, io, dir, "test-fixture-repo", buf)).project_code; } /// The durable config facts every sync-family run needs before it can build /// a peer: §9.1's project code and draft-fossil-submanifests-00 §8.1's /// seen-submanifest indicator and emission setting. Read together because they /// come out of the same config frame — a second open to fetch another field /// would be one more way for them to disagree about which frame they came from. const RepoSyncIdentity = struct { /// §9.1 — the adopted project code, or the zero placeholder. project_code: []const u8, /// submanifests §8.1 — has this repository ever stored a §7.2-gated /// artifact? False for every repository predating this draft, and the /// second thing that arms the §7.2 send gate (see `Server.seen_submanifest`). seen_submanifest: bool, /// submanifests §8.1 — the emission setting. The client twin of /// `serveContext`'s server-side refresh: it is the OTHER half of the §7.2 /// arming disjunction (`Client.mayHoldSubmanifests`). Today it is /// belt-and-braces, because any repository actually holding a gated artifact /// also has `seen_submanifest` set — the two are written in one `log.commit` /// with the first tree. It is threaded anyway so the field is never read /// stale: leaving it at its `false` default would silently disarm the gate /// for any future surface that commits mid-session, where the setting is on /// but the indicator has not been written yet. enable_submanifests: bool, /// §9.1 — this instance's own SERVERCODE, derived from the checkout's /// absolute path (`deriveServerCode`); informational on the wire and /// never adopted from a peer. By value so the client's borrow has no /// lifetime to manage beyond the identity itself. server_code: zit.sync.HashHex, }; fn repoSyncIdentity( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, abs: []const u8, buf: *[zit.sync.max_projectcode_len]u8, ) error{ OutOfMemory, Storage }!RepoSyncIdentity { var file = cli.openStoreRead(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return .{ .project_code = sync_projectcode, .seen_submanifest = false, .enable_submanifests = zit.sync.submanifest_emission_default, .server_code = zit.sync.deriveServerCode(&.{abs}), }; defer oc.deinit(gpa); const seen = oc.config.seen_submanifest; const enabled = oc.config.enable_submanifests; const server_code = zit.sync.deriveServerCode(&.{abs}); const code = oc.config.project_code; if (code.len == 0 or code.len > buf.len) return .{ .project_code = sync_projectcode, .seen_submanifest = seen, .enable_submanifests = enabled, .server_code = server_code, }; @memcpy(buf[0..code.len], code); return .{ .project_code = buf[0..code.len], .seen_submanifest = seen, .enable_submanifests = enabled, .server_code = server_code, }; } /// The number of content-bearing (non-phantom) artifacts in `store` — the held /// set (§3.3 r24). A pull measures its yield as the held count after convergence /// minus the count before, so "already up to date" is an exact zero, never a guess. fn contentCount(store: repostate.ArtifactStore) usize { // `held`, never `content`: on the log-backed client store a // `content()` probe MATERIALIZES the rid's plaintext — asking it of every // rid re-inflates the whole repository to produce a number, which is the // exact whole-store cost this store exists to remove. `held` answers from // the index, and `persistStore` counts `written` by the same predicate, // so the `written >= held_before` arithmetic stays aligned. return store.heldCount(); } // --------------------------------------------------------------------------- // `zit push URL` — the mirror of pull: UPLOAD this repository's artifacts into an // EXISTING served repo. It opens the cwd's `.zit`, announces what it holds so the // peer requests only what it lacks (sync §3.2/§3.3 igot/gimme), and uploads the // missing artifacts as `file` cards, driving to §3.7 termination. The peer's // `zit serve` persists what it receives; nothing changes on this side (§3.3 r24). // Anonymous only. // --------------------------------------------------------------------------- /// What one push run reports: the number of artifacts uploaded and the round /// count it took. `sent` reads the client's own upload counter, so it is the /// exact count of `file` cards emitted — never an estimate. /// submanifests §7.2 — `refused` counts artifacts the §7.2 gate declined to /// upload because the peer's announced version predates sub-manifest support. /// Carried alongside `sent` so a run that withheld an artifact can never be /// reported as a success: the draft rules out silent omission. const PushOutcome = struct { rounds: usize, sent: usize, refused: usize = 0, uv_skipped: u64 = 0 }; /// `zit push URL` — open the current repository, announce its holdings to the /// peer at URL, and upload every artifact the peer lacks until §3.7 termination. /// Thin wiring over the same deep modules pull uses: `transport` (HTTP/IO), /// `sync.Client` + `runSync` (sans-IO §3.7 loop). With URL omitted the saved /// remote (`zit remote`) is used; a repo with neither is a usage error. pub fn runPush( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parsePushArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "push", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "push", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(push_help) catch {}; return .ok; } // sync §6.3 — a login name is a whitespace-free token (the shared-secret // derivation embeds it between "/" separators). Reject a malformed --user // as a usage failure rather than reaching the derivation's assertion. if (parsed.user) |u| { if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) { streams.err.writeAll("zit push: --user must not contain whitespace\n") catch {}; return .failure; } } // Locate the current repository before any network I/O: a push outside a repo // is a usage failure, not a wasted connection. const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "push")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), error.NoRemote => return noRemoteError(streams, "push"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "push", url, e); defer tr.close(); // The client's holdings view — log-backed and lazy, the igot // announcement plus the upload source: the peer requests only what it // lacks, and only those artifacts' content materializes. var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), }; defer holdings.deinit(io); var admission = holdings.admission(); // Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once // and suppress spontaneous requests for elided phantoms. Both the // classification and the wrapper outlive the run below. var sync_cone: ?zit.cone.Cone = null; defer if (sync_cone) |*c| c.deinit(gpa); var sync_classification: ?SparseClassification = null; defer if (sync_classification) |*c| c.deinit(gpa); var sync_elided: RepoElidedSet = undefined; if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| { sync_cone = c; sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch return stateOom(streams, "push"); sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() }; admission.elided = sync_elided.shunSet(); } // §9.1 — announce this repository's adopted project code in the push card // (same identity gate as pull; a real peer refuses a mismatch). The buffer // outlives the run — the client borrows it. // The cluster index: what the held clusters already announce // (never announced again individually) and what they promised that this // repository never collected (seeded as phantoms, so the run asks for it). // The clusters' plaintext is memoised by the holdings adapter while it // resolves; drop it — the index keeps only names and bits. var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "push"); defer clusters.deinit(); holdings.index.releaseTransient(); var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), }; // sync §6 — resolve the login credential (null for an anonymous push). The // password plaintext is confined to `pw_buf`/argv/env; only the derived // 40-hex secret (§6.3) is signed with and — on success — persisted. var pw_buf: [max_password_len]u8 = undefined; // Defense-in-depth: once the 40-hex secret is derived the plaintext in // pw_buf is spent, so scrub it on every exit from this scope — success and // error alike — rather than leave it resident until the stack is reused. defer std.crypto.secureZero(u8, &pw_buf); var user_buf: [logstore.max_config_str]u8 = undefined; const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf, &user_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), error.NoPassword => { streams.err.writeAll("zit push: no password supplied for --user\n") catch {}; return .failure; }, }; var peer_error: PeerErrorCopy = .{}; const stderr_tty = std.Io.File.stderr().isTty(io) catch false; var progress: CloneProgress = .{ .err = streams.err }; const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null; defer progress.finishLine(); if (resolved) |r| armBasicAuth(&tr, r.credential.user, r.plaintext) catch return stateOom(streams, "push"); const outcome = pushRun(gpa, &tr, holdings.store(), identity, if (resolved) |r| r.credential else null, admission, &peer_error, sink, &clusters) catch |e| { // A peer that stated its refusal is quoted rather than flattened into // the generic mapping — the whole point of a server-side refusal is that the operator learns // WHICH file and WHICH rule without a second round trip. if (e == error.FatalProtocolError) { if (peer_error.text()) |text| { streams.err.print("zit push: the peer refused: {s}\n", .{text}) catch {}; return .failure; } } return syncRunFail(streams, "push", url, e, tr.last_status, basicStateOf(&tr)); }; // DECISION-3 — persist the derived secret (never the plaintext) at 0600 only // after the peer accepted the authenticated push, and only when it was // freshly derived: a reused stored secret is already durable. if (resolved) |r| { if (r.persist) { storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), else => return repoReadFail(streams, "push"), }; } } // submanifests §7.2 — "the sender MUST refuse to transmit that artifact, // with an explicit error -- never by converting, downgrading, or silently // omitting it." A client cannot put an `error` card in a REQUEST (it is // outside the §3.9 strict-accept request vocabulary), so the explicit error // is this: the operator is told exactly what was withheld and why, and the // run fails. Reported before the success line, because there was none. if (outcome.refused > 0) { streams.err.print( "zit push: {d} artifacts were withheld: {s}\n", .{ outcome.refused, zit.sync.submanifest_peer_version_error_text }, ) catch {}; return .failure; } if (outcome.sent == 0) { streams.out.print("push to {s}: already in sync\n", .{url}) catch {}; } else { streams.out.print("push to {s}: sent {d} artifact{s} in {d} round{s}\n", .{ url, outcome.sent, plural(outcome.sent), outcome.rounds, plural(outcome.rounds) }) catch {}; reportUvSkipped(streams, "push", outcome.uv_skipped); } return .ok; } /// The upper bound on an interactively-typed or `$ZIT_PASSWORD` password. A /// login password far shorter than this in practice; the fixed bound keeps the /// no-echo read on the stack with no unbounded allocation (TigerStyle). const max_password_len: usize = 1024; /// sync §6.3 — a resolved push credential and whether its secret was freshly /// derived (so it should be persisted after a successful push). const ResolvedCredential = struct { credential: zit.sync.Credential, persist: bool, /// The plaintext password when this run had one (--password, the /// environment, or the prompt): what an HTTP Basic challenge is answered /// with. A stored login has none, and a Basic challenge then fails with /// a message naming the flag. Borrows argv, the environment, or pw_buf. plaintext: ?[]const u8 = null, }; /// The causes `resolvePushCredential` can surface: store read failure, OOM, or /// a `--user` with no obtainable password (`NoPassword`). Closed set so the /// caller switches exhaustively. const ResolveError = error{ OutOfMemory, Storage, NoPassword }; /// sync §6.3/DECISION-3 — turn `push`'s `--user`/`--password` into a signing /// credential. An empty `--user` is an anonymous push (null result). An /// ABSENT `--user` falls back to the login a previous authenticated push /// stored: that credential is reused as-is and this branch never /// prompts — there is no operator at a keyboard by definition, since it is /// how autosync pushes — so with nothing stored the push is anonymous. That is /// fossil's shape: a remembered login authenticates every later sync. /// Otherwise the 40-hex secret is resolved, in order, from: a stored credential /// for this exact login (reused verbatim, already durable), `--password`, /// `$ZIT_PASSWORD`, or a no-echo terminal prompt. A freshly derived secret is /// flagged `persist`; the plaintext lives only in `pw_buf`/argv/env and is /// never returned or stored. `user_buf` holds a stored login's name for the /// returned credential to borrow; it must outlive the credential. /// Give the transport the plaintext this run holds, so a peer that gates the /// READ with an HTTP Basic challenge (a forge in front of a private /// repository) can be answered; nothing is sent unless the peer asks. fn armBasicAuth(tr: *transport.Transport, user: []const u8, plaintext: ?[]const u8) error{OutOfMemory}!void { const pw = plaintext orelse return; try tr.setBasicAuth(user, pw); } /// The user named in a remote URL's `user@host` component, or null. A /// `user:password@host` form yields the user only; the password part of a URL /// is never read from here (the saved remote strips it by construction). pub fn urlUser(url: []const u8) ?[]const u8 { const scheme_end = std.mem.indexOf(u8, url, "://") orelse return null; const rest = url[scheme_end + 3 ..]; const authority_end = std.mem.indexOfScalar(u8, rest, '/') orelse rest.len; const authority = rest[0..authority_end]; const at = std.mem.lastIndexOfScalar(u8, authority, '@') orelse return null; const userinfo = authority[0..at]; const colon = std.mem.indexOfScalar(u8, userinfo, ':') orelse userinfo.len; const user = userinfo[0..colon]; return if (user.len == 0) null else user; } test "urlUser: the user@ component, without any password, or null" { try std.testing.expectEqualStrings("alice", urlUser("https://[email protected]/r/p").?); try std.testing.expectEqualStrings("alice", urlUser("https://alice:[email protected]:8443/r/p").?); try std.testing.expect(urlUser("https://h.example/r/p") == null); try std.testing.expect(urlUser("https://h.example/r/p@ath") == null); try std.testing.expect(urlUser("") == null); } fn resolvePushCredential( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, streams: cli.Streams, project_code: []const u8, parsed: cli.PushArgs, amb: cli.Ambient, pw_buf: *[max_password_len]u8, user_buf: *[logstore.max_config_str]u8, ) ResolveError!?ResolvedCredential { // --user, then the remote URL's user@, then the login a previous // authenticated round stored; none of the three means anonymous. const explicit = parsed.user != null; const user = parsed.user orelse urlUser(parsed.url) orelse { const stored = (try storedLogin(gpa, io, dir)) orelse return null; @memcpy(user_buf[0..stored.user_len], stored.user()); return .{ .credential = .{ .user = user_buf[0..stored.user_len], .secret_hex = stored.secret_hex }, .persist = false, }; }; if (user.len == 0) return null; // an empty --user is an anonymous push std.debug.assert(!zit.sync.containsFossilWhitespace(user)); // pre-validated in runPush // A stored secret for this exact login is reused as-is: no re-derive, no // prompt, no re-persist. Only consulted when no explicit password is given. if (parsed.password == null and amb.push_password == null) { if (try storedSecretFor(gpa, io, dir, user)) |sec| { return .{ .credential = .{ .user = user, .secret_hex = sec }, .persist = false }; } } // Otherwise derive from a plaintext password. Precedence: --password, then // $ZIT_PASSWORD, then an interactive no-echo prompt. A user named only by // the URL is a hint, not a demand: with no password to be had, the round // goes anonymous and says so, so a private repository's 404 at least // points at the credential it wanted. --user with no password is an error. const password: []const u8 = parsed.password orelse amb.push_password orelse (readPasswordNoEcho(streams, user, pw_buf) orelse { if (explicit) return error.NoPassword; streams.err.print("zit: no password for '{s}' from the remote URL (set ZIT_PASSWORD or pass --user/--password); continuing anonymously\n", .{user}) catch {}; return null; }); const secret = zit.sync.deriveSharedSecret(project_code, user, password); return .{ .credential = .{ .user = user, .secret_hex = secret }, .persist = true, .plaintext = password }; } /// The login a successful authenticated push left in this repo's config /// (DECISION-3): the name and its 40-hex secret, both copied out — the config /// frame is freed here — so the value outlives the read. const StoredLogin = struct { user_buf: [logstore.max_config_str]u8, user_len: usize, secret_hex: [40]u8, fn user(self: *const StoredLogin) []const u8 { return self.user_buf[0..self.user_len]; } }; /// The stored login, or null when none is stored. Read-only. fn storedLogin( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, ) error{ OutOfMemory, Storage }!?StoredLogin { var file = cli.openStoreRead(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return null; defer oc.deinit(gpa); if (oc.config.login_secret_hex.len != 40) return null; const name = oc.config.login_user; if (name.len == 0 or name.len > logstore.max_config_str) return null; var out: StoredLogin = .{ .user_buf = undefined, .user_len = name.len, .secret_hex = undefined }; @memcpy(out.user_buf[0..name.len], name); @memcpy(&out.secret_hex, oc.config.login_secret_hex[0..40]); return out; } /// DECISION-3 — the 40-hex secret stored for `user` in this repo's config, or /// null when none is stored or it belongs to a different login. fn storedSecretFor( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, user: []const u8, ) error{ OutOfMemory, Storage }!?[40]u8 { const stored = (try storedLogin(gpa, io, dir)) orelse return null; if (!std.mem.eql(u8, stored.user(), user)) return null; return stored.secret_hex; } /// DECISION-3 — persist the derived login credential into the repo config, /// preserving every other field, then tighten the repository file to 0600 so /// the stored secret is owner-only at rest. The permission tightening is /// best-effort: the secret is already durable, so a filesystem that refuses /// chmod does not fail the push (perms + FDE are the boundary, not the sole /// guarantee — per DECISION-3). NEVER writes the plaintext password. fn storePushCredential( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, credential: zit.sync.Credential, ) PersistError!void { std.debug.assert(credential.secret_hex.len == 40); // The field-preserving RMW writes the credential without disturbing the // project code or a saved remote (a bare `putConfig` here would drop them). try updateConfig(gpa, io, dir, .{ .login = credential }); // Owner-only now that the file carries a credential (best-effort; see above). var file = cli.openStoreAppend(io, dir) catch return; defer file.close(io); file.setPermissions(io, File.Permissions.fromMode(0o600)) catch {}; } /// CLI/IO seam — prompt on stderr and read one password line from the terminal /// with echo disabled, returning the trimmed line in `buf` (never its newline), /// or null on a read failure or an empty line. Echo is restored on every exit /// path; when stdin is not a terminal (`tcgetattr` fails) the line is read with /// no termios toggle. Not exercised by the automated gates (they pass /// `--password`/`$ZIT_PASSWORD`) — this is the interactive path. fn readPasswordNoEcho( streams: cli.Streams, user: []const u8, buf: *[max_password_len]u8, ) ?[]const u8 { // No terminal, no prompt: a script, an autosync, or a test would otherwise // see a prompt it cannot answer and a read error underneath it. _ = std.posix.tcgetattr(std.posix.STDIN_FILENO) catch return null; streams.err.print("Password for {s}: ", .{user}) catch {}; streams.err.flush() catch {}; const fd = std.posix.STDIN_FILENO; const restore: ?std.posix.termios = blk: { var t = std.posix.tcgetattr(fd) catch break :blk null; const saved = t; t.lflag.ECHO = false; std.posix.tcsetattr(fd, .FLUSH, t) catch break :blk null; break :blk saved; }; defer if (restore) |saved| { std.posix.tcsetattr(fd, .FLUSH, saved) catch {}; streams.err.writeAll("\n") catch {}; streams.err.flush() catch {}; }; const n = std.posix.read(fd, buf) catch return null; if (n == 0) return null; var line = buf[0..n]; // Trim a single trailing CRLF/LF; interior bytes (including spaces) are kept. if (line.len > 0 and line[line.len - 1] == '\n') line = line[0 .. line.len - 1]; if (line.len > 0 and line[line.len - 1] == '\r') line = line[0 .. line.len - 1]; if (line.len == 0) return null; return line; } /// Drives a push client (§3.2/§3.3) to convergence over `tr`, uploading the /// artifacts the peer lacks; returns the upload count and round count. The client /// announces `ms`'s holdings via igot and uploads what the peer gimme's (§3.3 /// r24: push sends content but never obtains it). Sans-IO on this side of /// `tr.exchange`. Its closed error set is the driver's own causes unioned with /// the transport's, so `runPush` switches it exhaustively. fn pushRun( gpa: std.mem.Allocator, tr: *transport.Transport, holdings: repostate.ArtifactStore, identity: RepoSyncIdentity, credential: ?zit.sync.Credential, admission: RepoAdmission, /// A server that REFUSED states why in its error card. Copied out /// here because the client owns the buffer and is deinit'd on the way out, /// so the caller cannot reach it afterwards; the clone path does the same /// for the sparse-origin refusal. peer_error_out: *PeerErrorCopy, sink: ?sync_driver.RoundSink, clusters: ?*const clusterindex.Resolved, ) sync_driver.RunError(*transport.Transport)!PushOutcome { // A push client still RECEIVES — §3.3 answers the server's gimme-driven // `file` cards in the same session — so `admission` is load-bearing here too. var client = syncClient(.push, &identity, credential, admission); defer client.deinit(gpa); if (clusters) |c| { client.clustered = &c.clustered; try client.seedClusterPhantoms(gpa, holdings, c.missing); } errdefer peer_error_out.set(client.peerErrorText()); const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null); recordSyncStats(stats); // Read the upload counter before `deinit` runs (the return expression is // evaluated first): the exact number of `file` cards this session emitted. return .{ .rounds = stats.rounds, .sent = client.content_sent, .refused = client.submanifests_refused, .uv_skipped = client.uv_skipped }; } // --------------------------------------------------------------------------- // `zit sync [URL]` — BIDIRECTIONAL convergence with a peer in ONE session. // Unlike pull-then-push, a single `operation=.sync` client announces igot, // requests gimme, AND uploads on one §3.7 exchange (§3.2 round structure: a sync // request carries both push and pull each round) — not a third copy of the driver // loop over two transport sessions. // Opens the cwd's `.zit`, resolves the URL (arg → saved remote), may // authenticate (sync uploads), persists what it receives like pull, and reports // both directions. A saved `zit remote` lets the URL be omitted. // --------------------------------------------------------------------------- /// What one sync run reports: uploads (`sent`) and the round count. Received is /// measured by the caller as the persisted-count growth (as pull does), so it is /// not carried here. /// submanifests §7.2 — see `PushOutcome.refused`. const SyncOutcome = struct { rounds: usize, sent: usize, refused: usize = 0, uv_skipped: u64 = 0 }; pub fn runSync( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseSyncArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "sync", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "sync", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(sync_help) catch {}; return .ok; } // sync §6.3 — a login name is whitespace-free (mirrors push). Reject a // malformed --user as usage before the secret derivation's assertion. if (parsed.user) |u| { if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) { streams.err.writeAll("zit sync: --user must not contain whitespace\n") catch {}; return .failure; } } // Locate the current repository before any network I/O. const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "sync")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.NoRemote => return noRemoteError(streams, "sync"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "sync", url, e); defer tr.close(); // The client's holdings view — log-backed and lazy: the igot // announcement, the upload source, and the baseline for the received // count. The held set can only grow (§3.3 storage only adds content). var holdings = ClientHoldings.open(gpa, io, root_dir) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), }; defer holdings.deinit(io); var admission = holdings.admission(); // Sparse-cone RFC section 5.2 rule 3: under a store cone, classify once // and suppress spontaneous requests for elided phantoms. Both the // classification and the wrapper outlive the run below. var sync_cone: ?zit.cone.Cone = null; defer if (sync_cone) |*c| c.deinit(gpa); var sync_classification: ?SparseClassification = null; defer if (sync_classification) |*c| c.deinit(gpa); var sync_elided: RepoElidedSet = undefined; if (zit.repocache.readStoreCone(gpa, io, root_dir)) |c| { sync_cone = c; sync_classification = classifyKeepRids(gpa, holdings.store(), &sync_cone.?) catch return stateOom(streams, "sync"); sync_elided = .{ .classification = &sync_classification.?, .store = holdings.store() }; admission.elided = sync_elided.shunSet(); admission.cone_members = sync_cone.?.members; admission.cone_not_members = sync_cone.?.not_members; } const held_before = contentCount(holdings.store()); // §9.1 — announce this repository's adopted project code (a real peer refuses // a mismatch). The buffer outlives the run — the client borrows it. // The cluster index: what the held clusters already announce // (never announced again individually) and what they promised that this // repository never collected (seeded as phantoms, so the run asks for it). // The clusters' plaintext is memoised by the holdings adapter while it // resolves; drop it — the index keeps only names and bits. var clusters = loadClusterIndex(gpa, io, root_dir, &holdings.index.log, holdings.store()) catch return stateOom(streams, "sync"); defer clusters.deinit(); holdings.index.releaseTransient(); var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), }; // sync §6 — resolve the login credential (null for anonymous). A sync // uploads, so it may authenticate exactly as push does; the plaintext lives // only in `pw_buf`/argv/env and is scrubbed on every exit from this scope. var pw_buf: [max_password_len]u8 = undefined; defer std.crypto.secureZero(u8, &pw_buf); var user_buf: [logstore.max_config_str]u8 = undefined; const resolved = resolvePushCredential(gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf, &user_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.NoPassword => { streams.err.writeAll("zit sync: no password supplied for --user\n") catch {}; return .failure; }, }; const stderr_tty = std.Io.File.stderr().isTty(io) catch false; var progress: CloneProgress = .{ .err = streams.err }; const sink: ?sync_driver.RoundSink = if (stderr_tty) .{ .ctx = &progress, .emit = CloneProgress.emit } else null; defer progress.finishLine(); if (resolved) |r| armBasicAuth(&tr, r.credential.user, r.plaintext) catch return stateOom(streams, "sync"); const outcome = syncRun(gpa, &tr, holdings.store(), identity, if (resolved) |r| r.credential else null, admission, sink, &clusters) catch |e| return syncRunFail(streams, "sync", url, e, tr.last_status, basicStateOf(&tr)); // Sparse-cone RFC section 6.3 applies to EVERY receive: a dense peer // volunteers content the client never announced, so the filter runs at // persist. Reclassified over the post-run store, so a brand-new commit's // blobs classify by their own F cards -- new in-cone content persists, // new out-of-cone content elides with the rest. var post_classification: ?SparseClassification = null; defer if (post_classification) |*c| c.deinit(gpa); if (sync_cone) |*c| { post_classification = classifyKeepRids(gpa, holdings.store(), c) catch return stateOom(streams, "sync"); } const persisted = persistStore(gpa, io, root_dir, holdings.store(), if (post_classification) |*c| &c.keep else null, 0) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.Corrupt => return renderStorage(streams, root.abs, "the peer sent a malformed artifact name"), }; const written = persisted.written; std.debug.assert(written >= held_before); const received = written - held_before; // DECISION-3 — persist a freshly derived secret only after the peer accepted // the authenticated session, and only when it was freshly derived (mirrors push). if (resolved) |r| { if (r.persist) { storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), else => return repoReadFail(streams, "sync"), }; } } // submanifests §7.2 — "the sender MUST refuse to transmit that artifact, // with an explicit error -- never by converting, downgrading, or silently // omitting it." A client cannot put an `error` card in a REQUEST (it is // outside the §3.9 strict-accept request vocabulary), so the explicit error // is this: the operator is told exactly what was withheld and why, and the // run fails. Reported before the success line, because there was none. if (outcome.refused > 0) { streams.err.print( "zit sync: {d} artifacts were withheld: {s}\n", .{ outcome.refused, zit.sync.submanifest_peer_version_error_text }, ) catch {}; return .failure; } // submanifests §7.1/§7.3 — see `runPull`: convergence against the peer is // not the same fact as a complete tree closure. if (persisted.submanifest_gaps > 0) { streams.err.print( "zit sync: {d} sub-manifest references are missing; some check-ins cannot be enumerated\n", .{persisted.submanifest_gaps}, ) catch {}; return .failure; } if (outcome.sent == 0 and received == 0) { streams.out.print("sync with {s}: already in sync\n", .{url}) catch {}; } else { streams.out.print("sync with {s}: sent {d}, received {d} in {d} rounds\n", .{ url, outcome.sent, received, outcome.rounds }) catch {}; reportUvSkipped(streams, "sync", outcome.uv_skipped); } return .ok; } /// Drives a bidirectional sync client (§3.2 round structure — both directions on /// ONE §3.7 session) to convergence over `tr`; returns the upload count and round /// count. /// The received count is the caller's persisted-count growth. Sans-IO on this /// side of `tr.exchange`; its closed error set is switched exhaustively by /// `runSync`. The sibling of `pullRun`/`pushRun`, differing only in the /// operation tag (`.sync`, which drives igot + gimme + upload on one session). fn syncRun( gpa: std.mem.Allocator, tr: *transport.Transport, holdings: repostate.ArtifactStore, identity: RepoSyncIdentity, credential: ?zit.sync.Credential, admission: RepoAdmission, sink: ?sync_driver.RoundSink, clusters: ?*const clusterindex.Resolved, ) sync_driver.RunError(*transport.Transport)!SyncOutcome { var client = syncClient(.sync, &identity, credential, admission); defer client.deinit(gpa); if (clusters) |c| { client.clustered = &c.clustered; try client.seedClusterPhantoms(gpa, holdings, c.missing); } const stats = try sync_driver.runSyncProgress(gpa, &client, holdings, tr, max_stalled_sync_rounds, sink, null); recordSyncStats(stats); return .{ .rounds = stats.rounds, .sent = client.content_sent, .refused = client.submanifests_refused, .uv_skipped = client.uv_skipped }; } // --------------------------------------------------------------------------- // `zit remote [URL | --unset]` — read, set, or clear the single default peer URL // (sync §4/§6). Bare prints the saved remote; a URL saves it; --unset clears it. // A local, non-synced config value (`config.remote_url`) — anonymous only, no // credential in the URL — that lets pull/push/sync omit their URL argument. Not // a git-style named-remote registry: one value, matching the single-value config. // --------------------------------------------------------------------------- /// `zit settings [NAME [VALUE]]` — the local-settings surface, holding /// exactly one setting today: `autosync`. Bare lists; NAME prints; NAME VALUE /// sets. The value lives in the config frame like `remote_url` — local, never /// synced — inverted in storage (the flag records OFF) so absence means the /// fossil default, on. pub fn runSettings( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseSettingsArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "settings", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "settings", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(settings_help) catch {}; return .ok; } const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "settings")) orelse { main.reportNoRoot(streams, io, "settings", amb.repo_override); return .failure; }; var root_dir = root.dir; defer root_dir.close(io); const zset = zit.settings; // The tier the operator named. `--global` selects WHERE a write lands and // which stored value a bare NAME reports; it never changes the fact that a // listing reports the value in EFFECT. const tier: Tier = if (parsed.global) .global else .repository; if (parsed.name) |name| { const setting = zset.Setting.byName(name) orelse { var names_buf: [256]u8 = undefined; streams.err.print( "zit settings: unknown setting '{s}' (known: {s})\n", .{ name, zset.knownNames(&names_buf) }, ) catch {}; return .failure; }; if (parsed.value) |value| { return writeSetting(gpa, io, arena, root_dir, streams, amb, setting, value, tier); } return showSetting(gpa, io, arena, root_dir, streams, amb, setting, parsed.changed, parsed.value_only); } inline for (comptime std.enums.values(zset.Setting)) |setting| { const exit = showSetting(gpa, io, arena, root_dir, streams, amb, setting, parsed.changed, parsed.value_only); if (exit != .ok) return exit; } return .ok; } /// Write one setting into the tier the operator named. /// /// Validation happens BEFORE either tier is touched, so a rejected value never /// leaves a half-written global file or a config frame carrying a value the /// reader will refuse. fn writeSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, streams: cli.Streams, amb: cli.Ambient, setting: zit.settings.Setting, value: []const u8, tier: Tier, ) cli.Exit { const zset = zit.settings; switch (setting.kind()) { .boolean => if (zset.parseBool(value) == null and !(setting == .autosync and std.mem.eql(u8, value, "pullonly"))) { streams.err.print( "zit settings: {s} takes 'on' or 'off'{s}\ntry 'zit settings --help'\n", .{ setting.name(), if (setting == .autosync) " (or 'pullonly')" else "" }, ) catch {}; return .usage; }, .string => if (tier == .repository and value.len > logstore.max_config_str) { // Refusing beats truncating: a truncated glob list silently stops // protecting files, and a truncated command silently runs a // different program. if (setting == .ignore_glob) { streams.err.print( "zit settings: ignore-glob is limited to {d} bytes; use the versioned {s} file for longer lists\n", .{ logstore.max_config_str, zit.ignore.versioned_path }, ) catch {}; } else { streams.err.print( "zit settings: {s} is limited to {d} bytes\n", .{ setting.name(), logstore.max_config_str }, ) catch {}; } return .failure; }, } if (tier == .global) { writeGlobalSetting(gpa, io, arena, amb, setting, value) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "settings"), error.NoGlobalHome => { streams.err.writeAll( "zit settings: no global settings location (set $ZIT_CONFIG, $XDG_CONFIG_HOME or $HOME)\n", ) catch {}; return .failure; }, else => { streams.err.writeAll("zit settings: cannot write the global settings file\n") catch {}; return .failure; }, }; streams.out.print("{s} {s}\n", .{ setting.name(), value }) catch {}; // A global write that is shadowed locally changes nothing an operator // can see, which is precisely the confusion the provenance work exists // to end -- so say so at the moment it happens. const eff = resolveSetting(gpa, io, arena, root_dir, amb, setting) catch return .ok; if (eff.source != .global) { streams.err.print( "note: the {s} tier still supplies {s} here, so this global value is not in effect\n", .{ eff.source.label(), setting.name() }, ) catch {}; } _ = noteVersionedShadow(io, root_dir, streams, setting, eff); if (setting == .ignore_glob) noteGitignorePresent(io, root_dir, streams); return .ok; } const patch = repositoryPatch(setting, value) orelse { streams.err.print( "zit settings: {s} cannot be stored in this repository yet\n", .{setting.name()}, ) catch {}; return .failure; }; updateConfig(gpa, io, root_dir, patch) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "settings"), else => return repoReadFail(streams, "settings"), }; streams.out.print("{s} {s}\n", .{ setting.name(), value }) catch {}; // The value just written IS the repository tier, so the shadow question is // answerable without re-opening the store. const r = resolveSettingWith(io, arena, root_dir, amb, setting, value) catch return .ok; _ = noteVersionedShadow(io, root_dir, streams, setting, r); if (setting == .ignore_glob) noteGitignorePresent(io, root_dir, streams); return .ok; } /// The config-frame patch for a repository-tier write, or null for a setting /// the frame does not yet carry. fn repositoryPatch(setting: zit.settings.Setting, value: []const u8) ?ConfigPatch { const on = zit.settings.parseBool(value); return switch (setting) { // Both booleans record the explicit-set bit alongside the value, which // is what makes "explicitly the default" distinguishable from untouched. .autosync => if (std.mem.eql(u8, value, "pullonly")) .{ .autosync_off = false, .autosync_set = true, .autosync_mode = "pullonly" } else .{ .autosync_off = !(on orelse return null), .autosync_set = true, .autosync_mode = "" }, .editor => .{ .editor = value }, .clearsign => .{ .clearsign = on orelse return null, .clearsign_set = true }, .ignore_glob => .{ .ignore_glob = value }, .pgp_command => .{ .pgp_command = value }, .keep_glob => .{ .keep_glob = value }, .manifest => .{ .manifest = value }, .before_commit => .{ .before_commit = value }, .binary_glob => .{ .binary_glob = value }, .crlf_glob => .{ .crlf_glob = value }, .crnl_glob => .{ .crnl_glob = value }, .clean_glob => .{ .clean_glob = value }, .empty_dirs => .{ .empty_dirs = value }, .encoding_glob => .{ .encoding_glob = value }, // Boolean-valued, stored as the operator's own text: `settings // dotfiles yes` round-trips as `yes`, and `settings.isTruth` is the // one thing that reads it. .dotfiles => .{ .dotfiles = value }, .mimetypes => .{ .mimetypes = value }, .attachment_size_limit => .{ .attachment_size_limit = value }, }; } /// Read-modify-write the operator-scope settings file, creating its directory /// when absent. Read-modify-write rather than append so a second edit of the /// same setting replaces the first instead of leaving a dead line above it. fn writeGlobalSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, amb: cli.Ambient, setting: zit.settings.Setting, value: []const u8, ) !void { const zset = zit.settings; var path_buf: [1024]u8 = undefined; const g = zset.globalPath(amb.environ, &path_buf) orelse return error.NoGlobalHome; const cwd = std.Io.Dir.cwd(); const existing = cwd.readFileAlloc(io, g.path, arena, .limited(zset.max_versioned_bytes)) catch ""; const next = try zset.globalSet(gpa, existing, setting, value); defer gpa.free(next); if (g.dir.len != 0) cwd.createDirPath(io, g.dir) catch {}; var f = cwd.createFile(io, g.path, .{ .truncate = true }) catch return error.Storage; defer f.close(io); f.writePositionalAll(io, next, 0) catch return error.Storage; } /// Which stored tier a `zit settings` invocation addresses. const Tier = enum { repository, global }; /// Print one setting as `NAME VALUE` plus the tier that supplied it. /// /// The provenance column is the point. zit has four places a value can /// live, and a display that showed only the value would let an operator edit /// the shadowed tier -- twice -- while the effective answer never moved. The /// note is on stderr so `zit settings NAME` remains a clean value on stdout for /// a script. fn showSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, streams: cli.Streams, amb: cli.Ambient, setting: zit.settings.Setting, only_changed: bool, value_only: bool, ) cli.Exit { const r = resolveSetting(gpa, io, arena, root_dir, amb, setting) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "settings"), else => return repoReadFail(streams, "settings"), }; if (only_changed and r.source == .default) return .ok; // --changed if (value_only) { streams.out.print("{s}\n", .{r.value}) catch {}; // --value } else { streams.out.print("{s} {s}\n", .{ setting.name(), r.value }) catch {}; } // Only say where it came from when that is not the obvious answer: naming // the tier for every default would bury the two lines that matter. One // fact, one line: when a versioned file is shadowing a stored value the // note SAYS so and replaces the provenance line, and when `.no-warn` // silences that, the provenance line still names the file that is live. if (!setting.versionable()) noteVersionedIgnored(io, root_dir, streams, setting); if (noteVersionedShadow(io, root_dir, streams, setting, r)) return .ok; if (r.source == .versioned) { const dir = r.versioned_dir orelse .zit; streams.err.print( "note: {s} is set in {s}/{s}\n", .{ setting.name(), dir.dirName(), setting.name() }, ) catch {}; } else if (r.source != .default and r.source != .repository) { streams.err.print( "note: {s} is set in the {s} tier\n", .{ setting.name(), r.source.label() }, ) catch {}; } return .ok; } /// Gather every tier's answer and apply `settings.resolve`. The reads live here /// -- the model module stays sans-IO so precedence is a pure, tested function. /// Public so the COMMIT path resolves signing settings through the same /// precedence a listing reports. They must never disagree: a `zit settings` /// that says `clearsign on` while the commit quietly does not sign is worse /// than having no global tier at all. pub fn resolveSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, amb: cli.Ambient, setting: zit.settings.Setting, ) !zit.settings.Resolved { const repository = try repositorySetting(gpa, io, arena, root_dir, setting); return resolveSettingWith(io, arena, root_dir, amb, setting, repository); } /// The effective `crlf-glob`, with fossil's older `crnl-glob` as the /// fallback SPELLING of the same setting. fossil reads /// `db_get("crlf-glob", db_get("crnl-glob",""))` (checkin.c), so `crnl-glob` /// is consulted only when no tier supplies `crlf-glob` at all -- a stored /// empty `crlf-glob` is a value and wins, exactly as it does for every other /// glob list. /// /// The COMMIT gate only. The wire twin (`sync.PushPolicy`, built from the /// serve-side snapshot) keeps reading `crlf-glob` alone: fossil's server never /// gates content, so there is no parity to mirror there. /// `config` is the caller's already-read frame, as for `localSetting`: the /// commit gate holds the store's write lock and reads its config once, so both /// spellings answer from that one frame rather than re-opening the store twice. pub fn resolveCrlfGlob( io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, amb: cli.Ambient, config: ?*const logstore.Config, ) error{OutOfMemory}!zit.settings.Resolved { const crlf = try localSetting(io, arena, root_dir, amb, config, .crlf_glob); if (crlf.source != .default) return crlf; return localSetting(io, arena, root_dir, amb, config, .crnl_glob); } /// `resolveSetting` with the REPOSITORY tier supplied by the caller instead of /// read here. `repositorySetting` opens the store and pays a full frame-header /// scan to reach one config frame -- fine for a one-shot verb, ruinous in the /// serve loop, where it turned every round of a large-store clone into seconds /// of dead re-scanning. The server passes the values its per-round /// snapshot already read; every other tier is a cheap local read and stays /// here, so the resolution order cannot fork between the two paths. pub fn resolveSettingWith( io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, amb: cli.Ambient, setting: zit.settings.Setting, repository: ?[]const u8, ) !zit.settings.Resolved { const zset = zit.settings; var c: zset.Candidates = .{}; // The versioned file is read ONLY for a versionable setting. `resolve` // enforces the same rule, so this is belt and braces on the boundary that // keeps a cloned repository from choosing what this machine executes. if (setting.versionable()) { // fossil's spelling first, zit's as the fallback -- a mirrored // Fossil repository's own policy is honoured without adopting zit's // directory name, and when both exist the fossil file wins. // Two buffers: the candidates are materialized before the loop, and a // shared buffer would leave the first slice pointing at the second's // bytes. var fossil_buf: [128]u8 = undefined; var zit_buf: [128]u8 = undefined; const candidates = .{ setting.fossilVersionedPath(&fossil_buf), setting.versionedPath(&zit_buf) }; const dirs = [_]zset.VersionedDir{ .fossil, .zit }; inline for (candidates, 0..) |maybe_path, i| { if (c.versioned == null) { if (maybe_path) |path| { if (root_dir.readFileAlloc(io, path, arena, .limited(zset.max_versioned_bytes))) |text| { // fossil's `db_get_versioned` strips `#` lines // and trims before anything reads the value, so every // versionable setting -- in either directory -- gets // the same treatment from the one reader. c.versioned = zset.stripCommentLines(text); c.versioned_dir = dirs[i]; } else |_| {} } } } } return resolveSettingTiers(io, arena, amb, setting, &c, repository); } /// `resolveSettingWith` with the versioned tier read from /// the check-in `rid` instead of the working tree: what `open` needs, since /// its tree does not exist yet. Same two file names in the same order, the /// same comment stripping, the same tiers below; fossil's `db_get_versioned` /// reads the file out of the check-in through `historical_blob` too. fn checkinSetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, repo: *Repository, rid: Repository.Rid, amb: cli.Ambient, config: ?*const logstore.Config, setting: zit.settings.Setting, ) !zit.settings.Resolved { const zset = zit.settings; var c: zset.Candidates = .{}; if (setting.versionable()) { var fossil_buf: [128]u8 = undefined; var zit_buf: [128]u8 = undefined; const candidates = .{ setting.fossilVersionedPath(&fossil_buf), setting.versionedPath(&zit_buf) }; const dirs = [_]zset.VersionedDir{ .fossil, .zit }; inline for (candidates, 0..) |maybe_path, i| { if (c.versioned == null) { if (maybe_path) |path| { if (checkinFileText(gpa, arena, repo, rid, path)) |text| { c.versioned = zset.stripCommentLines(text); c.versioned_dir = dirs[i]; } } } } } const repository = if (config) |cf| try configValue(arena, cf, setting) else null; return resolveSettingTiers(io, arena, amb, setting, &c, repository); } /// The content of `path` in check-in `rid`, copied into `arena` and bounded /// like a versioned file on disk; null when the check-in has no such file or /// it cannot be read (the tiers below then answer, as for an unreadable file). fn checkinFileText(gpa: std.mem.Allocator, arena: std.mem.Allocator, repo: *Repository, rid: Repository.Rid, path: []const u8) ?[]u8 { const card = (repo.fileAt(gpa, rid, path) catch return null) orelse return null; defer Repository.freeFileCard(gpa, card); const hash = card.hash orelse return null; const bytes = (repo.blob(gpa, hash) catch return null) orelse return null; defer gpa.free(bytes); if (bytes.len > zit.settings.max_versioned_bytes) return null; return arena.dupe(u8, bytes) catch null; } /// The environment, repository and global tiers on top of an already-read /// versioned candidate -- the tail both readers above share, so the order the /// tiers are consulted in cannot fork between them. fn resolveSettingTiers( io: std.Io, arena: std.mem.Allocator, amb: cli.Ambient, setting: zit.settings.Setting, c: *zit.settings.Candidates, repository: ?[]const u8, ) !zit.settings.Resolved { if (setting.envVar()) |name| c.environment = amb.environ.getPosix(name); c.repository = repository; c.global = try globalSetting(io, arena, amb, setting); return zit.settings.resolve(setting, c.*); } /// Serialize and atomically store the content log's scan index (the same /// best-effort refresh `Repository.logStore` performs): a failure costs the /// next open one scan, nothing else. fn refreshSettingLogIndex(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, log: *const logstore.LogStore) void { const idx = log.serializeIndex(gpa) catch return; defer gpa.free(idx); zit.repocache.storeLogIndex(io, dir, idx); } /// Open a read-only log over `sf`, adopting the persisted scan index when the /// checkout's store cache holds a fresh one — the CLI pattern, shared so /// every read verb stops paying the full frame-header scan (diff and /// status carried an O(store) floor of ~0.2s per invocation on a /// 148k-artifact store from exactly this). A refused index (absent, stale, /// corrupt) falls back to the scan and refreshes the sidecar, so the next /// open is cheap again. The sidecar lives with the STORE, never a worktree, /// so the cache directory resolves through the checkout's store pointer. pub fn openLogAdoptingIndex( gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, sf: storage.StorageFile, /// Whether a scan (or a tail-extended adoption) may persist the refreshed /// sidecar. `.read_only` exists for verbs PINNED as tree-untouching — /// extras' B-extras-16 requires that a run create nothing, `.zit-cache` /// included — which still adopt a sidecar someone else already wrote. sidecar: enum { refresh_on_scan, read_only }, ) logstore.OpenError!logstore.LogStore { var resolved = checkout.storeDirOf(io, root_dir) catch null; defer if (resolved) |*r| r.close(io); const cache_dir = if (resolved) |r| r else root_dir; indexed: { const idx = zit.repocache.tryLoadLogIndex(gpa, io, cache_dir) orelse break :indexed; // An adopted store TAKES `idx` and frees it in `deinit` -- its // entry hashes are slices of it. Every refusal path leaves it ours. var taken = false; defer if (!taken) gpa.free(idx); var extended = false; const adopted = logstore.LogStore.openReadOnlyIndexed(gpa, sf, idx, &extended) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => break :indexed, }; var log = adopted orelse break :indexed; taken = true; if (extended and sidecar == .refresh_on_scan) refreshSettingLogIndex(gpa, io, cache_dir, &log); return log; } var log = try logstore.LogStore.openReadOnly(gpa, sf); // The scan just paid for the product the index persists. if (sidecar == .refresh_on_scan) refreshSettingLogIndex(gpa, io, cache_dir, &log); return log; } /// Open this checkout's store read-only for a one-shot config read, adopting /// the persisted scan index when the cache holds a fresh one (the CLI /// half). The plain `openReadOnly` walks every frame header, which `open` and /// `update` were paying once per setting on every materialization. A /// refused index (absent, stale, corrupt) falls back to the scan and refreshes /// the sidecar, as `Repository.logStore` does. The sidecar lives with the /// STORE, never a worktree, so the cache directory is resolved through the /// checkout's store pointer first. const OpenedStoreLog = struct { file: std.Io.File, fs: storage.FileStorage, log: logstore.LogStore, fn close(self: *OpenedStoreLog, gpa: std.mem.Allocator, io: std.Io) void { self.log.deinit(gpa); self.file.close(io); gpa.destroy(self); } }; /// Heap-allocated: the log holds a pointer into `fs`, so the record must /// never move after `storageFile()` is taken. fn openStoreLogForRead(gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir) !*OpenedStoreLog { var file = cli.openStoreRead(io, root_dir) catch return error.Storage; errdefer file.close(io); const st = file.stat(io) catch return error.Storage; const out = gpa.create(OpenedStoreLog) catch return error.OutOfMemory; errdefer gpa.destroy(out); out.* = .{ .file = file, .fs = storage.FileStorage.init(io, file, st.size), .log = undefined }; out.log = openLogAdoptingIndex(gpa, io, root_dir, out.fs.storageFile(), .refresh_on_scan) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; return out; } /// This repository's stored answer, or null when it has no opinion. The /// booleans consult their explicit-set bit: without it a repository would /// always have an opinion and the global tier could never be reached. fn repositorySetting( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, setting: zit.settings.Setting, ) !?[]const u8 { const opened = try openStoreLogForRead(gpa, io, root_dir); defer opened.close(gpa, io); var oc = (opened.log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return null; defer oc.deinit(gpa); return configValue(arena, &oc.config, setting); } /// One config frame's answer for one setting, or null when the frame has no /// opinion. PURE: the caller supplies the frame, so a verb that has already /// read `Config` once resolves every name it needs without a second store /// open (now the rule rather than the exception). The arena /// dupe is the only way this can fail. pub fn configValue( arena: std.mem.Allocator, config: *const logstore.Config, setting: zit.settings.Setting, ) error{OutOfMemory}!?[]const u8 { return switch (setting) { .autosync => if (config.autosync_set) (if (config.autosync_off) "off" else if (std.mem.eql(u8, config.autosync_mode, "pullonly")) "pullonly" else "on") else null, .editor => if (config.editor.len == 0) null else try arena.dupe(u8, config.editor), .clearsign => if (config.clearsign_set) (if (config.clearsign) "on" else "off") else null, // A stored empty string is "no opinion" for these: the config frame has // no separate absent marker for its string fields, and an operator who // wants the empty value spells it by clearing, which is the same thing. .ignore_glob => if (config.ignore_glob.len == 0) null else try arena.dupe(u8, config.ignore_glob), .pgp_command => if (config.pgp_command.len == 0) null else try arena.dupe(u8, config.pgp_command), .keep_glob => if (config.keep_glob.len == 0) null else try arena.dupe(u8, config.keep_glob), .manifest => if (config.manifest.len == 0) null else try arena.dupe(u8, config.manifest), .before_commit => if (config.before_commit.len == 0) null else try arena.dupe(u8, config.before_commit), .binary_glob => if (config.binary_glob.len == 0) null else try arena.dupe(u8, config.binary_glob), .crlf_glob => if (config.crlf_glob.len == 0) null else try arena.dupe(u8, config.crlf_glob), .crnl_glob => if (config.crnl_glob.len == 0) null else try arena.dupe(u8, config.crnl_glob), .clean_glob => if (config.clean_glob.len == 0) null else try arena.dupe(u8, config.clean_glob), .empty_dirs => if (config.empty_dirs.len == 0) null else try arena.dupe(u8, config.empty_dirs), .encoding_glob => if (config.encoding_glob.len == 0) null else try arena.dupe(u8, config.encoding_glob), .dotfiles => if (config.dotfiles.len == 0) null else try arena.dupe(u8, config.dotfiles), .mimetypes => if (config.mimetypes.len == 0) null else try arena.dupe(u8, config.mimetypes), .attachment_size_limit => if (config.attachment_size_limit.len == 0) null else try arena.dupe(u8, config.attachment_size_limit), }; } /// Open this checkout's store, read its config ONCE, and hand back the owned /// result; null on any failure, exactly as every reader of an optional /// repository value degrades today. `.read_only`, because the listing verbs /// are pinned tree-untouching (B-extras-16) and must never mint a sidecar. /// The `OwnedConfig` owns its backing bytes, so the log and its file are /// closed here and the config outlives them. pub fn readRepositoryConfig( gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, ) ?logstore.OwnedConfig { var file = cli.openStoreRead(io, root_dir) catch return null; defer file.close(io); const st = file.stat(io) catch return null; var fs = zit.storage.FileStorage.init(io, file, st.size); // Indexed open: this config read rides every extras/clean/ // addremove listing, each of which was paying its own full header scan. var log = openLogAdoptingIndex(gpa, io, root_dir, fs.storageFile(), .read_only) catch return null; defer log.deinit(gpa); return log.readConfig(gpa) catch null; } /// Resolve one setting from a config the caller already holds. The whole job /// is to make "resolve a name from the frame in my hand" one call, so a verb /// that reads `Config` once can answer for every name it needs. /// /// The error set is written out rather than inferred: `resolveSettingWith`'s /// own failures are all `catch return null`, so `configValue`'s arena dupe is /// the only thing left that can fail. The day a read there stops degrading, /// the compiler names it at this boundary. pub fn localSetting( io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, amb: cli.Ambient, config: ?*const logstore.Config, setting: zit.settings.Setting, ) error{OutOfMemory}!zit.settings.Resolved { const repository = if (config) |c| try configValue(arena, c, setting) else null; return resolveSettingWith(io, arena, root_dir, amb, setting, repository); } /// The operator-scope answer, or null when there is no global file or it says /// nothing about this setting. A missing file is never an error: no global /// tier is the state of every installation that never set one. fn globalSetting( io: std.Io, arena: std.mem.Allocator, amb: cli.Ambient, setting: zit.settings.Setting, ) !?[]const u8 { const zset = zit.settings; var path_buf: [1024]u8 = undefined; const g = zset.globalPath(amb.environ, &path_buf) orelse return null; const text = std.Io.Dir.cwd().readFileAlloc(io, g.path, arena, .limited(zset.max_versioned_bytes)) catch return null; const found = zset.globalLookup(text, setting) orelse return null; return found; } /// The local settings `zit settings` edits, as one table rather than an /// if-chain per name. Every one of them is repo-LOCAL and never crosses the /// wire: `autosync` and `ignore-glob` because they describe this working /// copy's habits, `clearsign` and `pgp-command` because fossil classifies the /// latter SENSITIVE and puts the former in no config group. The settings /// that DO propagate are a separate mechanism — versioned `.zit-settings/` /// files and the sync §9.3 config groups — not this verb. const LocalSetting = enum { autosync, ignore_glob, clearsign, pgp_command, const known_names = "autosync, ignore-glob, clearsign, pgp-command"; const Kind = enum { boolean, string }; fn wireName(self: LocalSetting) []const u8 { return switch (self) { .autosync => "autosync", .ignore_glob => "ignore-glob", .clearsign => "clearsign", .pgp_command => "pgp-command", }; } fn byName(name: []const u8) ?LocalSetting { inline for (comptime std.enums.values(LocalSetting)) |s| { if (std.mem.eql(u8, name, s.wireName())) return s; } return null; } fn kind(self: LocalSetting) Kind { return switch (self) { .autosync, .clearsign => .boolean, .ignore_glob, .pgp_command => .string, }; } /// This setting's stored value as display text, borrowed from `current`. fn read(self: LocalSetting, current: Settings) []const u8 { return switch (self) { // autosync is stored as the DEVIATION, so the flag reads inverted. .autosync => if (current.autosync_off) "off" else "on", .clearsign => if (current.clearsign) "on" else "off", .ignore_glob => current.ignore_glob, .pgp_command => current.pgp_command, }; } /// The patch that sets this setting to `value`, or null when `value` is /// not one this setting accepts. fn patch(self: LocalSetting, value: []const u8) ?ConfigPatch { switch (self) { .autosync, .clearsign => { const on = if (std.mem.eql(u8, value, "on")) true else if (std.mem.eql(u8, value, "off")) false else return null; // Inverted in storage for autosync (absent = fossil's default, // on); plain for clearsign (absent = fossil's default, off). return switch (self) { .autosync => .{ .autosync_off = !on }, .clearsign => .{ .clearsign = on }, else => unreachable, }; }, .ignore_glob, .pgp_command => { // The config frame caps every string field; refusing beats // truncating, which would silently change what runs or what // is ignored. if (value.len > logstore.max_config_str) return null; return switch (self) { .ignore_glob => .{ .ignore_glob = value }, .pgp_command => .{ .pgp_command = value }, else => unreachable, }; }, } } fn reportBadValue(self: LocalSetting, streams: cli.Streams, value: []const u8) void { switch (self.kind()) { .boolean => streams.err.print( "zit settings: {s} takes 'on' or 'off'\ntry 'zit settings --help'\n", .{self.wireName()}, ) catch {}, .string => { _ = value; if (self == .ignore_glob) { streams.err.print( "zit settings: ignore-glob is limited to {d} bytes; use the versioned {s} file for longer lists\n", .{ logstore.max_config_str, zit.ignore.versioned_path }, ) catch {}; } else { streams.err.print( "zit settings: {s} is limited to {d} bytes\n", .{ self.wireName(), logstore.max_config_str }, ) catch {}; } }, } } fn print(self: LocalSetting, streams: cli.Streams, value: []const u8) void { streams.out.print("{s} {s}\n", .{ self.wireName(), value }) catch {}; } }; /// Every local setting's value, read in ONE store open. Four separate readers /// would open, scan and close the log four times for a bare `zit settings`. const Settings = struct { autosync_off: bool = false, clearsign: bool = false, ignore_glob: []const u8 = "", pgp_command: []const u8 = "", }; /// Backing store for the two string settings a `Settings` borrows: the config /// frame's own buffer dies with the read, so the values are copied out. const SettingsBuf = [2 * logstore.max_config_str]u8; fn readSettings( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, buf: *SettingsBuf, ) !Settings { var file = cli.openStoreRead(io, dir) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return .{}; defer oc.deinit(gpa); const glob = oc.config.ignore_glob; const pgp = oc.config.pgp_command; std.debug.assert(glob.len + pgp.len <= buf.len); @memcpy(buf[0..glob.len], glob); @memcpy(buf[glob.len..][0..pgp.len], pgp); return .{ .autosync_off = oc.config.autosync_off, .clearsign = oc.config.clearsign, .ignore_glob = buf[0..glob.len], .pgp_command = buf[glob.len..][0..pgp.len], }; } /// fossil's posture: when the working tree carries the versioned /// ignore file, the sweeps use IT, and a display of the local setting must /// say so — a value that reads as live while being shadowed is how an /// operator edits the wrong one twice. fn noteVersionedShadow( io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams, setting: zit.settings.Setting, r: zit.settings.Resolved, ) bool { const shadowed = r.shadowed orelse return false; const dir = r.versioned_dir orelse return false; if (noWarnPresent(io, root_dir, setting)) return false; streams.err.print( "note: {s}/{s} overrides the {s} setting in the {s} tier; create {s}/{s}.no-warn to silence\n", .{ dir.dirName(), setting.name(), setting.name(), shadowed.label(), dir.dirName(), setting.name() }, ) catch {}; return true; } /// Whether either settings directory holds `<name>.no-warn`. /// /// EITHER silences the note, whichever directory holds the winning file: /// fossil has one directory and cannot pin this, and zit picks the permissive /// rule because the two directories are one setting with two spellings -- a /// silence the operator asked for should not evaporate because a later commit /// chose the other spelling. fn noWarnPresent(io: std.Io, root_dir: std.Io.Dir, setting: zit.settings.Setting) bool { inline for (comptime std.enums.values(zit.settings.VersionedDir)) |d| { var buf: [160]u8 = undefined; if (std.fmt.bufPrint(&buf, "{s}/{s}.no-warn", .{ d.dirName(), setting.name() })) |path| { if (root_dir.statFile(io, path, .{ .follow_symlinks = false })) |_| return true else |_| {} } else |_| {} } return false; } /// One line for a versioned file that cannot supply its /// setting. `autosync` describes this machine's relationship to the network, /// so no tracked file may set it; fossil ignores such a file silently, and zit /// says so once rather than leaving the operator to wonder why their committed /// value does nothing. fn noteVersionedIgnored( io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams, setting: zit.settings.Setting, ) void { std.debug.assert(!setting.versionable()); inline for (comptime std.enums.values(zit.settings.VersionedDir)) |d| { var buf: [160]u8 = undefined; if (std.fmt.bufPrint(&buf, "{s}/{s}", .{ d.dirName(), setting.name() })) |path| { if (root_dir.statFile(io, path, .{ .follow_symlinks = false })) |_| { // Two reasons a file cannot supply a setting, and the note says // which: network posture belongs to the machine, and an // executable-tier value a clone could set would be remote code // execution. const why: []const u8 = switch (setting.tier()) { .local => "a local setting", .executable => "never supplied by a tracked file", .data, .consented => unreachable, // versionable; not this path }; streams.err.print( "note: {s} is versioned and ignored; {s} is {s}\n", .{ path, setting.name(), why }, ) catch {}; } else |_| {} } else |_| {} } } /// `.zitignore` is not a settings file -- it is a second DIALECT that applies /// alongside whatever `ignore-glob` resolves to, so its presence is /// worth one line but is not a shadowing. fn noteGitignorePresent(io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams) void { if (root_dir.statFile(io, zit.ignore.gitignore_path, .{ .follow_symlinks = false })) |_| { streams.err.print( "note: {s} is present; its patterns apply in addition to the ignore-glob setting\n", .{zit.ignore.gitignore_path}, ) catch {}; } else |_| {} } /// The autosync hook: a best-effort pull (before commit/update/merge) /// or push (after a successful commit) against the saved remote. Silent no-op /// when autosync is off or no remote is saved (a nag would punish the /// single-repository workflow); any failure is ONE warning line and the /// caller proceeds — autosync must never turn a working offline operation /// into an error (fossil's posture). The exchange itself is the real verb /// runner, so behavior, diagnostics, and persistence never fork from a /// hand-run `zit pull`/`zit push`. pub const AutosyncOp = enum { pull, push }; pub fn autosyncPull(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, verb: []const u8) void { autosync(gpa, io, arena, streams, verb, .pull, null); } pub fn autosyncPush(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, verb: []const u8, amb: cli.Ambient) void { autosync(gpa, io, arena, streams, verb, .push, amb); } fn autosync( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, streams: cli.Streams, verb: []const u8, op: AutosyncOp, amb: ?cli.Ambient, ) void { { const root = (cli.discoverRoot(arena, io) catch return) orelse return; var root_dir = root.dir; defer root_dir.close(io); // Once per invocation: the pull hook runs before the push one on // every verb that has both, so this is the first hook to fire. if (op == .pull) noteVersionedIgnored(io, root_dir, streams, .autosync); const off = readAutosyncOff(gpa, io, root_dir) catch return; if (off) return; // `autosync pullonly`: pull as usual, never push. if (op == .push) { if (amb) |a| { const r = resolveSetting(gpa, io, arena, root_dir, a, .autosync) catch return; if (std.mem.eql(u8, r.value, "pullonly")) return; } } var url_buf: [logstore.max_config_str]u8 = undefined; _ = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch return; } // The exchange reports on STDERR only: the hosting verb owns stdout (a // commit's contract is ONE check-in line there), and autosync is ambient // narration, not the verb's answer. const side_streams: cli.Streams = .{ .out = streams.err, .err = streams.err }; const rc = switch (op) { .pull => runPull(gpa, io, arena, &.{}, side_streams, amb orelse .{ .now_epoch = 0, .user_env = null }), .push => runPush(gpa, io, arena, &.{}, side_streams, amb.?), }; if (rc != .ok) { streams.err.print( "zit {s}: autosync {s} failed; continuing locally\n", .{ verb, @tagName(op) }, ) catch {}; } } /// The stored autosync deviation, read off the config frame. Storage faults /// propagate; a repository with no config frame reads as the default (on). fn readAutosyncOff(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !bool { const opened = try openStoreLogForRead(gpa, io, dir); defer opened.close(gpa, io); var oc = (opened.log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return false; defer oc.deinit(gpa); return oc.config.autosync_off; } pub fn runRemote( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, repo_override: ?[]const u8, ) cli.Exit { const parsed = switch (cli.parseRemoteArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "remote", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "remote", .extra_argument, tok), .not_a_url => |tok| { streams.err.print("zit remote: '{s}' is not an http:// or https:// URL\nzit remote: no subcommands: 'zit remote' shows the saved remote, 'zit remote URL' sets it, 'zit remote --unset' clears it\n", .{tok}) catch {}; return .usage; }, }; if (parsed.help) { streams.out.writeAll(remote_help) catch {}; return .ok; } const root = (cli.resolveRoot(arena, io, repo_override) catch return stateOom(streams, "remote")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); if (parsed.unset) { // Overlay remote_url with empty — clears the saved value, every other // config field preserved by the field-preserving RMW. updateConfig(gpa, io, root_dir, .{ .remote_url = "" }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), else => return repoReadFail(streams, "remote"), }; streams.out.writeAll("remote cleared\n") catch {}; return .ok; } if (parsed.url.len > 0) { // A URL over the config-field cap cannot be stored (the putConfig assert); // reject it as usage rather than trip the assertion. if (parsed.url.len > logstore.max_config_str) { streams.err.print("zit remote: URL too long (max {d} bytes)\n", .{logstore.max_config_str}) catch {}; return .failure; } updateConfig(gpa, io, root_dir, .{ .remote_url = parsed.url }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), else => return repoReadFail(streams, "remote"), }; streams.out.print("remote set to {s}\n", .{parsed.url}) catch {}; return .ok; } // Bare `zit remote` — print the saved remote (reusing resolveRemoteUrl's read // with an empty arg; NoRemote means none is saved). var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), error.Storage => return repoReadFail(streams, "remote"), error.NoRemote => { streams.out.writeAll("no remote set\n") catch {}; return .ok; }, }; streams.out.print("{s}\n", .{url}) catch {}; return .ok; } test "serve §3.3: a pushed file card is persisted durably into the served repository's log" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A fresh, empty on-disk repository (header only) — the "server" a peer pushes // new content into. Built through the real LogStore, not seeded from fossil. const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.commit(); } // A pushed full-content file card for a 64-hex (SHA3-256 width) artifact the // server lacks (sync §7.1: `file HASH SIZE\n` + payload; §4.1 zlib container). // The name is the artifact's real SHA3-256 (computed by our own hasher, not // fossil) so the server's §7.7 commit-hook recompute accepts it. const push_body = "pushed artifact bytes\n"; // 22 bytes var name_buf: [64]u8 = undefined; const push_hash = zit.artifact.artifactName(.sha3_256, push_body, &name_buf); const request_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ push_hash, push_body.len, push_body }); defer gpa.free(request_plain); const request = try zit.container.encode(gpa, request_plain, .default); defer gpa.free(request); var digest_buf: [digest_len]u8 = undefined; const push_digest = decodeWireName(push_hash, &digest_buf).?; const server = zit.sync.Server{ .version = serve_mod.server_version }; // Answer the push twice through the real `serve.Responder` seam over a SINGLE // open handle (§3.8): the second answer re-reads the current on-disk log from // the same handle and sees the first push's frame, so it is an idempotent // re-push (content is addressed by hash — no dup frame). A stale startup // snapshot would instead re-append the artifact; this handle is never reopened. { var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); for (0..2) |_| { const reply = try serveAnswer(&ctx, gpa, request, null, null, true); gpa.free(reply.body); } } // THE PROOF: re-open the log from disk in a fresh handle; the pushed artifact // is present with its exact content — durably stored, not discarded — and // appears exactly once (the idempotent re-push added no second frame). var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var fs = storage.FileStorage.init(io, file, st.size); var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile()); defer log.deinit(gpa); const rid = log.lookup(push_digest) orelse return error.PushedArtifactMissing; const stored = (try log.content(gpa, rid)).?; defer gpa.free(stored); try std.testing.expectEqualStrings(push_body, stored); const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var artifacts: usize = 0; for (rids) |r| { if (log.kindOf(r) != .config) artifacts += 1; } try std.testing.expectEqual(@as(usize, 1), artifacts); } // --- D4: client verbatim retention + the §7.7 persist hook ------------------ /// Create a fresh, empty (header-only, committed) `.zit` in `dir` so a /// `persistStore` test can persist into it, exactly like a just-cloned repo. fn d4SeedEmptyRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.commit(); } /// Reopen the persisted `.zit` in `dir` for reading. `fs_out` is a caller-owned /// `FileStorage` that MUST outlive the returned `LogStore` (its `storageFile()` /// vtable borrows `fs_out`); the caller closes `fs_out.file` and deinits `log`. fn d4OpenLog(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, fs_out: *storage.FileStorage) !logstore.LogStore { const file = try dir.openFile(io, repo_filename, .{ .mode = .read_write }); const st = try file.stat(io); fs_out.* = storage.FileStorage.init(io, file, st.size); return try logstore.LogStore.open(gpa, fs_out.storageFile(), 0); } fn d4SeedConfiguredRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.putConfig(.{ .hash_policy = 9, .default_branch = "stable", .project_id = "project", .project_code = "init-code", .login_user = "operator", .login_secret_hex = "0" ** 40, .remote_url = "https://example.test/remote", .seen_submanifest = true, .enable_submanifests = false, .submanifest_k = 3, .submanifest_auto_retune = false, .submanifest_settled_cost_shape = [_]u8{0x42} ** 32, }); try seed.commit(); } test "with no --user the resolver reuses the stored login, never prompts, and an empty --user still means anonymous" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedConfiguredRepo(gpa, io, tmp.dir); // stores login "operator" / "0"*40 var sink: [256]u8 = undefined; var w = Io.Writer.fixed(&sink); const s: cli.Streams = .{ .out = &w, .err = &w }; const amb: cli.Ambient = .{ .now_epoch = 0, .user_env = null }; var pw_buf: [max_password_len]u8 = undefined; var user_buf: [logstore.max_config_str]u8 = undefined; // Absent --user: the stored login stands in, verbatim, and nothing is // written to either stream — this is the branch autosync takes, and a // prompt here would block a commit. const r = (try resolvePushCredential(gpa, io, tmp.dir, s, "init-code", .{ .url = "" }, amb, &pw_buf, &user_buf)) orelse return error.AnonymousDespiteStoredLogin; try std.testing.expectEqualStrings("operator", r.credential.user); try std.testing.expectEqualStrings("0" ** 40, &r.credential.secret_hex); try std.testing.expect(!r.persist); try std.testing.expectEqual(@as(usize, 0), w.buffered().len); // An explicit empty --user is the way to push anonymously regardless. try std.testing.expect((try resolvePushCredential(gpa, io, tmp.dir, s, "init-code", .{ .url = "", .user = "" }, amb, &pw_buf, &user_buf)) == null); // Nothing stored — no secret, and separately a secret with no name, which // is not a login either — absent --user is anonymous, and still no prompt. for ([_][2][]const u8{ .{ "", "" }, .{ "", "0" ** 40 } }) |login| { var bare = std.testing.tmpDir(.{}); defer bare.cleanup(); { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try bare.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.putConfig(.{ .hash_policy = 9, .default_branch = "trunk", .project_id = "project", .project_code = "init-code", .login_user = login[0], .login_secret_hex = login[1], .remote_url = "", .seen_submanifest = false, .enable_submanifests = false, .submanifest_k = 0, .submanifest_auto_retune = false, .submanifest_settled_cost_shape = [_]u8{0} ** 32, }); try seed.commit(); } try std.testing.expect((try resolvePushCredential(gpa, io, bare.dir, s, "init-code", .{ .url = "" }, amb, &pw_buf, &user_buf)) == null); try std.testing.expectEqual(@as(usize, 0), w.buffered().len); } } test "sync clone persistence: open-log artifacts and one combined config overlay preserve the wrapper result" { const gpa = std.testing.allocator; const io = std.testing.io; var wrapped_dir = std.testing.tmpDir(.{}); defer wrapped_dir.cleanup(); var open_dir = std.testing.tmpDir(.{}); defer open_dir.cleanup(); try d4SeedConfiguredRepo(gpa, io, wrapped_dir.dir); try d4SeedConfiguredRepo(gpa, io, open_dir.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const content = "one content-bearing clone artifact"; var name_buf: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, content, &name_buf); const container = try zit.container.encode(gpa, content, .default); defer gpa.free(container); const rid = try ms.store().put(name, content); try ms.store().retain(rid, .{ .full = container }); const patch: ConfigPatch = .{ .project_code = "origin-code", .enable_submanifests = true, .submanifest_k = 5, .submanifest_auto_retune = true, }; const wrapped = try persistStore(gpa, io, wrapped_dir.dir, ms.store(), null, 0); try updateConfig(gpa, io, wrapped_dir.dir, patch); var open_fs: storage.FileStorage = undefined; var open_log = try d4OpenLog(gpa, io, open_dir.dir, &open_fs); defer open_fs.file.close(io); defer open_log.deinit(gpa); const opened = try persistStoreOpen(gpa, &open_log, ms.store(), null, 0, null); try updateConfigOpen(gpa, &open_log, patch); try open_log.commit(); try std.testing.expectEqual(wrapped.written, opened.written); try std.testing.expectEqual(wrapped.submanifest_gaps, opened.submanifest_gaps); try std.testing.expectEqual(@as(u64, 3), (try open_log.logKey()).frame_count); var wrapped_fs: storage.FileStorage = undefined; var wrapped_log = try d4OpenLog(gpa, io, wrapped_dir.dir, &wrapped_fs); defer wrapped_fs.file.close(io); defer wrapped_log.deinit(gpa); var wrapped_cfg = (try wrapped_log.readConfig(gpa)).?; defer wrapped_cfg.deinit(gpa); var opened_cfg = (try open_log.readConfig(gpa)).?; defer opened_cfg.deinit(gpa); try std.testing.expectEqualDeep(wrapped_cfg.config, opened_cfg.config); try std.testing.expectEqualStrings("origin-code", opened_cfg.config.project_code); try std.testing.expectEqualStrings("operator", opened_cfg.config.login_user); try std.testing.expectEqualStrings("https://example.test/remote", opened_cfg.config.remote_url); try std.testing.expect(opened_cfg.config.seen_submanifest); try std.testing.expect(opened_cfg.config.enable_submanifests); try std.testing.expectEqual(@as(u8, 5), opened_cfg.config.submanifest_k); try std.testing.expect(opened_cfg.config.submanifest_auto_retune); try std.testing.expectEqualDeep([_]u8{0x42} ** 32, opened_cfg.config.submanifest_settled_cost_shape.?); var digest_buf: [digest_len]u8 = undefined; const stored = (try open_log.content(gpa, open_log.lookup(decodeWireName(name, &digest_buf).?).?)).?; defer gpa.free(stored); try std.testing.expectEqualStrings(content, stored); } test "sync clone config: absent origin settings append no frame and preserve init values" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedConfiguredRepo(gpa, io, tmp.dir); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); const patch: ConfigPatch = .{}; try std.testing.expect(!patch.any()); if (patch.any()) try updateConfigOpen(gpa, &log, patch); try log.commit(); try std.testing.expectEqual(@as(u64, 1), (try log.logKey()).frame_count); var cfg = (try log.readConfig(gpa)).?; defer cfg.deinit(gpa); try std.testing.expectEqual(@as(u8, 9), cfg.config.hash_policy); try std.testing.expectEqualStrings("stable", cfg.config.default_branch); try std.testing.expectEqualStrings("project", cfg.config.project_id); try std.testing.expectEqualStrings("init-code", cfg.config.project_code); try std.testing.expectEqualStrings("operator", cfg.config.login_user); try std.testing.expectEqualStrings("https://example.test/remote", cfg.config.remote_url); try std.testing.expect(cfg.config.seen_submanifest); try std.testing.expect(!cfg.config.enable_submanifests); try std.testing.expectEqual(@as(u8, 3), cfg.config.submanifest_k); try std.testing.expect(!cfg.config.submanifest_auto_retune); try std.testing.expectEqualDeep([_]u8{0x42} ** 32, cfg.config.submanifest_settled_cost_shape.?); } test "streaming persist writes the byte-identical log and releases every persisted entry" { const gpa = std.testing.allocator; const io = std.testing.io; // Two stores fed the same reverse-delta chain (older-as-delta-of-newer, // fossil's wire shape), one persisted batched and one streamed in // increments with release. The logs must match byte for byte — streaming // reorders nothing — and the streamed store must end empty of payload: // every entry held, none resident. const links = 6; var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| gpa.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(gpa, "streamed chain body: one long shared prefix kept across every revision -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } const seed = struct { fn one(a: std.mem.Allocator, s: repostate.ArtifactStore, i: usize, bs: *const [links][]u8, ns: *const [links][64]u8, nl: *const [links]usize) !void { const rid = try s.put(ns[i][0..nl[i]], bs[i]); if (i == links - 1) { const c = try zit.container.encode(a, bs[i], .default); defer a.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(a, bs[i + 1], bs[i]); defer a.free(dr); const c = try zit.container.encode(a, dr, .default); defer a.free(c); try s.retain(rid, .{ .delta = .{ .src = ns[i + 1][0..nl[i + 1]], .container = c, .target_size = bs[i].len } }); } } }.one; // Batched reference. var tmp_a = std.testing.tmpDir(.{}); defer tmp_a.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_a.dir); var ms_a = repostate.MemStore.init(gpa); defer ms_a.deinit(); for (0..links) |i| try seed(gpa, ms_a.store(), i, &bodies, &names, &nlen); { var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); const sum = try persistStoreOpen(gpa, &log, ms_a.store(), null, 0, null); try std.testing.expectEqual(@as(usize, links), sum.written); try log.commit(); } // Streamed: seed in two "rounds", draining after each, releasing as it goes. var tmp_b = std.testing.tmpDir(.{}); defer tmp_b.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_b.dir); var ms_b = repostate.MemStore.init(gpa); defer ms_b.deinit(); { var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp_b.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var pass = try PersistPass.init(gpa, &log, ms_b.store(), null, &ms_b, 0, null); defer pass.deinit(); for (0..3) |i| try seed(gpa, ms_b.store(), i, &bodies, &names, &nlen); try pass.drainNew(); // rids 1..3 are deltas of unseen bases: all defer for (3..links) |i| try seed(gpa, ms_b.store(), i, &bodies, &names, &nlen); try pass.drainNew(); // the full form lands and the whole chain drains const sum = try pass.finish(); try std.testing.expectEqual(@as(usize, links), sum.written); try std.testing.expectEqual(@as(usize, 0), sum.submanifest_gaps); try log.commit(); } // Every streamed entry is now frame-backed and released: held, no bytes. for (1..links + 1) |rid_us| { const rid: repostate.Rid = @intCast(rid_us); try std.testing.expect(ms_b.store().held(rid)); try std.testing.expect(ms_b.store().wire(rid) == null); try std.testing.expect(ms_b.store().content(rid) == null); } // The two logs are byte-identical: streaming changed when frames were // written, never what or in what order. const a_bytes = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 20)); defer gpa.free(a_bytes); const b_bytes = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 20)); defer gpa.free(b_bytes); try std.testing.expectEqualSlices(u8, a_bytes, b_bytes); } test "a basis interned as a phantom before the cursor passed it is revisited and persisted" { const gpa = std.testing.allocator; const io = std.testing.io; // The fossil-peer streaming shape: a delta cfile arrives naming an unseen // DELTASRC, which interns the basis as a PHANTOM at a low rid; the basis // content lands at that same rid in a later round, behind the drain's // cursor. The pass must come back for it — a drain that only moves forward // leaves the basis unwritten and its waiters unresolvable. var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s_store = ms.store(); const base_body = "phantom-first basis body: the long shared prefix every revision keeps\n"; const delta_body = "phantom-first basis body: the long shared prefix every revision keeps, edited\n"; var base_name: [64]u8 = undefined; const bn = zit.artifact.artifactName(.sha3_256, base_body, &base_name).len; var delta_name: [64]u8 = undefined; const dn = zit.artifact.artifactName(.sha3_256, delta_body, &delta_name).len; // Round 1: the basis is interned as a phantom (rid 1), then the delta // arrives as content (rid 2). const base_rid = try s_store.intern(base_name[0..bn]); const delta_rid = try s_store.put(delta_name[0..dn], delta_body); { const dr = try zit.delta.create(gpa, base_body, delta_body); defer gpa.free(dr); const c = try zit.container.encode(gpa, dr, .default); defer gpa.free(c); try s_store.retain(delta_rid, .{ .delta = .{ .src = base_name[0..bn], .container = c, .target_size = delta_body.len } }); } var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var pass = try PersistPass.init(gpa, &log, s_store, null, &ms, 0, null); defer pass.deinit(); try pass.drainNew(); // cursor passes the phantom; the delta defers on it // Round 2: the basis content lands at the SAME rid, behind the cursor. { const got = try s_store.put(base_name[0..bn], base_body); try std.testing.expectEqual(base_rid, got); const c = try zit.container.encode(gpa, base_body, .default); defer gpa.free(c); try s_store.retain(base_rid, .{ .full = c }); } try pass.drainNew(); // the revisit sweep drives it, unblocking the delta const sum = try pass.finish(); try std.testing.expectEqual(@as(usize, 2), sum.written); try log.commit(); // Both artifacts are durable frames, and both entries were released. var d: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(base_name[0..bn], &d).?) != null); try std.testing.expect(log.lookup(decodeWireName(delta_name[0..dn], &d).?) != null); try std.testing.expect(s_store.wire(base_rid) == null and s_store.held(base_rid)); try std.testing.expect(s_store.wire(delta_rid) == null and s_store.held(delta_rid)); } test "sync persistStoreOpen: malformed artifact retains the wrapper's corrupt error and writes no frame" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const content = "content that the wire name does not actually name\n"; const lying_name = "0" ** 64; const rid = try ms.store().put(lying_name, content); const container = try zit.container.encode(gpa, content, .default); defer gpa.free(container); try ms.store().retain(rid, .{ .full = container }); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); try std.testing.expectError(error.Corrupt, persistStoreOpen(gpa, &log, ms.store(), null, 0, null)); try log.commit(); var digest_buf: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null); } test "sync §7.7: persistStore recomputes the artifact hash and aborts on a lying container, writing no frame" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A received full cfile whose wire NAME does not name its content (§7.7 — // a clone skips the transfer-layer pre-storage check, so a lying container // reaches the persist commit hook). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const content = "content that the wire name does not actually name\n"; const lying_name = "0" ** 64; // not SHA3-256(content) const rid = try s.put(lying_name, content); const container = try zit.container.encode(gpa, content, .default); defer gpa.free(container); try s.retain(rid, .{ .full = container }); // §7.7 L1469-1471 — recompute and abort before ANY frame is written. try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, ms.store(), null, 0)); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var digest_buf: [digest_len]u8 = undefined; const digest = decodeWireName(lying_name, &digest_buf).?; try std.testing.expect(log.lookup(digest) == null); // no frame landed } test "sync §7.2/§4.1: persistStore stores received full and delta wire forms verbatim; a re-serve preserves the delta container byte-for-byte" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Model a clone reply in §8.2 RID order: a full base cfile then a delta // cfile against it. Names are the real SHA3-256 so the §7.7 hook accepts. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const base = "the base artifact with a long shared run so the delta stays a compact copy payload distinct in size from the reconstructed target"; const target = base ++ " plus a small trailing mutation"; var bn: [64]u8 = undefined; var tn: [64]u8 = undefined; const base_name = zit.artifact.artifactName(.sha3_256, base, &bn); const target_name = zit.artifact.artifactName(.sha3_256, target, &tn); const base_container = try zit.container.encode(gpa, base, .default); defer gpa.free(base_container); const base_rid = try s.put(base_name, base); try s.retain(base_rid, .{ .full = base_container }); const raw = try zit.delta.create(gpa, base, target); defer gpa.free(raw); const delta_container = try zit.container.encode(gpa, raw, .default); defer gpa.free(delta_container); const ts = try zit.delta.targetSize(raw); const delta_rid = try s.put(target_name, target); try s.retain(delta_rid, .{ .delta = .{ .src = base_name, .container = delta_container, .target_size = ts } }); const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 2), written.written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The target persisted as a `.delta` frame that reconstructs to the original. var tdig: [digest_len]u8 = undefined; const target_digest = decodeWireName(target_name, &tdig).?; const trid = log.lookup(target_digest).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(trid)); const got = (try log.content(gpa, trid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(target, got); // Clone-of-clone: a re-serve snapshot emits the SAME delta container bytes // (§7.2 retransmitted verbatim) — the compactness survives the second hop. var serve_ms = repostate.MemStore.init(gpa); defer serve_ms.deinit(); try loadContentStore(gpa, &log, &serve_ms); const serve_rid = serve_ms.store().lookup(target_name).?; const wf = serve_ms.store().wire(serve_rid).?; try std.testing.expect(wf.delta != null); try std.testing.expectEqualSlices(u8, delta_container, wf.container); try std.testing.expectEqual(@as(u64, target.len), wf.delta.?.target_size); } /// A tree manifest naming `subs` (submanifests §3.6), plus the §6.2 Z card, so /// the fixture is a well-formed card sequence rather than a bare S run. fn treeManifestBytes(gpa: std.mem.Allocator, subs: []const []const u8) ![]u8 { var body: std.ArrayListUnmanaged(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, "C tree\nD 2026-07-27T12:00:00.000\n"); // §3.4 — the S run ascends by filename; `a`, `b`, ... keeps it legal. for (subs, 0..) |h, i| { try body.print(gpa, "S {c}.txt {s}\n", .{ @as(u8, 'a') + @as(u8, @intCast(i)), h }); } try body.appendSlice(gpa, "U u\n"); var digest: [16]u8 = undefined; std.crypto.hash.Md5.hash(body.items, &digest, .{}); return std.fmt.allocPrint(gpa, "{s}Z {x}\n", .{ body.items, &digest }); } test "submanifests §7.1: the persist commit hook phantomizes every absent S reference" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Two leaves that were never received — the state a clone from a peer whose // OWN closure is incomplete lands in. Nothing on the receive path can see // this: a clone stores wire-backed and inflates nothing. const leaf_a = "F a.txt " ++ ("1" ** 64) ++ "\n"; const leaf_b = "F b.txt " ++ ("2" ** 64) ++ "\n"; var an: [64]u8 = undefined; var bn: [64]u8 = undefined; const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an); const name_b = zit.artifact.artifactName(.sha3_256, leaf_b, &bn); const tree = try treeManifestBytes(gpa, &.{ name_a, name_b }); defer gpa.free(tree); var tn: [64]u8 = undefined; const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const rid = try s.put(tree_name, tree); const container = try zit.container.encode(gpa, tree, .default); defer gpa.free(container); try s.retain(rid, .{ .full = container }); // §7.1: "Storing a structural artifact that carries S cards whose referenced // sub-manifests are absent MUST create a phantom for each missing reference." const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), summary.written); try std.testing.expectEqual(@as(usize, 2), summary.submanifest_gaps); } test "persist scoped past the holdings writes nothing and re-walks only received" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); // Five full-form blobs (the already-durable holdings), named by true hash. inline for (.{ "alpha", "bravo", "charlie", "delta", "echo" }) |body| { var nm: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, body, &nm); const rid = try s.put(name, body); const c = try zit.container.encode(gpa, body, .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } const first = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 5), first.written); // A pull that received nothing: scope past all five holdings. The walk // never begins, so nothing is persisted and nothing is re-visited (this // was O(store) — every holding read once from disk to be discarded). const noop = try persistStore(gpa, io, tmp.dir, ms.store(), null, 5); try std.testing.expectEqual(@as(usize, 0), noop.written); // Three artifacts arrive (rids 6..8); scope past the original five. Only the // new three are persisted — a mirror's incremental pull no longer re-walks // the millions it already holds. inline for (.{ "foxtrot", "golf", "hotel" }) |body| { var nm: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, body, &nm); const rid = try s.put(name, body); const c = try zit.container.encode(gpa, body, .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } const incr = try persistStore(gpa, io, tmp.dir, ms.store(), null, 5); try std.testing.expectEqual(@as(usize, 3), incr.written); // A full re-persist (start 0) early-returns every artifact and reaches the // same eight-artifact total, so scoping removed work, never content. const full = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 8), full.written); } test "submanifests §7.1/§8.1: the S-reference gap survives a RESTART, rebuilt from the durable indicator" { // §7.1's phantom is an interned entry with no frame, "exactly as an absent // delta source creates a source phantom" — and `LogStore.intern` states the // consequence outright: "Phantoms never reach disk." So a restart loses // them, and the only thing that can make them correct across one is a // REBUILD from the durable content-bearing frames. // // This test is that invariant: run 1 mints the gap, the process ends, run 2 // re-derives the identical gap over a store loaded from `.zit` alone. Remove // the rebuild (`rescan_persisted`) and run 2 reports ZERO gaps — a // repository that quietly declares an unenumerable check-in converged. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); const leaf_a = "F a.txt " ++ ("1" ** 64) ++ "\n"; const leaf_b = "F b.txt " ++ ("2" ** 64) ++ "\n"; var an: [64]u8 = undefined; var bn: [64]u8 = undefined; const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an); const name_b = zit.artifact.artifactName(.sha3_256, leaf_b, &bn); const tree = try treeManifestBytes(gpa, &.{ name_a, name_b }); defer gpa.free(tree); var tn: [64]u8 = undefined; const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn); // RUN 1 — the tree lands; neither sub-manifest is held. { var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(tree_name, tree); const first = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), first.written); try std.testing.expectEqual(@as(usize, 2), first.submanifest_gaps); } // RESTART — nothing survives but the bytes in `.zit`. var ms2 = repostate.MemStore.init(gpa); defer ms2.deinit(); { var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // §8.1 — the indicator IS durable: "record a local seen-submanifest // indicator". It is the one byte the rebuild rests on. var oc = (try log.readConfig(gpa)) orelse return error.NoConfig; defer oc.deinit(gpa); try std.testing.expect(oc.config.seen_submanifest); // …and the phantoms are NOT durable, asserted rather than assumed — // this is the premise the whole rebuild exists to answer. var adig: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(name_a, &adig).?) == null); // The snapshot a fresh process starts every verb from. try loadContentStore(gpa, &log, &ms2); } // Only the tree came back: the store a second run works from holds no trace // of the gap. try std.testing.expectEqual(@as(usize, 1), ms2.store().count()); // RUN 2 — the gap is re-derived, identically, from the durable frames. const second = try persistStore(gpa, io, tmp.dir, ms2.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), second.written); try std.testing.expectEqual(@as(usize, 2), second.submanifest_gaps); } test "submanifests §8.1: a repository predating this draft records no indicator and rescans nothing" { // The compatibility half of the rebuild: the indicator is false, so the // rescan is unreachable and a second persist over the same store is exactly // the sweep it always was — no config frame appended, no gap invented. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); const flat = "C msg\nD 2026-07-27T12:00:00.000\nF a.txt " ++ ("1" ** 64) ++ "\nU u\n"; var fn_buf: [64]u8 = undefined; const flat_name = zit.artifact.artifactName(.sha3_256, flat, &fn_buf); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(flat_name, flat); _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); const again = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 0), again.submanifest_gaps); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // No indicator was written at all: `d4SeedEmptyRepo` stores no config, and // nothing above gave it a reason to start one. try std.testing.expect((try log.readConfig(gpa)) == null); } test "submanifests §7.1: a persisted tree whose sub-manifests are held reports no gap" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); const leaf_a = "F a.txt " ++ ("1" ** 64) ++ "\n"; var an: [64]u8 = undefined; const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an); const tree = try treeManifestBytes(gpa, &.{name_a}); defer gpa.free(tree); var tn: [64]u8 = undefined; const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); // The sub-manifest is stored at a LOWER rid than the tree in one direction // and a higher one in the other; the gap verdict must not depend on which, // which is why it is taken once, after the whole sweep. _ = try s.put(tree_name, tree); _ = try s.put(name_a, leaf_a); const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 2), summary.written); try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps); // The §7.1 phantom was minted for the leaf BEFORE the leaf's own frame was // written (the tree persists first), so this asserts the phantom-upgrade // path storage actually takes: the frame must read back byte-for-byte, not // merely exist. A §7.1 duty that corrupted a frame would be worse than an // unmet one. var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var ldig: [digest_len]u8 = undefined; const leaf_digest = decodeWireName(name_a, &ldig).?; const lrid = log.lookup(leaf_digest).?; try std.testing.expect(!log.isPhantom(lrid)); const got = (try log.content(gpa, lrid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(leaf_a, got); } test "submanifests §7.1: an artifact predating this draft costs the persist hook no gap and no phantom" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // The backwards-compatibility guarantee as a test: a flat manifest carries // no S card, so §7.1's duty over it is empty and the persisted bytes and // frame count are exactly what they were before this rule existed. const flat = "C msg\nD 2026-07-27T12:00:00.000\nF a.txt " ++ ("1" ** 64) ++ "\nU u\n"; var fn_buf: [64]u8 = undefined; const flat_name = zit.artifact.artifactName(.sha3_256, flat, &fn_buf); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(flat_name, flat); const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), summary.written); try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var artifacts: usize = 0; for (rids) |r| { if (log.kindOf(r) != .config) artifacts += 1; } try std.testing.expectEqual(@as(usize, 1), artifacts); } test "submanifests §7.1: a malformed S run mints no phantom at the persist hook" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // §3.2: a third token on S is rejected. Validate-then-apply means the FIRST // (well-formed) S card mints nothing either — a half-read artifact never // leaves a half-minted phantom set behind. const bad = "C tree\nS a.txt " ++ ("1" ** 64) ++ "\nS b.txt " ++ ("2" ** 64) ++ " extra\nU u\n"; var bn: [64]u8 = undefined; const bad_name = zit.artifact.artifactName(.sha3_256, bad, &bn); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(bad_name, bad); const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), summary.written); try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps); } test "sync §3.3: persistStore falls back to full form when a retained delta's basis is absent, never a phantom-based delta" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Only the delta is present; its basis was never received/stored (a legal // §3.3 outcome that must not persist a delta over a phantom base). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const base = "an absent basis whose long shared run would make a compact delta if it were here at all"; const target = base ++ " mutated once"; var bn: [64]u8 = undefined; var tn: [64]u8 = undefined; const base_name = zit.artifact.artifactName(.sha3_256, base, &bn); const target_name = zit.artifact.artifactName(.sha3_256, target, &tn); const raw = try zit.delta.create(gpa, base, target); defer gpa.free(raw); const dc = try zit.container.encode(gpa, raw, .default); defer gpa.free(dc); const ts = try zit.delta.targetSize(raw); const rid = try s.put(target_name, target); try s.retain(rid, .{ .delta = .{ .src = base_name, .container = dc, .target_size = ts } }); const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 1), written.written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The target persisted as a FULL (.zlib) frame — the fallback, not a delta. var tdig: [digest_len]u8 = undefined; const target_digest = decodeWireName(target_name, &tdig).?; const trid = log.lookup(target_digest).?; try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(trid)); const got = (try log.content(gpa, trid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(target, got); // No phantom basis frame was minted for the absent DELTASRC. var bdig: [digest_len]u8 = undefined; const base_digest = decodeWireName(base_name, &bdig).?; try std.testing.expect(log.lookup(base_digest) == null); } test "sync §7.7/§3.3: persistStore stores a retained delta chain up to the read bound and falls back to full form beyond it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const cap: usize = logstore.max_delta_chain; // the READ bound (32) const n: usize = cap + 2; // links 0..cap+1: link cap+1's basis sits at depth cap // A chain grown one byte per link so each delta is tiny. Link i's basis is // link i-1 (§8.2 RID order); the persist path stores a `.delta` iff the base // depth is under `max_delta_chain`, so link cap+1 falls back to full form. var content: std.ArrayList(u8) = .empty; defer content.deinit(gpa); try content.appendSlice(gpa, "chain-root-artifact-with-a-long-shared-run-for-compact-forward-deltas"); var prev_name: [64]u8 = undefined; var prev_name_len: usize = 0; var deep_name: [64]u8 = undefined; // link cap (deepest stored delta) var deep_len: usize = 0; var last_name: [64]u8 = undefined; // link cap+1 (the fallback) var last_len: usize = 0; var i: usize = 0; while (i < n) : (i += 1) { try content.append(gpa, 'a' + @as(u8, @intCast(i % 26))); const cur = content.items; const prev = cur[0 .. cur.len - 1]; var nbuf: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, cur, &nbuf); const rid = try s.put(name, cur); if (i == 0) { const c = try zit.container.encode(gpa, cur, .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const raw = try zit.delta.create(gpa, prev, cur); defer gpa.free(raw); const c = try zit.container.encode(gpa, raw, .default); defer gpa.free(c); const tsz = try zit.delta.targetSize(raw); try s.retain(rid, .{ .delta = .{ .src = prev_name[0..prev_name_len], .container = c, .target_size = tsz } }); } @memcpy(prev_name[0..name.len], name); prev_name_len = name.len; if (i == cap) { @memcpy(deep_name[0..name.len], name); deep_len = name.len; } if (i == n - 1) { @memcpy(last_name[0..name.len], name); last_len = name.len; } } _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // Link cap is the deepest `.delta` the write bound allows; link cap+1 fell // back to a full `.zlib` frame yet still reconstructs (never a Corrupt read). var ddig: [digest_len]u8 = undefined; const deep_rid = log.lookup(decodeWireName(deep_name[0..deep_len], &ddig).?).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(deep_rid)); var ldig: [digest_len]u8 = undefined; const last_rid = log.lookup(decodeWireName(last_name[0..last_len], &ldig).?).?; try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(last_rid)); const got = (try log.content(gpa, last_rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(content.items, got); } test "sync §7.2/§7.4/§3.3: a dangling reverse-delta cfile persists as a verbatim .delta frame that re-serves byte-identically" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Fossil's clone stream: a reverse delta (the OLDER revision stored as a // delta of the NEWER one) arrives BEFORE its basis, so it is dangling. The // client retains the §4.1 container through the pending queue and, when the // basis lands, persists it as a compact `.delta` — not a full re-compression. const newer = "artifact NEWER: a long shared body kept verbatim so the reverse delta reconstructing the older revision stays a compact copy op"; const older = "artifact NEWER: a long shared body kept verbatim so the reverse delta reconstructing the older revision stays a compact copy op minus tail"; var nn: [64]u8 = undefined; var on: [64]u8 = undefined; const newer_name = zit.artifact.artifactName(.sha3_256, newer, &nn); const older_name = zit.artifact.artifactName(.sha3_256, older, &on); const newer_container = try zit.container.encode(gpa, newer, .default); defer gpa.free(newer_container); const raw = try zit.delta.create(gpa, newer, older); // OLDER = delta ∘ NEWER defer gpa.free(raw); const delta_container = try zit.container.encode(gpa, raw, .default); defer gpa.free(delta_container); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var client = zit.sync.Client{ .operation = .clone, .state = .clone_round0 }; defer client.deinit(gpa); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, "clone_seqno 0\npush 1111111111111111 aa87ea41b0bb05ba528f7a3bdba700d7e23d6318\n"); // Reverse (dangling) order: the delta cfile precedes its basis cfile. const dh = try std.fmt.allocPrint(gpa, "cfile {s} {s} {d} {d}\n", .{ older_name, newer_name, older.len, delta_container.len }); defer gpa.free(dh); try body.appendSlice(gpa, dh); try body.appendSlice(gpa, delta_container); try body.append(gpa, '\n'); const bh = try std.fmt.allocPrint(gpa, "cfile {s} {d} {d}\n", .{ newer_name, newer.len, newer_container.len }); defer gpa.free(bh); try body.appendSlice(gpa, bh); try body.appendSlice(gpa, newer_container); try body.append(gpa, '\n'); _ = try client.consumeReply(gpa, ms.store(), body.items); const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); try std.testing.expectEqual(@as(usize, 2), written.written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // §3.3 — the older revision persisted as a `.delta`, not a full re-compression, // and reconstructs the original bytes (delta ∘ basis). var od: [digest_len]u8 = undefined; const older_rid = log.lookup(decodeWireName(older_name, &od).?).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(older_rid)); const got = (try log.content(gpa, older_rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(older, got); // §7.2 — a clone-of-clone re-serve re-emits the SAME delta container byte-for-byte. var serve_ms = repostate.MemStore.init(gpa); defer serve_ms.deinit(); try loadContentStore(gpa, &log, &serve_ms); const srid = serve_ms.store().lookup(older_name).?; const wf = serve_ms.store().wire(srid).?; try std.testing.expect(wf.delta != null); try std.testing.expectEqualSlices(u8, delta_container, wf.container); try std.testing.expectEqualStrings(newer_name, wf.delta.?.src_hash); } test "sync §7.2/§3.3: persistStore stores a whole reverse-delta chain, ordering every basis frame before the delta that points at it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A reverse-delta chain in fossil's storage shape: revision i is stored as a // delta of the NEWER revision i+1, the newest stored full. The store rids run // oldest..newest (put order), so EVERY delta's basis sits at a HIGHER rid than // the delta — the worst case for physical ordering. persist's worklist must // still write each basis frame before its delta so `delta_of` points backward; // naive rid-order would degrade all but the oldest link to full form. const links = 6; var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| gpa.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(gpa, "chain body: one long shared prefix kept across every revision for compact reverse deltas -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } // put oldest..newest (store rids 1..links); older i is a reverse delta of newer i+1. for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == links - 1) { const c = try zit.container.encode(gpa, bodies[i], .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(gpa, bodies[i + 1], bodies[i]); // older from newer defer gpa.free(dr); const c = try zit.container.encode(gpa, dr, .default); defer gpa.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } }); } } _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The newest link is full; every older link is a `.delta` whose basis frame is // physically earlier (a LOWER log rid = a lower offset), and all reconstruct. for (0..links) |i| { var d: [digest_len]u8 = undefined; const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?; const rgot = (try log.content(gpa, rid)).?; defer gpa.free(rgot); try std.testing.expectEqualStrings(bodies[i], rgot); if (i == links - 1) { try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(rid)); } else { try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(rid)); var bd: [digest_len]u8 = undefined; const brid = log.lookup(decodeWireName(names[i + 1][0..nlen[i + 1]], &bd).?).?; try std.testing.expect(brid < rid); // basis frame precedes the reverse-delta } } } test "sync §7.7: persistStore verifies a retained delta's reconstruction and refuses a lying delta container, writing no frame" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A retained delta whose stored plaintext does NOT hash to its wire name (a // clone skips the transfer-layer pre-storage check, §7.7 r6, so a lying // delta reaches the persist commit hook). The §7.7 recompute runs BEFORE the // wire-form branch, so a delta-form entry is rejected exactly like a full one. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const content = "content that the wire name does not actually name, delivered as a delta\n"; const lying_name = "0" ** 64; // not SHA3-256(content) const some_basis = "1" ** 64; // an absent basis; verify fails before it is even consulted const raw = try zit.delta.create(gpa, "a plausible basis blob", content); defer gpa.free(raw); const dc = try zit.container.encode(gpa, raw, .default); defer gpa.free(dc); const rid = try s.put(lying_name, content); try s.retain(rid, .{ .delta = .{ .src = some_basis, .container = dc, .target_size = content.len } }); try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, ms.store(), null, 0)); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var digest_buf: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null); // no frame landed } fn persistReverseChainNoLeak(allocator: std.mem.Allocator) !void { const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(allocator, io, tmp.dir); // A small reverse-delta chain built on the (failing) allocator so injected // OOM is exercised in both the store build and the persist worklist. var ms = repostate.MemStore.init(allocator); defer ms.deinit(); const s = ms.store(); const links = 4; var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| allocator.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(allocator, "reverse chain oom: one shared body across revisions -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == links - 1) { const c = try zit.container.encode(allocator, bodies[i], .default); defer allocator.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(allocator, bodies[i + 1], bodies[i]); defer allocator.free(dr); const c = try zit.container.encode(allocator, dr, .default); defer allocator.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } }); } } _ = try persistStore(allocator, io, tmp.dir, ms.store(), null, 0); } test "sync §7.2/§3.3: persistStore drains the reverse-delta worklist without leaking under allocation failure" { try std.testing.checkAllAllocationFailures(std.testing.allocator, persistReverseChainNoLeak, .{}); } /// Drives `persistArtifact` over a FORWARD delta chain (each link a delta of the /// immediately older, already-persisted lower-rid link) in rid order, so a test /// owns the shared basis memo and can read its hit counter. Every basis precedes /// its delta, so `allow_defer=false` never defers and no worklist is needed — /// mirrors `persistStore`'s open/commit around a plain rid-order drive. fn persistForwardChain( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, ms: *repostate.MemStore, cache: *logstore.PlaintextCache, ) !void { var file = try cli.openStoreAppend(io, dir); defer file.close(io); const st = try file.stat(io); var fs = storage.FileStorage.init(io, file, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer log.deinit(gpa); const store = ms.store(); const n = store.count(); var srefs: SubManifestRefs = .{}; defer srefs.deinit(gpa); var rid: repostate.Rid = 1; while (rid <= n) : (rid += 1) { switch (try persistArtifact(gpa, ms.store(), &log, rid, false, false, cache, &srefs, false, null)) { .persisted => {}, .deferred => unreachable, // forward chain: no basis is ever absent } } try log.commit(); } test "sync §7.2/§7.4: the basis memo reuses each basis (hits) and reconstructs a byte-identical store to the uncached path" { const gpa = std.testing.allocator; const io = std.testing.io; const links = 8; // A forward delta chain: link 0 is full, every later link is a delta that // reconstructs its body FROM the older body (its basis a lower, already- // persisted rid) — the case where the memo turns each basis reconstruction // into a lookup for the delta built on it. Store owns its copies (retain // dupes), but the bodies/names back the reconstructed-content assertions. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| gpa.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(gpa, "forward chain shared prefix across every revision -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == 0) { const c = try zit.container.encode(gpa, bodies[i], .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(gpa, bodies[i - 1], bodies[i]); // newer body FROM its older basis defer gpa.free(dr); const c = try zit.container.encode(gpa, dr, .default); defer gpa.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i - 1][0..nlen[i - 1]], .container = c, .target_size = bodies[i].len } }); } } // (A) a normal memo; (B) a zero-budget memo that can hold nothing — the exact // uncached reconstruction. Same store, two freshly-seeded repos. var tmp_a = std.testing.tmpDir(.{}); defer tmp_a.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_a.dir); var cache_a = logstore.PlaintextCache.init(); defer cache_a.deinit(gpa); try persistForwardChain(gpa, io, tmp_a.dir, &ms, &cache_a); var tmp_b = std.testing.tmpDir(.{}); defer tmp_b.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_b.dir); var cache_b = logstore.PlaintextCache.init(); cache_b.max_bytes = 0; // never memoises: forces a fresh reconstruction each time defer cache_b.deinit(gpa); try persistForwardChain(gpa, io, tmp_b.dir, &ms, &cache_b); // The memo reused every basis exactly once (one hit per delta link); the // zero-budget run reused nothing — the two ways the same bytes get built. try std.testing.expectEqual(@as(u64, links - 1), cache_a.hits); try std.testing.expectEqual(@as(u64, 0), cache_b.hits); // Byte-identical store: the memo changes only the work, never the bytes. const bytes_a = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(bytes_a); const bytes_b = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(bytes_b); try std.testing.expectEqualSlices(u8, bytes_b, bytes_a); // Every link reconstructs to its exact body (§7.7 chain correctness). var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); for (0..links) |i| { var d: [digest_len]u8 = undefined; const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?; const got = (try log.content(gpa, rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(bodies[i], got); } } test "sync §9.1: repoProjectCode returns the stored code, else the zero placeholder" { const gpa = std.testing.allocator; const io = std.testing.io; const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); const seed = struct { fn withCode(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, code: []const u8) !void { var f = try dir.createFile(i, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(i); var fs = storage.FileStorage.init(i, f, 0); var log = try logstore.LogStore.open(g, fs.storageFile(), hk); defer log.deinit(g); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = code }); try log.commit(); } }; // (a) a repo whose config stores a real 40-hex project code: it is returned // verbatim — the code the pull/push card announces (§9.1), never the zeros // fossil would refuse. var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const code = "a1b2c3d4" ** 5; // 40 hex chars try seed.withCode(gpa, io, tmp.dir, hash_kind, code); var buf: [zit.sync.max_projectcode_len]u8 = undefined; const got = try repoProjectCode(gpa, io, tmp.dir, &buf); try std.testing.expectEqualStrings(code, got); try std.testing.expect(!std.mem.eql(u8, got, sync_projectcode)); // (b) a repo predating the stored code (empty project_code): the placeholder // is the honest fallback (no adopted identity to announce yet). var tmp2 = std.testing.tmpDir(.{}); defer tmp2.cleanup(); try seed.withCode(gpa, io, tmp2.dir, hash_kind, ""); var buf2: [zit.sync.max_projectcode_len]u8 = undefined; const got2 = try repoProjectCode(gpa, io, tmp2.dir, &buf2); try std.testing.expectEqualStrings(sync_projectcode, got2); } test "sync §9.1: the pull request card carries the adopted project code, not the placeholder" { const gpa = std.testing.allocator; // The pull client, constructed exactly as `pullRun` does, with a stored code. const code = "a1b2c3d4" ** 5; // a real (non-zero) 40-hex project code var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = sync_servercode, .project_code = code, .version = .{ .version = sync_client_version }, }; defer client.deinit(gpa); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var buf: std.ArrayListUnmanaged(u8) = .empty; defer buf.deinit(gpa); const request = try client.buildRequest(gpa, ms.store(), &buf); // §9.1 ABNF `pull-card = "pull" SP servercode SP projectcode`: the adopted // code MUST be the card's PROJECTCODE (a real peer refuses a mismatch). const expected = "pull " ++ sync_servercode ++ " " ++ ("a1b2c3d4" ** 5) ++ "\n"; try std.testing.expect(std.mem.indexOf(u8, request, expected) != null); // And the all-zero placeholder is NOT the project code the card announces. const rejected = "pull " ++ sync_servercode ++ " " ++ sync_projectcode ++ "\n"; try std.testing.expect(std.mem.indexOf(u8, request, rejected) == null); } test "serve §3.8: a per-request store-load on one serve handle sees a later append (no restart)" { const gpa = std.testing.allocator; const io = std.testing.io; const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const append = struct { fn artifactFrame(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, digest: []const u8, bytes: []const u8) !void { var f = dir.openFile(i, repo_filename, .{ .mode = .read_write }) catch try dir.createFile(i, repo_filename, .{ .read = true }); defer f.close(i); const st = try f.stat(i); var fs = storage.FileStorage.init(i, f, st.size); var log = try logstore.LogStore.open(g, fs.storageFile(), hk); defer log.deinit(g); _ = try log.put(digest, bytes); try log.commit(); } }; // Seed the repo holding exactly one content artifact (a 32-byte raw digest). const digest_a = "a" ** digest_len; try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_a, "content a\n"); // Open ONE read handle — the handle `serve` holds for its whole lifetime. var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); // First per-request read: one content artifact. var index1 = try LogIndexStore.openFromFile(gpa, io, handle, (try handle.stat(io)).size); defer index1.destroy(); const n1 = contentCount(index1.store()); try std.testing.expectEqual(@as(usize, 1), n1); // A SEPARATE writer appends a second artifact while the handle stays open — // exactly a `zit commit` running alongside a live `zit serve`. const digest_b = "b" ** digest_len; try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_b, "content b\n"); // Second per-request read on the SAME handle sees the new artifact: proof the // server re-reads the current on-disk log per request (§3.8). A startup // snapshot would still report n1 — no restart is needed here. var index2 = try LogIndexStore.openFromFile(gpa, io, handle, (try handle.stat(io)).size); defer index2.destroy(); try std.testing.expectEqual(n1 + 1, contentCount(index2.store())); } /// A test-only `storage.StorageFile` that wraps another and tallies the bytes /// every `pread` actually returned — the instrument for "how many times does a /// prime read the log?". Every other vtable method delegates verbatim, so a log /// opened over it behaves identically while its read traffic is measured. const CountingStorage = struct { inner: storage.StorageFile, read_bytes: usize = 0, fn storageFile(self: *CountingStorage) storage.StorageFile { return .{ .context = self, .vtable = &vtable }; } const vtable = storage.StorageFile.VTable{ .append = appendImpl, .pread = preadImpl, .sync = syncImpl, .size = sizeImpl, .truncate = truncateImpl, }; fn appendImpl(ctx: *anyopaque, bytes: []const u8) storage.StorageError!void { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); return self.inner.append(bytes); } fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); const n = try self.inner.pread(offset, buf); self.read_bytes += n; return n; } fn syncImpl(ctx: *anyopaque) storage.StorageError!void { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); return self.inner.sync(); } fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); return self.inner.size(); } fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void { const self: *CountingStorage = @ptrCast(@alignCast(ctx)); return self.inner.truncate(len); } }; /// Seed a `.zit` under `dir` holding a full+delta+manifest+config mix (via /// `packSeedDeltaStore`) and return its committed byte size — the fixture the /// serve-prime read-traffic and index-equivalence tests both build over. fn seedServeRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !u64 { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer log.deinit(gpa); var tip: [64]u8 = undefined; try packSeedDeltaStore(gpa, &log, &tip); return log.size(); } test "serve prime reads the served log at most once (no second full-container copy)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); var fs = storage.FileStorage.init(io, handle, file_size); var counting = CountingStorage{ .inner = fs.storageFile() }; // Build the serve base snapshot over the counting storage and measure how // many bytes priming read. `openReadOnly`'s scan already whole-frame-CRC'd // every byte once (the warm-start CRC concern); an index prime adds NOTHING on top, so // total prime reads stay at roughly one pass over the log. The copy prime // (`loadContentStore`) read every container a SECOND time — this bound fails // for it, which is exactly the cost the log-offset index removes. var index = try LogIndexStore.open(gpa, counting.storageFile()); defer index.destroy(); try std.testing.expect(counting.read_bytes <= file_size + file_size / 2); } test "log-backed index answers count/hashOf/held/content/wire identically to the MemStore snapshot" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); // Reference: the whole-container MemStore snapshot `loadContentStore` builds. var ref_handle = try tmp.dir.openFile(io, repo_filename, .{}); defer ref_handle.close(io); var ref_fs = storage.FileStorage.init(io, ref_handle, file_size); var ref_log = try logstore.LogStore.openReadOnly(gpa, ref_fs.storageFile()); defer ref_log.deinit(gpa); var ref = repostate.MemStore.init(gpa); defer ref.deinit(); try loadContentStore(gpa, &ref_log, &ref); const rs = ref.store(); // Subject: the log-backed index over the same file. var idx_handle = try tmp.dir.openFile(io, repo_filename, .{}); defer idx_handle.close(io); var index = try LogIndexStore.openFromFile(gpa, io, idx_handle, file_size); defer index.destroy(); const is = index.store(); try std.testing.expectEqual(rs.count(), is.count()); try std.testing.expect(rs.count() > 0); var rid: repostate.Rid = 1; while (rid <= rs.count()) : (rid += 1) { // Identity, held-ness, and lookup round-trip. try std.testing.expectEqualStrings(rs.hashOf(rid), is.hashOf(rid)); try std.testing.expectEqual(rs.held(rid), is.held(rid)); try std.testing.expectEqual(@as(?repostate.Rid, rid), is.lookup(rs.hashOf(rid))); // Plaintext byte-for-byte. const rc = rs.content(rid); const ic = is.content(rid); try std.testing.expectEqual(rc == null, ic == null); if (rc) |rb| try std.testing.expectEqualSlices(u8, rb, ic.?); // Wire form: container bytes, and the delta arm's DELTASRC + target size. const rw = rs.wire(rid); const iw = is.wire(rid); try std.testing.expectEqual(rw == null, iw == null); if (rw) |rwf| { try std.testing.expectEqualSlices(u8, rwf.container, iw.?.container); try std.testing.expectEqual(rwf.delta == null, iw.?.delta == null); if (rwf.delta) |rd| { try std.testing.expectEqualStrings(rd.src_hash, iw.?.delta.?.src_hash); try std.testing.expectEqual(rd.target_size, iw.?.delta.?.target_size); } } } } test "appendix honours intern/put/retain for hashes the log does not carry" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); var index = try LogIndexStore.openFromFile(gpa, io, handle, file_size); defer index.destroy(); const s = index.store(); // The serve base is contractually read-only — every write lands in the // per-request OverlayStore (ticket constraint 3), so these appendix arms never // fire in production. They exist so the store honours the whole ArtifactStore // interface instead of trapping on a write; exercise them directly here so that // completeness is not dead-untested surface. const persisted = s.count(); try std.testing.expect(persisted > 0); // A hash the log cannot hold (all-0xAB digest, hex-named as the wire does). var d1: [digest_len]u8 = undefined; @memset(&d1, 0xAB); var h1_buf: [2 * digest_len]u8 = undefined; const h1 = digestToWireName(&d1, &h1_buf); try std.testing.expectEqual(@as(?repostate.Rid, null), s.lookup(h1)); // intern mints an appendix phantom ABOVE the persisted rids: not held (no // content yet), count grows by one, identity round-trips, and it is idempotent. const r1 = try s.intern(h1); try std.testing.expect(r1 > persisted); try std.testing.expectEqual(r1, try s.intern(h1)); try std.testing.expectEqual(@as(?repostate.Rid, r1), s.lookup(h1)); try std.testing.expectEqualStrings(h1, s.hashOf(r1)); try std.testing.expect(!s.held(r1)); try std.testing.expectEqual(@as(?[]const u8, null), s.content(r1)); try std.testing.expectEqual(persisted + 1, s.count()); // put upgrades that phantom in place: held flips true, content reads back, and // the persisted count is unchanged (an off-log hash never touches the index). const body = "off-log body bytes"; try std.testing.expectEqual(r1, try s.put(h1, body)); try std.testing.expect(s.held(r1)); try std.testing.expectEqualSlices(u8, body, s.content(r1).?); try std.testing.expectEqual(persisted + 1, s.count()); // retain on a PERSISTED rid is the advisory no-op (sync §7.2 first-form-wins; // the log IS the retained form): it returns without error and ignores ingest. try s.retain(1, .{ .full = "ignored — a persisted rid keeps its log form" }); // None of the appendix traffic left persisted-round memos resident. try std.testing.expectEqual(@as(usize, 0), index.residentBytes()); } test "index materialises only the container a round reads and releases it between rounds" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); var index = try LogIndexStore.openFromFile(gpa, io, handle, file_size); defer index.destroy(); const s = index.store(); // Right after the prime the index holds ZERO container/plaintext bytes — the // whole point: it did not copy the store into RAM. try std.testing.expectEqual(@as(usize, 0), index.residentBytes()); // Find one wire-bearing rid (a zlib/delta frame). var target: ?repostate.Rid = null; var rid: repostate.Rid = 1; while (rid <= s.count()) : (rid += 1) { if (s.wire(rid) != null) { target = rid; break; } } try std.testing.expect(target != null); // Reading it materialised only its container; drop the round's memos. try std.testing.expect(index.residentBytes() > 0); index.releaseTransient(); try std.testing.expectEqual(@as(usize, 0), index.residentBytes()); // A single round's residency is EXACTLY the one container it emitted — not // the whole store's containers (which the old snapshot held resident). const one = s.wire(target.?).?; try std.testing.expectEqual(one.container.len, index.residentBytes()); } /// A test-only `storage.StorageFile` that reads faithfully until `fail_reads` is /// flipped, then fails every `pread` — the instrument for a fault that strikes /// AFTER `openReadOnly` has scanned and whole-frame-CRC'd the log, exactly the /// post-scan bitrot / transient EIO the on-demand serve reads must not swallow /// silently. Every other method delegates verbatim. const FaultAfterScanStorage = struct { inner: storage.StorageFile, fail_reads: bool = false, fn storageFile(self: *FaultAfterScanStorage) storage.StorageFile { return .{ .context = self, .vtable = &vtable }; } const vtable = storage.StorageFile.VTable{ .append = appendImpl, .pread = preadImpl, .sync = syncImpl, .size = sizeImpl, .truncate = truncateImpl, }; fn appendImpl(ctx: *anyopaque, bytes: []const u8) storage.StorageError!void { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); return self.inner.append(bytes); } fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); if (self.fail_reads) return error.Io; return self.inner.pread(offset, buf); } fn syncImpl(ctx: *anyopaque) storage.StorageError!void { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); return self.inner.sync(); } fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); return self.inner.size(); } fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void { const self: *FaultAfterScanStorage = @ptrCast(@alignCast(ctx)); return self.inner.truncate(len); } }; test "a post-scan read fault on a held rid is recorded, not silently swallowed" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const file_size = try seedServeRepo(gpa, io, tmp.dir); var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); var fs = storage.FileStorage.init(io, handle, file_size); var faulting = FaultAfterScanStorage{ .inner = fs.storageFile() }; // Open succeeds: the scan CRCs the whole log BEFORE any fault is armed. var index = try LogIndexStore.open(gpa, faulting.storageFile()); defer index.destroy(); const s = index.store(); // From here every on-demand container/plaintext read faults (post-scan EIO). faulting.fail_reads = true; // Walk the store as the §8.2 clone pager does: a held rid whose wire() AND // content() both return null is one the pager silently skips while still // reporting the sequence complete — a clone finishing MISSING an artifact. var silently_omitted = false; var rid: repostate.Rid = 1; while (rid <= s.count()) : (rid += 1) { if (!s.held(rid)) continue; if (s.wire(rid) == null and s.content(rid) == null) silently_omitted = true; } // The fault genuinely makes a held rid unemittable (the bug's precondition)... try std.testing.expect(silently_omitted); // ...and the store RECORDED it, so the serve layer can convert the round to an // error instead of completing a broken clone. A copy-based prime could not // reach this state: a read fault surfaced at prime, not silently at serve. try std.testing.expect(index.read_fault); } test "serveAnswer converts a recorded read fault into a storage error (no silent short clone)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var hex_buf: [2 * digest_len]u8 = undefined; try serveSeedRepo(gpa, io, tmp.dir, 4, &hex_buf); const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const clone_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 1\n", .{}); defer gpa.free(clone_plain); const clone_req = try zit.container.encode(gpa, clone_plain, .default); defer gpa.free(clone_req); // A healthy round builds the cached snapshot and completes normally. const ok = try serveAnswer(&ctx, gpa, clone_req, null, null, true); gpa.free(ok.body); // Simulate a post-scan read fault having struck THIS round — the test above // proves the store records exactly this. serveAnswer MUST refuse to hand back // a reply built while an artifact read faulted: otherwise the §8.2 pager's // silent skip ships a clone missing an artifact and reports it complete. ctx.snapshot.?.read_fault = true; try std.testing.expectError(error.ServerStorage, serveAnswer(&ctx, gpa, clone_req, null, null, true)); // The flag is cleared with the round's memos, so an immediately retried round // over the now-healthy snapshot succeeds again (a transient fault self-heals). const recovered = try serveAnswer(&ctx, gpa, clone_req, null, null, true); gpa.free(recovered.body); } /// Build a full-form 3-check-in chain over an evolving file into `log`, plus a /// live config frame; returns the tip check-in's 64-hex name. Every blob and /// manifest lands as a FULL frame — pack's re-delta target. fn packSeedFullStore(gpa: std.mem.Allocator, log: *logstore.LogStore, tip_out: *[64]u8) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); var root_name: [64]u8 = undefined; var mid_name: [64]u8 = undefined; const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const b1 = try packSeedBlob(log, v1); try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "f.txt", &b1, &root_name); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); const b2 = try packSeedBlob(log, v2); try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "f.txt", &b2, &mid_name); const v3 = try packEvolvingBlob(gpa, 3); defer gpa.free(v3); const b3 = try packSeedBlob(log, v3); try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &mid_name, "f.txt", &b3, tip_out); try log.commit(); } /// Store `content` under its sha3-256 name, delta-encoded against `base` when /// one is given (the evolving blobs delta to a fraction, so the frame lands as a /// `.delta`), and return its rid. Models a fossil-clone store that RETAINED /// compact wire deltas rather than full frames. fn packSeedDeltaBlob(log: *logstore.LogStore, content: []const u8, base: ?logstore.Rid) !logstore.Rid { var digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(content, &digest, .{}); return log.putArtifact(&digest, content, base); } /// Build a clone-shaped store whose successive blobs are RETAINED as `.delta` /// frames (the compact wire form), under DISTINCT filenames so pack's /// same-path-blob lineage hint finds NO base for them — exactly the case where /// the old pack reconstructed every delta to a full frame and inflated the /// store. `.zit` therefore holds real delta frames whose bases pack cannot /// re-derive, so a correct pack must preserve them verbatim. Returns the tip /// check-in's 64-hex name. fn packSeedDeltaStore(gpa: std.mem.Allocator, log: *logstore.LogStore, tip_out: *[64]u8) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); var root_name: [64]u8 = undefined; var mid_name: [64]u8 = undefined; const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const r1 = try packSeedDeltaBlob(log, v1, null); // first version: full frame (base of the chain) var b1: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v1, &b1); try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "f1.txt", &b1, &root_name); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); const r2 = try packSeedDeltaBlob(log, v2, r1); // delta of v1 var b2: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v2, &b2); try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "f2.txt", &b2, &mid_name); const v3 = try packEvolvingBlob(gpa, 3); defer gpa.free(v3); _ = try packSeedDeltaBlob(log, v3, r2); // delta of v2 var b3: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v3, &b3); try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &mid_name, "f3.txt", &b3, tip_out); try log.commit(); } /// Seed a store holding a wire delta that pack CANNOT beat by re-deltification, /// so ONLY the §6.1 monotonic-shrink guard keeps the store from regressing. `B` /// is a one-line edit of `A`, stored as a COMPACT zit delta against `A` — the /// smallest form pack could ever reproduce. But `max_base_candidates` unrelated /// filler blobs sit between `A` and `B`, so pack's bounded backward candidate /// window (§5.2) saturates on the fillers and never reaches `A`; with no lineage /// hint either (bare blobs, no F-card path), pack's best re-encoding of `B` is a /// full zlib frame STRICTLY LARGER than the stored A-based delta. The pre-guard /// pack replaced the delta with that larger full frame (the +33% clone /// regression in miniature); the guard keeps the existing smaller frame. Fills /// A's and B's 64-hex names. fn packSeedUnbeatableWireDelta( gpa: std.mem.Allocator, log: *logstore.LogStore, a_name: *[64]u8, b_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // A: B's ideal base, stored full (zlib). const a = try packEvolvingBlob(gpa, 1); defer gpa.free(a); var a_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(a, &a_digest, .{}); const a_rid = try log.putZlib(&a_digest, a); _ = zit.artifact.artifactName(.sha3_256, a, a_name); // `max_base_candidates` unrelated fillers between A and B: they saturate B's // backward candidate window so A is out of reach when pack re-encodes B. var f: usize = 0; while (f < logstore.max_base_candidates) : (f += 1) { var fbuf: std.ArrayList(u8) = .empty; defer fbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try fbuf.print(gpa, "filler {d} unrelated row {d}\n", .{ f, i }); var fdig: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(fbuf.items, &fdig, .{}); _ = try log.putZlib(&fdig, fbuf.items); } // B: a one-line edit of A, stored as a COMPACT zit delta against A (the // smallest form pack could reproduce — and only against A, which pack's // window cannot reach here). const b = try packEvolvingBlob(gpa, 2); defer gpa.free(b); var b_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{}); _ = zit.artifact.artifactName(.sha3_256, b, b_name); const raw = try zit.delta.create(gpa, a, b); defer gpa.free(raw); const container = try zit.container.encode(gpa, raw, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&b_digest, a_rid, container); try log.commit(); } /// Assert the §6.1 per-frame monotonicity: NO artifact's packed frame is larger /// on disk than its source frame. Pack keeps the smaller of {existing frame, new /// encoding} for every artifact, so a repack is non-worsening frame by frame. /// This holds universally over THESE fixtures because their chains stay below the /// read cap; the one path that can enlarge a frame — `copySrcFrame`'s §3.3 /// deep-chain fallback re-storing a kept delta as full zlib — needs a base at /// `max_delta_chain` depth, which the seeds never build. fn packExpectNoFrameLarger(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void { const src_rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); for (src_rids) |sr| { const dr = dst.lookup(src.hashOf(sr)) orelse return error.PackedArtifactMissing; try std.testing.expect(dst.frameSpanOf(dr) <= src.frameSpanOf(sr)); } } /// Count the `.delta` frames among a store's persisted artifacts. fn packDeltaFrameCount(gpa: std.mem.Allocator, log: *logstore.LogStore) !usize { const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var n: usize = 0; for (rids) |r| { if (log.kindOf(r) == .delta) n += 1; } return n; } /// Assert that every artifact in `src` reconstructs byte-identically in `dst` /// and the two artifact SETS are equal (no drop, no addition). fn packExpectContentEqual(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void { const src_rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); const dst_rids = try dst.ridsInPersistedOrder(gpa); defer gpa.free(dst_rids); try std.testing.expectEqual(src_rids.len, dst_rids.len); for (src_rids) |sr| { const digest = src.hashOf(sr); const dr = dst.lookup(digest) orelse return error.PackedArtifactMissing; const a = (try src.content(gpa, sr)).?; defer gpa.free(a); const b = (try dst.content(gpa, dr)).?; defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); } } test "pack: the packed store's artifact set + every reconstructed content byte-equal the original (config preserved)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // No drop: the packed content-frame count equals the source's, and every // artifact reconstructs byte-for-byte. try std.testing.expectEqual(src.entries.items.len, stats.artifacts); try packExpectContentEqual(gpa, &src, &dst); // The live config survives as exactly one frame with identical fields. var soc = (try src.readConfig(gpa)).?; defer soc.deinit(gpa); var doc = (try dst.readConfig(gpa)).?; defer doc.deinit(gpa); try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy); try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch); try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id); try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code); } test "pack re-deltas full-form frames (delta count rises, .zit shrinks) with byte-identical content" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // The source is all full-form: not one delta frame. { const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); for (rids) |r| try std.testing.expect(src.kindOf(r) != .delta); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // The two child blobs (v2 against v1, v3 against v2) re-delta against the // same path's parent blob, so the delta count rises and the store shrinks — // with content preserved exactly. try std.testing.expect(stats.deltas >= 2); try std.testing.expect(stats.bytes_after < stats.bytes_before); try packExpectContentEqual(gpa, &src, &dst); } test "pack preserves wire deltas — a store with delta frames packs to <= source size (regression for the 8.65x inflation bug)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedDeltaStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // The source genuinely holds retained wire deltas (the compact form pack // must not discard) whose bases pack's lineage hint cannot re-derive. const src_deltas = try packDeltaFrameCount(gpa, &src); try std.testing.expect(src_deltas >= 2); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // MONOTONIC: the packed store is never larger than the source. The old pack // reconstructed each delta to a full zlib frame and inflated the store; a // correct pack preserves the compact deltas verbatim. try std.testing.expect(stats.bytes_after <= stats.bytes_before); // Every source delta survives AS a delta frame — none reconstructed to full // (the optional re-delta may add further deltas, so the total only grows). { const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); for (rids) |sr| { if (src.kindOf(sr) != .delta) continue; const dr = dst.lookup(src.hashOf(sr)).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr)); } } try std.testing.expect(try packDeltaFrameCount(gpa, &dst) >= src_deltas); // No drop, every reconstruction byte-exact. try packExpectContentEqual(gpa, &src, &dst); } test "pack copies a delta frame verbatim re-linked to its base's new rid, content byte-identical" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedDeltaStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // For every delta frame in the source, the packed store holds it as a delta // whose stored container is byte-identical (verbatim re-serve) and whose base // names the SAME artifact (re-linked to the base's new rid). Content matches. const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); var checked: usize = 0; for (rids) |sr| { if (src.kindOf(sr) != .delta) continue; const digest = src.hashOf(sr); const dr = dst.lookup(digest).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr)); var sw = (try src.wire(gpa, sr)).?; defer sw.deinit(gpa); var dw = (try dst.wire(gpa, dr)).?; defer dw.deinit(gpa); // Verbatim container bytes and the same base hash (re-link preserves the // logical base while its rid/offset differs in the packed generation). try std.testing.expectEqualSlices(u8, sw.delta.container, dw.delta.container); try std.testing.expectEqualSlices(u8, sw.delta.baseHash(), dw.delta.baseHash()); const a = (try src.content(gpa, sr)).?; defer gpa.free(a); const b = (try dst.content(gpa, dr)).?; defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); checked += 1; } try std.testing.expect(checked >= 2); } /// The rid naming the 64-hex `name` in `log`, or null when absent (test-only /// name→digest→rid convenience over `lookup`). fn packRidByName(log: *logstore.LogStore, name: *const [64]u8) ?logstore.Rid { var d_buf: [digest_len]u8 = undefined; const d = hexToDigest(name, &d_buf) orelse return null; return log.lookup(d); } /// Seed a store modeling a peer-supplied (verbatim wire) delta whose sender /// chose a POOR base. `A` (a large file) and `C` (an unrelated large blob) are /// stored full; `B` (a one-line edit of `A`) is stored as a hand-crafted §4.1 /// delta container against `C` — a large, suboptimal delta the sender picked. /// `A` precedes `B` chronologically and is B's ideal base, so pack's best-of-N /// re-deltification (§6.1) must re-base `B` onto `A` rather than trust the /// sender's `C`. Fills the 64-hex names of A/B/C. fn packSeedSuboptimalWireDelta( gpa: std.mem.Allocator, log: *logstore.LogStore, a_name: *[64]u8, b_name: *[64]u8, c_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // A: the ideal base for B (near-identical predecessor), stored full. const a = try packEvolvingBlob(gpa, 1); defer gpa.free(a); a_name.* = try packSeedBlob(log, a); // C: an unrelated large blob (a poor base for B), stored full. var cbuf: std.ArrayList(u8) = .empty; defer cbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try cbuf.print(gpa, "unrelated content row {d}\n", .{i}); const c = cbuf.items; c_name.* = try packSeedBlob(log, c); const c_rid = packRidByName(log, c_name).?; // B: a one-line edit of A. delta(A,B) is tiny; delta(C,B) is large. const b = try packEvolvingBlob(gpa, 2); defer gpa.free(b); _ = zit.artifact.artifactName(.sha3_256, b, b_name); var b_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{}); // Hand-craft the verbatim wire delta against the POOR base C — models a peer // that chose C as base, retained at rest by `putDeltaContainer`. const raw = try zit.delta.create(gpa, c, b); defer gpa.free(raw); const container = try zit.container.encode(gpa, raw, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&b_digest, c_rid, container); try log.commit(); } test "pack §6.1: a verbatim wire delta is re-deltified against the size-minimizing base, not the sender's" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var a_name: [64]u8 = undefined; var b_name: [64]u8 = undefined; var c_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSuboptimalWireDelta(gpa, &wlog, &a_name, &b_name, &c_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source really holds B as a delta whose base is C (the // sender's suboptimal choice) — the exact frame the old pack copied verbatim. { const sb = packRidByName(&src, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb)); var sw = (try src.wire(gpa, sb)).?; defer sw.deinit(gpa); var cbin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&cbin, &c_name); try std.testing.expectEqualSlices(u8, &cbin, sw.delta.baseHash()); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // After pack, B is still a delta, but its base is now A (the size-minimizing // candidate) — the sender's base C is no longer trusted at rest (§6.1/§3). const db = packRidByName(&dst, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db)); var dw = (try dst.wire(gpa, db)).?; defer dw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash()); // Every artifact still reconstructs byte-for-byte and the store shrank (a // tiny A-based delta replaced the large C-based one). try packExpectContentEqual(gpa, &src, &dst); try std.testing.expect(stats.bytes_after < stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); try std.testing.expectEqual(src.size(), stats.bytes_before); } /// Seed the §6.2 same-path scenario: two versions of ONE tracked file whose ideal /// base (the earlier version) sits OUTSIDE pack's bounded chronological window. /// `v1` of "tracked.txt" is stored full, then `max_base_candidates` unrelated /// filler blobs saturate the backward candidate window, then `v2` (a one-line /// edit of `v1`) is stored full. With ONLY the chronological window (step §6.1) /// pack cannot reach `v1` from `v2` — the fillers fill every slot — so `v2` stays /// a full frame and the store does not shrink. The mlink-analog same-path walk /// (§6.2) reads both check-in manifests, learns tracked.txt: [v1, v2], and offers /// `v1` as `v2`'s priority candidate, so pack re-deltifies `v2` onto `v1`. Both /// check-ins are FULL frames (pack's re-delta target); fills v1/v2's 64-hex names. fn packSeedSamePathOutOfWindow( gpa: std.mem.Allocator, log: *logstore.LogStore, v1_name: *[64]u8, v2_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // v1 of tracked.txt (full), and the root check-in that names it. const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); v1_name.* = try packSeedBlob(log, v1); var root_name: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &root_name); // `max_base_candidates` unrelated fillers between v1 and v2: they saturate // v2's bounded backward window so v1 is out of chronological reach. var f: usize = 0; while (f < logstore.max_base_candidates) : (f += 1) { var fbuf: std.ArrayList(u8) = .empty; defer fbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try fbuf.print(gpa, "samepath filler {d} unrelated row {d}\n", .{ f, i }); _ = try packSeedBlob(log, fbuf.items); } // v2 of tracked.txt (a one-line edit of v1), stored FULL, and its check-in. const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); v2_name.* = try packSeedBlob(log, v2); var tip_name: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "tracked.txt", v2_name, &tip_name); try log.commit(); } test "pack §6.2: a same-path predecessor beyond the chronological window becomes v2's base (store shrinks)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var v1_name: [64]u8 = undefined; var v2_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSamePathOutOfWindow(gpa, &wlog, &v1_name, &v2_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source holds v2 as a NON-delta (full) frame — the case // step §6.1's chronological window alone cannot improve (v1 is out of reach). { const sv2 = packRidByName(&src, &v2_name).?; try std.testing.expect(src.kindOf(sv2) != .delta); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // After pack, v2 is a delta whose base is the same-path predecessor v1 — the // §6.2 mlink analog reached a base the chronological window could not. const dv2 = packRidByName(&dst, &v2_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dv2)); var dw = (try dst.wire(gpa, dv2)).?; defer dw.deinit(gpa); var v1bin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&v1bin, &v1_name); try std.testing.expectEqualSlices(u8, &v1bin, dw.delta.baseHash()); // The store shrank (a tiny v1-based delta replaced the full v2 frame), every // artifact still reconstructs byte-for-byte, and no frame grew. try std.testing.expect(stats.bytes_after < stats.bytes_before); try packExpectContentEqual(gpa, &src, &dst); try packExpectNoFrameLarger(gpa, &src, &dst); } /// Seed the §6.2 same-path REVERT scenario: one tracked file whose content goes /// A→B→A across three check-ins — the third restores a byte-identical earlier /// blob. Because the store is content-addressed, the reverted version resolves to /// the SAME blob rid as v1, so the pack manifest walk records that rid twice, /// non-consecutively, for one path. Exercises `deriveCandidateHints`'s /// distinct-rid compaction: a blob must never be offered as its own same-path /// predecessor. Fills v1/v2's 64-hex names. fn packSeedSamePathRevert( gpa: std.mem.Allocator, log: *logstore.LogStore, v1_name: *[64]u8, v2_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); // v1 of tracked.txt (full) + its root check-in. v1_name.* = try packSeedBlob(log, v1); var c1: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &c1); // v2 (a one-line edit) + its check-in. v2_name.* = try packSeedBlob(log, v2); var c2: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &c1, "tracked.txt", v2_name, &c2); // Revert: tracked.txt returns to v1's byte-identical content. The blob is // already stored (content-addressed), so this check-in re-references v1's rid // — the non-consecutive duplicate that recordPathBlob's consecutive-only dedup // lets through. var c3: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &c2, "tracked.txt", v1_name, &c3); try log.commit(); } test "pack §6.2: a same-path revert (A→B→A) re-references one blob rid twice and packs cleanly (no self-predecessor panic)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var v1_name: [64]u8 = undefined; var v2_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSamePathRevert(gpa, &wlog, &v1_name, &v2_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); // Before the distinct-rid compaction in deriveCandidateHints this panicked on // a self-predecessor (v1 sorted adjacent to its own duplicate ⇒ equal offsets // fail the strict offset-below assert). Now the pack completes and every // artifact reconstructs byte-for-byte with no frame enlarged. const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); try packExpectContentEqual(gpa, &src, &dst); try packExpectNoFrameLarger(gpa, &src, &dst); try std.testing.expectEqual(dst.size(), stats.bytes_after); try std.testing.expectEqual(src.size(), stats.bytes_before); } test "pack §6.1 monotonic guard: a wire delta pack cannot beat is KEPT, not replaced by a larger frame (no regression)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var a_name: [64]u8 = undefined; var b_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedUnbeatableWireDelta(gpa, &wlog, &a_name, &b_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source holds B as a compact delta whose base is A. { const sb = packRidByName(&src, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb)); var sw = (try src.wire(gpa, sb)).?; defer sw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, sw.delta.baseHash()); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); // (a) decode-all byte-identical pre/post. try packExpectContentEqual(gpa, &src, &dst); // (b) NO frame is larger post-pack than its pre-pack encoding. try packExpectNoFrameLarger(gpa, &src, &dst); // (c) the whole store did not regress on this wire-delta-heavy fixture. try std.testing.expect(stats.bytes_after <= stats.bytes_before); // Accounting exact: reported sizes are the real store sizes. try std.testing.expectEqual(src.size(), stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); // B stayed a delta (the compact A-based frame was kept, re-linked to A's new // rid) — the guard refused to replace it with the larger full zlib pack's // window-only re-encoding would have produced. const db = packRidByName(&dst, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db)); var dw = (try dst.wire(gpa, db)).?; defer dw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash()); // The kept frame is no larger than the source's (equal — copied verbatim). try std.testing.expect(dst.frameSpanOf(db) <= src.frameSpanOf(packRidByName(&src, &b_name).?)); } /// Seed a store exercising EVERY frame class pack must handle in one place: a /// raw frame, a zlib frame, a locally-created delta, a verbatim wire delta, an /// interned phantom (no frame on disk), and the config singleton. Fills the /// content-bearing artifacts' 64-hex names plus the phantom's binary digest. fn packSeedMixedStore( gpa: std.mem.Allocator, log: *logstore.LogStore, raw_name: *[64]u8, zlib_name: *[64]u8, delta_name: *[64]u8, wire_name: *[64]u8, phantom_digest: *[digest_len]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // raw: a unique blob large enough to be a real (if unrelated) base. var rbuf: std.ArrayList(u8) = .empty; defer rbuf.deinit(gpa); var i: usize = 0; while (i < 120) : (i += 1) try rbuf.print(gpa, "raw-only content row {d}\n", .{i}); const rbytes = rbuf.items; raw_name.* = try packSeedBlob(log, rbytes); // `put` => .raw // zlib: a large evolving blob stored as a full zlib container. const z = try packEvolvingBlob(gpa, 1); defer gpa.free(z); var z_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(z, &z_digest, .{}); const z_rid = try log.putZlib(&z_digest, z); _ = zit.artifact.artifactName(.sha3_256, z, zlib_name); // local delta: an edit of the zlib blob, delta-encoded against it. const d = try packEvolvingBlob(gpa, 2); defer gpa.free(d); var d_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(d, &d_digest, .{}); _ = try log.putArtifact(&d_digest, d, z_rid); _ = zit.artifact.artifactName(.sha3_256, d, delta_name); // wire delta: a hand-crafted §4.1 delta container against the raw blob (a // poor base) — the verbatim-retained peer form. const w = try packEvolvingBlob(gpa, 3); defer gpa.free(w); var w_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(w, &w_digest, .{}); _ = zit.artifact.artifactName(.sha3_256, w, wire_name); const r_rid = packRidByName(log, raw_name).?; const raw_delta = try zit.delta.create(gpa, rbytes, w); defer gpa.free(raw_delta); const container = try zit.container.encode(gpa, raw_delta, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&w_digest, r_rid, container); // phantom: an interned forward reference that never reaches disk. std.crypto.hash.sha3.Sha3_256.hash("absent forward-referenced artifact", phantom_digest, .{}); _ = try log.intern(phantom_digest); try log.commit(); } test "pack §6.1/§3: a mixed raw/zlib/delta/wire-delta/phantom/config store round-trips, config+phantom preserved, accounting exact" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var raw_name: [64]u8 = undefined; var zlib_name: [64]u8 = undefined; var delta_name: [64]u8 = undefined; var wire_name: [64]u8 = undefined; var phantom_digest: [digest_len]u8 = undefined; // A phantom never reaches disk, so we pack the in-memory writable store // directly (a reopen would drop it) — the case pack must tolerate without // choking or emitting a garbage frame. var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedMixedStore(gpa, &wlog, &raw_name, &zlib_name, &delta_name, &wire_name, &phantom_digest); // The seeded store really holds one frame of each class plus the phantom. try std.testing.expectEqual(logstore.FrameKind.raw, wlog.kindOf(packRidByName(&wlog, &raw_name).?)); try std.testing.expectEqual(logstore.FrameKind.zlib, wlog.kindOf(packRidByName(&wlog, &zlib_name).?)); try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &delta_name).?)); try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &wire_name).?)); try std.testing.expect(wlog.isPhantom(wlog.lookup(&phantom_digest).?)); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), wlog.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &wlog, &dst, Filter.keep_all, &.{}); // Decode-all byte-identical pre/post, and the artifact SETS match (the // phantom is excluded from both, so counts still agree). try packExpectContentEqual(gpa, &wlog, &dst); // §6.1 per-frame monotonicity: no frame grew in the repack. try packExpectNoFrameLarger(gpa, &wlog, &dst); // Config preserved verbatim in fields. var soc = (try wlog.readConfig(gpa)).?; defer soc.deinit(gpa); var doc = (try dst.readConfig(gpa)).?; defer doc.deinit(gpa); try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy); try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch); try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id); try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code); // Phantom preserved as a contentless forward reference: it carries no bytes, // so pack writes no frame for it (a phantom never reaches disk) and it is not // resurrected as a garbage artifact in the packed store. try std.testing.expect(dst.lookup(&phantom_digest) == null); // Accounting exact: the reported sizes are the actual store sizes and the // artifact tally is the source's real (non-phantom) frame count. try std.testing.expectEqual(wlog.size(), stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); const src_rids = try wlog.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); try std.testing.expectEqual(src_rids.len, stats.artifacts); } test "pack §7.7: packLog aborts on a source frame whose content does not name it (no packed frame written)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A frame keyed by a name that is NOT sha3-256 of its bytes: `put` stores // bytes verbatim under any key, so this models a corrupt/lying source frame // that survives `openReadOnly` (its CRC + embedded-hash label are intact). const lying_name = [_]u8{0xab} ** digest_len; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); _ = try wlog.put(&lying_name, "content these bytes do not name"); try wlog.commit(); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); try std.testing.expectError(error.Corrupt, packLog(gpa, &src, &dst, Filter.keep_all, &.{})); try std.testing.expect(dst.lookup(&lying_name) == null); // no frame landed on the abort } test "pack is idempotent: pack∘pack is byte-identical (0 additional re-deltas)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } // First pack: source (full) → L1. { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var f1 = try tmp.dir.createFile(io, "pack1", .{ .read = true, .truncate = true }); defer f1.close(io); var f1fs = storage.FileStorage.init(io, f1, 0); var dst = try logstore.LogStore.open(gpa, f1fs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); } // Second pack: L1 → L2, byte-for-byte equal, and no further re-deltaing. var l1fs: storage.FileStorage = undefined; var l1 = try packOpenReadOnly2(gpa, io, tmp.dir, "pack1", &l1fs); defer l1fs.file.close(io); defer l1.deinit(gpa); var f2 = try tmp.dir.createFile(io, "pack2", .{ .read = true, .truncate = true }); defer f2.close(io); var f2fs = storage.FileStorage.init(io, f2, 0); var l2 = try logstore.LogStore.open(gpa, f2fs.storageFile(), l1.hash_kind); defer l2.deinit(gpa); const stats2 = try packLog(gpa, &l1, &l2, Filter.keep_all, &.{}); try std.testing.expectEqual(l1.size(), l2.size()); const a = try tmp.dir.readFileAlloc(io, "pack1", gpa, .limited(1 << 24)); defer gpa.free(a); const b = try tmp.dir.readFileAlloc(io, "pack2", gpa, .limited(1 << 24)); defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); // byte-identical: pack∘pack == pack // An already-packed store still reports its deltas (they were re-created // identically), and packing it does not shrink further. try std.testing.expectEqual(l1.size(), stats2.bytes_after); } /// Reopen a named log file in `dir` read-only (a sibling to `packOpenReadOnly` /// that takes an arbitrary name, for the multi-generation idempotence test). fn packOpenReadOnly2(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, name: []const u8, fs_out: *storage.FileStorage) !logstore.LogStore { const f = try dir.openFile(io, name, .{}); const st = try f.stat(io); fs_out.* = storage.FileStorage.init(io, f, st.size); return logstore.LogStore.openReadOnly(gpa, fs_out.storageFile()); } test "pack: packLog never writes the source (a crash before the rename leaves the original byte-identical)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } const before = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(before); // Pack into a separate temp (the new generation). packLog only ever reads // the source; the atomic swap in `runPack` is the sole mutation of `.zit`. { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var f = try tmp.dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = true }); defer f.close(io); var ffs = storage.FileStorage.init(io, f, 0); var dst = try logstore.LogStore.open(gpa, ffs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); } // No rename happened (the "crash"): the original `.zit` is byte-for-byte // unchanged and still opens. const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(after); try std.testing.expectEqualSlices(u8, before, after); } /// The allocation-failure probe: pack a prebuilt source `.zit` into a fresh temp /// destination, using the failing allocator throughout. Every OOM point must /// unwind cleanly (no leak) and a fully-funded run must succeed. fn packOomProbe(alloc: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(alloc, io, dir, &sfs); defer sfs.file.close(io); defer src.deinit(alloc); var f = try dir.createFile(io, "oomdst", .{ .read = true, .truncate = true }); defer f.close(io); var ffs = storage.FileStorage.init(io, f, 0); var dst = try logstore.LogStore.open(alloc, ffs.storageFile(), src.hash_kind); defer dst.deinit(alloc); _ = try packLog(alloc, &src, &dst, Filter.keep_all, &.{}); } test "pack: checkAllAllocationFailures on packLog (every OOM path unwinds cleanly)" { const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(std.testing.allocator, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(std.testing.allocator); try packSeedFullStore(std.testing.allocator, &wlog, &tip); } try std.testing.checkAllAllocationFailures(std.testing.allocator, packOomProbe, .{ io, tmp.dir }); } test "pack∘derive == from-scratch derive: the packed store derives the identical repo view" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var src_view: zit.repoview.RepoView = undefined; try src_view.load(gpa, &src); defer src_view.deinit(gpa); var pfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer pfile.close(io); var pfs = storage.FileStorage.init(io, pfile, 0); var dst = try logstore.LogStore.open(gpa, pfs.storageFile(), src.hash_kind); _ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{}); dst.deinit(gpa); var rfs: storage.FileStorage = undefined; var packed_log = try packOpenReadOnly2(gpa, io, tmp.dir, "packed", &rfs); defer rfs.file.close(io); defer packed_log.deinit(gpa); var packed_view: zit.repoview.RepoView = undefined; try packed_view.load(gpa, &packed_log); defer packed_view.deinit(gpa); // Same derived view: main branch, artifact count, and the leaf set + order // (§7.5) are identical — pack changed only the storage layout. try std.testing.expectEqualStrings(src_view.main_branch, packed_view.main_branch); try std.testing.expectEqual(src_view.store.store().count(), packed_view.store.store().count()); var la = src_view.state.leaves(); var lb = packed_view.state.leaves(); while (la.next()) |ra| { const rb = lb.next() orelse return error.LeafCountMismatch; try std.testing.expectEqualStrings(src_view.hashOf(ra), packed_view.hashOf(rb)); } try std.testing.expect(lb.next() == null); } test "renderPackStats: 'packed' when the store shrank, 'already packed' otherwise" { var buf: [128]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 1695, .deltas = 1373, .dropped = 0, .shuns_recorded = 0, .bytes_before = 9_680_000, .bytes_after = 920_000 }); try std.testing.expectEqualStrings( "packed: 1695 artifacts walked, 1373 re-deltaed, 9680000 -> 920000 bytes\n", w.buffered(), ); } { var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 3, .deltas = 2, .dropped = 0, .shuns_recorded = 0, .bytes_before = 4096, .bytes_after = 4096 }); try std.testing.expectEqualStrings( "already packed: 3 artifacts walked, 2 re-deltaed, 4096 -> 4096 bytes\n", w.buffered(), ); } } test "renderPackStats: a reclaiming run names the dropped count; a run that drops nothing stays silent" { // Reclamation is the one pack outcome an operator must not have to infer // from a shrinking byte count: bytes shrink on an ordinary re-deltify too. // So the clause appears exactly when `dropped > 0`, and nowhere else. var buf: [128]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 3, .shuns_recorded = 0, .bytes_before = 8192, .bytes_after = 4096 }); try std.testing.expectEqualStrings( "packed: 12 artifacts walked, 4 re-deltaed, 3 shunned reclaimed, 8192 -> 4096 bytes\n", w.buffered(), ); } { // `artifacts` counts the source frames WALKED, so a run that dropped // every one of them still reports 12 walked and 12 reclaimed. var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 12, .deltas = 0, .dropped = 12, .shuns_recorded = 0, .bytes_before = 8192, .bytes_after = 64 }); try std.testing.expectEqualStrings( "packed: 12 artifacts walked, 0 re-deltaed, 12 shunned reclaimed, 8192 -> 64 bytes\n", w.buffered(), ); } { // A reclaim that GREW the log is still "packed". Reclaiming an artifact // other artifacts were deltified against moves its content into one of // them, so the log can legitimately end up larger — and "already packed" // on a run that just irreversibly deleted content would be the most // misleading line this verb could print. var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 7, .deltas = 5, .dropped = 1, .shuns_recorded = 0, .bytes_before = 5760, .bytes_after = 6248 }); try std.testing.expectEqualStrings( "packed: 7 artifacts walked, 5 re-deltaed, 1 shunned reclaimed, 5760 -> 6248 bytes\n", w.buffered(), ); } } test "rewriteAndSwap: a run the concurrency guard aborts leaves .zit untouched and records NO tombstone" { // THE "ONE DURABLE TRANSITION" INVARIANT, made observable. `zit shun --force` // must never leave a tombstone behind without also having removed the content // it names; the mechanism is that the rows are written into the TEMP and only // the rename makes anything durable. This drives the abort seam that is a live // guard rather than an injected fault — the no-repo-lock concurrency guard: a // concurrent writer extends `.zit` between our scan and the rename, so `validateTail` // fails after `packLog` has already written every row into the temp. If the // rows had been appended to the live log first, they would still be here. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const doomed_body = "content the operator named in this invocation " ** 4; var doomed: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(doomed_body, &doomed, .{}); { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wlog.deinit(gpa); _ = try packSeedBlob(&wlog, doomed_body); _ = try packSeedBlob(&wlog, "content with nothing against it " ** 4); try wlog.commit(); } // The scan the rewrite will be built from, and the tail key that pins it. var rfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &rfs); defer src.deinit(gpa); const src_key = try src.logKey(); // The concurrent writer. One appended frame is exactly what the guard exists // to detect; the snapshot in `src` is now stale. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); const st = try f.stat(io); var wfs = storage.FileStorage.init(io, f, st.size); var wlog = try logstore.LogStore.open(gpa, wfs.storageFile(), 0); defer wlog.deinit(gpa); _ = try packSeedBlob(&wlog, "a frame a concurrent writer appended " ** 3); try wlog.commit(); } const extended = try storeSnapshot(gpa, io, tmp.dir); defer gpa.free(extended); var set: NameSet = .{}; defer set.deinit(gpa); try set.add(gpa, &doomed); const rows = [_]logstore.ShunRecord{ .{ .hash = &doomed, .epoch = 1753000077, .reason = "takedown notice 4417" }, }; try std.testing.expectError(error.TailChanged, rewriteAndSwap( gpa, io, tmp.dir, &src, src_key, set.filter(), &rows, )); // THE PINS, tombstone FIRST because it is the one this test exists for: the // aborted run left no row, `.zit` is the concurrent writer's version byte for // byte (our stale rewrite never landed), the content the run intended to // remove is intact, and the temp is gone. var vfs: storage.FileStorage = undefined; var after_log = try packOpenReadOnly(gpa, io, tmp.dir, &vfs); defer after_log.deinit(gpa); try std.testing.expectEqual(@as(usize, 0), after_log.shunList().len); try std.testing.expect(!after_log.isShunned(&doomed)); try std.testing.expect(after_log.lookup(&doomed) != null); { const after = try storeSnapshot(gpa, io, tmp.dir); defer gpa.free(after); try std.testing.expectEqualSlices(u8, extended, after); } if (tmp.dir.openFile(io, pack_tmp_filename, .{})) |f| { var mf = f; mf.close(io); return error.TempSurvivedTheAbort; } else |_| {} } test "rewriteAndSwap: a run the concurrency guard aborts leaves .zit untouched and records NO tombstone (competing RENAME)" { // THE SIBLING SHAPE, and the one the caller's own file handle cannot see. The // test above simulates a concurrent writer that APPENDS to `.zit`; here a // competing rewrite (`zit pack`, or another `zit shun --force`) renames a whole // new generation over the name while our snapshot's handle stays open. That // handle then names an ORPHAN inode whose size and tail can never move again, // so a guard that re-stats it agrees with its own key forever and this run // renames a pre-competitor snapshot over the winner — silently restoring // content the winner shunned, with both runs exiting 0. Reopening `.zit` by // NAME is what turns it into `error.TailChanged`. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const doomed_body = "content the operator named in this invocation " ** 4; var doomed: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(doomed_body, &doomed, .{}); { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wlog.deinit(gpa); _ = try packSeedBlob(&wlog, doomed_body); _ = try packSeedBlob(&wlog, "content with nothing against it " ** 4); try wlog.commit(); } // The scan the rewrite will be built from, and the tail key that pins it. This // handle is the one the competing rename is about to orphan. var rfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &rfs); defer src.deinit(gpa); const src_key = try src.logKey(); // The competing rewrite: a NEW generation built beside `.zit` and renamed over // it. Nothing touches the inode `src` is reading, which is the whole point. { var f = try tmp.dir.createFile(io, "gen2", .{ .read = true, .truncate = true }); defer f.close(io); var gfs = storage.FileStorage.init(io, f, 0); var glog = try logstore.LogStore.open(gpa, gfs.storageFile(), src.hash_kind); defer glog.deinit(gpa); _ = try packSeedBlob(&glog, "the generation the competing rewrite produced " ** 3); try glog.commit(); } try tmp.dir.rename("gen2", tmp.dir, repo_filename, io); const winner = try storeSnapshot(gpa, io, tmp.dir); defer gpa.free(winner); var set: NameSet = .{}; defer set.deinit(gpa); try set.add(gpa, &doomed); const rows = [_]logstore.ShunRecord{ .{ .hash = &doomed, .epoch = 1753000077, .reason = "takedown notice 4417" }, }; try std.testing.expectError(error.TailChanged, rewriteAndSwap( gpa, io, tmp.dir, &src, src_key, set.filter(), &rows, )); // The winner's generation survives byte for byte, and the aborted run recorded // nothing — the loser did not resurrect its own snapshot's contents. { const after = try storeSnapshot(gpa, io, tmp.dir); defer gpa.free(after); try std.testing.expectEqualSlices(u8, winner, after); } var vfs: storage.FileStorage = undefined; var after_log = try packOpenReadOnly(gpa, io, tmp.dir, &vfs); defer after_log.deinit(gpa); try std.testing.expectEqual(@as(usize, 0), after_log.shunList().len); try std.testing.expect(after_log.lookup(&doomed) == null); if (tmp.dir.openFile(io, pack_tmp_filename, .{})) |f| { var mf = f; mf.close(io); return error.TempSurvivedTheAbort; } else |_| {} } /// Read `dir/.zit` whole (caller frees) — the byte-identical oracle for a run /// that must have changed nothing. fn storeSnapshot(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) ![]u8 { var file = try dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); const buf = try gpa.alloc(u8, @intCast(st.size)); errdefer gpa.free(buf); const n = try file.readPositionalAll(io, buf, 0); try std.testing.expectEqual(buf.len, n); return buf; } // --- `zit shun`'s pure renderers ------------------------------------------- // // Every operator-visible byte this verb emits is produced by a function of data // alone, for the same reason `renderPackStats` is: the exact wording of a // refusal that precedes irreversible deletion is the part most worth pinning, // and pinning it through a subprocess would make the e2e tests the only place // it is stated. const shun_name_a = "1c04b7" ++ "0" ** 58; const shun_name_b = "7bd1e0" ++ "1" ** 58; const shun_name_c = "9f2a1c" ++ "2" ** 58; test "renderShunRefusal: the counts and the word refusing share the first line, then one line per hash" { // The register is `clean`'s (`cleanRefuse`): an operator must not have to read // six lines of inventory to discover that nothing happened. var buf: [1024]u8 = undefined; var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = .manifest, .disposition = .remove }, .{ .name = shun_name_b, .kind = null, .disposition = .remove }, .{ .name = shun_name_c, .disposition = .absent }, }, 0, false); try std.testing.expectEqualStrings( "zit shun: 3 hashes would be shunned in this repository; 2 artifacts would be permanently removed; refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " checkin a check-in; its descendants become unresolvable\n" ++ " " ++ shun_name_b ++ " file file content; every check-in listing it loses that file\n" ++ " " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --force <HASH>...' to proceed\n", w.buffered(), ); } test "renderShunRefusal: singular agreement, an already-shunned line, and the nothing-held wording" { var buf: [1024]u8 = undefined; { // One target, held here: "1 hash", "1 artifact". var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = .wiki, .disposition = .remove }, .{ .name = shun_name_b, .disposition = .already }, }, 0, false); try std.testing.expectEqualStrings( "zit shun: 1 hash would be shunned in this repository; 1 artifact would be permanently removed; refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " wiki structural content; what it renders stops rendering\n" ++ " " ++ shun_name_b ++ " already shunned; content already gone\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --force <HASH>...' to proceed\n", w.buffered(), ); } { // A purely pre-emptive run: the tombstones are still irreversible, so the // refusal stands, but nothing would be removed and the line says so // instead of claiming a removal that is not going to happen. var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{.{ .name = shun_name_c, .disposition = .absent }}, 0, false); try std.testing.expectEqualStrings( "zit shun: 1 hash would be shunned in this repository; nothing you named holds content to remove; refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --force <HASH>...' to proceed\n", w.buffered(), ); } } test "renderShunRefusal: a reclaim-only run, and the collateral an earlier --defer left behind" { // THE CONSENT GATE MUST NAME EVERYTHING THE REWRITE DESTROYS, including what the // operator did not name. `renderShunStats` is honest after the fact that // `dropped` can exceed `recorded`; a refusal that only counted this // invocation's own targets would have the operator consent to one deletion and // get several. var buf: [1024]u8 = undefined; var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = null, .disposition = .reclaim }, }, 2, false); try std.testing.expectEqualStrings( "zit shun: no new hash would be recorded; 1 artifact would be permanently removed; " ++ "the same rewrite also removes 2 artifacts shunned earlier and not yet reclaimed; " ++ "refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " file already shunned; content still here and would be removed\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --force <HASH>...' to proceed\n", w.buffered(), ); } test "renderShunRefusal: a --defer run is described as a --defer run, and suggests --defer --force" { // THE CONSENT GATE MUST DESCRIBE THE OPERATION THE OPERATOR ASKED FOR. A // `--defer` run removes nothing now, so it must not say "would be permanently // removed"; it must not attribute rewrite collateral to a run that performs no // rewrite; and — the reason this is pinned byte for byte — the command it // suggests must be the deferred one. An operator who copies `zit shun --force` // out of a `--defer` refusal escalates a tombstone into a full-store reclaim of // the whole union set, including collateral this run never named. var buf: [1024]u8 = undefined; { // `deferred_held` is 2 here and the clause must NOT appear: nothing on this // path reclaims those rows. var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = .manifest, .disposition = .remove }, .{ .name = shun_name_b, .kind = null, .disposition = .remove }, .{ .name = shun_name_c, .disposition = .absent }, }, 2, true); try std.testing.expectEqualStrings( "zit shun: 3 hashes would be shunned in this repository; " ++ "the bytes of 2 artifacts stay on disk and stay reconstructible until 'zit pack'; " ++ "refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " checkin a check-in; its descendants become unresolvable\n" ++ " " ++ shun_name_b ++ " file file content; every check-in listing it loses that file\n" ++ " " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --defer --force <HASH>...' to proceed\n", w.buffered(), ); } { // Singular agreement on the deferred clause, and the `.reclaim` line: under // `--defer` the content stays rather than being removed. var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{ .{ .name = shun_name_a, .kind = .wiki, .disposition = .reclaim }, }, 0, true); try std.testing.expectEqualStrings( "zit shun: no new hash would be recorded; " ++ "the bytes of 1 artifact stay on disk and stay reconstructible until 'zit pack'; " ++ "refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_a ++ " wiki already shunned; content still here and stays\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --defer --force <HASH>...' to proceed\n", w.buffered(), ); // The escalating suggestion must be ABSENT, not merely reworded: this is the // one substring an operator is most likely to paste. try std.testing.expect(std.mem.indexOf(u8, w.buffered(), "run 'zit shun --force") == null); } { // Nothing named holds content: the wording is the same on both paths, and // only the suggested command differs. var w: std.Io.Writer = .fixed(&buf); renderShunRefusal(&w, &.{.{ .name = shun_name_c, .disposition = .absent }}, 0, true); try std.testing.expectEqualStrings( "zit shun: 1 hash would be shunned in this repository; " ++ "nothing you named holds content to remove; refusing without --force:\n" ++ "\n" ++ " " ++ shun_name_c ++ " recorded only (not present in this repository)\n" ++ "\n" ++ "This is irreversible: zit undo does not recover a shun, and the content\n" ++ "cannot be re-fetched from a peer.\n" ++ "run 'zit shun --defer --force <HASH>...' to proceed\n", w.buffered(), ); } } test "shunConsequence/shunTypeLabel: one arm per consequence, total over ArtifactType" { // The per-type diagnostic, pinned type by type. The switch is total, so this // test and the compiler together guarantee a type added later cannot ship // with no warning attached to its removal. try std.testing.expectEqualStrings("file", shunTypeLabel(null)); try std.testing.expectEqualStrings( "file content; every check-in listing it loses that file", shunConsequence(null), ); try std.testing.expectEqualStrings("checkin", shunTypeLabel(.manifest)); try std.testing.expectEqualStrings( "a check-in; its descendants become unresolvable", shunConsequence(.manifest), ); try std.testing.expectEqualStrings("cluster", shunTypeLabel(.cluster)); try std.testing.expectEqualStrings("sync housekeeping only; regenerable", shunConsequence(.cluster)); // The six that all stop rendering share one arm; their LABELS still differ, // so the operator can tell a ticket from a forum post. const rendering = "structural content; what it renders stops rendering"; inline for (.{ .control, .wiki, .ticket, .attachment, .event, .forum }) |k| { try std.testing.expectEqualStrings(rendering, shunConsequence(k)); } try std.testing.expectEqualStrings("tag", shunTypeLabel(.control)); try std.testing.expectEqualStrings("technote", shunTypeLabel(.event)); try std.testing.expectEqualStrings("forumpost", shunTypeLabel(.forum)); } test "renderShunBlockers: the unconditional stops, one line each, with no --force offered" { // The current-checkout stop is the only refusal in this verb with no // override, so its text must NOT mention --force — an operator who reads a // flag out of it will try the flag and be refused twice. var buf: [1024]u8 = undefined; var w: std.Io.Writer = .fixed(&buf); renderShunBlockers(&w, &.{ .{ .name = shun_name_a, .kind = .manifest, .disposition = .current_checkout }, .{ .name = "nothex", .disposition = .not_a_name }, .{ .name = "1c04", .disposition = .unresolved }, .{ .name = "7bd1", .disposition = .ambiguous }, // A permissive target contributes nothing to this block. .{ .name = shun_name_c, .disposition = .remove }, }); try std.testing.expectEqualStrings( "zit shun: " ++ shun_name_a ++ " is the check-in this checkout is on; refusing\n" ++ "run 'zit update' to a different check-in first\n" ++ "zit shun: 'nothex' is not an artifact name\n" ++ "zit shun: no artifact matches '1c04'; give the complete name to shun a hash this repository does not hold\n" ++ "zit shun: hash '7bd1' is ambiguous\n", w.buffered(), ); try std.testing.expect(std.mem.indexOf(u8, w.buffered(), "--force") == null); } test "renderShunStats/renderShunDeferred: the two success lines" { var buf: [256]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderShunStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 3, .shuns_recorded = 3, .bytes_before = 481232, .bytes_after = 402118, }); try std.testing.expectEqualStrings( "shunned: 3 hashes recorded, 3 artifacts removed, 12 -> 9 remaining, 481232 -> 402118 bytes\n", w.buffered(), ); } { // `dropped` legitimately EXCEEDS `recorded`: the same rewrite reclaims // shuns deferred by earlier runs, and an operator must be told that // rather than left to infer it from the byte count. var w: std.Io.Writer = .fixed(&buf); renderShunStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 4, .shuns_recorded = 1, .bytes_before = 8192, .bytes_after = 4096, }); try std.testing.expectEqualStrings( "shunned: 1 hash recorded, 4 artifacts removed, 12 -> 8 remaining, 8192 -> 4096 bytes\n", w.buffered(), ); } { // THE RECORDED COUNT IS THE PACK'S, NOT THE CALLER'S. `shuns_recorded` is // measured off the new generation's own shun table, so this line cannot // over-report a takedown even if a caller's intent and the pack's result ever // diverged. Two tombstones, and this rewrite reclaimed nothing. var w: std.Io.Writer = .fixed(&buf); renderShunStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 0, .shuns_recorded = 2, .bytes_before = 8192, .bytes_after = 8320, }); try std.testing.expectEqualStrings( "shunned: 2 hashes recorded, 0 artifacts removed, 12 -> 12 remaining, 8192 -> 8320 bytes\n", w.buffered(), ); } { // --defer promises no byte count: nothing on that path measured one. var w: std.Io.Writer = .fixed(&buf); renderShunDeferred(&w, 2, 2); try std.testing.expectEqualStrings( "shunned: 2 hashes recorded; run 'zit pack' to reclaim the space\n", w.buffered(), ); } { var w: std.Io.Writer = .fixed(&buf); renderShunDeferred(&w, 1, 1); try std.testing.expectEqualStrings( "shunned: 1 hash recorded; run 'zit pack' to reclaim the space\n", w.buffered(), ); } { // Re-deferring a hash already deferred: no row to add, and yet content still // to lose. "0 hashes recorded" would read as a no-op over live bytes. var w: std.Io.Writer = .fixed(&buf); renderShunDeferred(&w, 0, 1); try std.testing.expectEqualStrings( "shunned: no new hash to record; 1 artifact still holds content; run 'zit pack' to reclaim the space\n", w.buffered(), ); } { // The purely pre-emptive success line: no rewrite ran, so it claims no byte // count AND does not claim a removal. var w: std.Io.Writer = .fixed(&buf); renderShunRecordedOnly(&w, 1); try std.testing.expectEqualStrings( "shunned: 1 hash recorded; none is held here, so nothing was removed\n", w.buffered(), ); } } test "renderShunList: persisted order, the recorded date, and the reason only when given" { var buf: [512]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderShunList(&w, &.{}); try std.testing.expectEqualStrings("the shun table is empty\n", w.buffered()); } { // 1753660800 = 2025-07-28T00:00:00Z; 1752192000 = 2025-07-11T00:00:00Z. // A row with no reason prints no trailing separator at all. var w: std.Io.Writer = .fixed(&buf); renderShunList(&w, &.{ .{ .hash = "\x9f\x2a\x1c" ++ "\x00" ** 29, .epoch = 1753660800, .reason = "legal takedown ref #4417" }, .{ .hash = "\x1c\x04\xb7" ++ "\x00" ** 17, .epoch = 1753660800, .reason = "" }, .{ .hash = "\x7b\xd1\xe0" ++ "\x00" ** 29, .epoch = 1752192000, .reason = "committed credential" }, }); try std.testing.expectEqualStrings( "9f2a1c" ++ "00" ** 29 ++ " 2025-07-28 legal takedown ref #4417\n" ++ "1c04b7" ++ "00" ** 17 ++ " 2025-07-28\n" ++ "7bd1e0" ++ "00" ** 29 ++ " 2025-07-11 committed credential\n", w.buffered(), ); } } /// How many rows the store in `dir` holds in its shun table. fn shunRowsIn(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !usize { var fs: storage.FileStorage = undefined; var log = try packOpenReadOnly(gpa, io, dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); return log.shunList().len; } test "shunAppendRows: the append aborts on a changed tail and on a held temp lock, recording nothing" { // THE APPEND PATH'S TWO CONCURRENCY GUARDS, driven directly because neither is // reachable from the CLI in one process: both windows open between the moment // `runShun` classified its targets and the moment this function commits. // // The TAIL guard is what makes the append path's success lines honest. They are // claims about STORE STATE — "none is held here, so nothing was removed" — and a // `zit commit` landing a named hash in that window falsifies them: the tombstone // would be recorded, exit 0, and an operator under a takedown order told nothing // was held while the bytes were live and unreclaimed. // // The LOCK guard is what stops the row being written to an ORPHANED INODE. A // `zit pack` renaming its new generation over `.zit` after our `openFile` takes // the append with it, and the run would still report success — a deferred shun // that silently does not exist, having just told the operator to run `zit pack` // to finish it. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } const digest_a = [_]u8{0xab} ** digest_len; const digest_b = [_]u8{0xcd} ** digest_len; const rows_a = [_]logstore.ShunRecord{.{ .hash = &digest_a, .epoch = 1_753_660_800, .reason = "" }}; const rows_b = [_]logstore.ShunRecord{.{ .hash = &digest_b, .epoch = 1_753_660_800, .reason = "" }}; // The key the runner would have captured off its read-only snapshot. var key_1: logstore.LogStore.LogKey = undefined; { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); key_1 = try src.logKey(); } // Control: with a current key the append lands, so the guards below are failing // for the reason under test rather than because this path never works. try shunAppendRows(gpa, io, tmp.dir, key_1, &rows_a); try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir)); // No temp left behind: the lock file is removed on the success path too. try std.testing.expectError(error.FileNotFound, tmp.dir.openFile(io, pack_tmp_filename, .{})); // That append is itself the "concurrent writer": `key_1` is now stale. const before = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(before); try std.testing.expectError(error.TailChanged, shunAppendRows(gpa, io, tmp.dir, key_1, &rows_b)); // NOTHING WRITTEN: the check precedes the first `putShun`. try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir)); { const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(after); try std.testing.expectEqualSlices(u8, before, after); } var key_2: logstore.LogStore.LogKey = undefined; { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); key_2 = try src.logKey(); } { // Stand in for the concurrent rewrite by holding the temp lock, which makes // this deterministic rather than a race. var held = try tmp.dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = false, .lock = .exclusive, .lock_nonblocking = true, }); defer held.close(io); try std.testing.expectError(error.RewriteBusy, shunAppendRows(gpa, io, tmp.dir, key_2, &rows_b)); try std.testing.expectEqual(@as(usize, 1), try shunRowsIn(gpa, io, tmp.dir)); const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(after); try std.testing.expectEqualSlices(u8, before, after); } tmp.dir.deleteFile(io, pack_tmp_filename) catch {}; // Released and current: the same call now lands. try shunAppendRows(gpa, io, tmp.dir, key_2, &rows_b); try std.testing.expectEqual(@as(usize, 2), try shunRowsIn(gpa, io, tmp.dir)); } /// Render one `ApplyError` through `openApplyFail` into `buf`. fn openFailText(buf: []u8, err: update.ApplyError) []const u8 { var w = Io.Writer.fixed(buf); var sink: [1]u8 = undefined; var wnull = Io.Writer.fixed(&sink); const s: cli.Streams = .{ .out = &wnull, .err = &w }; _ = openApplyFail(s, err); return w.buffered(); } // Same guard as the `zit update`, `zit merge` and `zit bisect` mappers. // `Corrupt` alleges the stored bytes do not match the name they are stored // under; `StateRecordInvalid` says only that an artifact was unusable as a // state record. A clone of an intact fossil repository reaches the second // without anything being damaged, so it must never be answered with a // corruption claim. test "open: the two read-path failures do not share a diagnostic" { var a: [256]u8 = undefined; var b: [256]u8 = undefined; const corrupt = openFailText(&a, error.Corrupt); const state = openFailText(&b, error.StateRecordInvalid); try std.testing.expect(!std.mem.eql(u8, corrupt, state)); try std.testing.expect(std.mem.indexOf(u8, corrupt, "corrupt") != null); try std.testing.expect(std.mem.indexOf(u8, state, "corrupt") == null); } // --------------------------------------------------------------------------- // `zit serve` must answer one sync round in time bounded by the // REQUEST, not by the served store. Two per-request costs made a clone // O(rounds x store): counting held artifacts by forcing a full inflate of every // artifact, and rebuilding the entire in-memory snapshot from the log on every // request. The guards below pin both as SCALING properties measured with work // counters (plaintext bytes materialised, snapshot builds performed) — never // wall-clock thresholds, which flake and prove nothing about the mechanism. // --------------------------------------------------------------------------- /// Seed `ms` with `n` distinct wire-backed artifacts exactly as /// `loadContentStore` does for a served repository: one §4.1 full container per /// artifact, with NO plaintext materialised. `ms.plaintextBytesHeld()` is 0 on /// return, so any later non-zero reading is inflation this store did not ask for. fn serveSeedWireStore(gpa: std.mem.Allocator, ms: *repostate.MemStore, n: usize) !void { var i: usize = 0; while (i < n) : (i += 1) { var body_buf: [64]u8 = undefined; const body = try std.fmt.bufPrint(&body_buf, "issue nine artifact {d}\n", .{i}); var name_buf: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, body, &name_buf); const container = try zit.container.encode(gpa, body, .default); defer gpa.free(container); _ = try ms.putWire(name, container); } std.debug.assert(ms.plaintextBytesHeld() == 0); } test "serve: the held-artifact count serve uses never materialises plaintext (constant in store size)" { const gpa = std.testing.allocator; // Two store sizes: the work this count performs must not grow with n. The // count VALUE alone would pass before and after the fix — it is the // materialised-plaintext reading that observes the defect, so that is what // is asserted, at both sizes, as the scaling statement. for ([_]usize{ 8, 64 }) |n| { var ms = repostate.MemStore.init(gpa); defer ms.deinit(); try serveSeedWireStore(gpa, &ms, n); try std.testing.expectEqual(n, ms.store().heldCount()); try std.testing.expectEqual(@as(usize, 0), ms.plaintextBytesHeld()); } } test "serve: a held artifact whose container does not inflate is still counted as servable" { const gpa = std.testing.allocator; var ms = repostate.MemStore.init(gpa); defer ms.deinit(); // §4.1 shape (4-byte length prefix) with a payload that is not a valid zlib // stream: `held()==true`, `content()==null` — the case `held()`'s contract at // repostate.zig names. `respond` retransmits it verbatim and `persistStore` // counts it, so serve's own count must agree with them. const bogus = "\x00\x00\x00\x0anot-zlib-bytes"; _ = try ms.putWire("f" ** 64, bogus); try std.testing.expectEqual(@as(usize, 1), ms.store().heldCount()); try std.testing.expect(ms.store().held(1)); try std.testing.expect(ms.store().content(1) == null); } /// Append one content frame (raw `digest` naming `bytes`) to the `.zit` in /// `dir`, creating the repository on first call — a separate writer's committed /// append, exactly what a `zit commit` alongside a live `zit serve` performs. fn serveAppendFrame(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, digest: []const u8, bytes: []const u8) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch try dir.createFile(io, repo_filename, .{ .read = true }); defer f.close(io); const st = try f.stat(io); var fs = storage.FileStorage.init(io, f, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer log.deinit(gpa); _ = try log.put(digest, bytes); try log.commit(); } /// A §4.1-framed read-only request naming `hash` in a `gimme` card. fn serveGimmeRequest(gpa: std.mem.Allocator, card: []const u8, hash: []const u8) ![]u8 { const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\n{s} {s}\n", .{ card, hash }); defer gpa.free(plain); return zit.container.encode(gpa, plain, .default); } test "serve: repeated rounds over an unchanged log build the snapshot once (constant in round count)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const digest_a = "a" ** digest_len; try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n"); var hex_buf: [2 * digest_len]u8 = undefined; const hex_a = digestToWireName(digest_a, &hex_buf); const request = try serveGimmeRequest(gpa, "gimme", hex_a); defer gpa.free(request); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); // Eight rounds, log unchanged: the snapshot is a function of the log, not of // the request, so the number of builds must not grow with the round count. for (0..8) |_| { const reply = try serveAnswer(&ctx, gpa, request, null, null, true); gpa.free(reply.body); } try std.testing.expectEqual(@as(u64, 1), ctx.loads); } test "serve: a responder's phantom interning cannot leak into the next round" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const digest_a = "a" ** digest_len; try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n"); var hex_buf: [2 * digest_len]u8 = undefined; const hex_a = digestToWireName(digest_a, &hex_buf); // §3.3 — an `igot` for a hash the server lacks makes `respond` intern a // phantom into whatever store it answers over. That mutation must not // survive into the reused snapshot. const unknown_hex = "c" ** (2 * digest_len); const igot_req = try serveGimmeRequest(gpa, "igot", unknown_hex); defer gpa.free(igot_req); const gimme_req = try serveGimmeRequest(gpa, "gimme", hex_a); defer gpa.free(gimme_req); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); // NOTE ON THIS GUARD'S ROLE: it is green against the pre-fix code (which // rebuilds per request and so cannot leak) — the round-count guard above is // the red one. This guard exists because the obvious WRONG way to satisfy // that guard — answer directly over one shared mutable snapshot — passes it // and truncates real clones, since the phantoms and memoisations `respond` // writes change what later rounds serve. Without this assertion that // implementation ships. var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const r1 = try serveAnswer(&ctx, gpa, igot_req, null, null, true); gpa.free(r1.body); const snap = ctx.snapshot.?.store(); try std.testing.expectEqual(@as(usize, 1), snap.count()); // the one on-disk artifact, nothing more try std.testing.expect(snap.lookup(unknown_hex) == null); // the phantom did not land here const r2 = try serveAnswer(&ctx, gpa, gimme_req, null, null, true); defer gpa.free(r2.body); try std.testing.expectEqual(@as(u64, 1), ctx.loads); // still one build: the log never changed // The reply is byte-identical to the one a server with no prior round produces. var fresh_file = try tmp.dir.openFile(io, repo_filename, .{}); defer fresh_file.close(io); var fresh_ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &fresh_file, .server = &server }; defer fresh_ctx.dropSnapshot(); const expected = try serveAnswer(&fresh_ctx, gpa, gimme_req, null, null, true); defer gpa.free(expected.body); try std.testing.expectEqualSlices(u8, expected.body, r2.body); } test "serve §3.8: a reused snapshot still sees a concurrent commit's append" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const digest_a = "a" ** digest_len; const digest_b = "b" ** digest_len; try serveAppendFrame(gpa, io, tmp.dir, digest_a, "content a\n"); var hex_a_buf: [2 * digest_len]u8 = undefined; var hex_b_buf: [2 * digest_len]u8 = undefined; const hex_a = digestToWireName(digest_a, &hex_a_buf); const hex_b = digestToWireName(digest_b, &hex_b_buf); const req_a = try serveGimmeRequest(gpa, "gimme", hex_a); defer gpa.free(req_a); const req_b = try serveGimmeRequest(gpa, "gimme", hex_b); defer gpa.free(req_b); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const r1 = try serveAnswer(&ctx, gpa, req_a, null, null, true); gpa.free(r1.body); try std.testing.expectEqual(@as(usize, 1), ctx.snapshot.?.store().count()); // A SEPARATE writer appends while the serve handle stays open. try serveAppendFrame(gpa, io, tmp.dir, digest_b, "content b\n"); const r2 = try serveAnswer(&ctx, gpa, req_b, null, null, true); gpa.free(r2.body); // The log grew, so the snapshot was rebuilt and the new artifact is servable: // a cache that never invalidates would still report one artifact and one load. try std.testing.expectEqual(@as(u64, 2), ctx.loads); try std.testing.expectEqual(@as(usize, 2), ctx.snapshot.?.store().count()); try std.testing.expect(ctx.snapshot.?.store().lookup(hex_b) != null); } /// Seed `dir`'s `.zit` with `n` distinct compressible artifacts, returning the /// wire (hex) name of the first. Compressible so the log retains a §4.1 /// container per frame — the shape a served repository actually has, and the /// only shape in which "answering a round does not inflate the store" has /// anything to say. fn serveSeedRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, n: usize, first_hex: *[2 * digest_len]u8) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch try dir.createFile(io, repo_filename, .{ .read = true }); defer f.close(io); const st = try f.stat(io); var fs = storage.FileStorage.init(io, f, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer log.deinit(gpa); var i: usize = 0; while (i < n) : (i += 1) { var body: [512]u8 = undefined; @memset(&body, 'a' + @as(u8, @intCast(i % 26))); var digest: [digest_len]u8 = undefined; @memset(&digest, 0); std.mem.writeInt(u64, digest[0..8], i + 1, .big); _ = try log.putZlib(&digest, &body); if (i == 0) _ = digestToWireName(&digest, first_hex); } try log.commit(); } test "serve: answering a round inflates only what the round emits (constant in store size)" { const gpa = std.testing.allocator; const io = std.testing.io; // The same read round against two store sizes. The reply is identical, so // the plaintext the server inflates to produce it must be identical too: any // dependence on n is the server paying for artifacts the round never names. // This is the assertion the whole change exists to make true — the pre-fix // held-count walked `content()` over every rid, and so did inflate the store. var inflated: [2]usize = undefined; for ([_]usize{ 4, 40 }, 0..) |n, k| { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var hex_buf: [2 * digest_len]u8 = undefined; try serveSeedRepo(gpa, io, tmp.dir, n, &hex_buf); const request = try serveGimmeRequest(gpa, "gimme", &hex_buf); defer gpa.free(request); const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); for (0..3) |_| { const reply = try serveAnswer(&ctx, gpa, request, null, null, true); gpa.free(reply.body); // Nothing the round materialised is still resident: the cached index // reads containers on demand and frees them between rounds, never // holding the repository's uncompressed history (or its containers). try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.residentBytes()); } const emitted = ctx.snapshot.?.materialized_bytes; // A CLONE round is the worst case for residency: it emits every artifact // and proves each one's declared size against the plaintext, so it // legitimately inflates the whole store. What must not survive it is the // plaintext — a snapshot that outlives the round would otherwise end up // holding the entire repository uncompressed, for the process's life. const clone_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 1\n", .{}); defer gpa.free(clone_plain); const clone_req = try zit.container.encode(gpa, clone_plain, .default); defer gpa.free(clone_req); const clone_reply = try serveAnswer(&ctx, gpa, clone_req, null, null, true); gpa.free(clone_reply.body); try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.residentBytes()); try std.testing.expect(ctx.snapshot.?.materialized_bytes >= emitted); try std.testing.expectEqual(@as(usize, n), ctx.snapshot.?.store().count()); inflated[k] = emitted; } try std.testing.expectEqual(inflated[0], inflated[1]); // ... and the constant is the round's own emission, not the store's: a // 40-artifact store holds 20480 bytes of plaintext, so an equality that // happened to hold at "everything" would still be caught here. try std.testing.expect(inflated[1] < 4 * 512); } test "serve: the pushing round does not rebuild inside its deadline; maintain rebuilds between connections" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.commit(); } const push_body = "pushed artifact bytes\n"; var name_buf: [64]u8 = undefined; const push_hash = zit.artifact.artifactName(.sha3_256, push_body, &name_buf); const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ push_hash, push_body.len, push_body }); defer gpa.free(plain); const push_req = try zit.container.encode(gpa, plain, .default); defer gpa.free(push_req); const read_req = try serveGimmeRequest(gpa, "gimme", push_hash); defer gpa.free(read_req); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const r0 = try serveAnswer(&ctx, gpa, read_req, null, null, true); gpa.free(r0.body); try std.testing.expectEqual(@as(u64, 1), ctx.loads); // The push makes the snapshot stale, but answering it must NOT pay the // O(store) rebuild: the reply is written inside a connection deadline, and // The failure is exactly the whole-repository rebuild landing inside that // budget. The rebuild is deferred to `maintain`, which the accept loop runs // after the connection closes — outside every deadline. So the pushing round // leaves `loads` untouched. const r1 = try serveAnswer(&ctx, gpa, push_req, null, null, true); gpa.free(r1.body); try std.testing.expectEqual(@as(u64, 1), ctx.loads); // no rebuild inside the round // The snapshot is momentarily stale — it predates the push — yet the push is // already durable on disk: `persistStore` ran inside the round. try std.testing.expect(ctx.snapshot != null); try std.testing.expect(ctx.snapshot.?.store().lookup(push_hash) == null); // The accept loop's post-connection maintenance hook does the O(store) // rebuild, between connections, with no connection open to charge it to. ServeCtx.maintain(&ctx, gpa); try std.testing.expectEqual(@as(u64, 2), ctx.loads); try std.testing.expect(ctx.snapshot.?.store().lookup(push_hash) != null); // The rebuilt snapshot describes the grown log exactly (the append-only stat // gate that `baseSnapshot` already carries). const st = try file.stat(io); try std.testing.expectEqual(st.size, ctx.snapshot_log_size); // A later read round is stat-gated to no rebuild, and still serves the push. const r2 = try serveAnswer(&ctx, gpa, read_req, null, null, true); defer gpa.free(r2.body); try std.testing.expectEqual(@as(u64, 2), ctx.loads); } // --------------------------------------------------------------------------- // sync §3.3 L527-530 / §8.8 L1777-1779 — the durable side of the phantom-scan // shun filter: the repository shun table, snapshotted off the log and answered // as a wire-name question. `RepoShunSet` lives above with `LogIndexStore`; these // tests pin the key domain and the end-to-end suppression. // --------------------------------------------------------------------------- /// The shunned artifact's digest and its complete 64-hex wire name. 's' is 0x73, /// so the name is the byte pattern spelled out — no runtime encode in the vector. const shun_test_digest = "s" ** digest_len; const shun_test_name = "73" ** digest_len; /// A second artifact the repo actually holds ('h' is 0x68), to prove the filter /// suppresses only the shunned name. const shun_test_held_digest = "h" ** digest_len; const shun_test_held_name = "68" ** digest_len; /// A third shunned digest whose hex spelling contains LETTER nibbles, so the /// uppercase-spelling question below is a real one. const shun_test_letter_digest = "\xab" ** digest_len; const shun_test_letter_name = "ab" ** digest_len; comptime { std.debug.assert(shun_test_name.len == 2 * digest_len); std.debug.assert(logstore.isCompleteDigest(shun_test_digest.len)); } /// Seed `dir`'s repository with one held artifact and one shun row naming an /// artifact it does NOT hold — exactly the state a `zit shun` on a hash the peer /// still offers leaves behind. fn seedShunnedRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); _ = try log.put(shun_test_held_digest, "held content\n"); try log.putShun(shun_test_digest, "operator request", 1); try log.putShun(shun_test_letter_digest, "operator request", 1); try log.commit(); } test "sync §3.3 L527-530: a repository shun row is a durable wire-name hit, keyed exactly" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedShunnedRepo(gpa, io, tmp.dir); var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const shun = &holdings.index.shun; const set = holdings.admission().shun; // The complete name is the key: a hit. try std.testing.expect(set.contains(shun_test_name)); // The held artifact is not shunned. try std.testing.expect(!set.contains(shun_test_held_name)); // A 12-hex PREFIX of the shunned name is a DIFFERENT key, not a hit. The // lookup never truncates, pads, or prefix-matches — a prefix must not be // able to masquerade as a shun-table hit. Nor may an incomplete name reach // `LogStore.isShunned`, whose caller contract (src/logstore.zig L1746-1757) // binds this boundary: the width check is a CHECKED rejection, not an // assert, because an assert is nothing in ReleaseFast and these names come // off the wire. // // These OUT-OF-DOMAIN widths are probed against `containsWireName` directly, // not through `zit.sync.ShunSet.contains`: the release-checked rule lives // here, in the implementation that owns it, while the wrapper asserts the // §3.2 width its own key domain promises. Going through the wrapper would // pin the assert instead of the rejection these lines mean to pin. try std.testing.expect(!RepoShunSet.containsWireName(shun, shun_test_name[0..12])); try std.testing.expect(!RepoShunSet.containsWireName(shun, shun_test_name[0 .. 2 * digest_len - 1])); // odd length try std.testing.expect(!RepoShunSet.containsWireName(shun, "")); // empty try std.testing.expect(!RepoShunSet.containsWireName(shun, "ab" ** 48)); // 96 hex — not one of the two forms // Correct WIDTH, not hex: in the wrapper's key domain, so probed through it. try std.testing.expect(!set.contains("z" ** (2 * digest_len))); // not hex // An UPPERCASE spelling of the shunned name decodes to the same digest and // so answers TRUE. artifact-format §3.3 L370-377 makes the token invalid but // lets an implementation accept it at name validation and fail it at // verification; over-suppressing a name that can never verify is the safe // direction. Pinned so the fail-closed direction is a decision, not an // accident of `hexToBytes` accepting case. try std.testing.expect(set.contains(shun_test_letter_name)); var upper: [2 * digest_len]u8 = undefined; for (shun_test_letter_name, 0..) |c, i| upper[i] = std.ascii.toUpper(c); try std.testing.expect(!std.mem.eql(u8, &upper, shun_test_letter_name)); // the case really differs try std.testing.expect(set.contains(&upper)); } test "sync §3.3 L527-530: an OOM inside the shun-set load leaves no borrowed key in the owned-key map" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedShunnedRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var fs = storage.FileStorage.init(io, file, st.size); var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile()); defer log.deinit(gpa); try std.testing.expect(log.shunList().len >= 2); // `load` dupes each row's digest because `shunList` borrows the STORE's own // bytes, and the dupe must PRECEDE the insert: insert-then-replace leaves the // borrowed slice live as a key on the one error path, after which `deinit` // frees the store's digest and `log.deinit` frees it again. A manual sweep // (not `checkAllAllocationFailures`) because the obligation is about which // POINTER survives the unwind, not just about leaking: the store is // allocated from the UNWRAPPED allocator, so a key that is not `load`'s own // dupe is freed through the wrap and the double free lands on // `testing.allocator`, which catches it. `fail_index` indexes `load`'s // allocations alone for the same reason. var i: usize = 0; while (i < 64) : (i += 1) { var fa = std.testing.FailingAllocator.init(gpa, .{ .fail_index = i }); var shun = RepoShunSet.init(fa.allocator()); defer shun.deinit(); if (shun.load(&log)) |_| { // The fail point lay at or past the last allocation: no failure was // forced, so every allocation site is now covered and the set is // whole. try std.testing.expect(shun.digests.count() >= 2); break; } else |e| { try std.testing.expectEqual(error.OutOfMemory, e); } } else return error.SweepNeverCompleted; } test "sync §3.3 L527-530: a pull client carrying the repository shun set creates the phantom but never gimme's it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedShunnedRepo(gpa, io, tmp.dir); var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); // The pull client, constructed exactly as `pullRun` does. var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = sync_servercode, .project_code = sync_projectcode, .version = .{ .version = sync_client_version }, .shun = holdings.admission().shun, }; defer client.deinit(gpa); // The peer announces BOTH artifacts. §3.3 r24(1) is unconditional on the // shun table, so the shunned one still becomes a phantom. const reply = "pragma server-version 1\nigot " ++ shun_test_name ++ "\n"; const obs = try client.consumeReply(gpa, holdings.store(), reply); try std.testing.expect(obs.phantom_created); try std.testing.expectEqual(@as(usize, 1), client.phantoms.count()); // …and the request never asks for it, in any round. var buf: std.ArrayListUnmanaged(u8) = .empty; defer buf.deinit(gpa); var round: usize = 0; while (round < 3) : (round += 1) { const request = try client.buildRequest(gpa, holdings.store(), &buf); try std.testing.expect(std.mem.indexOf(u8, request, "gimme " ++ shun_test_name) == null); // The held artifact is still announced — nothing else changed. try std.testing.expect(std.mem.indexOf(u8, request, "igot " ++ shun_test_held_name) != null); } } /// Does the decoded reply body carry `gimme NAME` as a whole card line? fn replyHasGimme(gpa: std.mem.Allocator, reply_container: []const u8, name: []const u8) !bool { const body = try zit.container.decode(gpa, reply_container, @intCast(logstore.max_payload)); defer gpa.free(body); var it = std.mem.splitScalar(u8, body, '\n'); while (it.next()) |line| { if (line.len == 6 + name.len and std.mem.startsWith(u8, line, "gimme ") and std.mem.eql(u8, line[6..], name)) return true; } return false; } test "sync §3.3 L527-530 / §8.8 L1777-1779: a served reply never gimme's a shunned phantom, and a shun row written between rounds is honoured by the next" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A repo holding one artifact and NO shun row yet. const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer seed.deinit(gpa); _ = try seed.put(shun_test_held_digest, "held content\n"); try seed.commit(); } // A client `igot` for a hash this repo lacks: §3.3 r24(1) L511-514 makes the // server a receiver, so `respond` interns the phantom and would beg for it. const igot_req = try serveGimmeRequest(gpa, "igot", shun_test_name); defer gpa.free(igot_req); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); // Round 1, no shun row: the reply asks for it. This is the behaviour the // filter must change, so the test is red on an unfiltered server. { const r = try serveAnswer(&ctx, gpa, igot_req, null, null, true); defer gpa.free(r.body); try std.testing.expect(try replyHasGimme(gpa, r.body, shun_test_name)); } // The operator shuns it BETWEEN rounds — an append to the same log. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); const st = try f.stat(io); var fs = storage.FileStorage.init(io, f, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putShun(shun_test_digest, "operator request", 1); try log.commit(); } // Round 2 honours it with no restart: the append grew the log, so // `baseSnapshot`'s stat gate rebuilds the index — and the rebuild re-loads // the shun table (`LogIndexStore.shun`). That is the freshness mechanism this // set rides; nothing here is memoised past a build. { const r = try serveAnswer(&ctx, gpa, igot_req, null, null, true); defer gpa.free(r.body); try std.testing.expect(!try replyHasGimme(gpa, r.body, shun_test_name)); // The unshunned holding is still advertised: only the shunned name went quiet. const body = try zit.container.decode(gpa, r.body, @intCast(logstore.max_payload)); defer gpa.free(body); try std.testing.expect(std.mem.indexOf(u8, body, "igot " ++ shun_test_held_name) != null); } try std.testing.expectEqual(@as(u64, 2), ctx.loads); // one build per log state, not per round } // --------------------------------------------------------------------------- // sync §9.5 L1988-1992 — the SEND side of the shun filter. // // "A subsequent read-authorized transfer path requests igot for every // eligible holding after the Section 3.3 shun, phantom, private, and // peer-has filters" // // `LogIndexStore`'s enumeration is that "every eligible holding", so the shun // filter belongs in the enumeration itself: a shunned holding is masked, which // makes the server neither `igot`-announce it (the §3.2 walk's `held()` gate // answers false) nor serve it in a clone batch (no wire form, no content), and // makes an inbound `gimme` for it land on §3.3 L532-535's *"If absent, silently // skip"* with no code on that path — the name is simply not in `by_name`. // // Which of those three the MASK is load-bearing for, stated honestly: the igot // announcement and the `gimme` miss are new here — nothing else suppressed them. // The clone batch was already covered by `LogStore.wire`/`content`'s own rid-keyed // read mask, which the pager gates on; the mask's contribution there is that the // pager's cursor stays contiguous over the survivors (see the clone-pager test). // // The mask is a MASK, not a removal, and §8.2 L1504-1506 is why: // // "SEQNO is a server blob-table RID. Iterate from supplied RID through // current maximum, silently skipping gaps; clients MUST NOT infer count // or size from range." // // Dropping the shunned frame from the enumeration would renumber every survivor // above it. The server is stateless (§8.9 L1783-1784) — the client's SEQNO *is* the // clone cursor — and a shun row appended between two rounds of one clone grows // the log, which is exactly what trips `ServeCtx.baseSnapshot`'s append-only stat // gate and forces a rebuild. Under renumbering the resuming client's cursor would // then step over a survivor and still be told the sequence was complete: a // silently short clone, the failure class `read_fault` exists to prevent. Keeping // the slot and answering it as a §8.2 GAP makes adapter-rid identity a pure // function of the log, independent of the shun table. // --------------------------------------------------------------------------- /// Five artifact digests in persisted order. Distinct byte fills, so each wire /// name is the pattern spelled out and no vector needs a runtime encode. const send_shun_digests = [5][]const u8{ "\x11" ** digest_len, "\x22" ** digest_len, "\x33" ** digest_len, "\x44" ** digest_len, "\x55" ** digest_len, }; const send_shun_names = [5][]const u8{ "11" ** digest_len, "22" ** digest_len, "33" ** digest_len, "44" ** digest_len, "55" ** digest_len, }; /// The FIRST, a MIDDLE and the LAST of the five are shunned, so an off-by-one or /// a misindexed mask cannot pass: the survivors are interleaved among them. fn sendShunIsShunned(i: usize) bool { return i % 2 == 0; } /// Seed `dir`'s repository holding all five `send_shun_digests` in order, with a /// live config frame BEFORE the first and another BETWEEN the second and third, /// and shun rows naming the first, middle and last. /// /// The interleaving is what an off-by-one cannot survive: shunning the FIRST, a /// MIDDLE and the LAST slot means a mask built by any walk other than the /// enumeration's own loop names one artifact and suppresses another, and the /// alignment test below reads that straight off the log. The config frames are /// here for a WEAKER reason than they look: they never enter /// `ridsInPersistedOrder` at all (a config frame is not interned — `LogStore.scan` /// only records its offset), so they exercise the `.config` guards' *presence*, not /// a live skip; they do not by themselves make a second walk misalign. fn seedSendShunRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); _ = try log.put(send_shun_digests[0], "content zero\n"); _ = try log.put(send_shun_digests[1], "content one\n"); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "cccc" }); _ = try log.put(send_shun_digests[2], "content two\n"); _ = try log.put(send_shun_digests[3], "content three\n"); _ = try log.put(send_shun_digests[4], "content four\n"); for (send_shun_digests, 0..) |d, i| { if (sendShunIsShunned(i)) try log.putShun(d, "operator request", 1); } try log.commit(); } test "sync §9.5 L1988-1992: a shunned holding is masked out of the adapter enumeration and its name is absent from by_name" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // §8.2 L1504-1506 — the rid space is the LOG's, not the survivors': all five // artifact frames still hold their slot (config frames were never in it). try std.testing.expectEqual(@as(usize, 5), index.rids.len); try std.testing.expectEqual(@as(usize, 5), s.count()); for (send_shun_names, 0..) |name, i| { const rid: repostate.Rid = @intCast(i + 1); if (sendShunIsShunned(i)) { // Not resolvable by name: the §3.2 gimme arm keys on `lookup`, so // this IS §3.3 L532-535's "If absent, silently skip". try std.testing.expect(index.by_name.get(name) == null); try std.testing.expect(s.lookup(name) == null); // Not announced (§3.2's igot walk gates on `held`), and nothing to // send in either form. try std.testing.expect(!s.held(rid)); try std.testing.expect(s.content(rid) == null); try std.testing.expect(s.wire(rid) == null); } else { try std.testing.expectEqual(@as(?repostate.Rid, rid), s.lookup(name)); try std.testing.expect(s.held(rid)); try std.testing.expect(s.content(rid) != null); } } // A masked frame is a policy gap, never a read fault: the round must not be // converted to `error.ServerStorage` by `serveAnswer`. try std.testing.expect(!index.read_fault); } /// Does the decoded reply body carry `VERB NAME` as a whole card line? fn replyHasCardFor(body: []const u8, verb: []const u8, name: []const u8) bool { var it = std.mem.splitScalar(u8, body, '\n'); while (it.next()) |line| { if (line.len != verb.len + 1 + name.len) continue; if (!std.mem.startsWith(u8, line, verb)) continue; if (line[verb.len] != ' ') continue; if (std.mem.eql(u8, line[verb.len + 1 ..], name)) return true; } return false; } test "sync §9.5 L1988-1992: a served round igot-announces the survivors and never a masked holding" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); // A round that makes the server run its §3.2 inventory walk over every rid. // The announced hash is one this repo does not hold, so nothing else about // the request steers which holdings the walk reaches. const req = try serveGimmeRequest(gpa, "igot", shun_test_name); defer gpa.free(req); const reply = try serveAnswer(&ctx, gpa, req, null, null, true); defer gpa.free(reply.body); const body = try zit.container.decode(gpa, reply.body, @intCast(logstore.max_payload)); defer gpa.free(body); for (send_shun_names, 0..) |name, i| { if (sendShunIsShunned(i)) { // §9.5 L1988-1992 — not among "every eligible holding": the walk's // `held()` gate answers false, so no igot names it. try std.testing.expect(!replyHasCardFor(body, "igot", name)); // Nor does it fall out of the walk as a PHANTOM to beg for. The // enumeration now hands the walk a non-held rid, and it is `Server.shun` // — the §3.3 L527-530 arm this repository's own table feeds through // `serveRound` — that keeps the reply quiet. That check is not dead // after this slice; the mask is what makes it load-bearing. try std.testing.expect(!replyHasCardFor(body, "gimme", name)); } else { try std.testing.expect(replyHasCardFor(body, "igot", name)); } } // A masked holding is not an error condition on any card. try std.testing.expect(std.mem.indexOf(u8, body, "error") == null); } test "sync §9.5 L1988-1992: the masked enumeration stays aligned — adapter rid r names rids[r-1] with shunned first, middle and last" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // `rids`, `names`, `masked`, `content_memo` and `wire_memo` are PARALLEL // arrays and `by_name` maps a wire name to that same index. Filtering or // masking one out of step with the others makes adapter rid r name a // DIFFERENT artifact's bytes — silent corruption on the wire, not an error. // Pin the whole alignment against the log itself, entry by entry. var shunned_seen: usize = 0; for (index.rids, 0..) |lr, a| { var buf: [2 * digest_len]u8 = undefined; const expect_name = digestToWireName(index.log.hashOf(lr), &buf); const rid: repostate.Rid = @intCast(a + 1); try std.testing.expectEqualStrings(expect_name, index.names[a]); try std.testing.expectEqualStrings(expect_name, s.hashOf(rid)); try std.testing.expectEqualStrings(send_shun_names[a], expect_name); // The mask indexes the SAME slot as `rids[a]` — the invariant a mask // built over any second walk over the log breaks (verified: a mask // written one slot out of step turns this expectation red). try std.testing.expectEqual(index.shun.digests.contains(index.log.hashOf(lr)), index.masked[a]); if (index.masked[a]) { shunned_seen += 1; try std.testing.expect(index.by_name.get(expect_name) == null); } else { try std.testing.expectEqual(@as(?repostate.Rid, rid), index.by_name.get(expect_name)); } } // The fixture really does interleave: three shunned, two survivors. try std.testing.expectEqual(@as(usize, 3), shunned_seen); try std.testing.expectEqual(@as(usize, 2), index.by_name.count()); } test "sync §9.5 L1988-1992 / §7.7 L1437-1440: a write against a masked wire name does not unmask it — no second identity, still not held, still not servable" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // The masked name is absent from `by_name`, so every write door below misses the // dedup key the survivors hit and would otherwise fall through to the appendix — // minting a SECOND, HELD entry for a name §9.5 says this server does not send, // and putting it back on the §3.2 walk. The mask outranks the write instead. const masked_name = send_shun_names[0]; try std.testing.expect(index.masked[0]); const before = s.count(); try std.testing.expectEqual(@as(repostate.Rid, 1), try s.intern(masked_name)); try std.testing.expectEqual(@as(repostate.Rid, 1), try s.put(masked_name, "content zero\n")); // Even bytes that DISAGREE with the frame: the mask is not a content check, and // this path must not read (nor assert on) a masked frame's plaintext. try std.testing.expectEqual(@as(repostate.Rid, 1), try s.put(masked_name, "different bytes\n")); try s.retain(1, .{ .full = "not a real container" }); // Nothing was created and nothing was unmasked. try std.testing.expectEqual(before, s.count()); try std.testing.expectEqual(@as(usize, 2), index.by_name.count()); try std.testing.expect(!s.held(1)); try std.testing.expect(s.content(1) == null); try std.testing.expect(s.wire(1) == null); // §3.3 L532-535 still applies to an inbound `gimme` for it: absent, silently // skipped, no second rid for the walk to trip over. try std.testing.expect(s.lookup(masked_name) == null); try std.testing.expect(!index.read_fault); // NEGATIVE SPACE: the appendix arm is intact for a name the log genuinely does // not carry — the guard above narrows the masked case only. const absent_name = "99" ** digest_len; const appendix_rid = try s.intern(absent_name); try std.testing.expectEqual(@as(repostate.Rid, 6), appendix_rid); try std.testing.expectEqual(before + 1, s.count()); try std.testing.expect(!s.held(appendix_rid)); } test "sync §7.7 L1444-1450 / §9.5 L1988-1992: the adapter owns its content mask — the frame's plaintext is still on disk and still readable through the maintenance door" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // §9.5's filter is a POLICY answer about what this server SENDS, not a deletion: // the frame, its bytes and its checksums are untouched on disk. §7.7 L1444-1450 // depends on exactly that — "For an admitted target whose DELTASRC name is in // the repository shun table, the receiver does not apply admission independently // to that source: content still held locally MAY serve as the delta basis" — so // "still held locally" is a property this slice must not have quietly destroyed. // `LogStore.contentUnmasked` is the maintenance door that reads under no mask // (`pack.zig`'s reader), and it is the only way to distinguish "the bytes are // gone" from "the send side refuses to hand them out". try std.testing.expect(index.masked[0]); const on_disk = try index.log.contentUnmasked(gpa, index.rids[0], null); defer if (on_disk) |b| gpa.free(b); try std.testing.expectEqualStrings("content zero\n", on_disk.?); // ...and the store still refuses, in both forms, without calling it a fault. try std.testing.expect(s.content(1) == null); try std.testing.expect(s.wire(1) == null); try std.testing.expect(!index.read_fault); // Nothing was materialised on the way: a refusal is answered from the mask, so // a masked frame is never inflated into the round's memo (which is also what // keeps a shunned artifact's plaintext out of a server's memory). try std.testing.expectEqual(@as(usize, 0), index.materialized_bytes); try std.testing.expectEqual(@as(usize, 0), index.residentBytes()); // NEGATIVE SPACE: the maintenance door is not a blanket unmask of the store — // a SURVIVOR reads identically through both, so the asymmetry above is the // mask's doing and not an artefact of which reader was called. const survivor = try index.log.contentUnmasked(gpa, index.rids[1], null); defer if (survivor) |b| gpa.free(b); try std.testing.expectEqualStrings(survivor.?, s.content(2).?); } test "sync §9.5 L1988-1992 / §7.7 L1437-1440: a served round's writes land in the round-local overlay — the base enumeration's write doors are never reached" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // `seedSendShunRepo`'s digests are synthetic, so no peer could legitimately push // one back — the responder hashes the payload and would reject the name long // before reaching any store door. This fixture names its two artifacts by their // REAL content address, so the push below is one a real peer could send. const shunned_body = "shunned artifact bytes\n"; const survivor_body = "surviving artifact bytes\n"; var shunned_buf: [64]u8 = undefined; var survivor_buf: [64]u8 = undefined; const shunned_name = zit.artifact.artifactName(.sha3_256, shunned_body, &shunned_buf); const survivor_name = zit.artifact.artifactName(.sha3_256, survivor_body, &survivor_buf); var shunned_dbuf: [digest_len]u8 = undefined; var survivor_dbuf: [digest_len]u8 = undefined; const shunned_digest = hexToDigest(shunned_name, &shunned_dbuf) orelse return error.BadTestName; const survivor_digest = hexToDigest(survivor_name, &survivor_dbuf) orelse return error.BadTestName; const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer seed.deinit(gpa); try seed.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); _ = try seed.put(shunned_digest, shunned_body); _ = try seed.put(survivor_digest, survivor_body); try seed.putShun(shunned_digest, "operator request", 1); try seed.commit(); } const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); // A peer pushes the very name this repository has shunned. const push_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ shunned_name, shunned_body.len, shunned_body }); defer gpa.free(push_plain); const push_req = try zit.container.encode(gpa, push_plain, .default); defer gpa.free(push_req); { const reply = try serveAnswer(&ctx, gpa, push_req, null, null, true); defer gpa.free(reply.body); // THE PROPERTY. `putImpl`/`internImpl` deliberately NARROW the store contract // for a masked name — they answer a rid whose `content` is null and whose // `held` is false — and the vtable's `error{OutOfMemory}` cannot signal that. // What keeps the narrowing safe is that a served round never reaches those // doors at all: `respond` writes into a per-request `repostate.OverlayStore` // that is discarded at round end. `appendix` is where any write that DID // reach this base would land, so its emptiness is that claim, measured. const base = ctx.snapshot.?; try std.testing.expectEqual(@as(usize, 0), base.appendix.store().count()); try std.testing.expectEqual(@as(usize, 2), base.store().count()); // ...and the push did not unmask the name it carried. try std.testing.expect(base.masked[0]); try std.testing.expect(base.store().lookup(shunned_name) == null); try std.testing.expect(!base.store().held(1)); try std.testing.expect(base.store().content(1) == null); try std.testing.expect(!base.read_fault); // The survivor is untouched by any of it. try std.testing.expectEqual(@as(?repostate.Rid, 2), base.store().lookup(survivor_name)); } // NEGATIVE SPACE: it is not the MASK that keeps the base's doors shut, it is the // overlay. A push of a name this repository has never seen — the case that has a // real appendix arm waiting for it — must land in the round-local store just the // same. (This round's `baseSnapshot` rebuilds over the grown log first, so the // snapshot read below is the one this round actually served from.) const fresh_body = "a name this repository has never held\n"; var fresh_buf: [64]u8 = undefined; const fresh_name = zit.artifact.artifactName(.sha3_256, fresh_body, &fresh_buf); const fresh_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ fresh_name, fresh_body.len, fresh_body }); defer gpa.free(fresh_plain); const fresh_req = try zit.container.encode(gpa, fresh_plain, .default); defer gpa.free(fresh_req); const reply2 = try serveAnswer(&ctx, gpa, fresh_req, null, null, true); defer gpa.free(reply2.body); try std.testing.expectEqual(@as(usize, 0), ctx.snapshot.?.appendix.store().count()); } /// The `clone_seqno NEXT` cursor a clone reply carries (§8.2 L1517-1520: "An /// accepting clone reply emits zero or more clone cfile cards in RID order … then /// clone_seqno NEXT, push SERVERCODE PROJECTCODE, and final comment. … NEXT zero /// signals exhaustion."), read off the decoded body. fn cloneSeqnoOf(body: []const u8) !u64 { var it = std.mem.splitScalar(u8, body, '\n'); while (it.next()) |line| { if (!std.mem.startsWith(u8, line, "clone_seqno ")) continue; return std.fmt.parseInt(u64, line["clone_seqno ".len..], 10) catch return error.BadCloneSeqno; } return error.NoCloneSeqno; } /// Does the decoded reply body carry a `cfile` card for `name`? Keyed on the card /// prefix plus the separating space, so a name that is a prefix of another cannot /// answer for it. fn replyHasCfile(gpa: std.mem.Allocator, body: []const u8, name: []const u8) !bool { const needle = try std.fmt.allocPrint(gpa, "cfile {s} ", .{name}); defer gpa.free(needle); return std.mem.indexOf(u8, body, needle) != null; } // WHAT THIS TEST PINS, and what it does NOT. The masked frames' absence from the // clone stream is NOT this slice's doing: the pager gates on `store.wire()` then // `store.content()`, and `LogStore.wire`/`content` already answered null for a // shunned entry through their own rid-keyed read mask before the enumeration was // masked at all — so this test passes with `LogIndexStore`'s exclusion removed // (confirmed by mutation). What it does pin, and what nothing else pins, is that // the mask keeps the pager's persisted-order cursor CONTIGUOUS over the survivors: // each survivor arrives exactly once across paged batches and the sequence // terminates at NEXT 0. A mask written out of step with `rids` turns it red. test "sync §8.2 L1504-1506 / §9.5 L1988-1992: the clone pager walks the survivors contiguously across paged batches and terminates at NEXT 0" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); // A one-artifact batch cap forces the pager to page, so the persisted-order // cursor is exercised at every masked frame rather than swallowed by a single // full-store batch. const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, .clone_batch_max = 1, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); var delivered = [_]usize{0} ** 5; var seqno: u64 = 1; var rounds: usize = 0; while (rounds < 16) : (rounds += 1) { const plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nclone 3 {d}\n", .{seqno}); defer gpa.free(plain); const req = try zit.container.encode(gpa, plain, .default); defer gpa.free(req); const reply = try serveAnswer(&ctx, gpa, req, null, null, true); defer gpa.free(reply.body); // §8.1 L1489-1491 / §4.1 L885-887 — a clone reply is the RAW card/cfile stream under // `application/x-fossil-uncompressed`, never a zlib container. try std.testing.expectEqualStrings("application/x-fossil-uncompressed", reply.content_type); const body = reply.body; for (send_shun_names, 0..) |name, i| { if (try replyHasCfile(gpa, body, name)) delivered[i] += 1; } const next = try cloneSeqnoOf(body); if (next == 0) break; // A LOOP GUARD, not a protocol requirement: §8.2 L1521-1523 says the // opposite — "The client records a digit-only NEXT and does not validate // that a positive value advances from the supplied SEQNO". A pager that // returned a non-advancing cursor would spin this test to its round cap // instead of failing on the thing it is here to measure, so the guard // states the pager's own contract, with no normative claim behind it. try std.testing.expect(next > seqno); seqno = next; } else return error.CloneNeverExhausted; // THE PROOF: every survivor arrived EXACTLY ONCE across the paged batches, and // the sequence reached NEXT 0. The zero counts for the masked slots are carried // by the log's own read mask (see the note above the test), not by this slice; // the "exactly once" and the termination are the mask's contribution — a slot // masked out of step with `rids` drops or duplicates a survivor here. for (delivered, 0..) |n, i| { const want: usize = if (sendShunIsShunned(i)) 0 else 1; try std.testing.expectEqual(want, n); } } test "sync §9.5 L1988-1992: a shun row written between clone rounds is honoured by the next round WITHOUT renumbering the survivors" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A repo holding all five artifacts and NO shun row yet. const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer seed.deinit(gpa); try seed.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); for (send_shun_digests, 0..) |d, i| { var body_buf: [32]u8 = undefined; _ = try seed.put(d, try std.fmt.bufPrint(&body_buf, "distinct content {d}\n", .{i})); } try seed.commit(); } const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const clone_plain = "pragma client-version 20000\nclone 3 1\n"; const clone_req = try zit.container.encode(gpa, clone_plain, .default); defer gpa.free(clone_req); // Round 1, no shun row: every artifact is served, and artifact 3 sits at // adapter rid 3. This is the behaviour the mask must change, so the test is // red on an unfiltered server. { const reply = try serveAnswer(&ctx, gpa, clone_req, null, null, true); defer gpa.free(reply.body); const body = reply.body; // §8.1 L1489-1491 / §4.1 L885-887 — uncompressed clone stream for (send_shun_names) |name| try std.testing.expect(try replyHasCfile(gpa, body, name)); try std.testing.expectEqual(@as(?repostate.Rid, 3), ctx.snapshot.?.store().lookup(send_shun_names[2])); try std.testing.expectEqual(@as(?repostate.Rid, 5), ctx.snapshot.?.store().lookup(send_shun_names[4])); } // The operator shuns the MIDDLE artifact between rounds — an append, which is // exactly what trips `baseSnapshot`'s append-only stat gate and rebuilds. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); const st = try f.stat(io); var fs = storage.FileStorage.init(io, f, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putShun(send_shun_digests[1], "operator request", 1); try log.commit(); } // Round 2 honours it with no restart — AND the survivors above it keep their // adapter rids. A dense filter would shift artifact 3 to rid 2 and artifact 5 // to rid 4, so a client resuming this clone at cursor 3 would step over // artifact 3 entirely and still be told NEXT 0 (§8.2 L1504-1506 licenses the // gap; it does not license the shift). { const reply = try serveAnswer(&ctx, gpa, clone_req, null, null, true); defer gpa.free(reply.body); const body = reply.body; // §8.1 L1489-1491 / §4.1 L885-887 — uncompressed clone stream try std.testing.expect(!try replyHasCfile(gpa, body, send_shun_names[1])); for (send_shun_names, 0..) |name, i| { if (i == 1) continue; try std.testing.expect(try replyHasCfile(gpa, body, name)); } try std.testing.expectEqual(@as(usize, 5), ctx.snapshot.?.store().count()); try std.testing.expectEqual(@as(?repostate.Rid, 3), ctx.snapshot.?.store().lookup(send_shun_names[2])); try std.testing.expectEqual(@as(?repostate.Rid, 5), ctx.snapshot.?.store().lookup(send_shun_names[4])); // THE MASK ITSELF, at THIS layer. The absent `cfile` above does not prove it: // `LogStore.wire`/`content` answer null for a shunned entry through their own // read mask, so the clone stream would omit the frame even with the // enumeration unmasked. The rebuilt index is where a shun table read once at // startup instead of per generation shows up, so assert on it directly. try std.testing.expect(ctx.snapshot.?.masked[1]); try std.testing.expect(ctx.snapshot.?.store().lookup(send_shun_names[1]) == null); try std.testing.expect(!ctx.snapshot.?.store().held(2)); } try std.testing.expectEqual(@as(u64, 2), ctx.loads); // one build per log state } test "serve §3.8 / sync §9.5 L1988-1992: the snapshot cache key is not the size alone — a same-length write to the log invalidates it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try seedSendShunRepo(gpa, io, tmp.dir); const server = zit.sync.Server{ .version = serve_mod.server_version }; var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const req = try serveGimmeRequest(gpa, "igot", shun_test_name); defer gpa.free(req); // Two read rounds over an untouched log: built once, then stat-gated. This is // the property the mtime term must NOT cost — a read round still does not // rebuild, so the cache is still a cache. for (0..2) |_| { const reply = try serveAnswer(&ctx, gpa, req, null, null, true); gpa.free(reply.body); } try std.testing.expectEqual(@as(u64, 1), ctx.loads); const before = try file.stat(io); // A write that leaves the log BYTE-IDENTICAL and exactly as long: the last byte // is lifted off and put straight back. Nothing a `zit` writer does today looks // like this — they all append — but "every writer only appends" is an assumption // about all present and future writers, not a checked property, and since this // snapshot began carrying the §3.3 shun table a missed rebuild no longer costs // stale CONTENT (self-correcting) but a stale SUPPRESSION TABLE: an artifact the // operator shunned served on, with nothing failing and nothing logged. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); var fs = storage.FileStorage.init(io, f, before.size); var last: [1]u8 = undefined; try std.testing.expectEqual(@as(usize, 1), try fs.storageFile().pread(before.size - 1, &last)); try fs.storageFile().truncate(before.size - 1); var reopened = storage.FileStorage.init(io, f, before.size - 1); try reopened.storageFile().append(&last); try reopened.storageFile().sync(); } // THE PREMISE, asserted rather than assumed: the log really is the same length // and the filesystem really did move the modification time. A filesystem that // could not express this scenario fails here, loudly, instead of letting the // conclusion below pass vacuously. const after = try file.stat(io); try std.testing.expectEqual(before.size, after.size); try std.testing.expect(after.mtime.nanoseconds != before.mtime.nanoseconds); // THE CONCLUSION: the size-only gate would have returned the stale snapshot and // its stale shun table. The size+mtime key rebuilds. { const reply = try serveAnswer(&ctx, gpa, req, null, null, true); gpa.free(reply.body); } try std.testing.expectEqual(@as(u64, 2), ctx.loads); try std.testing.expectEqual(after.mtime.nanoseconds, ctx.snapshot_log_mtime_ns); // ...and having rebuilt, it settles again: the key is stable, not a rebuild on // every round. { const reply = try serveAnswer(&ctx, gpa, req, null, null, true); gpa.free(reply.body); } try std.testing.expectEqual(@as(u64, 2), ctx.loads); } test "sync §7.2 L1217-1218 / §9.5 L1988-1992: a surviving delta whose basis is masked is served in FULL form, never as a dangling DELTASRC" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // `zit shun --defer` records the row without rewriting the generation, so a // SURVIVING frame in the current generation can still be a delta rooted on // the shunned one. Handing that delta's wire form out would emit a `cfile` // whose DELTASRC is never sent: the peer stores a permanently dangling delta // and gimme's a basis it will be silently skipped on — CORRUPT, not absent. const base_plain = "the quick brown fox jumps over the lazy dog, and then again once more"; const child_plain = "the quick brown fox jumps over the lazy dog, and then twice more, at last"; const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); const base_rid = try log.put(send_shun_digests[0], base_plain); const d = try zit.delta.create(gpa, base_plain, child_plain); defer gpa.free(d); _ = try log.putDelta(send_shun_digests[1], base_rid, d); try log.putShun(send_shun_digests[0], "operator request", 1); try log.commit(); } var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); { var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); const s = index.store(); // The premise: the child really is a delta frame rooted on the masked one. try std.testing.expectEqual(logstore.FrameKind.delta, index.log.kindOf(index.rids[1])); // The masked basis is a §8.2 gap... try std.testing.expect(!s.held(1)); try std.testing.expect(s.wire(1) == null); // ...so the child's DELTA wire form is withheld — the responder falls back // to compressing `content()` on the fly (§7.2), which reconstructs against // the raw log and is unaffected by the name-level mask. try std.testing.expect(s.held(2)); try std.testing.expect(s.wire(2) == null); try std.testing.expectEqualStrings(child_plain, s.content(2).?); // Withholding a wire form for policy is not a read fault. try std.testing.expect(!index.read_fault); } // End to end through a clone round: the child arrives, the basis does not, and // the shunned NAME appears nowhere in the reply — no DELTASRC to dangle on. const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const clone_req = try zit.container.encode(gpa, "pragma client-version 20000\nclone 3 1\n", .default); defer gpa.free(clone_req); const reply = try serveAnswer(&ctx, gpa, clone_req, null, null, true); defer gpa.free(reply.body); const body = reply.body; // §8.1 L1489-1491 / §4.1 L885-887 — uncompressed clone stream try std.testing.expect(try replyHasCfile(gpa, body, send_shun_names[1])); try std.testing.expect(std.mem.indexOf(u8, body, send_shun_names[0]) == null); } test "sync §3.3 L527-530: the CLIENT's holdings snapshot carries the same masked-basis guard — a surviving delta on a masked base is held as full content" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // The same `zit shun --defer` shape as the serve-side test above, asked of // the CLIENT's builder: `ClientHoldings`, which every `zit pull`/`push`/ // `sync` client announces from. Serving the delta wire form would give the // snapshot an entry that `held()` calls true but whose DELTASRC names a // masked basis — so §3.2's igot walk would announce a name the gimme send // gate can never answer, and a push would spin the round with nothing to // send while reporting success. The artifact itself is NOT shunned; only // its basis is. const base_plain = "the quick brown fox jumps over the lazy dog, and then again once more"; const child_plain = "the quick brown fox jumps over the lazy dog, and then twice more, at last"; const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = "" }); const base_rid = try log.put(send_shun_digests[0], base_plain); const d = try zit.delta.create(gpa, base_plain, child_plain); defer gpa.free(d); _ = try log.putDelta(send_shun_digests[1], base_rid, d); try log.putShun(send_shun_digests[0], "operator request", 1); try log.commit(); } var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const s = holdings.store(); // The masked basis is ABSENT from the name map — `lookup` answers null, // which IS §3.3 L532-535's "If absent, silently skip" on the gimme path — // and a peer-announced intern lands on the masked slot rather than minting // a second identity: held by nothing, readable as nothing. try std.testing.expect(s.lookup(send_shun_names[0]) == null); const base_rid = try s.intern(send_shun_names[0]); try std.testing.expect(!s.held(base_rid)); try std.testing.expect(s.content(base_rid) == null); // The survivor is held AND readable — held-ness and readability agree, which // is the property a recorded dangling delta would have broken. const child_rid = s.lookup(send_shun_names[1]).?; try std.testing.expect(s.held(child_rid)); try std.testing.expectEqualStrings(child_plain, s.content(child_rid).?); try std.testing.expect(s.wire(child_rid) == null); // full content, no DELTASRC to dangle on } // --------------------------------------------------------------------------- // sync §7.7 L1428-1432 — the hash-policy half of the receive-side admission // predicate, from the REPOSITORY's stored config to the receiver that asks it. // // "shun-sha1 applies exactly when the receiving repository's effective // hash-policy value at the start of processing that card is the shun-sha1 // value defined by [FossilArtifact], Section 7." // // The tests below exist because every §7.7 unit test in `sync_test.zig` sets // `Client.hash_policy` / `Server.hash_policy` on a struct literal. That pins the // RULE and proves nothing about the WIRING, so a regression that dropped // `effectiveHashPolicy` — or read the wrong config field — would ship green. // These two close that: one over the stored byte, one over the served round. // --------------------------------------------------------------------------- /// Seed `dir` with a repository whose config frame names `policy` and which /// holds nothing. The empty holdings matter: the push below must be the only /// thing that could ever land. fn s77SeedPolicyRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, policy: artifact.HashPolicy) !void { const hk: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var log = try logstore.LogStore.open(gpa, fs.storageFile(), hk); defer log.deinit(gpa); try log.putConfig(.{ .hash_policy = @intFromEnum(policy), .default_branch = "trunk", .project_id = "", .project_code = "", }); try log.commit(); } /// True when `dir`'s repository resolves `name` — read back off disk, after the /// served round has persisted, so this answers "did it LAND", not "was it /// staged". fn s77RepoHolds(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, name: []const u8) !bool { var file = try dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var index = try LogIndexStore.openFromFile(gpa, io, file, st.size); defer index.destroy(); return index.store().lookup(name) != null; } test "sync §7.7 L1428-1432: the SERVED round's admission predicate is read from the repository's stored hash-policy, not defaulted" { const gpa = std.testing.allocator; const io = std.testing.io; const content = "a SHA1-named artifact a peer pushes at us\n"; var nbuf: [64]u8 = undefined; const sha1_name = zit.artifact.artifactName(.sha1, content, &nbuf); try std.testing.expectEqual(@as(usize, 40), sha1_name.len); const plain = try std.fmt.allocPrint( gpa, "pragma client-version 20000\npush {s} {s}\nfile {s} {d}\n{s}", .{ serve_mod.server_servercode, serve_mod.server_projectcode, sha1_name, content.len, content }, ); defer gpa.free(plain); const request = try zit.container.encode(gpa, plain, .default); defer gpa.free(request); const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; // CONTROL: the same repository under `sha3` stores the pushed SHA1 name. If // this half ever stops holding, the shun-sha1 half below proves nothing. { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp.dir, .sha3); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const reply = try serveAnswer(&ctx, gpa, request, null, null, true); gpa.free(reply.body); try std.testing.expect(try s77RepoHolds(gpa, io, tmp.dir, sha1_name)); } // THE PROOF: the identical round against a repository whose stored config // names shun-sha1 discards it — with no error card (L1437-1440) — and the // ONLY difference between the two runs is that stored byte. A `serveRound` // that defaulted the policy instead of reading the snapshot would store here. { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1); var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const reply = try serveAnswer(&ctx, gpa, request, null, null, true); defer gpa.free(reply.body); const plain_reply = try zit.container.decode(gpa, reply.body, 1 << 20); defer gpa.free(plain_reply); try std.testing.expect(std.mem.indexOf(u8, plain_reply, "error ") == null); try std.testing.expect(!try s77RepoHolds(gpa, io, tmp.dir, sha1_name)); } } test "sync §7.7 L1428-1432: ClientHoldings carries the stored hash-policy into the one place a Client is built from repository state" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1); var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const admission = holdings.admission(); try std.testing.expectEqual(artifact.HashPolicy.shun_sha1, admission.hash_policy); // `syncClient` is the sole construction `pullRun`/`pushRun`/`syncRun` share, // so this one assertion covers all three verbs: a client that dropped either // half of `RepoAdmission` would apply half a filter with nothing failing. // A placeholder identity: this test is about `RepoAdmission`, and the // sub-manifest half of the same client is covered by the §7.2 tests. const identity: RepoSyncIdentity = .{ .project_code = "", .seen_submanifest = false, .enable_submanifests = false, .server_code = zit.sync.deriveServerCode(&.{"test-fixture-repo"}) }; for ([_]zit.sync.Operation{ .pull, .push, .sync }) |op| { const client = syncClient(op, &identity, null, admission); try std.testing.expectEqual(op, client.operation); try std.testing.expectEqual(artifact.HashPolicy.shun_sha1, client.hash_policy); // A 40-hex name is rejected by the very predicate this policy feeds. try std.testing.expectEqual( zit.sync.Admission.reject, zit.sync.admitInbound(client.hash_policy, client.shun, "a" ** 40), ); } // And the `.auto` repository next door admits the same name, so the // assertion above is reading the stored byte and not a constant. var tmp2 = std.testing.tmpDir(.{}); defer tmp2.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp2.dir, .auto); var holdings2 = try ClientHoldings.open(gpa, io, tmp2.dir); defer holdings2.deinit(io); const admission2 = holdings2.admission(); try std.testing.expectEqual(artifact.HashPolicy.auto, admission2.hash_policy); const c2 = syncClient(.pull, &identity, null, admission2); try std.testing.expectEqual( zit.sync.Admission.admit, zit.sync.admitInbound(c2.hash_policy, c2.shun, "a" ** 40), ); } test "sync §7.7 L1428-1432: a DAMAGED repository never yields a hash-policy — the admission predicate fails the load rather than defaulting" { // The property: no damaged repository may hand the receiver a policy value. // `readConfig` answers `Corrupt`/`Storage` for a config frame that will not // read or will not parse, and that is UNKNOWN, not "no defined policy": // answering `.auto` there would silently disable shun-sha1 and admit // SHA1-named content this repository is configured to refuse, with nothing // failing. §3.3's shun half already refuses to fail open (`RepoShunSet.load` // propagates, and `serveRound` returns `error.ServerStorage` rather than // serving tableless); the two halves of one predicate must not differ. // // HONESTY ABOUT WHAT THIS PINS: with the injury below, the fault is caught // UPSTREAM, at `LogStore.openReadOnly`, so this test does not by itself kill // a mutant that restores `effectiveHashPolicy`'s old blanket // `catch return .auto` — verified by applying that mutant, which leaves the // suite green. It is here as the end-to-end guard: the day the log open grows // tolerant of a damaged frame, the read below must still not answer `.auto`. // Pinning `effectiveHashPolicy`'s own catch would need a storage layer that // faults a single frame read, which this tree does not have. const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try s77SeedPolicyRepo(gpa, io, tmp.dir, .shun_sha1); // Damage the config frame in place: its default_branch bytes are on disk // verbatim, so overwriting them is a one-frame injury that leaves every other // frame — and the log's own scan — intact. { var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer f.close(io); const st = try f.stat(io); const buf = try gpa.alloc(u8, st.size); defer gpa.free(buf); var r = f.reader(io, buf); try r.interface.readSliceAll(buf); const at = std.mem.indexOf(u8, buf, "trunk") orelse return error.TestUnexpectedResult; var w = f.writer(io, &.{}); try w.seekTo(at); try w.interface.writeAll("\xff\xff\xff\xff\xff"); try w.interface.flush(); } try std.testing.expectError(error.Storage, ClientHoldings.open(gpa, io, tmp.dir)); } test "submanifests §8.1/§7.2: a repository that HOLDS a leaf arms its serve-side gate against a pre-S peer" { // The PRODUCTION wiring of the §7.2 gate, end to end through the durable // store: `persistStore` records §8.1's seen-submanifest indicator for a // stored SUBMANIFEST (which carries no S card at all, §4.4), the serve // snapshot re-reads it, and `serveRound` arms the gate with it. Every link // is load-bearing: // // * drop the `transferGatedArtifact` line in `phantomizeSubManifestRefs` // and the indicator stays false — a leaf is not "S-bearing" literally; // * drop `round_server.seen_submanifest = …` in `serveRound` and the gate // is unarmed, because §8.1's emission setting is false on every zit; // * drop `or self.seen_submanifest` from `Server.submanifestGateArmed` // and it is unarmed for the same reason. // // Each of the three leaves the leaf in the reply to a fossil-2.28 peer, // which §8 says can only hold it as "unreadable ... opaque content". const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var hex_buf: [2 * digest_len]u8 = undefined; try serveSeedRepo(gpa, io, tmp.dir, 1, &hex_buf); // §4.3: an F run with no D card is type SUBMANIFEST. The flat check-in // manifest beside it is everything that predates this draft. const leaf = "F a.txt " ++ ("3" ** 64) ++ "\n"; const flat = "C m\nD 2026-07-27T12:00:00.000\nF a.txt " ++ ("2" ** 64) ++ "\nU u\n"; var leaf_name: [2 * digest_len]u8 = undefined; var flat_name: [2 * digest_len]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, leaf, &leaf_name); _ = zit.artifact.artifactName(.sha3_256, flat, &flat_name); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); _ = try ms.store().put(&leaf_name, leaf); _ = try ms.store().put(&flat_name, flat); _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); // §8.1 — the indicator is DURABLE and covers §7.2's type shape, not only // its S-card shape. Read back through the same function the CLI uses. var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = try repoSyncIdentity(gpa, io, tmp.dir, "test-fixture-repo", &pc_buf); try std.testing.expect(identity.seen_submanifest); // …and the emission setting is the OTHER, independent half of the arming // disjunction: this repository holds a gated artifact it received, and has // never been told to emit one. A single flag could not tell the two apart. try std.testing.expect(!identity.enable_submanifests); // …and the served reply refuses the leaf to a peer announcing fossil 2.28, // while still handing it the artifact it can read. const server = zit.sync.Server{ .version = serve_mod.server_version, .self_servercode = serve_mod.server_servercode, .project_code = serve_mod.server_projectcode, }; try std.testing.expect(!server.submanifests_enabled); // §8.1's default, stated var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = &file, .server = &server }; defer ctx.dropSnapshot(); const plain = try std.fmt.allocPrint( gpa, "pragma client-version 22800\ngimme {s}\ngimme {s}\n", .{ &leaf_name, &flat_name }, ); defer gpa.free(plain); const request = try zit.container.encode(gpa, plain, .default); defer gpa.free(request); const reply = try serveAnswer(&ctx, gpa, request, null, null, true); defer gpa.free(reply.body); const body = try zit.container.decode(gpa, reply.body, zit.sync.reply_body_cap); defer gpa.free(body); // Built through the same writer the server uses (§11.1's escape encoding // applies to the text), so the expectation cannot drift from the emission. var card_buf: [2 * zit.sync.submanifest_peer_version_error_text.len + 16]u8 = undefined; const refusal = zit.sync.writeSignalCard(.err, zit.sync.submanifest_peer_version_error_text, &card_buf); try std.testing.expect(std.mem.indexOf(u8, body, refusal) != null); try std.testing.expect(std.mem.indexOf(u8, body, &leaf_name) == null); try std.testing.expect(std.mem.indexOf(u8, body, &flat_name) != null); } test "submanifests §7.2/§10.1: the CLI announces a DATED client-version so the peer's version can reach the gate" { // §10.1 r11/r12 — the server replies `pragma server-version` exactly when // the received client-version carried DATE and TIME, and §7.2's gate reads // "the remote peer's announced version". An undated pragma therefore leaves // `Client.peer.version` at 0 for the whole session and an armed client // refuses its own trees to an opted-in server: pullable, never pushable. // (`sync_driver_test` drives that consequence end to end; this pins the one // CLI constant the whole chain hangs off.) const gpa = std.testing.allocator; try std.testing.expect(sync_client_version_info.datetime != null); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = sync_servercode, .project_code = sync_projectcode, .version = sync_client_version_info, }; defer client.deinit(gpa); var buf: std.ArrayListUnmanaged(u8) = .empty; defer buf.deinit(gpa); const request = try client.buildRequest(gpa, ms.store(), &buf); // The emitted pragma carries at least §10.1 r11's five tokens. const eol = std.mem.indexOfScalar(u8, request, '\n').?; var tokens: usize = 0; var it = std.mem.tokenizeScalar(u8, request[0..eol], ' '); while (it.next()) |_| tokens += 1; try std.testing.expect(std.mem.startsWith(u8, request, "pragma client-version ")); try std.testing.expect(tokens >= zit.sync.min_datetime_tokens); try std.testing.expect(zit.sync.serverEmitsServerVersion(.{ .version = sync_client_version, .datetime = sync_client_version_info.datetime, })); } test "configValue reads every name from one frame, and empty means no opinion" { var arena_state: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); // An untouched frame has no opinion about anything, so the versioned and // global tiers stay reachable. const bare: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p" }; try std.testing.expect((try configValue(arena, &bare, .ignore_glob)) == null); try std.testing.expect((try configValue(arena, &bare, .autosync)) == null); try std.testing.expect((try configValue(arena, &bare, .clearsign)) == null); try std.testing.expect((try configValue(arena, &bare, .dotfiles)) == null); try std.testing.expect((try configValue(arena, &bare, .crnl_glob)) == null); const set: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p", .ignore_glob = "*.o", .crnl_glob = "*.bat", .clean_glob = "*.tmp", .empty_dirs = "logs", .encoding_glob = "*.utf16", .dotfiles = "yes", .mimetypes = "md text/markdown", .clearsign_set = true, .clearsign = false, .autosync_set = true, .autosync_off = true, }; try std.testing.expectEqualStrings("*.o", (try configValue(arena, &set, .ignore_glob)).?); try std.testing.expectEqualStrings("*.bat", (try configValue(arena, &set, .crnl_glob)).?); try std.testing.expectEqualStrings("*.tmp", (try configValue(arena, &set, .clean_glob)).?); try std.testing.expectEqualStrings("logs", (try configValue(arena, &set, .empty_dirs)).?); try std.testing.expectEqualStrings("*.utf16", (try configValue(arena, &set, .encoding_glob)).?); // Verbatim: the operator wrote `yes`, so the listing says `yes`. try std.testing.expectEqualStrings("yes", (try configValue(arena, &set, .dotfiles)).?); try std.testing.expectEqualStrings("md text/markdown", (try configValue(arena, &set, .mimetypes)).?); // The booleans answer through their explicit-set bit, as they always have. try std.testing.expectEqualStrings("off", (try configValue(arena, &set, .clearsign)).?); try std.testing.expectEqualStrings("off", (try configValue(arena, &set, .autosync)).?); } test "resolveCrlfGlob falls back to crnl-glob from one frame" { // The commit gate holds the store's write lock and reads its config once, // so both spellings are answered from that frame -- no second store open, // and no way for the two reads to see different bytes. const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); const amb: cli.Ambient = .{ .now_epoch = 0, .user_env = null }; // The older spelling answers while the newer one is absent. const older: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p", .crnl_glob = "*.txt", }; const from_crnl = try resolveCrlfGlob(io, arena, tmp.dir, amb, &older); try std.testing.expectEqualStrings("*.txt", from_crnl.value); try std.testing.expectEqual(zit.settings.Source.repository, from_crnl.source); // `crlf-glob` present REPLACES it, exactly as fossil's // `db_get("crlf-glob", db_get("crnl-glob",""))` does. const newer: logstore.Config = .{ .hash_policy = 0, .default_branch = "trunk", .project_id = "p", .crlf_glob = "*.c", .crnl_glob = "*.txt", }; const from_crlf = try resolveCrlfGlob(io, arena, tmp.dir, amb, &newer); try std.testing.expectEqualStrings("*.c", from_crlf.value); // No frame and no versioned file: nothing is declared, so the class is // not checked at all. const none = try resolveCrlfGlob(io, arena, tmp.dir, amb, null); try std.testing.expectEqual(zit.settings.Source.default, none.source); try std.testing.expectEqualStrings("", none.value); } test "clone observer forwards phase edges without owning measurement state" { const Probe = struct { count: usize = 0, phase: ClonePhase = .transport, event: ClonePhaseEvent = .begin, fn callback(context: *anyopaque, phase: ClonePhase, event: ClonePhaseEvent) void { const self: *@This() = @ptrCast(@alignCast(context)); self.count += 1; self.phase = phase; self.event = event; } }; var probe = Probe{}; const observer = CloneObserver{ .context = &probe, .callback = Probe.callback }; observeClonePhase(&observer, .persist, .end); try std.testing.expectEqual(@as(usize, 1), probe.count); try std.testing.expectEqual(ClonePhase.persist, probe.phase); try std.testing.expectEqual(ClonePhaseEvent.end, probe.event); } // --------------------------------------------------------------------------- // `zit config pull AREA`: the explicit configuration request side // --------------------------------------------------------------------------- /// `zit config pull shun [URL]` — fossil's `configuration pull AREA ?URL?`. /// /// Configuration carriage is otherwise CLONE-TIME ONLY, and deliberately so: a /// shun table that refreshed on every sync would let any peer suppress content /// in this repository on its own say-so. This verb is how an operator asks for /// it anyway — explicitly, naming the area, against a peer they chose. /// /// `shun` is the one area zit models. The rest of fossil's areas describe a web /// interface and a user table zit has no equivalent for; naming one of those is /// a clean refusal rather than a silent no-op, so an operator learns which /// areas exist rather than watching nothing happen. /// Apply the peer's user rows to this repository's table. Shares /// `runConfigFromFile`'s two modes and its one-marker rule: `overwrite` /// replaces, and the marker lands before the first row and only when there /// is a row to land, so a pull that returned nothing never wipes the table it /// could not replace. /// /// An EMPTY result is reported rather than treated as success-with-nothing: /// against fossil, "no rows" is what an under-privileged client sees, so an /// operator who forgot `--user` must not read it as "the peer has no users". fn applyPeerUsers( gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, rows: []const zit.sync.OwnedUserRow, overwrite: bool, streams: cli.Streams, ) cli.Exit { if (rows.len == 0) { streams.err.writeAll( "zit config: the peer offered no user rows\n" ++ "zit config: fossil serves the user area to an administrator only -- " ++ "pass --user=NAME (and --password=, or $ZIT_PASSWORD)\n", ) catch {}; return .failure; } var file = cli.openStoreAppend(io, root_dir) catch return repoReadFail(streams, "config"); defer file.close(io); const st = file.stat(io) catch return repoReadFail(streams, "config"); var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch return repoReadFail(streams, "config"); defer log.deinit(gpa); if (overwrite) log.putUserReset(0) catch return repoReadFail(streams, "config"); var skipped: usize = 0; for (rows) |row| { log.putUser(.{ .login = row.login, .pw = row.pw, .cap = row.cap, .info = row.info, .mtime = row.mtime, }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), error.LoginTooLong, error.FieldTooLong => { skipped += 1; continue; }, }; } log.commit() catch return repoReadFail(streams, "config"); const table = log.userTable(gpa) catch return stateOom(streams, "config"); defer gpa.free(table); streams.out.print("user: {d} row{s} {s} from the peer; {d} in the table", .{ rows.len, plural(rows.len), if (overwrite) "adopted" else "merged", table.len, }) catch {}; if (skipped != 0) streams.out.print("; {d} unusable and skipped", .{skipped}) catch {}; streams.out.writeAll("\n") catch {}; if (skipped != 0) return .failure; return .ok; } /// `zit config`'s three shapes, spelled as fossil spells them /// (`configuration pull|import|merge`). const ConfigSub = enum { pull, import, merge }; const config_usage_line: []const u8 = "zit config: usage: zit config pull shun [URL] | zit config import|merge FILE\n"; /// The largest configuration export this verb will read whole. A user table /// is a handful of rows per operator; a file past this bound is not a /// configuration export and is refused rather than allocated for. const max_config_file_bytes: usize = 8 * 1024 * 1024; /// Apply a fossil configuration export file: the output of /// `fossil configuration export AREA FILE`, which is byte-identical to what /// the sync protocol carries for the same area (`config /AREA SIZE\n` then /// SIZE payload bytes then a newline), plus `#` comment lines at the top. /// /// That identity is the whole reason this path exists: a repository's users /// reach zit with no SQLite reader, no network, and no fossil server — one /// `fossil configuration export` on the migration host, and the SAME parser /// the wire path will use. /// /// `replace` is fossil's `CONFIGSET_OVERWRITE`. Applied ONCE, before the /// first row lands, exactly as `configure_receive`'s `configHasBeenReset` /// guard does — a per-row reset would leave only the last row of the file. fn runConfigFromFile( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, path: []const u8, replace: bool, repo_override: ?[]const u8, streams: cli.Streams, ) cli.Exit { // The user/shun areas are store tables; `-R` redirects which // store they land in. The FILE is still read from the cwd (fossil's // `configuration import FILENAME`), so only the store is redirected. const root = (cli.resolveRoot(arena, io, repo_override) catch return stateOom(streams, "config")) orelse { main.reportNoRoot(streams, io, "config", repo_override); return .failure; }; var root_dir = root.dir; defer root_dir.close(io); const text = std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(max_config_file_bytes)) catch { streams.err.print("zit config: cannot read '{s}'\n", .{path}) catch {}; return .failure; }; defer gpa.free(text); var users: std.ArrayList(zit.sync.UserRow) = .empty; defer users.deinit(gpa); var value_arena = std.heap.ArenaAllocator.init(gpa); defer value_arena.deinit(); const va = value_arena.allocator(); var skipped: usize = 0; var other_areas: usize = 0; var pos: usize = 0; while (pos < text.len) { const line_end = std.mem.indexOfScalarPos(u8, text, pos, '\n') orelse text.len; const line = text[pos..line_end]; // `configuration export` writes a `#` banner naming the repository and // the export time; blank lines separate nothing but are harmless. if (line.len == 0 or line[0] == '#') { pos = line_end + 1; continue; } const card = parseConfigCardLine(line) orelse { streams.err.print("zit config: '{s}' is not a configuration export\n", .{path}) catch {}; return .failure; }; const payload = zit.sync.consumeConfigPayload(text, line_end + 1, card.size) catch { streams.err.print("zit config: '{s}' is truncated\n", .{path}) catch {}; return .failure; }; pos = payload.next; if (!std.mem.eql(u8, card.area, zit.sync.config_user_group)) { other_areas += 1; continue; } const row = zit.sync.parseUserConfigPayload(payload.bytes) catch { skipped += 1; continue; }; const parsed = unquoteUserRow(va, row) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Malformed => { skipped += 1; continue; }, }; users.append(gpa, parsed) catch return stateOom(streams, "config"); } if (other_areas != 0 and users.items.len == 0) { streams.err.print( "zit config: '{s}' carries no /user cards ({d} card{s} in areas zit does not model)\n", .{ path, other_areas, plural(other_areas) }, ) catch {}; return .failure; } var file = cli.openStoreAppend(io, root_dir) catch return repoReadFail(streams, "config"); defer file.close(io); const st = file.stat(io) catch return repoReadFail(streams, "config"); var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch return repoReadFail(streams, "config"); defer log.deinit(gpa); // The marker lands before the first row and only when there is a row to // land: an import whose file held nothing applicable must not wipe the // table it could not replace. if (replace and users.items.len != 0) { log.putUserReset(0) catch return repoReadFail(streams, "config"); } for (users.items) |row| { log.putUser(.{ .login = row.login, .pw = row.pw, .cap = row.cap, .info = row.info, .mtime = row.mtime, }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), // A row this store cannot key or hold is reported, never silently // dropped: an operator migrating an access-control table needs to // know a login did not come across. error.LoginTooLong, error.FieldTooLong => { skipped += 1; continue; }, }; } log.commit() catch return repoReadFail(streams, "config"); const table = log.userTable(gpa) catch return stateOom(streams, "config"); defer gpa.free(table); streams.out.print("user: {d} row{s} {s}; {d} in the table", .{ users.items.len, plural(users.items.len), if (replace) "imported" else "merged", table.len, }) catch {}; if (skipped != 0) streams.out.print("; {d} unreadable and skipped", .{skipped}) catch {}; if (other_areas != 0) streams.out.print("; {d} card{s} in other areas ignored", .{ other_areas, plural(other_areas) }) catch {}; streams.out.writeAll("\n") catch {}; if (skipped != 0) return .failure; return .ok; } /// One `config /AREA SIZE` card header line. const ConfigCardLine = struct { area: []const u8, size: u32 }; fn parseConfigCardLine(line: []const u8) ?ConfigCardLine { const prefix = "config "; if (!std.mem.startsWith(u8, line, prefix)) return null; const rest = line[prefix.len..]; const sp = std.mem.indexOfScalar(u8, rest, ' ') orelse return null; const area = rest[0..sp]; if (area.len == 0 or area[0] != '/') return null; const size = std.fmt.parseInt(u32, rest[sp + 1 ..], 10) catch return null; return .{ .area = area, .size = size }; } /// Unquote a wire user row's fields onto `arena`. A field fossil omitted is /// the empty string here — zit models "no password" as empty rather than as /// a distinct NULL, which is also what it emits back on the wire. fn unquoteUserRow(arena: std.mem.Allocator, row: zit.sync.QuotedUserRow) error{ OutOfMemory, Malformed }!zit.sync.UserRow { const mtime = std.fmt.parseInt(i64, row.mtime_text, 10) catch return error.Malformed; return .{ .login = try unquoteField(arena, row.quoted_login), .pw = try unquoteField(arena, row.quoted_pw), .cap = try unquoteField(arena, row.quoted_cap), .info = try unquoteField(arena, row.quoted_info), .mtime = mtime, }; } fn unquoteField(arena: std.mem.Allocator, quoted: ?[]const u8) error{ OutOfMemory, Malformed }![]const u8 { const q = quoted orelse return ""; const buf = try arena.alloc(u8, q.len); const out = zit.sync.fossilUnquote(q, buf) catch return error.Malformed; return out; } pub fn runConfig( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { var area: ?[]const u8 = null; var url_arg: []const u8 = ""; var sub: ?ConfigSub = null; var overwrite = false; var login_user: ?[]const u8 = null; var login_password: ?[]const u8 = null; for (verb_args) |tok| { if (std.mem.eql(u8, tok, "-h") or std.mem.eql(u8, tok, "--help")) { streams.out.writeAll(main.config_help) catch {}; return .ok; } else if (std.mem.eql(u8, tok, "--overwrite")) { overwrite = true; } else if (std.mem.startsWith(u8, tok, "--user=")) { login_user = tok["--user=".len..]; } else if (std.mem.startsWith(u8, tok, "--password=")) { login_password = tok["--password=".len..]; } else if (tok.len > 1 and tok[0] == '-') { return cli.usageError(streams, "config", .unknown_option, tok); } else if (sub == null) { sub = std.meta.stringToEnum(ConfigSub, tok) orelse return cli.usageError(streams, "config", .unknown_option, tok); } else if (area == null) { area = tok; } else if (url_arg.len == 0) { url_arg = tok; } else { return cli.usageError(streams, "config", .extra_argument, tok); } } if (sub == null or area == null) { streams.err.writeAll(config_usage_line) catch {}; return .usage; } // `import`/`merge` take a FILE, not an area and a URL: fossil's own // `configuration import|merge FILENAME`. The two differ in exactly one // thing, which is why they share a body: `import` REPLACES the local table // (fossil's `CONFIGSET_OVERWRITE` — `DELETE FROM user` then `REPLACE // INTO`), `merge` folds rows in newest-mtime-wins. switch (sub.?) { .import, .merge => { if (url_arg.len != 0) return cli.usageError(streams, "config", .extra_argument, url_arg); // The file forms take their mode from the SUBCOMMAND, as fossil's // do; accepting a contradicting --overwrite would give one // invocation two ways to say the same thing and one way to lie. if (overwrite) return cli.usageError(streams, "config", .unknown_option, "--overwrite"); if (login_user != null or login_password != null) { streams.err.writeAll("zit config: --user/--password apply to 'config pull', not to a file\n") catch {}; return .usage; } return runConfigFromFile(gpa, io, arena, area.?, sub.? == .import, amb.repo_override, streams); }, .pull => {}, } // Fossil matches area names by PREFIX (`configure_name_to_mask`), and so // does the server side of this exchange, so the CLI accepts the same. const want = area.?; // Fossil matches by PREFIX, and orders `shun` before `user` in // `aGroupName`, so an ambiguous prefix resolves the way fossil resolves it. const group: []const u8 = if (want.len != 0 and std.mem.startsWith(u8, "shun", want)) zit.sync.config_shun_group else if (want.len != 0 and std.mem.startsWith(u8, "user", want)) zit.sync.config_user_group else { streams.err.print("zit config: unknown configuration area '{s}' (known: shun, user)\n", .{want}) catch {}; return .failure; }; const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "config")) orelse { main.reportNoRoot(streams, io, "config", amb.repo_override); return .failure; }; var root_dir = root.dir; defer root_dir.close(io); var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, url_arg, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), error.NoRemote => return noRemoteError(streams, "config"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "config", url, e); defer tr.close(); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); // §9.1 — the pull card carries this repository's adopted project code, and a // real peer refuses a mismatch, so the identity is loaded even though this // exchange moves no artifacts. The buffer outlives the run (the client // borrows it). var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const identity = repoSyncIdentity(gpa, io, root_dir, root.abs, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), }; // A pull-shaped session that asks for one group. The in-memory store stays // EMPTY on purpose: this verb moves CONFIGURATION, never artifacts, so an // operator asking for a shun table cannot be handed a repository's worth of // content as a side effect. // fossil serves the user area to an ADMINISTRATOR only // (`configure.c`: `m &= ~(CONFIGSET_USER|CONFIGSET_SCRIBER)` for a client // without Admin), so a `/user` pull is credentialed or it comes back // empty. The §6.3 derivation is `resolvePushCredential`'s, reused rather // than restated: one place turns a password into a signing secret. var pw_buf: [max_password_len]u8 = undefined; defer std.crypto.secureZero(u8, &pw_buf); var user_buf: [logstore.max_config_str]u8 = undefined; const resolved = resolvePushCredential( gpa, io, root_dir, streams, identity.project_code, .{ .url = url, .user = login_user, .password = login_password }, amb, &pw_buf, &user_buf, ) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), error.Storage => return repoReadFail(streams, "config"), error.NoPassword => { streams.err.writeAll("zit config: no password supplied for --user\n") catch {}; return .failure; }, }; var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = &identity.server_code, .project_code = identity.project_code, .version = sync_client_version_info, .request_config = true, .config_group = group, .credential = if (resolved) |r| r.credential else null, }; defer client.deinit(gpa); _ = sync_driver.runSync(gpa, &client, ms.store(), &tr, max_stalled_sync_rounds) catch |e| return syncRunFail(streams, "config", url, e, tr.last_status, basicStateOf(&tr)); if (std.mem.eql(u8, group, zit.sync.config_user_group)) { return applyPeerUsers(gpa, io, root_dir, client.peerUserRows(), overwrite, streams); } const rows = client.peerShunRows(); if (rows.len == 0) { streams.out.writeAll("no shun rows offered by the peer\n") catch {}; return .ok; } var file = cli.openStoreAppend(io, root_dir) catch return repoReadFail(streams, "config"); defer file.close(io); const st = file.stat(io) catch return repoReadFail(streams, "config"); var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch return repoReadFail(streams, "config"); defer log.deinit(gpa); // Count by the TABLE's own size, not by successful calls: `putShun` is // idempotent on a name already present, so counting calls would report rows // the operator already had as newly adopted. const before = log.shunList().len; var skipped: usize = 0; for (rows) |row| { var digest_buf: [32]u8 = undefined; if (row.hash.len != 40 and row.hash.len != 64) { skipped += 1; continue; } const digest = digest_buf[0 .. row.hash.len / 2]; _ = std.fmt.hexToBytes(digest, row.hash) catch { skipped += 1; continue; }; // `putShun` is idempotent on a name already present, so re-running this // verb converges rather than accumulating. log.putShun(digest, row.reason, row.epoch) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "config"), else => { skipped += 1; continue; }, }; } const added = log.shunList().len - before; log.commit() catch return repoReadFail(streams, "config"); streams.out.print("shun: {d} new row{s} adopted", .{ added, plural(added) }) catch {}; if (skipped != 0) streams.out.print("; {d} unreadable and skipped", .{skipped}) catch {}; streams.out.writeAll("\n") catch {}; return .ok; } test "the cluster sidecar is created by persist, adopted without a walk, rebuilt by one walk when lost, extended by a later persist" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Two blobs and a cluster naming one of them, persisted into the fresh log. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var names: [2][64]u8 = undefined; inline for (.{ "clustered blob", "loose blob" }, 0..) |body, i| { _ = zit.artifact.artifactName(.sha3_256, body, &names[i]); const rid = try ms.store().put(&names[i], body); const c = try zit.container.encode(gpa, body, .default); defer gpa.free(c); try ms.store().retain(rid, .{ .full = c }); } const cluster = try std.fmt.allocPrint(gpa, "M {s}\nZ 00000000000000000000000000000000\n", .{&names[0]}); defer gpa.free(cluster); var cn: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, cluster, &cn); { const rid = try ms.store().put(&cn, cluster); const c = try zit.container.encode(gpa, cluster, .default); defer gpa.free(c); try ms.store().retain(rid, .{ .full = c }); } _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0); // 1. Adopted: persist created the index, so a load walks nothing. { var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const before = clusterindex.walk_rids; var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store()); defer r.deinit(); try std.testing.expectEqual(before, clusterindex.walk_rids); try std.testing.expectEqual(@as(usize, 1), r.cluster_names.len); try std.testing.expectEqualStrings(&cn, r.cluster_names[0]); try std.testing.expect(r.clustered.isSet(holdings.store().lookup(&names[0]).?)); try std.testing.expect(!r.clustered.isSet(holdings.store().lookup(&names[1]).?)); try std.testing.expect(!r.clustered.isSet(holdings.store().lookup(&cn).?)); } // 2. Lost: one full walk rebuilds it; the load after that adopts again. try tmp.dir.deleteFile(io, zit.repocache.clusters_filename); { var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const before = clusterindex.walk_rids; var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store()); defer r.deinit(); try std.testing.expectEqual(before + 3, clusterindex.walk_rids); try std.testing.expectEqual(@as(usize, 1), r.cluster_names.len); const again = clusterindex.walk_rids; var r2 = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store()); defer r2.deinit(); try std.testing.expectEqual(again, clusterindex.walk_rids); } // 3. Extended: a later persist records the cluster it writes; no walk. const cluster2 = try std.fmt.allocPrint(gpa, "M {s}\nZ 00000000000000000000000000000000\n", .{&names[1]}); defer gpa.free(cluster2); var cn2: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, cluster2, &cn2); { const rid = try ms.store().put(&cn2, cluster2); const c = try zit.container.encode(gpa, cluster2, .default); defer gpa.free(c); try ms.store().retain(rid, .{ .full = c }); } _ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 3); { var holdings = try ClientHoldings.open(gpa, io, tmp.dir); defer holdings.deinit(io); const before = clusterindex.walk_rids; var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store()); defer r.deinit(); try std.testing.expectEqual(before, clusterindex.walk_rids); try std.testing.expectEqual(@as(usize, 2), r.cluster_names.len); try std.testing.expect(r.clustered.isSet(holdings.store().lookup(&names[1]).?)); } }