Artifact 9ffd01a40d9f3a8d276d99770ec94d96295a3ac0415406cbc7ccf1ff1bca77d6:
- File
src/cmd/sync.zig
— part of check-in
[f75dfdfbe7]
at
2026-07-23 11:53:26
on branch fix10-update-diag
— Split the update read-path diagnostics apart from the corruption claim
zit update, open, merge and bisect collapsed ApplyError.StateRecordInvalid and ApplyError.Corrupt onto one line, "cannot read repository: repository is corrupt". They are different claims. Corrupt means stored bytes do not match the name they are stored under. StateRecordInvalid means only that a stored artifact was not usable as a check-in state record, which says nothing about integrity.
An intact fossil clone reaches the second routinely: fossil stores and syncs artifacts it declines to crosslink itself. Assembly treats those as fatal while the read verbs skip them, so update accused an undamaged clone of corruption while timeline read the same store fine.
StateRecordInvalid now carries its own reason text, shared across the four verbs so they cannot drift or re-merge. The bisect mapper had a blanket else arm swallowing the whole set into that corruption line; it is now exhaustive over ApplyError.
Diagnostics only: assembly still rejects these artifacts, so update still exits 1 on such a clone. The tolerance fix is separate; the shape that triggers it, an anonymous wiki page with no U card, is pinned. (user: dmestas, size: 182584) [annotate] [blame] [check-ins using] [more...]
//! zit sync / network / store-transfer verbs (serve, clone, open, pack, pull, //! push, sync, remote) and the helpers and tests private to them. //! //! Extracted verbatim from src/main.zig (release step S2). Every declaration //! below is byte-identical to its main.zig source; only this import prologue is //! new. Helpers still shared with other verbs stay in main.zig and are aliased //! back here so the moved bodies call them unqualified. main.zig's verb-dispatch //! switch calls into this file (`cmd_sync.runServe` ...); its test block imports //! it so these inline tests keep running under `zig build test`. const std = @import("std"); const zit = @import("zit"); const cli = @import("../cli.zig"); const extras = @import("../extras.zig"); const logstore = @import("zit").logstore; const storage = @import("zit").storage; const update = @import("zit").update; const checkout = @import("zit").checkout; const Repository = @import("zit").Repository; const serve_mod = @import("../serve.zig"); const transport = @import("../transport.zig"); const sync_driver = @import("../sync_driver.zig"); const Io = std.Io; const File = Io.File; const repostate = zit.repostate; const managed = zit.managed; const artifact = zit.artifact; // Shared CLI helpers and test fixtures that stay in main.zig (other verbs use // them too); aliased back so the moved code and tests reference them by their // original unqualified names. const main = @import("../main.zig"); const runInit = main.runInit; const repo_filename = main.repo_filename; const renderStorage = main.renderStorage; const reasonOf = main.reasonOf; const stateOom = main.stateOom; const not_a_repo_msg = main.not_a_repo_msg; const repoReadFail = main.repoReadFail; const repoOpenFail = main.repoOpenFail; const openRepo = main.openRepo; const updateApplyFail = main.updateApplyFail; const digest_len = main.digest_len; const hexToDigest = main.hexToDigest; const digestToWireName = main.digestToWireName; const decodeWireName = main.decodeWireName; const open_help = main.open_help; const pack_help = main.pack_help; const serve_help = main.serve_help; const clone_help = main.clone_help; const pull_help = main.pull_help; const push_help = main.push_help; const sync_help = main.sync_help; const remote_help = main.remote_help; const packSeedBlob = main.packSeedBlob; const packSeedCheckin = main.packSeedCheckin; const packEvolvingBlob = main.packEvolvingBlob; const packOpenWritable = main.packOpenWritable; const packOpenReadOnly = main.packOpenReadOnly; /// `zit serve [--port N]` — put an HTTP face on the stateless sync responder. /// Opens `.zit` read-only and keeps it open for the server's lifetime; every /// request reads a fresh content view of it and is answered independently /// (sync §3.8). `std.http`/socket I/O lives entirely in `serve.zig`; this only /// wires the repository and the listen address. pub fn runServe( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { const parsed = switch (cli.parseServeArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "serve", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "serve", .extra_argument, tok), .bad_port => |tok| { streams.err.print("zit serve: invalid --port value '{s}'\n", .{tok}) catch {}; return .usage; }, }; if (parsed.help) { streams.out.writeAll(serve_help) catch {}; return .ok; } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "serve")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; var file = root_dir.openFile(io, repo_filename, .{}) catch { root_dir.close(io); return repoReadFail(streams, "serve"); }; defer file.close(io); const log_st = file.stat(io) catch { root_dir.close(io); return repoReadFail(streams, "serve"); }; var fs = storage.FileStorage.init(io, file, log_st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| { root_dir.close(io); switch (e) { error.OutOfMemory => streams.err.writeAll("zit serve: out of memory\n") catch {}, else => streams.err.print("zit serve: cannot read repository: {s}\n", .{reasonOf(e)}) catch {}, } return .failure; }; defer log.deinit(gpa); defer root_dir.close(io); // §9.1/§8.2 — emit the repository's stored project code in the clone // reply's `push` card, so a clone learns the shared project identity. A // repo predating the stored code, or an unreadable config, falls back to // the fixed placeholder (informational; servercode never gates a clone). var maybe_cfg: ?logstore.OwnedConfig = log.readConfig(gpa) catch null; defer if (maybe_cfg) |*oc| oc.deinit(gpa); const project_code: []const u8 = if (maybe_cfg) |oc| (if (oc.config.project_code.len > 0) oc.config.project_code else serve_mod.server_projectcode) else serve_mod.server_projectcode; const server = zit.sync.Server{ .version = serve_mod.server_version, .credential = null, .self_servercode = serve_mod.server_servercode, .project_code = project_code, }; var ctx = ServeCtx{ .io = io, .root_dir = root_dir, .file = file, .server = &server }; const responder = serve_mod.Responder{ .ctx = &ctx, .answerFn = serveAnswer }; const address = std.Io.net.IpAddress.parse("127.0.0.1", parsed.port) catch { streams.err.print("zit serve: invalid --port value '{d}'\n", .{parsed.port}) catch {}; return .usage; }; streams.err.print("zit serve: listening on http://127.0.0.1:{d}/xfer (Ctrl-C to stop)\n", .{parsed.port}) catch {}; streams.err.flush() catch {}; serve_mod.serve(io, gpa, address, responder, .{}) catch |e| switch (e) { error.Listen => { streams.err.print("zit serve: cannot bind 127.0.0.1:{d}\n", .{parsed.port}) catch {}; return .failure; }, error.Accept => { streams.err.writeAll("zit serve: connection accept failed\n") catch {}; return .failure; }, }; return .ok; } /// The per-request context threaded into `serve.Responder`: the open log to read /// a fresh store from, the server identity to answer with, and the injected IO + /// repository directory used to make pushed content durable (a push request /// ingests new artifacts into the per-request store, which must be persisted into /// the served repository's on-disk log — sync §3.3). const ServeCtx = struct { io: std.Io, root_dir: std.Io.Dir, /// The open read handle on the served `.zit`. `respond` snapshots the /// CURRENT on-disk log from it per request (§3.8): a fresh stat fixes the /// committed end-of-log, so a check-in appended by a separate `zit commit` /// while `serve` runs is advertised on the next request — no restart. file: File, server: *const zit.sync.Server, }; /// The `serve.Responder` seam: snapshot the repository into an in-memory store, /// answer the framed request over it, and — when the request pushed new content — /// make that content durable in the served repository's on-disk log (sync §3.3). /// /// `respond` stays sans-IO: it only `put`s pushed artifacts into the snapshot /// store it is given. The durable write is this serve-layer's job and reuses the /// idempotent `persistStore` (a re-push of an already-stored hash appends nothing /// — content is addressed by hash), the very path clone/pull persist through. A /// pure gimme/pull request grows the store by nothing, so `contentCount` stays /// flat and the log is never reopened; only a genuine push touches disk. /// /// The pushed bytes were size-validated by `respond`'s request scan before they /// entered the store, so `persistStore` only ever writes well-formed content, and /// a mid-write fault surfaces before `commit`, never as a partial log append. fn serveAnswer(ctx_ptr: *anyopaque, gpa: std.mem.Allocator, request_container: []const u8) serve_mod.AnswerError!serve_mod.XferReply { const ctx: *ServeCtx = @ptrCast(@alignCast(ctx_ptr)); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); try readServedStore(gpa, ctx.io, ctx.file, &ms); const held_before = contentCount(ms.store()); const reply = try serve_mod.handleXfer(gpa, ctx.server, ms.store(), request_container); errdefer gpa.free(reply.body); if (contentCount(ms.store()) > held_before) { _ = persistStore(gpa, ctx.io, ctx.root_dir, &ms) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; } return reply; } /// Snapshot every stored artifact from the persisted log into `ms`, skipping /// config frames (not artifacts). A content-bearing frame becomes a held entry; /// a content-absent frame becomes a phantom — exactly the held/phantom split /// `respond` walks (sync §3.2/§3.3). /// /// The log keys frames by the RAW binary digest, but the sync wire names an /// artifact by that digest's lowercase hex (sync §3.3 — a `file`/`igot`/`gimme` /// HASH is 40/64 hex characters). The store the responder walks must therefore /// hold the hex name, or `respond` would emit non-hex bytes that break the /// peer's card tokenizer. Every hash crossing this boundary is hex-encoded once, /// here — the single place binary-log identity becomes wire identity. fn loadContentStore( gpa: std.mem.Allocator, log: *logstore.LogStore, ms: *repostate.MemStore, ) serve_mod.AnswerError!void { const rids = log.ridsInPersistedOrder(gpa) catch return error.OutOfMemory; defer gpa.free(rids); for (rids) |rid| { if (log.kindOf(rid) == .config) continue; var name_buf: [2 * digest_len]u8 = undefined; const name = digestToWireName(log.hashOf(rid), &name_buf); // sync §4.1/§7.2 — copy the stored container into the snapshot verbatim // so the clone serves it without inflating here and re-compressing at // serve. `wire()` is null only for a `.raw` frame or a phantom; those // fall back to the plaintext path below. A `.zlib` frame carries the // full form; a `.delta` frame carries the stored delta container plus // its basis name and reconstructed size (the §7.2 USIZE, delta §3.2). var maybe_wire = log.wire(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; if (maybe_wire) |*w| { defer w.deinit(gpa); switch (w.*) { .full => |container| { _ = ms.putWire(name, container) catch return error.OutOfMemory; }, .delta => |d| { // The basis frame precedes this one in the log (delta_of < // offset), so it was already inserted at an earlier rid; its // wire name is the hex of the base's binary digest. var src_name_buf: [2 * digest_len]u8 = undefined; const src_name = digestToWireName(d.baseHash(), &src_name_buf); _ = ms.putWireDelta(name, d.container, src_name, d.target_size) catch return error.OutOfMemory; }, } continue; } const maybe = log.content(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; if (maybe) |bytes| { defer gpa.free(bytes); _ = ms.store().put(name, bytes) catch return error.OutOfMemory; } else { _ = ms.store().intern(name) catch return error.OutOfMemory; } } } /// §3.8 — read the CURRENT on-disk log from the open serve handle `file` into /// `ms` for one request. The server is stateless per request, so its holdings /// are re-read every time rather than snapshotted at startup: a fresh `stat` /// fixes the committed end-of-log (a separate `zit commit` process, or this /// server's own push persist, appends complete frames that grow the file), and /// `openReadOnly` scans exactly that snapshot through a positional read — the /// handle is never advanced, so re-reading it sees the new bytes. A concurrent /// append caught mid-write leaves a torn tail, which the read-only scanner /// rejects as corruption (never a torn artifact); the committed frames below it /// are read consistently, and the request retries once the write completes. The /// upshot: a commit made while `serve` runs is advertised on the next request, /// with no restart (the Direction-A incremental-pull gap). fn readServedStore( gpa: std.mem.Allocator, io: std.Io, file: File, ms: *repostate.MemStore, ) serve_mod.AnswerError!void { const st = file.stat(io) catch return error.ServerStorage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage, error.Corrupt => return error.ServerStorage, }; defer log.deinit(gpa); try loadContentStore(gpa, &log, ms); } // --------------------------------------------------------------------------- // `zit clone URL DIR` — the interop milestone: a real client converging against // a real `zit serve` over HTTP. Create a fresh repo at DIR, open an HTTP // transport to URL, pump the sans-IO clone client (`sync.Client`) to §3.7 // convergence through `sync_driver.runSync`, then persist the received // artifacts into DIR's on-disk log so the clone is durable. Anonymous only. // --------------------------------------------------------------------------- /// §9.1 — a fixed placeholder repository identity emitted in the client's /// pull/push operation cards. Neither a fresh clone nor a pulled repository /// carries a persistent SERVERCODE/PROJECTCODE of its own yet (real per-repo /// sync identity is future work); the stateless server echoes the peer's codes /// and never gates on a configured project code (sync §9.1), so any well-formed /// 16-hex pair carries a content-only exchange. Shared by clone and pull. const sync_servercode = "0000000000000000"; const sync_projectcode = "0000000000000000"; /// §10.1 — the release integer the sync client announces as `pragma /// client-version`. Undated (no server-version is solicited); a fixed build /// identity, matching `serve`'s own version constant style. const sync_client_version: u32 = 22700; /// An explicit upper bound on sync rounds (TigerStyle: every loop is bounded). /// A single-hop reconcile converges in a handful of rounds; this is deep slack, /// so a misbehaving peer yields `error.RoundLimitExceeded` instead of hanging. const max_sync_rounds: usize = 1024; /// `zit clone URL DIR` — thin wiring over three deep modules: `transport` /// (HTTP/IO), `sync.Client` + `sync_driver.runSync` (sans-IO §3.7 loop), and /// `LogStore` (persistence). The URL is used verbatim (sync §4 r1 — the client /// never appends /xfer). All-or-nothing failures surface an honest exit-1 /// diagnostic; a bad URL scheme is rejected before any repo is created. pub fn runClone( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { const parsed = switch (cli.parseCloneArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "clone", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "clone", .extra_argument, tok), .no_url => return cli.usageError(streams, "clone", .no_url, ""), .no_dir => return cli.usageError(streams, "clone", .no_dir, ""), }; if (parsed.help) { streams.out.writeAll(clone_help) catch {}; return .ok; } // Open the transport first: a malformed URL fails before any repo lands on // disk (nothing half-made to roll back). `open` performs no network I/O. var tr = transport.Transport.open(io, gpa, parsed.url) catch |e| return syncOpenFail(streams, "clone", parsed.url, e); defer tr.close(); // Create a fresh `.zit` at DIR by reusing the init sequence, quietly (its // own success line is suppressed; clone prints its own). A pre-existing repo // there is init's "already exists" error, propagated verbatim. const init_rc = runInit(gpa, io, arena, &.{ "-q", "--", parsed.dir }, streams); if (init_rc != .ok) return init_rc; // Converge an in-memory store against the peer, then persist it into DIR. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const outcome = cloneRun(gpa, &tr, &ms) catch |e| return syncRunFail(streams, "clone", e); var target = std.Io.Dir.cwd().openDir(io, parsed.dir, .{}) catch return renderStorage(streams, parsed.dir, "cannot reopen the clone target directory"); defer target.close(io); const written = persistStore(gpa, io, target, &ms) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"), }; // §9.1 — adopt the origin's project code as this repository's shared // identity (a later pull refuses a PROJECTCODE mismatch), overwriting the // local code `init` seeded (last config frame wins). A peer that sent no // push card leaves the local code in place. storeClonedProjectCode(gpa, io, target, outcome.projectCode()) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "clone"), error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"), error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"), }; // Eagerly build the derivation cache so the FIRST read verb after this clone // (`leaves`/`log`/…) takes the zero-parse image path instead of paying the // full cold derive. Run after the project-code frame above — the clone's last // log append — so the cache keys to the settled log. Best-effort (see // `Repository.warmCache`): a failure here leaves the clone correct, with the // first read deriving lazily as it did before; it never fails the clone. Repository.warmCache(gpa, io, target); streams.out.print("cloned {d} artifacts into {s} in {d} rounds\n", .{ written, parsed.dir, outcome.rounds }) catch {}; return .ok; } /// Render an `open` diagnostic to stderr and fail. Best-effort on the write: a /// diagnostic that cannot be printed must not change the exit code. fn openDiag(s: cli.Streams, msg: []const u8) cli.Exit { s.err.print("zit open: {s}\n", .{msg}) catch {}; return .failure; } /// Map a materialization fault to its `open` diagnostic, one arm per /// `update.ApplyError` variant and NO `else`: a new variant must fail this /// compile and force a decision here, exactly as it does in `updateApplyFail`. /// /// Only `InvalidCheckinId` and `CheckinNotFound` are genuinely unreachable — /// `open` hands the engine a hash it just resolved out of this repository's own /// derived state, never a caller-supplied name — and even those get honest /// prose rather than a corruption claim. Everything else is reachable on an /// intact repository: the content/manifest arms describe a repository that /// really is damaged, while `SymlinkUnsupported` / `InvalidPath` / /// `ReservedPath` are raised while planning the TARGET manifest, so a clone of /// any repository containing a symlink or a path this platform cannot represent /// reaches them with nothing corrupt at all. Reporting those as corruption /// would be a false accusation against a healthy repository; the prose mirrors /// `updateApplyFail`'s. fn openApplyFail(s: cli.Streams, err: update.ApplyError) cli.Exit { return switch (err) { error.OutOfMemory => openDiag(s, "out of memory"), error.InvalidCheckinId => openDiag(s, "cannot read repository: the resolved check-in ID is not a valid artifact name"), error.CheckinNotFound => openDiag(s, "cannot read repository: the resolved check-in is not present"), error.CheckinContentMissing => openDiag(s, "cannot read repository: check-in content is missing"), error.CheckinNameMismatch => openDiag(s, "cannot read repository: check-in content does not match its name"), error.TargetNotManifest => openDiag(s, "cannot read repository: the check-in is not a valid manifest"), error.BaselineNotFound => openDiag(s, "cannot read repository: the check-in baseline is not found"), error.BaselineContentMissing => openDiag(s, "cannot read repository: the check-in baseline content is missing"), error.BaselineNameMismatch => openDiag(s, "cannot read repository: the check-in baseline content does not match its name"), error.BaselineNotManifest => openDiag(s, "cannot read repository: the check-in baseline is not a valid manifest"), error.BaselineIsDelta => openDiag(s, "cannot read repository: the check-in baseline is not fully resolvable"), error.StateRecordInvalid => openDiag(s, "cannot read repository: " ++ update.state_record_invalid_reason), error.Corrupt => openDiag(s, "cannot read repository: repository is corrupt"), error.SymlinkUnsupported => openDiag(s, "the check-in contains an unsupported symlink entry"), error.InvalidPath => openDiag(s, "the check-in names an invalid file path"), error.ReservedPath => openDiag(s, "the check-in names a reserved file path"), error.WorktreeNotClean => openDiag(s, "working tree is not clean"), error.UnmanagedCollision => openDiag(s, "an untracked file or directory is in the way"), error.CurrentStateInvalid => openDiag(s, "current working-tree state is invalid"), error.Storage => openDiag(s, "cannot update working tree: input/output error"), error.TransactionUnavailable => openDiag(s, "cannot create a transaction directory"), error.RollbackFailed => openDiag(s, "open failed and could not be rolled back; the previous working-tree state is retained for recovery"), error.CleanupFailed => openDiag(s, "open committed, but the transaction directory could not be removed"), }; } /// `zit open` — the second half of `zit clone URL DIR`: turn a repository whose /// checkout is still UNBORN into a materialized working tree. Fossil's own UX is /// the same two steps (`fossil clone` then `fossil open`), and splitting them /// keeps `runClone` strictly repo-only. /// /// Thin wiring over the shipped machinery, deliberately: probe the checkout /// record, resolve the tip of the main branch out of the derived repo state, and /// hand materialization to the SAME `update.applyCheckin` engine every other /// working-tree move rides (journal-armed, transactional, rollback-safe). Only /// the tip resolution is open's own work, and it is load-bearing: `update` /// accepts nothing but a literal 40/64-hex check-in ID, so a freshly cloned tree /// has no way to name its own tip. /// /// It writes NO checkout record and NO managed set of its own. `init` — which /// `clone` delegates repo creation to — already wrote the unborn record; an /// absent managed set already reads as the empty set; and the update engine is /// the single writer of the born record and the managed set. Adding a second /// writer of that bootstrap state would only create two things to keep in sync. pub fn runOpen( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { const parsed = switch (cli.parseOpenArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "open", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "open", .extra_argument, tok), .repeated_option => |tok| return cli.usageError(streams, "open", .repeated_option, tok), .empty_value => |tok| return cli.usageError(streams, "open", .empty_value, tok), .bad_manifest => |tok| return cli.usageError(streams, "open", .bad_manifest, tok), }; if (parsed.help) { streams.out.writeAll(open_help) catch {}; return .ok; } // The full derive: resolving the main branch reads the tag tables, which the // `.fast_read` image path does not carry. var repo: Repository = undefined; var root_dir: std.Io.Dir = undefined; const rc = openRepo(gpa, io, arena, streams, "open", null, false, false, &repo, &root_dir); if (rc != .ok) return rc; defer root_dir.close(io); defer repo.deinit(gpa); // BORN-ness, not presence, is the "already open" signal. Every fresh // repository carries a checkout record — `init` writes an unborn one // (`version == ""`, branch `trunk`) as its step 8, and `clone` delegates to // `init` — so a presence-based gate would reject `zit clone && zit open` on // every clone, the primary path this verb exists to serve. Only a non-empty // version means a tree has actually been materialized against a check-in. var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined; const current = checkout.read(io, root_dir, &ck_buf) catch |e| switch (e) { error.Storage => return openDiag(streams, "cannot read the checkout state: input/output error"), error.Corrupt => return openDiag(streams, "the checkout state is corrupt"), }; if (current) |state| if (state.version.len != 0) { streams.err.print( "zit open: this working tree is already open at check-in '{s}'; use 'zit update' to change check-ins\n", .{state.version}, ) catch {}; return .failure; }; // The tip of the main branch (repo-state §9 current-trunk). const main_branch = repo.mainBranch(); // Switched, not an unbound `catch`: the success message below is an exit-0 // claim that the tree is already correct, and it is only honest for // `NotFound`. Naming the arm means a later Storage/OOM arm on `currentTrunk` // fails this compile instead of silently reporting a genuine failure as // success on a materialization verb. const tip = repo.state().currentTrunk(main_branch) catch |e| switch (e) { // A repository with no check-in on its main branch is ALREADY in its // correct end state — the unborn record and an empty tree. There is // nothing to materialize and nothing wrong, so this is reported, not // failed, and the record is left exactly as it was found. error.NotFound => { streams.out.print( "open: no check-in on '{s}' yet; the working tree is already in its correct (empty) state\n", .{main_branch}, ) catch {}; return .ok; }, // Settling an equal-mtime tie by ancestry allocates (repo-state // `MtimeMaxPick`). A failure here is a genuine failure — reporting it as // the exit-0 "already correct" message would be the lie the switch // above exists to prevent. error.OutOfMemory => return openDiag(streams, "out of memory"), }; // Both borrow the derived view, which outlives the update below. const target = repo.hashOf(tip); const branch = repo.branchOf(tip); // An assertion, not a check: unlike `update`, whose target is user-supplied // and therefore validated, open's target comes out of this repository's own // derived state. A malformed name here is a defect in the derivation, not // bad input, so it belongs in the negative space rather than in a // diagnostic path no correct run can reach. std.debug.assert(artifact.validateName(target)); const log = repo.logStore(gpa) catch |e| return repoOpenFail(streams, "open", e); // `open` is the ONE verb that SETS the manifest opt-in; absent flag records // "none", which is what an untouched checkout already carried. const manifest: update.ManifestOpt = .{ .set = parsed.manifest orelse "" }; update.applyCheckin(gpa, io, root_dir, log, target, manifest) catch |e| return openApplyFail(streams, e); streams.out.print("opened {s} on {s}\n", .{ target, branch }) catch {}; return .ok; } /// The fields a `updateConfig` overlay may change; a null field is left exactly /// as the stored config has it. Naming a single change site keeps the "copy the /// whole config, overlay one field" discipline in one place — a caller that /// means to change only the project code can never silently drop the credential /// or the saved remote (the class of bug the pre-`updateConfig` re-puts had). const ConfigPatch = struct { project_code: ?[]const u8 = null, login: ?zit.sync.Credential = null, remote_url: ?[]const u8 = null, }; /// Read the live repo config at `dir/.zit`, overlay `patch`, and append the /// result as the new live config (§E last-frame-wins), preserving every field /// the patch does not name. This is the SOLE read-modify-write of the repo /// config; storing a clone's project code, a push credential, or a saved remote /// all route through it. The repo must already carry a config frame (`init` /// always writes one); its absence is `Corrupt`. Shares `PersistError` with the /// artifact-write path so the caller maps one storage-diagnostic surface. fn updateConfig( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, patch: ConfigPatch, ) PersistError!void { var file = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse return error.Corrupt; // init always writes a config frame defer oc.deinit(gpa); // Copy every field, then overlay only the named ones. `next`'s string slices // borrow `oc.backing` / `patch`; both outlive the `putConfig` memcpy below. var next = oc.config; if (patch.project_code) |c| next.project_code = c; if (patch.login) |cred| { next.login_user = cred.user; // Point at the function-scoped `patch` storage, not the block-local // capture `cred`: `next` is read by `putConfig` after this block closes. next.login_secret_hex = &patch.login.?.secret_hex; } if (patch.remote_url) |u| next.remote_url = u; log.putConfig(next) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; log.commit() catch return error.Storage; } /// §9.1 — persist the origin's project code into the freshly cloned repo at /// `dir/.zit`, preserving every other field `init` wrote. A no-op when the peer /// sent no code. Thin wrapper over `updateConfig` (the field-preserving RMW). fn storeClonedProjectCode( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, project_code: []const u8, ) PersistError!void { if (project_code.len == 0) return; // peer sent none; keep the local init code return updateConfig(gpa, io, dir, .{ .project_code = project_code }); } /// Drives a modern-clone client (§3.8 CLONE_ROUND0 → EXCHANGE) to convergence /// over `tr`, filling `ms` with the peer's artifacts; returns the round count. /// Sans-IO on this side of `tr.exchange`: the state machine only moves /// `[]const u8` bytes. Its closed error set is the driver's own causes unioned /// with the transport's, so `runClone` switches the whole surface exhaustively. /// The result of a clone run: the round count plus the origin's project code /// (§9.1), copied out of the sans-IO client so the caller can persist the /// shared identity after the client is torn down. const CloneOutcome = struct { rounds: usize, project_code_buf: [zit.sync.max_projectcode_len]u8, project_code_len: usize, fn projectCode(self: *const CloneOutcome) []const u8 { return self.project_code_buf[0..self.project_code_len]; } }; fn cloneRun( gpa: std.mem.Allocator, tr: *transport.Transport, ms: *repostate.MemStore, ) sync_driver.RunError(*transport.Transport)!CloneOutcome { var client = zit.sync.Client{ .state = zit.sync.initialState(.clone), .operation = .clone, .self_servercode = sync_servercode, .project_code = sync_projectcode, .version = .{ .version = sync_client_version }, }; defer client.deinit(gpa); const stats = try sync_driver.runSync(gpa, &client, ms.store(), tr, max_sync_rounds); // §8.3/§9.1 — the peer's project code, learned from the clone reply's // `push` card, copied out before the client is deinitialized. var outcome = CloneOutcome{ .rounds = stats.rounds, .project_code_buf = undefined, .project_code_len = 0 }; const pc = client.peerProjectCode(); @memcpy(outcome.project_code_buf[0..pc.len], pc); outcome.project_code_len = pc.len; return outcome; } /// The closed error set of `persistStore`: memory, a storage write fault, or a /// corrupt/oversized peer artifact name that will not decode to a log key. const PersistError = error{ OutOfMemory, Storage, Corrupt }; /// The disposition of one artifact in `persistStore`'s ordered pass. const PersistOutcome = union(enum) { /// A non-phantom frame for this artifact now exists in the log — written /// here, or already present from a prior cascade / a pull re-persist. persisted, /// A reverse-delta whose basis is a content-bearing store entry not YET /// written to the log. It is deferred on the basis wire-hash and re-driven /// once that basis lands, so its `delta_of` offset points strictly backward. deferred: []const u8, }; /// Persist one artifact `rid` into `log`, choosing its stored form, and verify /// §7.7 integrity (the reconstructed content MUST hash to its wire name) BEFORE /// any durable write. A retained delta whose basis is already a held log frame /// under the read depth bound is stored VERBATIM as a `.delta`; a retained delta /// whose basis is a not-yet-written content-bearing store entry is DEFERRED /// (when `allow_defer`) so the basis frame is written first (reverse-delta /// ordering); every other case — no wire form, a full container, or an /// absent / phantom / over-deep basis — lands as the full form, never a /// phantom-based or unreadable delta. Idempotent: an artifact already holding a /// non-phantom frame returns `.persisted` without a re-write or re-verification /// (§7.7 was proven when it was first persisted). fn persistArtifact( gpa: std.mem.Allocator, ms: *repostate.MemStore, log: *logstore.LogStore, rid: repostate.Rid, allow_defer: bool, /// A bounded plaintext memo shared across `persistStore`'s whole sweep. Each /// artifact reconstructs its own plaintext exactly once here — for the §7.7 /// verify — and primes it under its just-written frame's offset, so a later /// reverse-delta naming this frame as basis reuses that plaintext instead of /// re-walking (re-inflating, re-applying) the chain. Bounded (LRU, fixed byte /// + entry caps): a cache miss just costs a fresh reconstruction, never a /// wrong answer, so the stored bytes are identical with or without it. cache: *logstore.PlaintextCache, ) PersistError!PersistOutcome { const store = ms.store(); const name = store.hashOf(rid); var digest_buf: [digest_len]u8 = undefined; const digest = decodeWireName(name, &digest_buf) orelse return error.Corrupt; if (log.lookup(digest)) |ex| { if (!log.isPhantom(ex)) return .persisted; } // A genuine phantom carries nothing durable — skip it (never a drop of held // content). `held` classifies without inflating a wire-backed entry. if (!store.held(rid)) return .persisted; // §7.7 L1332/L1336-1338 — the repository transaction-commit hook. Recompute // the artifact hash over the reconstructed bytes and abort BEFORE any frame is // written on a mismatch (a lying container inflates but does not name its // content). The verify plaintext is reconstructed against the LOG basis (a // read bounded by `max_delta_chain`) — NOT the in-memory wire chain, which, // now a clone stores every artifact wire-backed, is the full reverse-delta // history and would exceed the read depth bound. `persistStore` writes basis // frames before the deltas that name them (reverse-delta order), so a delta's // basis is already a log frame here. if (store.wire(rid)) |wf| { if (wf.delta) |d| { var base_buf: [digest_len]u8 = undefined; const base_digest = decodeWireName(d.src_hash, &base_buf) orelse return error.Corrupt; const maybe_base = log.lookup(base_digest); if (maybe_base == null or log.isPhantom(maybe_base.?)) { // The basis is not yet a log frame. Defer on it (reverse-delta // order: the newer basis frame must be written first) when it is a // held store entry that WILL be persisted. if (allow_defer) { if (store.lookup(d.src_hash)) |bs_rid| { if (store.held(bs_rid)) return .{ .deferred = d.src_hash }; } } // Basis absent and not deferrable (§3.3): fall back to FULL form // from the entry's OWN plaintext — a put+retain entry carries it // directly (never a phantom-based delta). In a converged clone this // is unreachable (every delta's basis is a written frame, handled // below); a wire-delta entry whose basis truly never arrived cannot // reconstruct — a hard error, never a fabricated artifact. const content = store.content(rid) orelse return error.Corrupt; if (!zit.artifact.verify(name, content)) return error.Corrupt; const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } const base_rid = maybe_base.?; const base_off = log.offsetOf(base_rid); // Reuse the basis plaintext memoised when the basis frame was // persisted — reverse-delta order writes every basis before the // delta that names it, so the memo is normally warm. Only a miss // (first use, or an LRU eviction) reconstructs it by walking the // chain; either way `base_plain` is owned and freed below. const base_plain = if (cache.get(base_off)) |hit| blk: { // §7.7 positive space: a memo entry is the exact plaintext that // named its wire hash at the basis's own persist, so the reused // bytes still hash to the basis name (debug-only assertion; the // hash is never recomputed on the hot path). std.debug.assert(zit.artifact.verify(d.src_hash, hit)); break :blk gpa.dupe(u8, hit) catch return error.OutOfMemory; } else blk: { const p = (log.contentCached(gpa, base_rid, cache) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse return error.Corrupt; // `contentCached` does not memoise a top-level full-form basis; // prime it here so sibling reverse-deltas on the same basis reuse // this reconstruction instead of repeating it. try cache.put(gpa, base_off, p); break :blk p; }; defer gpa.free(base_plain); const raw = zit.container.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(raw); const content = zit.delta.apply(gpa, base_plain, raw) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(content); if (!zit.artifact.verify(name, content)) return error.Corrupt; // Store the delta VERBATIM iff its basis is a held frame under the READ // depth bound; an over-deep basis falls back to full form (never an // unreadable chain). `delta_of` is the basis's backward offset — // strictly lower than this new frame, the invariant `putDeltaContainer` // asserts. if (log.depthOf(base_rid) < logstore.max_delta_chain) { const new_rid = log.putDeltaContainer(digest, base_rid, wf.container) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the just-verified plaintext so an older reverse-delta // naming this frame as basis reuses it (no chain re-walk). try cache.put(gpa, log.offsetOf(new_rid), content); } else { const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; try cache.put(gpa, log.offsetOf(new_rid), content); } return .persisted; } // §7.2 full form — decode to verify, then store the container verbatim. const content = zit.container.decode(gpa, wf.container, @intCast(logstore.max_payload)) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Corrupt, }; defer gpa.free(content); if (!zit.artifact.verify(name, content)) return error.Corrupt; const new_rid = log.putContainer(digest, wf.container) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the just-verified full-form plaintext so a reverse-delta naming // this frame as basis reuses it instead of re-inflating the container. try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } // §4.1 — no retained wire (a raw pull `put`): the plaintext is memoised in the // store, no chain to walk. Verify and compress it; the hash names the // UNCOMPRESSED bytes, so compression is address-transparent. const content = store.content(rid) orelse return error.Corrupt; if (!zit.artifact.verify(name, content)) return error.Corrupt; const new_rid = log.putZlib(digest, content) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, }; // Prime the plaintext so a reverse-delta naming this frame as basis reuses it. try cache.put(gpa, log.offsetOf(new_rid), content); return .persisted; } /// Persists every content-bearing artifact in the converged store `ms` into the /// on-disk log at `dir/.zit`, then fsyncs (durability before success). The /// store names artifacts by their wire hex; the log keys frames by the raw /// binary digest, so each name is decoded once here — the inverse of the /// hex-encode in `loadContentStore`. Idempotent per `LogStore.put` (an artifact /// already held returns its rid without a re-append), so a pull re-persisting a /// store that mixes prior holdings with freshly received content writes only the /// new frames. Returns the number of content-bearing artifacts in the store. /// Shared by clone (fresh repo) and pull (existing repo). fn persistStore( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, ms: *repostate.MemStore, ) PersistError!usize { var file = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); // The header of the just-created repo carries the hash kind; `open` reads it // from the non-empty file, so the passed value is a don't-care (0). var log = logstore.LogStore.open(gpa, fs.storageFile(), 0) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }; defer log.deinit(gpa); var written: usize = 0; const store = ms.store(); const n = store.count(); // Reverse-delta persist ordering (§7.2/§8.2). Fossil stores an OLDER // artifact as a delta of a NEWER one, so a delta cfile can land in the store // at a LOWER rid than the basis it points at, while the log's `delta_of` is a // BACKWARD offset that REQUIRES the basis frame to precede the delta. Persist // therefore drains a worklist: a delta whose basis is not yet a frame is // deferred (keyed by the basis wire-hash) and re-driven the instant that // basis lands. Each content-bearing artifact seeds the drain once in rid // order; each deferred delta is re-driven exactly once, so the pass is O(n) // and terminates — a chain bottoms out at fossil's full newest artifact, and // content addressing forbids a cycle. var waiting: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(repostate.Rid)) = .empty; defer { var it = waiting.valueIterator(); while (it.next()) |list| list.deinit(gpa); waiting.deinit(gpa); } var ready: std.ArrayListUnmanaged(repostate.Rid) = .empty; defer ready.deinit(gpa); // One bounded plaintext memo for the whole sweep: a basis reconstructed for // its own §7.7 verify is reused by every reverse-delta that names it, so a // delta family reconstructs each basis ~once rather than once per referencing // delta. Bounded (LRU byte + entry caps) — it never retains the whole repo. var cache = logstore.PlaintextCache.init(); defer cache.deinit(gpa); var seed: repostate.Rid = 1; while (seed <= n) : (seed += 1) { if (!store.held(seed)) continue; // residual phantom: not durable written += 1; // count each content-bearing artifact exactly once, at its rid try ready.append(gpa, seed); while (ready.pop()) |rid| { switch (try persistArtifact(gpa, ms, &log, rid, true, &cache)) { .persisted => { // `rid`'s frame now exists: re-drive any reverse-deltas that // deferred on it as their (now-written) basis. if (waiting.fetchRemove(store.hashOf(rid))) |kv| { var list = kv.value; defer list.deinit(gpa); try ready.appendSlice(gpa, list.items); } }, .deferred => |base_hash| { const gop = try waiting.getOrPut(gpa, base_hash); if (!gop.found_existing) gop.value_ptr.* = .empty; try gop.value_ptr.append(gpa, rid); }, } } } // Defensive drain: a deferred basis is itself content-bearing and therefore // persisted above (draining its waiters), so `waiting` is normally empty // here. Should a basis nonetheless be missing, its waiters land as full form // rather than be dropped — every content-bearing artifact stays durable. var leftover = waiting.iterator(); while (leftover.next()) |e| { for (e.value_ptr.items) |rid| { switch (try persistArtifact(gpa, ms, &log, rid, false, &cache)) { .persisted => {}, .deferred => unreachable, // allow_defer=false never defers } } } log.commit() catch return error.Storage; return written; } // --------------------------------------------------------------------------- // `zit pack` — repack the store into a compact new generation. // // On-disk storage is UNSPECIFIED by the RFCs (they fix the wire format and // content-addressed naming, §4.1 L1206-1208), so pack is free to reorganize // the log as long as every artifact's exact content and name survive and the // §7.2 verbatim wire bytes are still servable. It re-ingests every artifact, // in the source's offset order, into a FRESH log through `putArtifact` with a // lineage base hint — a check-in against its parent manifest, a file blob // against the same path's blob in the parent check-in (the commit path's own // §4.1/§7.2 compaction heuristic, applied offline). Offset order plus the // strictly-backward `delta_of` invariant means every base is already written // before the delta that names it, so no worklist is needed (unlike // `persistStore`, whose input is rid-ordered). // // The new log is written to a temporary file beside `.zit`, fsynced, and then // atomically renamed over `.zit`: a crash anywhere before the rename leaves the // original byte-identical (the temp is discarded next run). The `.zit-cache` // invalidates itself on the tail change and rebuilds on the next read verb. // --------------------------------------------------------------------------- /// pack's core store-to-store transform lives in the `zit` module (pack.zig) /// so the DST/fuzz harnesses drive the REAL pass in-process; the CLI wiring /// below (temp file, atomic rename, stats rendering) stays here. These aliases /// keep every call site and test reading `packLog`/`PackError`/`PackStats`. const PackError = zit.pack.PackError; const PackStats = zit.pack.PackStats; const packLog = zit.pack.packLog; /// The name of pack's transient new-generation file, beside `.zit`. A fixed /// name (never a random one) so a temp left by a crashed pack is simply /// overwritten by the next run rather than accumulating. Sourced from /// `extras.zig`, which owns the literal for the single never-tracked predicate. const pack_tmp_filename = extras.pack_tmp_filename; /// `zit pack` — repack the current repository's store into a compact new /// generation and atomically swap it in. Thin CLI wiring over `packLog`: parse /// (no positional), discover the checkout root, open `.zit` read-only (which /// whole-frame-CRC-verifies every frame — the integrity pre-flight) and capture /// its tail key, stream the packed log into a temp file, fsync, re-check the /// source tail (the no-repo-lock concurrency guard), then rename the temp over /// `.zit`. On any failure the temp is deleted and the original store is intact. pub fn runPack( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { const help = switch (cli.parseNoPositionalArgs(verb_args)) { .ok => |a| a.help, .unknown_option => |tok| return cli.usageError(streams, "pack", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "pack", .extra_argument, tok), }; if (help) { streams.out.writeAll(pack_help) catch {}; return .ok; } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "pack")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // Open the source read-only: `openReadOnly` scans and whole-frame-CRC-checks // every frame, so a corrupt source aborts here (no temp written). The tail // key pins the log we packed for the pre-rename concurrency re-check. var src_file = root_dir.openFile(io, repo_filename, .{}) catch return repoReadFail(streams, "pack"); defer src_file.close(io); const src_st = src_file.stat(io) catch return repoReadFail(streams, "pack"); var src_fs = storage.FileStorage.init(io, src_file, src_st.size); var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| { switch (e) { error.OutOfMemory => streams.err.writeAll("zit pack: out of memory\n") catch {}, else => streams.err.print("zit pack: cannot read repository: {s}\n", .{reasonOf(e)}) catch {}, } return .failure; }; defer src.deinit(gpa); const src_key = src.logKey() catch return packFail(streams, error.Corrupt); // New-generation file: a readable temp beside `.zit` (createFileAtomic opens // write-only, but pack reads its own delta bases back, so we roll our own // readable temp and rename it). `truncate` overwrites a temp left by a prior // crashed pack. Cleaned up on every path except the successful rename. var tmp_file = root_dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = true }) catch return packFail(streams, error.Storage); var renamed = false; defer { tmp_file.close(io); if (!renamed) root_dir.deleteFile(io, pack_tmp_filename) catch {}; } var dst_fs = storage.FileStorage.init(io, tmp_file, 0); var dst = logstore.LogStore.open(gpa, dst_fs.storageFile(), src.hash_kind) catch |e| switch (e) { error.OutOfMemory => return packFail(streams, error.OutOfMemory), error.Storage => return packFail(streams, error.Storage), error.Corrupt => return packFail(streams, error.Corrupt), }; const stats = packLog(gpa, &src, &dst) catch |e| { dst.deinit(gpa); return packFail(streams, e); }; dst.deinit(gpa); // `packLog` already fsynced; deinit only frees memory // No repo lock exists: if the source log changed under us since we scanned // it, abort rather than clobber a concurrent write with our stale snapshot. // O(1) (one stat + one header read); the original stays intact. if (!logstore.LogStore.validateTail(src_fs.storageFile(), src_key)) { streams.err.writeAll( "zit pack: repository changed during pack; aborted (original is intact)\n", ) catch {}; return .failure; } // The single durable transition: atomically replace `.zit` with the packed // log. A crash before this leaves the original untouched; after it, the // complete new log. There is no window where `.zit` is partial. root_dir.rename(pack_tmp_filename, root_dir, repo_filename, io) catch return packFail(streams, error.Storage); renamed = true; renderPackStats(streams.out, stats); return .ok; } /// Render pack's failure `err` as `zit pack: <reason>` on stderr, exit 1. The /// original store is always intact when this runs (every write went to the /// temp, which the caller's `defer` deletes). fn packFail(s: cli.Streams, err: PackError) cli.Exit { s.err.print("zit pack: {s}\n", .{reasonOf(err)}) catch {}; return .failure; } /// One human line summarizing a repack. "already packed" when the new log did /// not shrink (the store was already compact — pack still verified every frame, /// its integrity-scrub value). Split from `runPack` so the exact wording is unit /// tested without a filesystem. fn renderPackStats(out: *std.Io.Writer, stats: PackStats) void { const verb = if (stats.bytes_after < stats.bytes_before) "packed" else "already packed"; out.print("{s}: {d} artifacts, {d} re-deltaed, {d} -> {d} bytes\n", .{ verb, stats.artifacts, stats.deltas, stats.bytes_before, stats.bytes_after, }) catch {}; } /// Maps a transport-open failure (bad URL) to the `verb` diagnostic + exit 1. /// Shared by clone and pull — the only difference is the verb label. fn syncOpenFail(s: cli.Streams, verb: []const u8, url: []const u8, err: transport.TransportError) cli.Exit { switch (err) { error.InvalidUrl => s.err.print("zit {s}: '{s}' is not a valid URL\n", .{ verb, url }) catch {}, error.UnsupportedScheme => s.err.print("zit {s}: unsupported URL scheme in '{s}' (use http or https)\n", .{ verb, url }) catch {}, error.OutOfMemory => s.err.print("zit {s}: out of memory\n", .{verb}) catch {}, else => s.err.print("zit {s}: cannot open '{s}'\n", .{ verb, url }) catch {}, } return .failure; } /// Maps a sync run (transport + driver) failure to the `verb` diagnostic + exit /// 1. Exhaustive over the driver's own causes and the transport's, so the whole /// closed error set is accounted for (no `else`). Shared by clone and pull. fn syncRunFail(s: cli.Streams, verb: []const u8, err: sync_driver.RunError(*transport.Transport)) cli.Exit { const reason: []const u8 = switch (err) { error.OutOfMemory => "out of memory", error.RoundLimitExceeded => "the peer did not converge within the round limit", error.FatalProtocolError => "the peer sent a fatal protocol error", error.NotImplemented => "the peer used an unsupported protocol feature", error.RequestTooLarge => "the sync request exceeded the transport limit", error.ReplyTruncated, error.ReplyMalformed => "the peer sent a malformed reply", error.ReplyTooLarge => "the peer's reply exceeded the size limit", error.RedirectLimit => "too many HTTP redirects", error.InsecureRedirect => "the peer redirected HTTPS to plaintext HTTP", error.RedirectNotAllowed => "the peer redirected on a transport that forbids it", error.MissingLocation => "the peer sent a redirect with no location", error.InvalidUrl => "a redirect target was not a valid URL", error.UnsupportedScheme => "a redirect named an unsupported URL scheme", error.ProtocolError => "the peer answered with an unexpected HTTP status", error.ConnectionFailed => "cannot connect to the peer", }; s.err.print("zit {s}: {s}\n", .{ verb, reason }) catch {}; return .failure; } /// The causes `resolveRemoteUrl` can surface: a store read failure, OOM, or the /// repository having no URL to use (`NoRemote` — neither an explicit arg nor a /// saved remote). Closed set so the caller switches exhaustively. const RemoteResolveError = error{ OutOfMemory, Storage, NoRemote }; /// sync §4/§6 — resolve the peer URL for a sync-family verb: an explicit `arg` /// (non-empty) wins; otherwise the saved remote (`config.remote_url`) is used; /// a repository with neither is a usage error (`NoRemote`). The saved URL is /// copied into `buf` so it outlives the read-only config frame. `buf` need only /// live until `transport.Transport.open` (which dupes the URL). An out-of-spec /// stored URL longer than the config-field cap is treated as none. fn resolveRemoteUrl( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, arg: []const u8, buf: *[logstore.max_config_str]u8, ) RemoteResolveError![]const u8 { if (arg.len > 0) return arg; var file = dir.openFile(io, repo_filename, .{}) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.NoRemote; defer oc.deinit(gpa); const saved = oc.config.remote_url; if (saved.len == 0 or saved.len > buf.len) return error.NoRemote; @memcpy(buf[0..saved.len], saved); return buf[0..saved.len]; } /// sync §4/§6 — the usage error (exit 2) when a sync-family `verb` is given no /// URL and the repository has no saved remote to fall back to. Names the fix. /// Shared by pull, push, and sync so the wording lives in one place. fn noRemoteError(s: cli.Streams, verb: []const u8) cli.Exit { s.err.print("zit {s}: no repository URL given and no saved remote\n", .{verb}) catch {}; s.err.writeAll("set one with 'zit remote URL', or pass the URL explicitly\n") catch {}; return .usage; } // --------------------------------------------------------------------------- // `zit pull URL` — incremental sync into an EXISTING repository. The sibling of // clone: it reuses the same transport + `runSync` + persist machinery, but opens // the cwd's `.zit` (never creating one), announces what it already holds so the // peer sends only the missing artifacts (sync §3.2/§3.3 igot/gimme), and drives // to §3.7 termination. Receive-only (§3.3 r24: pull obtains but never uploads). // --------------------------------------------------------------------------- /// `zit pull URL` — open the current repository, converge it toward the peer at /// URL, and persist the newly received artifacts. Thin wiring over the same deep /// modules clone uses: `transport` (HTTP/IO), `sync.Client` + `runSync` (sans-IO /// §3.7 loop), and `LogStore` (persistence). With URL omitted the saved remote /// (`zit remote`) is used; a repo with neither is a usage error. Anonymous only. pub fn runPull( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { const parsed = switch (cli.parsePullArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "pull", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "pull", .extra_argument, tok), // An omitted URL is not an error: it falls back to the saved remote below. .no_url => cli.PullArgs{ .url = "" }, }; if (parsed.help) { streams.out.writeAll(pull_help) catch {}; return .ok; } // Locate the current repository before any network I/O: a pull outside a repo // is a usage failure, not a wasted connection. const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "pull")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), error.NoRemote => return noRemoteError(streams, "pull"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "pull", url, e); defer tr.close(); // Snapshot what this repository already holds into the in-memory store: those // entries become the client's `igot` announcement, so the peer sends only the // artifacts this side lacks (sync §3.3 — the incremental of the exchange). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); loadRepoHoldings(gpa, io, root_dir, &ms) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), }; const held_before = contentCount(ms.store()); // §9.1 — announce this repository's adopted project code in the pull card; // a real peer refuses a PROJECTCODE mismatch, so the placeholder cannot be // sent against fossil. The buffer outlives the run (the client borrows it). var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const project_code = repoProjectCode(gpa, io, root_dir, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), }; const outcome = pullRun(gpa, &tr, &ms, project_code) catch |e| return syncRunFail(streams, "pull", e); const written = persistStore(gpa, io, root_dir, &ms) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "pull"), error.Storage => return repoReadFail(streams, "pull"), error.Corrupt => return renderStorage(streams, root.abs, "the peer sent a malformed artifact name"), }; // A pull only adds content to what was already held (§3.3 r24: it never drops // an artifact), so the persisted count can never fall below the starting set. std.debug.assert(written >= held_before); const received = written - held_before; // The peer named artifacts this run never obtained. Everything is persisted // — a partial pull is still progress — but the exit status says so, because // "exit 0 with nothing said" is indistinguishable from a converged sync and // leaves a repository quietly out of date. This holds whether `received` is // zero or not: the invariant is convergence, not motion. if (outcome.unresolved > 0) { streams.err.print( "zit pull: the peer holds {d} artifacts this pull did not obtain\n", .{outcome.unresolved}, ) catch {}; return .failure; } if (received == 0) { streams.out.writeAll("already up to date\n") catch {}; } else { streams.out.print("received {d} artifacts in {d} rounds\n", .{ received, outcome.rounds }) catch {}; } return .ok; } /// Drives a pull client (§3.2/§3.3) to convergence over `tr`, filling `ms` with /// the artifacts the peer holds and this side lacked; returns the round count. /// The client announces `ms`'s holdings via igot and requests the rest via /// gimme (receive-only — §3.3 r24: pull obtains content but never uploads). /// Sans-IO on this side of `tr.exchange`. Its closed error set is the driver's /// own causes unioned with the transport's, so `runPull` switches it exhaustively. fn pullRun( gpa: std.mem.Allocator, tr: *transport.Transport, ms: *repostate.MemStore, project_code: []const u8, ) sync_driver.RunError(*transport.Transport)!PullOutcome { var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = sync_servercode, .project_code = project_code, .version = .{ .version = sync_client_version }, }; defer client.deinit(gpa); const stats = try sync_driver.runSync(gpa, &client, ms.store(), tr, max_sync_rounds); return .{ .rounds = stats.rounds, .unresolved = client.unresolvedPhantoms(ms.store()) }; } /// What one pull run reports: the round count, and how many artifacts the peer /// announced (§3.3 r24 — by igot, or as a member of a cluster it sent) that this /// side still lacks when §3.7 reached its clean terminal. A nonzero `unresolved` /// is a pull that stopped short of what it was told existed; the caller must not /// report success for it, however well-formed every card on the wire was. const PullOutcome = struct { rounds: usize, unresolved: usize }; /// Opens `dir/.zit` read-only and snapshots its stored artifacts into `ms` — the /// held/phantom split the pull client announces via `igot` (sync §3.2/§3.3). /// Reuses `loadContentStore` (the very snapshot `serve` builds per request), so /// the binary-log→wire-hex name encoding lives in exactly one place. The /// read-only handle is closed before `persistStore` reopens the log read-write. fn loadRepoHoldings( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, ms: *repostate.MemStore, ) error{ OutOfMemory, Storage }!void { var file = dir.openFile(io, repo_filename, .{}) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); loadContentStore(gpa, &log, ms) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; } /// sync §9.1 — the repository's stored project code, copied into `buf`, or the /// zero placeholder when the repo carries none (a repo predating the stored /// code) or an out-of-spec-length one. `init` derives a project code and a /// clone adopts the origin's (`storeClonedProjectCode`), so a real repo /// announces its adopted code here; the pull/push client emits it verbatim as /// the operation card's PROJECTCODE, and a real peer (fossil §9.1, ABNF /// `pull-card = "pull" SP servercode SP projectcode`) refuses a mismatch — so a /// zero placeholder would be rejected. Bounded by `max_projectcode_len`: a /// longer stored field is out of spec and never truncated into the wire. fn repoProjectCode( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, buf: *[zit.sync.max_projectcode_len]u8, ) error{ OutOfMemory, Storage }![]const u8 { var file = dir.openFile(io, repo_filename, .{}) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return sync_projectcode; defer oc.deinit(gpa); const code = oc.config.project_code; if (code.len == 0 or code.len > buf.len) return sync_projectcode; @memcpy(buf[0..code.len], code); return buf[0..code.len]; } /// The number of content-bearing (non-phantom) artifacts in `store` — the held /// set (§3.3 r24). A pull measures its yield as the held count after convergence /// minus the count before, so "already up to date" is an exact zero, never a guess. fn contentCount(store: repostate.ArtifactStore) usize { var n: usize = 0; const total = store.count(); var rid: repostate.Rid = 1; while (rid <= total) : (rid += 1) { if (store.content(rid) != null) n += 1; } return n; } // --------------------------------------------------------------------------- // `zit push URL` — the mirror of pull: UPLOAD this repository's artifacts into an // EXISTING served repo. It opens the cwd's `.zit`, announces what it holds so the // peer requests only what it lacks (sync §3.2/§3.3 igot/gimme), and uploads the // missing artifacts as `file` cards, driving to §3.7 termination. The peer's // `zit serve` persists what it receives; nothing changes on this side (§3.3 r24). // Anonymous only. // --------------------------------------------------------------------------- /// What one push run reports: the number of artifacts uploaded and the round /// count it took. `sent` reads the client's own upload counter, so it is the /// exact count of `file` cards emitted — never an estimate. const PushOutcome = struct { rounds: usize, sent: usize }; /// `zit push URL` — open the current repository, announce its holdings to the /// peer at URL, and upload every artifact the peer lacks until §3.7 termination. /// Thin wiring over the same deep modules pull uses: `transport` (HTTP/IO), /// `sync.Client` + `runSync` (sans-IO §3.7 loop). With URL omitted the saved /// remote (`zit remote`) is used; a repo with neither is a usage error. pub fn runPush( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parsePushArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "push", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "push", .extra_argument, tok), // An omitted URL is not an error: it falls back to the saved remote below. .no_url => cli.PushArgs{ .url = "" }, }; if (parsed.help) { streams.out.writeAll(push_help) catch {}; return .ok; } // sync §6.3 — a login name is a whitespace-free token (the shared-secret // derivation embeds it between "/" separators). Reject a malformed --user // as a usage failure rather than reaching the derivation's assertion. if (parsed.user) |u| { if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) { streams.err.writeAll("zit push: --user must not contain whitespace\n") catch {}; return .failure; } } // Locate the current repository before any network I/O: a push outside a repo // is a usage failure, not a wasted connection. const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "push")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), error.NoRemote => return noRemoteError(streams, "push"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "push", url, e); defer tr.close(); // Snapshot what this repository holds: those entries become the client's igot // announcement, so the peer requests (and this side uploads) only the artifacts // the peer lacks (sync §3.2/§3.3 — the incremental of the exchange). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); loadRepoHoldings(gpa, io, root_dir, &ms) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), }; // §9.1 — announce this repository's adopted project code in the push card // (same identity gate as pull; a real peer refuses a mismatch). The buffer // outlives the run — the client borrows it. var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const project_code = repoProjectCode(gpa, io, root_dir, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), }; // sync §6 — resolve the login credential (null for an anonymous push). The // password plaintext is confined to `pw_buf`/argv/env; only the derived // 40-hex secret (§6.3) is signed with and — on success — persisted. var pw_buf: [max_password_len]u8 = undefined; // Defense-in-depth: once the 40-hex secret is derived the plaintext in // pw_buf is spent, so scrub it on every exit from this scope — success and // error alike — rather than leave it resident until the stack is reused. defer std.crypto.secureZero(u8, &pw_buf); const resolved = resolvePushCredential(gpa, io, root_dir, streams, project_code, parsed, amb, &pw_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), error.Storage => return repoReadFail(streams, "push"), error.NoPassword => { streams.err.writeAll("zit push: no password supplied for --user\n") catch {}; return .failure; }, }; const outcome = pushRun(gpa, &tr, &ms, project_code, if (resolved) |r| r.credential else null) catch |e| return syncRunFail(streams, "push", e); // DECISION-3 — persist the derived secret (never the plaintext) at 0600 only // after the peer accepted the authenticated push, and only when it was // freshly derived: a reused stored secret is already durable. if (resolved) |r| { if (r.persist) { storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "push"), else => return repoReadFail(streams, "push"), }; } } if (outcome.sent == 0) { streams.out.writeAll("nothing to push\n") catch {}; } else { streams.out.print("sent {d} artifacts in {d} rounds\n", .{ outcome.sent, outcome.rounds }) catch {}; } return .ok; } /// The upper bound on an interactively-typed or `$ZIT_PASSWORD` password. A /// login password far shorter than this in practice; the fixed bound keeps the /// no-echo read on the stack with no unbounded allocation (TigerStyle). const max_password_len: usize = 1024; /// sync §6.3 — a resolved push credential and whether its secret was freshly /// derived (so it should be persisted after a successful push). const ResolvedCredential = struct { credential: zit.sync.Credential, persist: bool, }; /// The causes `resolvePushCredential` can surface: store read failure, OOM, or /// a `--user` with no obtainable password (`NoPassword`). Closed set so the /// caller switches exhaustively. const ResolveError = error{ OutOfMemory, Storage, NoPassword }; /// sync §6.3/DECISION-3 — turn `push`'s `--user`/`--password` into a signing /// credential. A null (or empty) `--user` is an anonymous push (null result). /// Otherwise the 40-hex secret is resolved, in order, from: a stored credential /// for this exact login (reused verbatim, already durable), `--password`, /// `$ZIT_PASSWORD`, or a no-echo terminal prompt. A freshly derived secret is /// flagged `persist`; the plaintext lives only in `pw_buf`/argv/env and is /// never returned or stored. fn resolvePushCredential( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, streams: cli.Streams, project_code: []const u8, parsed: cli.PushArgs, amb: cli.Ambient, pw_buf: *[max_password_len]u8, ) ResolveError!?ResolvedCredential { const user = parsed.user orelse return null; if (user.len == 0) return null; // an empty --user is an anonymous push std.debug.assert(!zit.sync.containsFossilWhitespace(user)); // pre-validated in runPush // A stored secret for this exact login is reused as-is: no re-derive, no // prompt, no re-persist. Only consulted when no explicit password is given. if (parsed.password == null and amb.push_password == null) { if (try storedSecretFor(gpa, io, dir, user)) |sec| { return .{ .credential = .{ .user = user, .secret_hex = sec }, .persist = false }; } } // Otherwise derive from a plaintext password. Precedence: --password, then // $ZIT_PASSWORD, then an interactive no-echo prompt. const password: []const u8 = parsed.password orelse amb.push_password orelse (readPasswordNoEcho(streams, user, pw_buf) orelse return error.NoPassword); const secret = zit.sync.deriveSharedSecret(project_code, user, password); return .{ .credential = .{ .user = user, .secret_hex = secret }, .persist = true }; } /// DECISION-3 — the 40-hex secret stored for `user` in this repo's config, or /// null when none is stored or it belongs to a different login. Read-only; the /// returned value is an owned copy (the config frame is freed here). fn storedSecretFor( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, user: []const u8, ) error{ OutOfMemory, Storage }!?[40]u8 { var file = dir.openFile(io, repo_filename, .{}) catch return error.Storage; defer file.close(io); const st = file.stat(io) catch return error.Storage; var fs = storage.FileStorage.init(io, file, st.size); var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; defer log.deinit(gpa); var oc = (log.readConfig(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return null; defer oc.deinit(gpa); if (oc.config.login_secret_hex.len != 40) return null; if (!std.mem.eql(u8, oc.config.login_user, user)) return null; var out: [40]u8 = undefined; @memcpy(&out, oc.config.login_secret_hex[0..40]); return out; } /// DECISION-3 — persist the derived login credential into the repo config, /// preserving every other field, then tighten the repository file to 0600 so /// the stored secret is owner-only at rest. The permission tightening is /// best-effort: the secret is already durable, so a filesystem that refuses /// chmod does not fail the push (perms + FDE are the boundary, not the sole /// guarantee — per DECISION-3). NEVER writes the plaintext password. fn storePushCredential( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, credential: zit.sync.Credential, ) PersistError!void { std.debug.assert(credential.secret_hex.len == 40); // The field-preserving RMW writes the credential without disturbing the // project code or a saved remote (a bare `putConfig` here would drop them). try updateConfig(gpa, io, dir, .{ .login = credential }); // Owner-only now that the file carries a credential (best-effort; see above). var file = dir.openFile(io, repo_filename, .{ .mode = .read_write }) catch return; defer file.close(io); file.setPermissions(io, File.Permissions.fromMode(0o600)) catch {}; } /// CLI/IO seam — prompt on stderr and read one password line from the terminal /// with echo disabled, returning the trimmed line in `buf` (never its newline), /// or null on a read failure or an empty line. Echo is restored on every exit /// path; when stdin is not a terminal (`tcgetattr` fails) the line is read with /// no termios toggle. Not exercised by the automated gates (they pass /// `--password`/`$ZIT_PASSWORD`) — this is the interactive path. fn readPasswordNoEcho( streams: cli.Streams, user: []const u8, buf: *[max_password_len]u8, ) ?[]const u8 { streams.err.print("Password for {s}: ", .{user}) catch {}; streams.err.flush() catch {}; const fd = std.posix.STDIN_FILENO; const restore: ?std.posix.termios = blk: { var t = std.posix.tcgetattr(fd) catch break :blk null; const saved = t; t.lflag.ECHO = false; std.posix.tcsetattr(fd, .FLUSH, t) catch break :blk null; break :blk saved; }; defer if (restore) |saved| { std.posix.tcsetattr(fd, .FLUSH, saved) catch {}; streams.err.writeAll("\n") catch {}; streams.err.flush() catch {}; }; const n = std.posix.read(fd, buf) catch return null; if (n == 0) return null; var line = buf[0..n]; // Trim a single trailing CRLF/LF; interior bytes (including spaces) are kept. if (line.len > 0 and line[line.len - 1] == '\n') line = line[0 .. line.len - 1]; if (line.len > 0 and line[line.len - 1] == '\r') line = line[0 .. line.len - 1]; if (line.len == 0) return null; return line; } /// Drives a push client (§3.2/§3.3) to convergence over `tr`, uploading the /// artifacts the peer lacks; returns the upload count and round count. The client /// announces `ms`'s holdings via igot and uploads what the peer gimme's (§3.3 /// r24: push sends content but never obtains it). Sans-IO on this side of /// `tr.exchange`. Its closed error set is the driver's own causes unioned with /// the transport's, so `runPush` switches it exhaustively. fn pushRun( gpa: std.mem.Allocator, tr: *transport.Transport, ms: *repostate.MemStore, project_code: []const u8, credential: ?zit.sync.Credential, ) sync_driver.RunError(*transport.Transport)!PushOutcome { var client = zit.sync.Client{ .state = zit.sync.initialState(.push), .operation = .push, .self_servercode = sync_servercode, .project_code = project_code, .version = .{ .version = sync_client_version }, // sync §6 — a named credential makes `buildRequest` emit the §6.1 login // card; null leaves the push anonymous (unchanged behavior). .credential = credential, }; defer client.deinit(gpa); const stats = try sync_driver.runSync(gpa, &client, ms.store(), tr, max_sync_rounds); // Read the upload counter before `deinit` runs (the return expression is // evaluated first): the exact number of `file` cards this session emitted. return .{ .rounds = stats.rounds, .sent = client.content_sent }; } // --------------------------------------------------------------------------- // `zit sync [URL]` — BIDIRECTIONAL convergence with a peer in ONE session. // Unlike pull-then-push, a single `operation=.sync` client announces igot, // requests gimme, AND uploads on one §3.7 exchange (§3.2 round structure: a sync // request carries both push and pull each round) — not a third copy of the driver // loop over two transport sessions. // Opens the cwd's `.zit`, resolves the URL (arg → saved remote), may // authenticate (sync uploads), persists what it receives like pull, and reports // both directions. A saved `zit remote` lets the URL be omitted. // --------------------------------------------------------------------------- /// What one sync run reports: uploads (`sent`) and the round count. Received is /// measured by the caller as the persisted-count growth (as pull does), so it is /// not carried here. const SyncOutcome = struct { rounds: usize, sent: usize }; pub fn runSync( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, amb: cli.Ambient, ) cli.Exit { const parsed = switch (cli.parseSyncArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "sync", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "sync", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(sync_help) catch {}; return .ok; } // sync §6.3 — a login name is whitespace-free (mirrors push). Reject a // malformed --user as usage before the secret derivation's assertion. if (parsed.user) |u| { if (u.len > 0 and zit.sync.containsFossilWhitespace(u)) { streams.err.writeAll("zit sync: --user must not contain whitespace\n") catch {}; return .failure; } } // Locate the current repository before any network I/O. const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "sync")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); // sync §4/§6 — the explicit URL wins; an omitted one falls back to the saved // remote; a repo with neither is a usage error. `url_buf` outlives `open`. var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, parsed.url, &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.NoRemote => return noRemoteError(streams, "sync"), }; var tr = transport.Transport.open(io, gpa, url) catch |e| return syncOpenFail(streams, "sync", url, e); defer tr.close(); // Snapshot holdings: the igot announcement and the baseline for the received // count. A sync both uploads what the peer lacks and receives what this side // lacks — the held set can only grow (§3.3 storage only adds content). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); loadRepoHoldings(gpa, io, root_dir, &ms) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), }; const held_before = contentCount(ms.store()); // §9.1 — announce this repository's adopted project code (a real peer refuses // a mismatch). The buffer outlives the run — the client borrows it. var pc_buf: [zit.sync.max_projectcode_len]u8 = undefined; const project_code = repoProjectCode(gpa, io, root_dir, &pc_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), }; // sync §6 — resolve the login credential (null for anonymous). A sync // uploads, so it may authenticate exactly as push does; the plaintext lives // only in `pw_buf`/argv/env and is scrubbed on every exit from this scope. var pw_buf: [max_password_len]u8 = undefined; defer std.crypto.secureZero(u8, &pw_buf); const resolved = resolvePushCredential(gpa, io, root_dir, streams, project_code, .{ .url = url, .user = parsed.user, .password = parsed.password }, amb, &pw_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.NoPassword => { streams.err.writeAll("zit sync: no password supplied for --user\n") catch {}; return .failure; }, }; const outcome = syncRun(gpa, &tr, &ms, project_code, if (resolved) |r| r.credential else null) catch |e| return syncRunFail(streams, "sync", e); const written = persistStore(gpa, io, root_dir, &ms) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), error.Storage => return repoReadFail(streams, "sync"), error.Corrupt => return renderStorage(streams, root.abs, "the peer sent a malformed artifact name"), }; std.debug.assert(written >= held_before); const received = written - held_before; // DECISION-3 — persist a freshly derived secret only after the peer accepted // the authenticated session, and only when it was freshly derived (mirrors push). if (resolved) |r| { if (r.persist) { storePushCredential(gpa, io, root_dir, r.credential) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "sync"), else => return repoReadFail(streams, "sync"), }; } } if (outcome.sent == 0 and received == 0) { streams.out.writeAll("already in sync\n") catch {}; } else { streams.out.print("sent {d}, received {d} in {d} rounds\n", .{ outcome.sent, received, outcome.rounds }) catch {}; } return .ok; } /// Drives a bidirectional sync client (§3.2 round structure — both directions on /// ONE §3.7 session) to convergence over `tr`; returns the upload count and round /// count. /// The received count is the caller's persisted-count growth. Sans-IO on this /// side of `tr.exchange`; its closed error set is switched exhaustively by /// `runSync`. The sibling of `pullRun`/`pushRun`, differing only in the /// operation tag (`.sync`, which drives igot + gimme + upload on one session). fn syncRun( gpa: std.mem.Allocator, tr: *transport.Transport, ms: *repostate.MemStore, project_code: []const u8, credential: ?zit.sync.Credential, ) sync_driver.RunError(*transport.Transport)!SyncOutcome { var client = zit.sync.Client{ .state = zit.sync.initialState(.sync), .operation = .sync, .self_servercode = sync_servercode, .project_code = project_code, .version = .{ .version = sync_client_version }, // sync §6 — a named credential emits the §6.1 login card; null is anonymous. .credential = credential, }; defer client.deinit(gpa); const stats = try sync_driver.runSync(gpa, &client, ms.store(), tr, max_sync_rounds); return .{ .rounds = stats.rounds, .sent = client.content_sent }; } // --------------------------------------------------------------------------- // `zit remote [URL | --unset]` — read, set, or clear the single default peer URL // (sync §4/§6). Bare prints the saved remote; a URL saves it; --unset clears it. // A local, non-synced config value (`config.remote_url`) — anonymous only, no // credential in the URL — that lets pull/push/sync omit their URL argument. Not // a git-style named-remote registry: one value, matching the single-value config. // --------------------------------------------------------------------------- pub fn runRemote( gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, verb_args: []const []const u8, streams: cli.Streams, ) cli.Exit { const parsed = switch (cli.parseRemoteArgs(verb_args)) { .ok => |p| p, .unknown_option => |tok| return cli.usageError(streams, "remote", .unknown_option, tok), .extra_argument => |tok| return cli.usageError(streams, "remote", .extra_argument, tok), }; if (parsed.help) { streams.out.writeAll(remote_help) catch {}; return .ok; } const root = (cli.discoverRoot(arena, io) catch return stateOom(streams, "remote")) orelse { streams.err.writeAll(not_a_repo_msg) catch {}; return .failure; }; var root_dir = root.dir; defer root_dir.close(io); if (parsed.unset) { // Overlay remote_url with empty — clears the saved value, every other // config field preserved by the field-preserving RMW. updateConfig(gpa, io, root_dir, .{ .remote_url = "" }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), else => return repoReadFail(streams, "remote"), }; streams.out.writeAll("remote cleared\n") catch {}; return .ok; } if (parsed.url.len > 0) { // A URL over the config-field cap cannot be stored (the putConfig assert); // reject it as usage rather than trip the assertion. if (parsed.url.len > logstore.max_config_str) { streams.err.print("zit remote: URL too long (max {d} bytes)\n", .{logstore.max_config_str}) catch {}; return .failure; } updateConfig(gpa, io, root_dir, .{ .remote_url = parsed.url }) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), else => return repoReadFail(streams, "remote"), }; streams.out.print("remote set to {s}\n", .{parsed.url}) catch {}; return .ok; } // Bare `zit remote` — print the saved remote (reusing resolveRemoteUrl's read // with an empty arg; NoRemote means none is saved). var url_buf: [logstore.max_config_str]u8 = undefined; const url = resolveRemoteUrl(gpa, io, root_dir, "", &url_buf) catch |e| switch (e) { error.OutOfMemory => return stateOom(streams, "remote"), error.Storage => return repoReadFail(streams, "remote"), error.NoRemote => { streams.out.writeAll("no remote set\n") catch {}; return .ok; }, }; streams.out.print("{s}\n", .{url}) catch {}; return .ok; } test "serve §3.3: a pushed file card is persisted durably into the served repository's log" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A fresh, empty on-disk repository (header only) — the "server" a peer pushes // new content into. Built through the real LogStore, not seeded from fossil. const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); { var f = try tmp.dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.commit(); } // A pushed full-content file card for a 64-hex (SHA3-256 width) artifact the // server lacks (sync §7.1: `file HASH SIZE\n` + payload; §4.1 zlib container). // The name is the artifact's real SHA3-256 (computed by our own hasher, not // fossil) so the server's §7.7 commit-hook recompute accepts it. const push_body = "pushed artifact bytes\n"; // 22 bytes var name_buf: [64]u8 = undefined; const push_hash = zit.artifact.artifactName(.sha3_256, push_body, &name_buf); const request_plain = try std.fmt.allocPrint(gpa, "pragma client-version 20000\nfile {s} {d}\n{s}", .{ push_hash, push_body.len, push_body }); defer gpa.free(request_plain); const request = try zit.container.encode(gpa, request_plain, .default); defer gpa.free(request); var digest_buf: [digest_len]u8 = undefined; const push_digest = decodeWireName(push_hash, &digest_buf).?; const server = zit.sync.Server{ .version = serve_mod.server_version }; // Answer the push twice through the real `serve.Responder` seam over a SINGLE // open handle (§3.8): the second answer re-reads the current on-disk log from // the same handle and sees the first push's frame, so it is an idempotent // re-push (content is addressed by hash — no dup frame). A stale startup // snapshot would instead re-append the artifact; this handle is never reopened. { var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); var ctx = ServeCtx{ .io = io, .root_dir = tmp.dir, .file = file, .server = &server }; for (0..2) |_| { const reply = try serveAnswer(&ctx, gpa, request); gpa.free(reply.body); } } // THE PROOF: re-open the log from disk in a fresh handle; the pushed artifact // is present with its exact content — durably stored, not discarded — and // appears exactly once (the idempotent re-push added no second frame). var file = try tmp.dir.openFile(io, repo_filename, .{}); defer file.close(io); const st = try file.stat(io); var fs = storage.FileStorage.init(io, file, st.size); var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile()); defer log.deinit(gpa); const rid = log.lookup(push_digest) orelse return error.PushedArtifactMissing; const stored = (try log.content(gpa, rid)).?; defer gpa.free(stored); try std.testing.expectEqualStrings(push_body, stored); const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var artifacts: usize = 0; for (rids) |r| { if (log.kindOf(r) != .config) artifacts += 1; } try std.testing.expectEqual(@as(usize, 1), artifacts); } // --- D4: client verbatim retention + the §7.7 persist hook ------------------ /// Create a fresh, empty (header-only, committed) `.zit` in `dir` so a /// `persistStore` test can persist into it, exactly like a just-cloned repo. fn d4SeedEmptyRepo(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var f = try dir.createFile(io, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(io); var fs = storage.FileStorage.init(io, f, 0); var seed = try logstore.LogStore.open(gpa, fs.storageFile(), hash_kind); defer seed.deinit(gpa); try seed.commit(); } /// Reopen the persisted `.zit` in `dir` for reading. `fs_out` is a caller-owned /// `FileStorage` that MUST outlive the returned `LogStore` (its `storageFile()` /// vtable borrows `fs_out`); the caller closes `fs_out.file` and deinits `log`. fn d4OpenLog(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, fs_out: *storage.FileStorage) !logstore.LogStore { const file = try dir.openFile(io, repo_filename, .{}); const st = try file.stat(io); fs_out.* = storage.FileStorage.init(io, file, st.size); return try logstore.LogStore.open(gpa, fs_out.storageFile(), 0); } test "sync §7.7: persistStore recomputes the artifact hash and aborts on a lying container, writing no frame" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A received full cfile whose wire NAME does not name its content (§7.7 — // a clone skips the transfer-layer pre-storage check, so a lying container // reaches the persist commit hook). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const content = "content that the wire name does not actually name\n"; const lying_name = "0" ** 64; // not SHA3-256(content) const rid = try s.put(lying_name, content); const container = try zit.container.encode(gpa, content, .default); defer gpa.free(container); try s.retain(rid, .{ .full = container }); // §7.7 L1336-1338 — recompute and abort before ANY frame is written. try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, &ms)); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var digest_buf: [digest_len]u8 = undefined; const digest = decodeWireName(lying_name, &digest_buf).?; try std.testing.expect(log.lookup(digest) == null); // no frame landed } test "sync §7.2/§4.1: persistStore stores received full and delta wire forms verbatim; a re-serve preserves the delta container byte-for-byte" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Model a clone reply in §8.2 RID order: a full base cfile then a delta // cfile against it. Names are the real SHA3-256 so the §7.7 hook accepts. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const base = "the base artifact with a long shared run so the delta stays a compact copy payload distinct in size from the reconstructed target"; const target = base ++ " plus a small trailing mutation"; var bn: [64]u8 = undefined; var tn: [64]u8 = undefined; const base_name = zit.artifact.artifactName(.sha3_256, base, &bn); const target_name = zit.artifact.artifactName(.sha3_256, target, &tn); const base_container = try zit.container.encode(gpa, base, .default); defer gpa.free(base_container); const base_rid = try s.put(base_name, base); try s.retain(base_rid, .{ .full = base_container }); const raw = try zit.delta.create(gpa, base, target); defer gpa.free(raw); const delta_container = try zit.container.encode(gpa, raw, .default); defer gpa.free(delta_container); const ts = try zit.delta.targetSize(raw); const delta_rid = try s.put(target_name, target); try s.retain(delta_rid, .{ .delta = .{ .src = base_name, .container = delta_container, .target_size = ts } }); const written = try persistStore(gpa, io, tmp.dir, &ms); try std.testing.expectEqual(@as(usize, 2), written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The target persisted as a `.delta` frame that reconstructs to the original. var tdig: [digest_len]u8 = undefined; const target_digest = decodeWireName(target_name, &tdig).?; const trid = log.lookup(target_digest).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(trid)); const got = (try log.content(gpa, trid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(target, got); // Clone-of-clone: a re-serve snapshot emits the SAME delta container bytes // (§7.2 retransmitted verbatim) — the compactness survives the second hop. var serve_ms = repostate.MemStore.init(gpa); defer serve_ms.deinit(); try loadContentStore(gpa, &log, &serve_ms); const serve_rid = serve_ms.store().lookup(target_name).?; const wf = serve_ms.store().wire(serve_rid).?; try std.testing.expect(wf.delta != null); try std.testing.expectEqualSlices(u8, delta_container, wf.container); try std.testing.expectEqual(@as(u64, target.len), wf.delta.?.target_size); } test "sync §3.3: persistStore falls back to full form when a retained delta's basis is absent, never a phantom-based delta" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Only the delta is present; its basis was never received/stored (a legal // §3.3 outcome that must not persist a delta over a phantom base). var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const base = "an absent basis whose long shared run would make a compact delta if it were here at all"; const target = base ++ " mutated once"; var bn: [64]u8 = undefined; var tn: [64]u8 = undefined; const base_name = zit.artifact.artifactName(.sha3_256, base, &bn); const target_name = zit.artifact.artifactName(.sha3_256, target, &tn); const raw = try zit.delta.create(gpa, base, target); defer gpa.free(raw); const dc = try zit.container.encode(gpa, raw, .default); defer gpa.free(dc); const ts = try zit.delta.targetSize(raw); const rid = try s.put(target_name, target); try s.retain(rid, .{ .delta = .{ .src = base_name, .container = dc, .target_size = ts } }); const written = try persistStore(gpa, io, tmp.dir, &ms); try std.testing.expectEqual(@as(usize, 1), written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The target persisted as a FULL (.zlib) frame — the fallback, not a delta. var tdig: [digest_len]u8 = undefined; const target_digest = decodeWireName(target_name, &tdig).?; const trid = log.lookup(target_digest).?; try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(trid)); const got = (try log.content(gpa, trid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(target, got); // No phantom basis frame was minted for the absent DELTASRC. var bdig: [digest_len]u8 = undefined; const base_digest = decodeWireName(base_name, &bdig).?; try std.testing.expect(log.lookup(base_digest) == null); } test "sync §7.7/§3.3: persistStore stores a retained delta chain up to the read bound and falls back to full form beyond it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const cap: usize = logstore.max_delta_chain; // the READ bound (32) const n: usize = cap + 2; // links 0..cap+1: link cap+1's basis sits at depth cap // A chain grown one byte per link so each delta is tiny. Link i's basis is // link i-1 (§8.2 RID order); the persist path stores a `.delta` iff the base // depth is under `max_delta_chain`, so link cap+1 falls back to full form. var content: std.ArrayList(u8) = .empty; defer content.deinit(gpa); try content.appendSlice(gpa, "chain-root-artifact-with-a-long-shared-run-for-compact-forward-deltas"); var prev_name: [64]u8 = undefined; var prev_name_len: usize = 0; var deep_name: [64]u8 = undefined; // link cap (deepest stored delta) var deep_len: usize = 0; var last_name: [64]u8 = undefined; // link cap+1 (the fallback) var last_len: usize = 0; var i: usize = 0; while (i < n) : (i += 1) { try content.append(gpa, 'a' + @as(u8, @intCast(i % 26))); const cur = content.items; const prev = cur[0 .. cur.len - 1]; var nbuf: [64]u8 = undefined; const name = zit.artifact.artifactName(.sha3_256, cur, &nbuf); const rid = try s.put(name, cur); if (i == 0) { const c = try zit.container.encode(gpa, cur, .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const raw = try zit.delta.create(gpa, prev, cur); defer gpa.free(raw); const c = try zit.container.encode(gpa, raw, .default); defer gpa.free(c); const tsz = try zit.delta.targetSize(raw); try s.retain(rid, .{ .delta = .{ .src = prev_name[0..prev_name_len], .container = c, .target_size = tsz } }); } @memcpy(prev_name[0..name.len], name); prev_name_len = name.len; if (i == cap) { @memcpy(deep_name[0..name.len], name); deep_len = name.len; } if (i == n - 1) { @memcpy(last_name[0..name.len], name); last_len = name.len; } } _ = try persistStore(gpa, io, tmp.dir, &ms); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // Link cap is the deepest `.delta` the write bound allows; link cap+1 fell // back to a full `.zlib` frame yet still reconstructs (never a Corrupt read). var ddig: [digest_len]u8 = undefined; const deep_rid = log.lookup(decodeWireName(deep_name[0..deep_len], &ddig).?).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(deep_rid)); var ldig: [digest_len]u8 = undefined; const last_rid = log.lookup(decodeWireName(last_name[0..last_len], &ldig).?).?; try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(last_rid)); const got = (try log.content(gpa, last_rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(content.items, got); } test "sync §7.2/§7.4/§3.3: a dangling reverse-delta cfile persists as a verbatim .delta frame that re-serves byte-identically" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // Fossil's clone stream: a reverse delta (the OLDER revision stored as a // delta of the NEWER one) arrives BEFORE its basis, so it is dangling. The // client retains the §4.1 container through the pending queue and, when the // basis lands, persists it as a compact `.delta` — not a full re-compression. const newer = "artifact NEWER: a long shared body kept verbatim so the reverse delta reconstructing the older revision stays a compact copy op"; const older = "artifact NEWER: a long shared body kept verbatim so the reverse delta reconstructing the older revision stays a compact copy op minus tail"; var nn: [64]u8 = undefined; var on: [64]u8 = undefined; const newer_name = zit.artifact.artifactName(.sha3_256, newer, &nn); const older_name = zit.artifact.artifactName(.sha3_256, older, &on); const newer_container = try zit.container.encode(gpa, newer, .default); defer gpa.free(newer_container); const raw = try zit.delta.create(gpa, newer, older); // OLDER = delta ∘ NEWER defer gpa.free(raw); const delta_container = try zit.container.encode(gpa, raw, .default); defer gpa.free(delta_container); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var client = zit.sync.Client{ .operation = .clone, .state = .clone_round0 }; defer client.deinit(gpa); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, "clone_seqno 0\npush 1111111111111111 aa87ea41b0bb05ba528f7a3bdba700d7e23d6318\n"); // Reverse (dangling) order: the delta cfile precedes its basis cfile. const dh = try std.fmt.allocPrint(gpa, "cfile {s} {s} {d} {d}\n", .{ older_name, newer_name, older.len, delta_container.len }); defer gpa.free(dh); try body.appendSlice(gpa, dh); try body.appendSlice(gpa, delta_container); try body.append(gpa, '\n'); const bh = try std.fmt.allocPrint(gpa, "cfile {s} {d} {d}\n", .{ newer_name, newer.len, newer_container.len }); defer gpa.free(bh); try body.appendSlice(gpa, bh); try body.appendSlice(gpa, newer_container); try body.append(gpa, '\n'); _ = try client.consumeReply(gpa, ms.store(), body.items); const written = try persistStore(gpa, io, tmp.dir, &ms); try std.testing.expectEqual(@as(usize, 2), written); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // §3.3 — the older revision persisted as a `.delta`, not a full re-compression, // and reconstructs the original bytes (delta ∘ basis). var od: [digest_len]u8 = undefined; const older_rid = log.lookup(decodeWireName(older_name, &od).?).?; try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(older_rid)); const got = (try log.content(gpa, older_rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(older, got); // §7.2 — a clone-of-clone re-serve re-emits the SAME delta container byte-for-byte. var serve_ms = repostate.MemStore.init(gpa); defer serve_ms.deinit(); try loadContentStore(gpa, &log, &serve_ms); const srid = serve_ms.store().lookup(older_name).?; const wf = serve_ms.store().wire(srid).?; try std.testing.expect(wf.delta != null); try std.testing.expectEqualSlices(u8, delta_container, wf.container); try std.testing.expectEqualStrings(newer_name, wf.delta.?.src_hash); } test "sync §7.2/§3.3: persistStore stores a whole reverse-delta chain, ordering every basis frame before the delta that points at it" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A reverse-delta chain in fossil's storage shape: revision i is stored as a // delta of the NEWER revision i+1, the newest stored full. The store rids run // oldest..newest (put order), so EVERY delta's basis sits at a HIGHER rid than // the delta — the worst case for physical ordering. persist's worklist must // still write each basis frame before its delta so `delta_of` points backward; // naive rid-order would degrade all but the oldest link to full form. const links = 6; var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| gpa.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(gpa, "chain body: one long shared prefix kept across every revision for compact reverse deltas -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } // put oldest..newest (store rids 1..links); older i is a reverse delta of newer i+1. for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == links - 1) { const c = try zit.container.encode(gpa, bodies[i], .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(gpa, bodies[i + 1], bodies[i]); // older from newer defer gpa.free(dr); const c = try zit.container.encode(gpa, dr, .default); defer gpa.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } }); } } _ = try persistStore(gpa, io, tmp.dir, &ms); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); // The newest link is full; every older link is a `.delta` whose basis frame is // physically earlier (a LOWER log rid = a lower offset), and all reconstruct. for (0..links) |i| { var d: [digest_len]u8 = undefined; const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?; const rgot = (try log.content(gpa, rid)).?; defer gpa.free(rgot); try std.testing.expectEqualStrings(bodies[i], rgot); if (i == links - 1) { try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(rid)); } else { try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(rid)); var bd: [digest_len]u8 = undefined; const brid = log.lookup(decodeWireName(names[i + 1][0..nlen[i + 1]], &bd).?).?; try std.testing.expect(brid < rid); // basis frame precedes the reverse-delta } } } test "sync §7.7: persistStore verifies a retained delta's reconstruction and refuses a lying delta container, writing no frame" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp.dir); // A retained delta whose stored plaintext does NOT hash to its wire name (a // clone skips the transfer-layer pre-storage check, §7.7 r6, so a lying // delta reaches the persist commit hook). The §7.7 recompute runs BEFORE the // wire-form branch, so a delta-form entry is rejected exactly like a full one. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); const content = "content that the wire name does not actually name, delivered as a delta\n"; const lying_name = "0" ** 64; // not SHA3-256(content) const some_basis = "1" ** 64; // an absent basis; verify fails before it is even consulted const raw = try zit.delta.create(gpa, "a plausible basis blob", content); defer gpa.free(raw); const dc = try zit.container.encode(gpa, raw, .default); defer gpa.free(dc); const rid = try s.put(lying_name, content); try s.retain(rid, .{ .delta = .{ .src = some_basis, .container = dc, .target_size = content.len } }); try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, &ms)); var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); var digest_buf: [digest_len]u8 = undefined; try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null); // no frame landed } fn persistReverseChainNoLeak(allocator: std.mem.Allocator) !void { const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); try d4SeedEmptyRepo(allocator, io, tmp.dir); // A small reverse-delta chain built on the (failing) allocator so injected // OOM is exercised in both the store build and the persist worklist. var ms = repostate.MemStore.init(allocator); defer ms.deinit(); const s = ms.store(); const links = 4; var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| allocator.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(allocator, "reverse chain oom: one shared body across revisions -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == links - 1) { const c = try zit.container.encode(allocator, bodies[i], .default); defer allocator.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(allocator, bodies[i + 1], bodies[i]); defer allocator.free(dr); const c = try zit.container.encode(allocator, dr, .default); defer allocator.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } }); } } _ = try persistStore(allocator, io, tmp.dir, &ms); } test "sync §7.2/§3.3: persistStore drains the reverse-delta worklist without leaking under allocation failure" { try std.testing.checkAllAllocationFailures(std.testing.allocator, persistReverseChainNoLeak, .{}); } /// Drives `persistArtifact` over a FORWARD delta chain (each link a delta of the /// immediately older, already-persisted lower-rid link) in rid order, so a test /// owns the shared basis memo and can read its hit counter. Every basis precedes /// its delta, so `allow_defer=false` never defers and no worklist is needed — /// mirrors `persistStore`'s open/commit around a plain rid-order drive. fn persistForwardChain( gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, ms: *repostate.MemStore, cache: *logstore.PlaintextCache, ) !void { var file = try dir.openFile(io, repo_filename, .{ .mode = .read_write }); defer file.close(io); const st = try file.stat(io); var fs = storage.FileStorage.init(io, file, st.size); var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0); defer log.deinit(gpa); const store = ms.store(); const n = store.count(); var rid: repostate.Rid = 1; while (rid <= n) : (rid += 1) { switch (try persistArtifact(gpa, ms, &log, rid, false, cache)) { .persisted => {}, .deferred => unreachable, // forward chain: no basis is ever absent } } try log.commit(); } test "sync §7.2/§7.4: the basis memo reuses each basis (hits) and reconstructs a byte-identical store to the uncached path" { const gpa = std.testing.allocator; const io = std.testing.io; const links = 8; // A forward delta chain: link 0 is full, every later link is a delta that // reconstructs its body FROM the older body (its basis a lower, already- // persisted rid) — the case where the memo turns each basis reconstruction // into a lookup for the delta built on it. Store owns its copies (retain // dupes), but the bodies/names back the reconstructed-content assertions. var ms = repostate.MemStore.init(gpa); defer ms.deinit(); const s = ms.store(); var bodies: [links][]u8 = undefined; var made: usize = 0; defer for (bodies[0..made]) |b| gpa.free(b); var names: [links][64]u8 = undefined; var nlen: [links]usize = undefined; for (0..links) |i| { bodies[i] = try std.fmt.allocPrint(gpa, "forward chain shared prefix across every revision -- revision {d}", .{i}); made += 1; nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len; } for (0..links) |i| { const rid = try s.put(names[i][0..nlen[i]], bodies[i]); if (i == 0) { const c = try zit.container.encode(gpa, bodies[i], .default); defer gpa.free(c); try s.retain(rid, .{ .full = c }); } else { const dr = try zit.delta.create(gpa, bodies[i - 1], bodies[i]); // newer body FROM its older basis defer gpa.free(dr); const c = try zit.container.encode(gpa, dr, .default); defer gpa.free(c); try s.retain(rid, .{ .delta = .{ .src = names[i - 1][0..nlen[i - 1]], .container = c, .target_size = bodies[i].len } }); } } // (A) a normal memo; (B) a zero-budget memo that can hold nothing — the exact // uncached reconstruction. Same store, two freshly-seeded repos. var tmp_a = std.testing.tmpDir(.{}); defer tmp_a.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_a.dir); var cache_a = logstore.PlaintextCache.init(); defer cache_a.deinit(gpa); try persistForwardChain(gpa, io, tmp_a.dir, &ms, &cache_a); var tmp_b = std.testing.tmpDir(.{}); defer tmp_b.cleanup(); try d4SeedEmptyRepo(gpa, io, tmp_b.dir); var cache_b = logstore.PlaintextCache.init(); cache_b.max_bytes = 0; // never memoises: forces a fresh reconstruction each time defer cache_b.deinit(gpa); try persistForwardChain(gpa, io, tmp_b.dir, &ms, &cache_b); // The memo reused every basis exactly once (one hit per delta link); the // zero-budget run reused nothing — the two ways the same bytes get built. try std.testing.expectEqual(@as(u64, links - 1), cache_a.hits); try std.testing.expectEqual(@as(u64, 0), cache_b.hits); // Byte-identical store: the memo changes only the work, never the bytes. const bytes_a = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(bytes_a); const bytes_b = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(bytes_b); try std.testing.expectEqualSlices(u8, bytes_b, bytes_a); // Every link reconstructs to its exact body (§7.7 chain correctness). var fs: storage.FileStorage = undefined; var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs); defer fs.file.close(io); defer log.deinit(gpa); for (0..links) |i| { var d: [digest_len]u8 = undefined; const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?; const got = (try log.content(gpa, rid)).?; defer gpa.free(got); try std.testing.expectEqualStrings(bodies[i], got); } } test "sync §9.1: repoProjectCode returns the stored code, else the zero placeholder" { const gpa = std.testing.allocator; const io = std.testing.io; const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); const seed = struct { fn withCode(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, code: []const u8) !void { var f = try dir.createFile(i, repo_filename, .{ .read = true, .exclusive = true }); defer f.close(i); var fs = storage.FileStorage.init(i, f, 0); var log = try logstore.LogStore.open(g, fs.storageFile(), hk); defer log.deinit(g); try log.putConfig(.{ .hash_policy = hk, .default_branch = "trunk", .project_id = "", .project_code = code }); try log.commit(); } }; // (a) a repo whose config stores a real 40-hex project code: it is returned // verbatim — the code the pull/push card announces (§9.1), never the zeros // fossil would refuse. var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const code = "a1b2c3d4" ** 5; // 40 hex chars try seed.withCode(gpa, io, tmp.dir, hash_kind, code); var buf: [zit.sync.max_projectcode_len]u8 = undefined; const got = try repoProjectCode(gpa, io, tmp.dir, &buf); try std.testing.expectEqualStrings(code, got); try std.testing.expect(!std.mem.eql(u8, got, sync_projectcode)); // (b) a repo predating the stored code (empty project_code): the placeholder // is the honest fallback (no adopted identity to announce yet). var tmp2 = std.testing.tmpDir(.{}); defer tmp2.cleanup(); try seed.withCode(gpa, io, tmp2.dir, hash_kind, ""); var buf2: [zit.sync.max_projectcode_len]u8 = undefined; const got2 = try repoProjectCode(gpa, io, tmp2.dir, &buf2); try std.testing.expectEqualStrings(sync_projectcode, got2); } test "sync §9.1: the pull request card carries the adopted project code, not the placeholder" { const gpa = std.testing.allocator; // The pull client, constructed exactly as `pullRun` does, with a stored code. const code = "a1b2c3d4" ** 5; // a real (non-zero) 40-hex project code var client = zit.sync.Client{ .state = zit.sync.initialState(.pull), .operation = .pull, .self_servercode = sync_servercode, .project_code = code, .version = .{ .version = sync_client_version }, }; defer client.deinit(gpa); var ms = repostate.MemStore.init(gpa); defer ms.deinit(); var buf: std.ArrayListUnmanaged(u8) = .empty; defer buf.deinit(gpa); const request = try client.buildRequest(gpa, ms.store(), &buf); // §9.1 ABNF `pull-card = "pull" SP servercode SP projectcode`: the adopted // code MUST be the card's PROJECTCODE (a real peer refuses a mismatch). const expected = "pull " ++ sync_servercode ++ " " ++ ("a1b2c3d4" ** 5) ++ "\n"; try std.testing.expect(std.mem.indexOf(u8, request, expected) != null); // And the all-zero placeholder is NOT the project code the card announces. const rejected = "pull " ++ sync_servercode ++ " " ++ sync_projectcode ++ "\n"; try std.testing.expect(std.mem.indexOf(u8, request, rejected) == null); } test "serve §3.8: a per-request store-load on one serve handle sees a later append (no restart)" { const gpa = std.testing.allocator; const io = std.testing.io; const hash_kind: u8 = @intFromEnum(zit.artifact.defaultPolicy(false, false)); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const append = struct { fn artifactFrame(g: std.mem.Allocator, i: std.Io, dir: std.Io.Dir, hk: u8, digest: []const u8, bytes: []const u8) !void { var f = dir.openFile(i, repo_filename, .{ .mode = .read_write }) catch try dir.createFile(i, repo_filename, .{ .read = true }); defer f.close(i); const st = try f.stat(i); var fs = storage.FileStorage.init(i, f, st.size); var log = try logstore.LogStore.open(g, fs.storageFile(), hk); defer log.deinit(g); _ = try log.put(digest, bytes); try log.commit(); } }; // Seed the repo holding exactly one content artifact (a 32-byte raw digest). const digest_a = "a" ** digest_len; try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_a, "content a\n"); // Open ONE read handle — the handle `serve` holds for its whole lifetime. var handle = try tmp.dir.openFile(io, repo_filename, .{}); defer handle.close(io); // First per-request read: one content artifact. var ms1 = repostate.MemStore.init(gpa); defer ms1.deinit(); try readServedStore(gpa, io, handle, &ms1); const n1 = contentCount(ms1.store()); try std.testing.expectEqual(@as(usize, 1), n1); // A SEPARATE writer appends a second artifact while the handle stays open — // exactly a `zit commit` running alongside a live `zit serve`. const digest_b = "b" ** digest_len; try append.artifactFrame(gpa, io, tmp.dir, hash_kind, digest_b, "content b\n"); // Second per-request read on the SAME handle sees the new artifact: proof the // server re-reads the current on-disk log per request (§3.8). A startup // snapshot would still report n1 — no restart is needed here. var ms2 = repostate.MemStore.init(gpa); defer ms2.deinit(); try readServedStore(gpa, io, handle, &ms2); try std.testing.expectEqual(n1 + 1, contentCount(ms2.store())); } /// Build a full-form 3-check-in chain over an evolving file into `log`, plus a /// live config frame; returns the tip check-in's 64-hex name. Every blob and /// manifest lands as a FULL frame — pack's re-delta target. fn packSeedFullStore(gpa: std.mem.Allocator, log: *logstore.LogStore, tip_out: *[64]u8) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); var root_name: [64]u8 = undefined; var mid_name: [64]u8 = undefined; const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const b1 = try packSeedBlob(log, v1); try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "f.txt", &b1, &root_name); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); const b2 = try packSeedBlob(log, v2); try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "f.txt", &b2, &mid_name); const v3 = try packEvolvingBlob(gpa, 3); defer gpa.free(v3); const b3 = try packSeedBlob(log, v3); try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &mid_name, "f.txt", &b3, tip_out); try log.commit(); } /// Store `content` under its sha3-256 name, delta-encoded against `base` when /// one is given (the evolving blobs delta to a fraction, so the frame lands as a /// `.delta`), and return its rid. Models a fossil-clone store that RETAINED /// compact wire deltas rather than full frames. fn packSeedDeltaBlob(log: *logstore.LogStore, content: []const u8, base: ?logstore.Rid) !logstore.Rid { var digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(content, &digest, .{}); return log.putArtifact(&digest, content, base); } /// Build a clone-shaped store whose successive blobs are RETAINED as `.delta` /// frames (the compact wire form), under DISTINCT filenames so pack's /// same-path-blob lineage hint finds NO base for them — exactly the case where /// the old pack reconstructed every delta to a full frame and inflated the /// store. `.zit` therefore holds real delta frames whose bases pack cannot /// re-derive, so a correct pack must preserve them verbatim. Returns the tip /// check-in's 64-hex name. fn packSeedDeltaStore(gpa: std.mem.Allocator, log: *logstore.LogStore, tip_out: *[64]u8) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); var root_name: [64]u8 = undefined; var mid_name: [64]u8 = undefined; const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const r1 = try packSeedDeltaBlob(log, v1, null); // first version: full frame (base of the chain) var b1: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v1, &b1); try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "f1.txt", &b1, &root_name); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); const r2 = try packSeedDeltaBlob(log, v2, r1); // delta of v1 var b2: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v2, &b2); try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "f2.txt", &b2, &mid_name); const v3 = try packEvolvingBlob(gpa, 3); defer gpa.free(v3); _ = try packSeedDeltaBlob(log, v3, r2); // delta of v2 var b3: [64]u8 = undefined; _ = zit.artifact.artifactName(.sha3_256, v3, &b3); try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &mid_name, "f3.txt", &b3, tip_out); try log.commit(); } /// Seed a store holding a wire delta that pack CANNOT beat by re-deltification, /// so ONLY the §6.1 monotonic-shrink guard keeps the store from regressing. `B` /// is a one-line edit of `A`, stored as a COMPACT zit delta against `A` — the /// smallest form pack could ever reproduce. But `max_base_candidates` unrelated /// filler blobs sit between `A` and `B`, so pack's bounded backward candidate /// window (§5.2) saturates on the fillers and never reaches `A`; with no lineage /// hint either (bare blobs, no F-card path), pack's best re-encoding of `B` is a /// full zlib frame STRICTLY LARGER than the stored A-based delta. The pre-guard /// pack replaced the delta with that larger full frame (the +33% clone /// regression in miniature); the guard keeps the existing smaller frame. Fills /// A's and B's 64-hex names. fn packSeedUnbeatableWireDelta( gpa: std.mem.Allocator, log: *logstore.LogStore, a_name: *[64]u8, b_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // A: B's ideal base, stored full (zlib). const a = try packEvolvingBlob(gpa, 1); defer gpa.free(a); var a_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(a, &a_digest, .{}); const a_rid = try log.putZlib(&a_digest, a); _ = zit.artifact.artifactName(.sha3_256, a, a_name); // `max_base_candidates` unrelated fillers between A and B: they saturate B's // backward candidate window so A is out of reach when pack re-encodes B. var f: usize = 0; while (f < logstore.max_base_candidates) : (f += 1) { var fbuf: std.ArrayList(u8) = .empty; defer fbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try fbuf.print(gpa, "filler {d} unrelated row {d}\n", .{ f, i }); var fdig: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(fbuf.items, &fdig, .{}); _ = try log.putZlib(&fdig, fbuf.items); } // B: a one-line edit of A, stored as a COMPACT zit delta against A (the // smallest form pack could reproduce — and only against A, which pack's // window cannot reach here). const b = try packEvolvingBlob(gpa, 2); defer gpa.free(b); var b_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{}); _ = zit.artifact.artifactName(.sha3_256, b, b_name); const raw = try zit.delta.create(gpa, a, b); defer gpa.free(raw); const container = try zit.container.encode(gpa, raw, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&b_digest, a_rid, container); try log.commit(); } /// Assert the §6.1 per-frame monotonicity: NO artifact's packed frame is larger /// on disk than its source frame. Pack keeps the smaller of {existing frame, new /// encoding} for every artifact, so a repack is non-worsening frame by frame. /// This holds universally over THESE fixtures because their chains stay below the /// read cap; the one path that can enlarge a frame — `copySrcFrame`'s §3.3 /// deep-chain fallback re-storing a kept delta as full zlib — needs a base at /// `max_delta_chain` depth, which the seeds never build. fn packExpectNoFrameLarger(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void { const src_rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); for (src_rids) |sr| { const dr = dst.lookup(src.hashOf(sr)) orelse return error.PackedArtifactMissing; try std.testing.expect(dst.frameSpanOf(dr) <= src.frameSpanOf(sr)); } } /// Count the `.delta` frames among a store's persisted artifacts. fn packDeltaFrameCount(gpa: std.mem.Allocator, log: *logstore.LogStore) !usize { const rids = try log.ridsInPersistedOrder(gpa); defer gpa.free(rids); var n: usize = 0; for (rids) |r| { if (log.kindOf(r) == .delta) n += 1; } return n; } /// Assert that every artifact in `src` reconstructs byte-identically in `dst` /// and the two artifact SETS are equal (no drop, no addition). fn packExpectContentEqual(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void { const src_rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); const dst_rids = try dst.ridsInPersistedOrder(gpa); defer gpa.free(dst_rids); try std.testing.expectEqual(src_rids.len, dst_rids.len); for (src_rids) |sr| { const digest = src.hashOf(sr); const dr = dst.lookup(digest) orelse return error.PackedArtifactMissing; const a = (try src.content(gpa, sr)).?; defer gpa.free(a); const b = (try dst.content(gpa, dr)).?; defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); } } test "pack: the packed store's artifact set + every reconstructed content byte-equal the original (config preserved)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst); // No drop: the packed content-frame count equals the source's, and every // artifact reconstructs byte-for-byte. try std.testing.expectEqual(src.entries.items.len, stats.artifacts); try packExpectContentEqual(gpa, &src, &dst); // The live config survives as exactly one frame with identical fields. var soc = (try src.readConfig(gpa)).?; defer soc.deinit(gpa); var doc = (try dst.readConfig(gpa)).?; defer doc.deinit(gpa); try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy); try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch); try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id); try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code); } test "pack re-deltas full-form frames (delta count rises, .zit shrinks) with byte-identical content" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // The source is all full-form: not one delta frame. { const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); for (rids) |r| try std.testing.expect(src.kindOf(r) != .delta); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst); // The two child blobs (v2 against v1, v3 against v2) re-delta against the // same path's parent blob, so the delta count rises and the store shrinks — // with content preserved exactly. try std.testing.expect(stats.deltas >= 2); try std.testing.expect(stats.bytes_after < stats.bytes_before); try packExpectContentEqual(gpa, &src, &dst); } test "pack preserves wire deltas — a store with delta frames packs to <= source size (regression for the 8.65x inflation bug)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedDeltaStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // The source genuinely holds retained wire deltas (the compact form pack // must not discard) whose bases pack's lineage hint cannot re-derive. const src_deltas = try packDeltaFrameCount(gpa, &src); try std.testing.expect(src_deltas >= 2); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst); // MONOTONIC: the packed store is never larger than the source. The old pack // reconstructed each delta to a full zlib frame and inflated the store; a // correct pack preserves the compact deltas verbatim. try std.testing.expect(stats.bytes_after <= stats.bytes_before); // Every source delta survives AS a delta frame — none reconstructed to full // (the optional re-delta may add further deltas, so the total only grows). { const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); for (rids) |sr| { if (src.kindOf(sr) != .delta) continue; const dr = dst.lookup(src.hashOf(sr)).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr)); } } try std.testing.expect(try packDeltaFrameCount(gpa, &dst) >= src_deltas); // No drop, every reconstruction byte-exact. try packExpectContentEqual(gpa, &src, &dst); } test "pack copies a delta frame verbatim re-linked to its base's new rid, content byte-identical" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedDeltaStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst); // For every delta frame in the source, the packed store holds it as a delta // whose stored container is byte-identical (verbatim re-serve) and whose base // names the SAME artifact (re-linked to the base's new rid). Content matches. const rids = try src.ridsInPersistedOrder(gpa); defer gpa.free(rids); var checked: usize = 0; for (rids) |sr| { if (src.kindOf(sr) != .delta) continue; const digest = src.hashOf(sr); const dr = dst.lookup(digest).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr)); var sw = (try src.wire(gpa, sr)).?; defer sw.deinit(gpa); var dw = (try dst.wire(gpa, dr)).?; defer dw.deinit(gpa); // Verbatim container bytes and the same base hash (re-link preserves the // logical base while its rid/offset differs in the packed generation). try std.testing.expectEqualSlices(u8, sw.delta.container, dw.delta.container); try std.testing.expectEqualSlices(u8, sw.delta.baseHash(), dw.delta.baseHash()); const a = (try src.content(gpa, sr)).?; defer gpa.free(a); const b = (try dst.content(gpa, dr)).?; defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); checked += 1; } try std.testing.expect(checked >= 2); } /// The rid naming the 64-hex `name` in `log`, or null when absent (test-only /// name→digest→rid convenience over `lookup`). fn packRidByName(log: *logstore.LogStore, name: *const [64]u8) ?logstore.Rid { const d = hexToDigest(name) orelse return null; return log.lookup(&d); } /// Seed a store modeling a peer-supplied (verbatim wire) delta whose sender /// chose a POOR base. `A` (a large file) and `C` (an unrelated large blob) are /// stored full; `B` (a one-line edit of `A`) is stored as a hand-crafted §4.1 /// delta container against `C` — a large, suboptimal delta the sender picked. /// `A` precedes `B` chronologically and is B's ideal base, so pack's best-of-N /// re-deltification (§6.1) must re-base `B` onto `A` rather than trust the /// sender's `C`. Fills the 64-hex names of A/B/C. fn packSeedSuboptimalWireDelta( gpa: std.mem.Allocator, log: *logstore.LogStore, a_name: *[64]u8, b_name: *[64]u8, c_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // A: the ideal base for B (near-identical predecessor), stored full. const a = try packEvolvingBlob(gpa, 1); defer gpa.free(a); a_name.* = try packSeedBlob(log, a); // C: an unrelated large blob (a poor base for B), stored full. var cbuf: std.ArrayList(u8) = .empty; defer cbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try cbuf.print(gpa, "unrelated content row {d}\n", .{i}); const c = cbuf.items; c_name.* = try packSeedBlob(log, c); const c_rid = packRidByName(log, c_name).?; // B: a one-line edit of A. delta(A,B) is tiny; delta(C,B) is large. const b = try packEvolvingBlob(gpa, 2); defer gpa.free(b); _ = zit.artifact.artifactName(.sha3_256, b, b_name); var b_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{}); // Hand-craft the verbatim wire delta against the POOR base C — models a peer // that chose C as base, retained at rest by `putDeltaContainer`. const raw = try zit.delta.create(gpa, c, b); defer gpa.free(raw); const container = try zit.container.encode(gpa, raw, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&b_digest, c_rid, container); try log.commit(); } test "pack §6.1: a verbatim wire delta is re-deltified against the size-minimizing base, not the sender's" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var a_name: [64]u8 = undefined; var b_name: [64]u8 = undefined; var c_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSuboptimalWireDelta(gpa, &wlog, &a_name, &b_name, &c_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source really holds B as a delta whose base is C (the // sender's suboptimal choice) — the exact frame the old pack copied verbatim. { const sb = packRidByName(&src, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb)); var sw = (try src.wire(gpa, sb)).?; defer sw.deinit(gpa); var cbin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&cbin, &c_name); try std.testing.expectEqualSlices(u8, &cbin, sw.delta.baseHash()); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst); // After pack, B is still a delta, but its base is now A (the size-minimizing // candidate) — the sender's base C is no longer trusted at rest (§6.1/§3). const db = packRidByName(&dst, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db)); var dw = (try dst.wire(gpa, db)).?; defer dw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash()); // Every artifact still reconstructs byte-for-byte and the store shrank (a // tiny A-based delta replaced the large C-based one). try packExpectContentEqual(gpa, &src, &dst); try std.testing.expect(stats.bytes_after < stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); try std.testing.expectEqual(src.size(), stats.bytes_before); } /// Seed the §6.2 same-path scenario: two versions of ONE tracked file whose ideal /// base (the earlier version) sits OUTSIDE pack's bounded chronological window. /// `v1` of "tracked.txt" is stored full, then `max_base_candidates` unrelated /// filler blobs saturate the backward candidate window, then `v2` (a one-line /// edit of `v1`) is stored full. With ONLY the chronological window (step §6.1) /// pack cannot reach `v1` from `v2` — the fillers fill every slot — so `v2` stays /// a full frame and the store does not shrink. The mlink-analog same-path walk /// (§6.2) reads both check-in manifests, learns tracked.txt: [v1, v2], and offers /// `v1` as `v2`'s priority candidate, so pack re-deltifies `v2` onto `v1`. Both /// check-ins are FULL frames (pack's re-delta target); fills v1/v2's 64-hex names. fn packSeedSamePathOutOfWindow( gpa: std.mem.Allocator, log: *logstore.LogStore, v1_name: *[64]u8, v2_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // v1 of tracked.txt (full), and the root check-in that names it. const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); v1_name.* = try packSeedBlob(log, v1); var root_name: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &root_name); // `max_base_candidates` unrelated fillers between v1 and v2: they saturate // v2's bounded backward window so v1 is out of chronological reach. var f: usize = 0; while (f < logstore.max_base_candidates) : (f += 1) { var fbuf: std.ArrayList(u8) = .empty; defer fbuf.deinit(gpa); var i: usize = 0; while (i < 200) : (i += 1) try fbuf.print(gpa, "samepath filler {d} unrelated row {d}\n", .{ f, i }); _ = try packSeedBlob(log, fbuf.items); } // v2 of tracked.txt (a one-line edit of v1), stored FULL, and its check-in. const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); v2_name.* = try packSeedBlob(log, v2); var tip_name: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "tracked.txt", v2_name, &tip_name); try log.commit(); } test "pack §6.2: a same-path predecessor beyond the chronological window becomes v2's base (store shrinks)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var v1_name: [64]u8 = undefined; var v2_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSamePathOutOfWindow(gpa, &wlog, &v1_name, &v2_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source holds v2 as a NON-delta (full) frame — the case // step §6.1's chronological window alone cannot improve (v1 is out of reach). { const sv2 = packRidByName(&src, &v2_name).?; try std.testing.expect(src.kindOf(sv2) != .delta); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst); // After pack, v2 is a delta whose base is the same-path predecessor v1 — the // §6.2 mlink analog reached a base the chronological window could not. const dv2 = packRidByName(&dst, &v2_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dv2)); var dw = (try dst.wire(gpa, dv2)).?; defer dw.deinit(gpa); var v1bin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&v1bin, &v1_name); try std.testing.expectEqualSlices(u8, &v1bin, dw.delta.baseHash()); // The store shrank (a tiny v1-based delta replaced the full v2 frame), every // artifact still reconstructs byte-for-byte, and no frame grew. try std.testing.expect(stats.bytes_after < stats.bytes_before); try packExpectContentEqual(gpa, &src, &dst); try packExpectNoFrameLarger(gpa, &src, &dst); } /// Seed the §6.2 same-path REVERT scenario: one tracked file whose content goes /// A→B→A across three check-ins — the third restores a byte-identical earlier /// blob. Because the store is content-addressed, the reverted version resolves to /// the SAME blob rid as v1, so the pack manifest walk records that rid twice, /// non-consecutively, for one path. Exercises `deriveCandidateHints`'s /// distinct-rid compaction: a blob must never be offered as its own same-path /// predecessor. Fills v1/v2's 64-hex names. fn packSeedSamePathRevert( gpa: std.mem.Allocator, log: *logstore.LogStore, v1_name: *[64]u8, v2_name: *[64]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); const v1 = try packEvolvingBlob(gpa, 1); defer gpa.free(v1); const v2 = try packEvolvingBlob(gpa, 2); defer gpa.free(v2); // v1 of tracked.txt (full) + its root check-in. v1_name.* = try packSeedBlob(log, v1); var c1: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &c1); // v2 (a one-line edit) + its check-in. v2_name.* = try packSeedBlob(log, v2); var c2: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &c1, "tracked.txt", v2_name, &c2); // Revert: tracked.txt returns to v1's byte-identical content. The blob is // already stored (content-addressed), so this check-in re-references v1's rid // — the non-consecutive duplicate that recordPathBlob's consecutive-only dedup // lets through. var c3: [64]u8 = undefined; try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &c2, "tracked.txt", v1_name, &c3); try log.commit(); } test "pack §6.2: a same-path revert (A→B→A) re-references one blob rid twice and packs cleanly (no self-predecessor panic)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var v1_name: [64]u8 = undefined; var v2_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedSamePathRevert(gpa, &wlog, &v1_name, &v2_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); // Before the distinct-rid compaction in deriveCandidateHints this panicked on // a self-predecessor (v1 sorted adjacent to its own duplicate ⇒ equal offsets // fail the strict offset-below assert). Now the pack completes and every // artifact reconstructs byte-for-byte with no frame enlarged. const stats = try packLog(gpa, &src, &dst); try packExpectContentEqual(gpa, &src, &dst); try packExpectNoFrameLarger(gpa, &src, &dst); try std.testing.expectEqual(dst.size(), stats.bytes_after); try std.testing.expectEqual(src.size(), stats.bytes_before); } test "pack §6.1 monotonic guard: a wire delta pack cannot beat is KEPT, not replaced by a larger frame (no regression)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var a_name: [64]u8 = undefined; var b_name: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedUnbeatableWireDelta(gpa, &wlog, &a_name, &b_name); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); // Precondition: the source holds B as a compact delta whose base is A. { const sb = packRidByName(&src, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb)); var sw = (try src.wire(gpa, sb)).?; defer sw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, sw.delta.baseHash()); } var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &src, &dst); // (a) decode-all byte-identical pre/post. try packExpectContentEqual(gpa, &src, &dst); // (b) NO frame is larger post-pack than its pre-pack encoding. try packExpectNoFrameLarger(gpa, &src, &dst); // (c) the whole store did not regress on this wire-delta-heavy fixture. try std.testing.expect(stats.bytes_after <= stats.bytes_before); // Accounting exact: reported sizes are the real store sizes. try std.testing.expectEqual(src.size(), stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); // B stayed a delta (the compact A-based frame was kept, re-linked to A's new // rid) — the guard refused to replace it with the larger full zlib pack's // window-only re-encoding would have produced. const db = packRidByName(&dst, &b_name).?; try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db)); var dw = (try dst.wire(gpa, db)).?; defer dw.deinit(gpa); var abin: [digest_len]u8 = undefined; _ = try std.fmt.hexToBytes(&abin, &a_name); try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash()); // The kept frame is no larger than the source's (equal — copied verbatim). try std.testing.expect(dst.frameSpanOf(db) <= src.frameSpanOf(packRidByName(&src, &b_name).?)); } /// Seed a store exercising EVERY frame class pack must handle in one place: a /// raw frame, a zlib frame, a locally-created delta, a verbatim wire delta, an /// interned phantom (no frame on disk), and the config singleton. Fills the /// content-bearing artifacts' 64-hex names plus the phantom's binary digest. fn packSeedMixedStore( gpa: std.mem.Allocator, log: *logstore.LogStore, raw_name: *[64]u8, zlib_name: *[64]u8, delta_name: *[64]u8, wire_name: *[64]u8, phantom_digest: *[digest_len]u8, ) !void { try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" }); // raw: a unique blob large enough to be a real (if unrelated) base. var rbuf: std.ArrayList(u8) = .empty; defer rbuf.deinit(gpa); var i: usize = 0; while (i < 120) : (i += 1) try rbuf.print(gpa, "raw-only content row {d}\n", .{i}); const rbytes = rbuf.items; raw_name.* = try packSeedBlob(log, rbytes); // `put` => .raw // zlib: a large evolving blob stored as a full zlib container. const z = try packEvolvingBlob(gpa, 1); defer gpa.free(z); var z_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(z, &z_digest, .{}); const z_rid = try log.putZlib(&z_digest, z); _ = zit.artifact.artifactName(.sha3_256, z, zlib_name); // local delta: an edit of the zlib blob, delta-encoded against it. const d = try packEvolvingBlob(gpa, 2); defer gpa.free(d); var d_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(d, &d_digest, .{}); _ = try log.putArtifact(&d_digest, d, z_rid); _ = zit.artifact.artifactName(.sha3_256, d, delta_name); // wire delta: a hand-crafted §4.1 delta container against the raw blob (a // poor base) — the verbatim-retained peer form. const w = try packEvolvingBlob(gpa, 3); defer gpa.free(w); var w_digest: [digest_len]u8 = undefined; std.crypto.hash.sha3.Sha3_256.hash(w, &w_digest, .{}); _ = zit.artifact.artifactName(.sha3_256, w, wire_name); const r_rid = packRidByName(log, raw_name).?; const raw_delta = try zit.delta.create(gpa, rbytes, w); defer gpa.free(raw_delta); const container = try zit.container.encode(gpa, raw_delta, .default); defer gpa.free(container); _ = try log.putDeltaContainer(&w_digest, r_rid, container); // phantom: an interned forward reference that never reaches disk. std.crypto.hash.sha3.Sha3_256.hash("absent forward-referenced artifact", phantom_digest, .{}); _ = try log.intern(phantom_digest); try log.commit(); } test "pack §6.1/§3: a mixed raw/zlib/delta/wire-delta/phantom/config store round-trips, config+phantom preserved, accounting exact" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var raw_name: [64]u8 = undefined; var zlib_name: [64]u8 = undefined; var delta_name: [64]u8 = undefined; var wire_name: [64]u8 = undefined; var phantom_digest: [digest_len]u8 = undefined; // A phantom never reaches disk, so we pack the in-memory writable store // directly (a reopen would drop it) — the case pack must tolerate without // choking or emitting a garbage frame. var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedMixedStore(gpa, &wlog, &raw_name, &zlib_name, &delta_name, &wire_name, &phantom_digest); // The seeded store really holds one frame of each class plus the phantom. try std.testing.expectEqual(logstore.FrameKind.raw, wlog.kindOf(packRidByName(&wlog, &raw_name).?)); try std.testing.expectEqual(logstore.FrameKind.zlib, wlog.kindOf(packRidByName(&wlog, &zlib_name).?)); try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &delta_name).?)); try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &wire_name).?)); try std.testing.expect(wlog.isPhantom(wlog.lookup(&phantom_digest).?)); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), wlog.hash_kind); defer dst.deinit(gpa); const stats = try packLog(gpa, &wlog, &dst); // Decode-all byte-identical pre/post, and the artifact SETS match (the // phantom is excluded from both, so counts still agree). try packExpectContentEqual(gpa, &wlog, &dst); // §6.1 per-frame monotonicity: no frame grew in the repack. try packExpectNoFrameLarger(gpa, &wlog, &dst); // Config preserved verbatim in fields. var soc = (try wlog.readConfig(gpa)).?; defer soc.deinit(gpa); var doc = (try dst.readConfig(gpa)).?; defer doc.deinit(gpa); try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy); try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch); try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id); try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code); // Phantom preserved as a contentless forward reference: it carries no bytes, // so pack writes no frame for it (a phantom never reaches disk) and it is not // resurrected as a garbage artifact in the packed store. try std.testing.expect(dst.lookup(&phantom_digest) == null); // Accounting exact: the reported sizes are the actual store sizes and the // artifact tally is the source's real (non-phantom) frame count. try std.testing.expectEqual(wlog.size(), stats.bytes_before); try std.testing.expectEqual(dst.size(), stats.bytes_after); const src_rids = try wlog.ridsInPersistedOrder(gpa); defer gpa.free(src_rids); try std.testing.expectEqual(src_rids.len, stats.artifacts); } test "pack §7.7: packLog aborts on a source frame whose content does not name it (no packed frame written)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); // A frame keyed by a name that is NOT sha3-256 of its bytes: `put` stores // bytes verbatim under any key, so this models a corrupt/lying source frame // that survives `openReadOnly` (its CRC + embedded-hash label are intact). const lying_name = [_]u8{0xab} ** digest_len; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); _ = try wlog.put(&lying_name, "content these bytes do not name"); try wlog.commit(); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer dfile.close(io); var dfs = storage.FileStorage.init(io, dfile, 0); var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind); defer dst.deinit(gpa); try std.testing.expectError(error.Corrupt, packLog(gpa, &src, &dst)); try std.testing.expect(dst.lookup(&lying_name) == null); // no frame landed on the abort } test "pack is idempotent: pack∘pack is byte-identical (0 additional re-deltas)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } // First pack: source (full) → L1. { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var f1 = try tmp.dir.createFile(io, "pack1", .{ .read = true, .truncate = true }); defer f1.close(io); var f1fs = storage.FileStorage.init(io, f1, 0); var dst = try logstore.LogStore.open(gpa, f1fs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst); } // Second pack: L1 → L2, byte-for-byte equal, and no further re-deltaing. var l1fs: storage.FileStorage = undefined; var l1 = try packOpenReadOnly2(gpa, io, tmp.dir, "pack1", &l1fs); defer l1fs.file.close(io); defer l1.deinit(gpa); var f2 = try tmp.dir.createFile(io, "pack2", .{ .read = true, .truncate = true }); defer f2.close(io); var f2fs = storage.FileStorage.init(io, f2, 0); var l2 = try logstore.LogStore.open(gpa, f2fs.storageFile(), l1.hash_kind); defer l2.deinit(gpa); const stats2 = try packLog(gpa, &l1, &l2); try std.testing.expectEqual(l1.size(), l2.size()); const a = try tmp.dir.readFileAlloc(io, "pack1", gpa, .limited(1 << 24)); defer gpa.free(a); const b = try tmp.dir.readFileAlloc(io, "pack2", gpa, .limited(1 << 24)); defer gpa.free(b); try std.testing.expectEqualSlices(u8, a, b); // byte-identical: pack∘pack == pack // An already-packed store still reports its deltas (they were re-created // identically), and packing it does not shrink further. try std.testing.expectEqual(l1.size(), stats2.bytes_after); } /// Reopen a named log file in `dir` read-only (a sibling to `packOpenReadOnly` /// that takes an arbitrary name, for the multi-generation idempotence test). fn packOpenReadOnly2(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, name: []const u8, fs_out: *storage.FileStorage) !logstore.LogStore { const f = try dir.openFile(io, name, .{}); const st = try f.stat(io); fs_out.* = storage.FileStorage.init(io, f, st.size); return logstore.LogStore.openReadOnly(gpa, fs_out.storageFile()); } test "pack: packLog never writes the source (a crash before the rename leaves the original byte-identical)" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } const before = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(before); // Pack into a separate temp (the new generation). packLog only ever reads // the source; the atomic swap in `runPack` is the sole mutation of `.zit`. { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var f = try tmp.dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = true }); defer f.close(io); var ffs = storage.FileStorage.init(io, f, 0); var dst = try logstore.LogStore.open(gpa, ffs.storageFile(), src.hash_kind); defer dst.deinit(gpa); _ = try packLog(gpa, &src, &dst); } // No rename happened (the "crash"): the original `.zit` is byte-for-byte // unchanged and still opens. const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24)); defer gpa.free(after); try std.testing.expectEqualSlices(u8, before, after); } /// The allocation-failure probe: pack a prebuilt source `.zit` into a fresh temp /// destination, using the failing allocator throughout. Every OOM point must /// unwind cleanly (no leak) and a fully-funded run must succeed. fn packOomProbe(alloc: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void { var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(alloc, io, dir, &sfs); defer sfs.file.close(io); defer src.deinit(alloc); var f = try dir.createFile(io, "oomdst", .{ .read = true, .truncate = true }); defer f.close(io); var ffs = storage.FileStorage.init(io, f, 0); var dst = try logstore.LogStore.open(alloc, ffs.storageFile(), src.hash_kind); defer dst.deinit(alloc); _ = try packLog(alloc, &src, &dst); } test "pack: checkAllAllocationFailures on packLog (every OOM path unwinds cleanly)" { const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(std.testing.allocator, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(std.testing.allocator); try packSeedFullStore(std.testing.allocator, &wlog, &tip); } try std.testing.checkAllAllocationFailures(std.testing.allocator, packOomProbe, .{ io, tmp.dir }); } test "pack∘derive == from-scratch derive: the packed store derives the identical repo view" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var tip: [64]u8 = undefined; { var wfs: storage.FileStorage = undefined; var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs); defer wfs.file.close(io); defer wlog.deinit(gpa); try packSeedFullStore(gpa, &wlog, &tip); } var sfs: storage.FileStorage = undefined; var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs); defer sfs.file.close(io); defer src.deinit(gpa); var src_view: zit.repoview.RepoView = undefined; try src_view.load(gpa, &src); defer src_view.deinit(gpa); var pfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true }); defer pfile.close(io); var pfs = storage.FileStorage.init(io, pfile, 0); var dst = try logstore.LogStore.open(gpa, pfs.storageFile(), src.hash_kind); _ = try packLog(gpa, &src, &dst); dst.deinit(gpa); var rfs: storage.FileStorage = undefined; var packed_log = try packOpenReadOnly2(gpa, io, tmp.dir, "packed", &rfs); defer rfs.file.close(io); defer packed_log.deinit(gpa); var packed_view: zit.repoview.RepoView = undefined; try packed_view.load(gpa, &packed_log); defer packed_view.deinit(gpa); // Same derived view: main branch, artifact count, and the leaf set + order // (§7.5) are identical — pack changed only the storage layout. try std.testing.expectEqualStrings(src_view.main_branch, packed_view.main_branch); try std.testing.expectEqual(src_view.store.store().count(), packed_view.store.store().count()); var la = src_view.state.leaves(); var lb = packed_view.state.leaves(); while (la.next()) |ra| { const rb = lb.next() orelse return error.LeafCountMismatch; try std.testing.expectEqualStrings(src_view.hashOf(ra), packed_view.hashOf(rb)); } try std.testing.expect(lb.next() == null); } test "renderPackStats: 'packed' when the store shrank, 'already packed' otherwise" { var buf: [128]u8 = undefined; { var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 1695, .deltas = 1373, .bytes_before = 9_680_000, .bytes_after = 920_000 }); try std.testing.expectEqualStrings( "packed: 1695 artifacts, 1373 re-deltaed, 9680000 -> 920000 bytes\n", w.buffered(), ); } { var w: std.Io.Writer = .fixed(&buf); renderPackStats(&w, .{ .artifacts = 3, .deltas = 2, .bytes_before = 4096, .bytes_after = 4096 }); try std.testing.expectEqualStrings( "already packed: 3 artifacts, 2 re-deltaed, 4096 -> 4096 bytes\n", w.buffered(), ); } } /// Render one `ApplyError` through `openApplyFail` into `buf`. fn openFailText(buf: []u8, err: update.ApplyError) []const u8 { var w = Io.Writer.fixed(buf); var sink: [1]u8 = undefined; var wnull = Io.Writer.fixed(&sink); const s: cli.Streams = .{ .out = &wnull, .err = &w }; _ = openApplyFail(s, err); return w.buffered(); } // Same guard as the `zit update`, `zit merge` and `zit bisect` mappers. // `Corrupt` alleges the stored bytes do not match the name they are stored // under; `StateRecordInvalid` says only that an artifact was unusable as a // state record. A clone of an intact fossil repository reaches the second // without anything being damaged, so it must never be answered with a // corruption claim. test "open: the two read-path failures do not share a diagnostic" { var a: [256]u8 = undefined; var b: [256]u8 = undefined; const corrupt = openFailText(&a, error.Corrupt); const state = openFailText(&b, error.StateRecordInvalid); try std.testing.expect(!std.mem.eql(u8, corrupt, state)); try std.testing.expect(std.mem.indexOf(u8, corrupt, "corrupt") != null); try std.testing.expect(std.mem.indexOf(u8, state, "corrupt") == null); }