//! `zit clone`: create a repository from a served one, converge over the
//! transport, persist what arrived and adopt the origin's identity.
const std = @import("std");
const ClientHoldings = zit.sync_session.ClientHoldings;
const CloneProgress = @import("sync_dialogue.zig").CloneProgress;
const ConeMode = zit.sync_session.ConeMode;
const ConfigPatch = zit.repo_config.ConfigPatch;
const File = Io.File;
const HorizonUrl = zit.sync_session.HorizonUrl;
const Io = std.Io;
const LogIndexStore = zit.log_index_store.LogIndexStore;
const PeerErrorCopy = zit.sync_session.PeerErrorCopy;
const PersistError = zit.repo_config.PersistError;
const PersistLockedError = zit.repo_config.PersistLockedError;
const PersistPass = zit.receive.PersistPass;
const PersistSummary = zit.receive.PersistSummary;
const Repository = @import("zit").Repository;
const SparseClassification = zit.sparse.SparseClassification;
const SubManifestRefs = zit.receive.SubManifestRefs;
const artifact = zit.artifact;
const classifyKeepRids = zit.sparse.classifyKeepRids;
const cli = @import("zit").cli;
const clone_help = help_pages.clone_help;
const clusterindex = @import("zit").clusterindex;
const cmd_init = @import("init.zig");
const coneMode = zit.sync_session.coneMode;
const d4OpenLog = @import("sync_test_support.zig").d4OpenLog;
const d4SeedConfiguredRepo = @import("sync_test_support.zig").d4SeedConfiguredRepo;
const d4SeedEmptyRepo = @import("sync_test_support.zig").d4SeedEmptyRepo;
const decodeWireName = zit.digest_name.decodeWireName;
const diagnostics = @import("../cli/diagnostics.zig");
const digest_len = zit.digest_name.digest_len;
const forksAfter = @import("sync_dialogue.zig").forksAfter;
const help_pages = @import("../cli/help.zig");
const loadClusterIndex = zit.receive.loadClusterIndex;
const logstore = @import("zit").logstore;
const max_password_len = @import("sync_dialogue.zig").max_password_len;
const max_stalled_sync_rounds = zit.sync_session.max_stalled_sync_rounds;
const notePlaintext = @import("sync_dialogue.zig").notePlaintext;
const openLogAdoptingIndex = zit.log_index_store.openLogAdoptingIndex;
const persistArtifact = zit.receive.persistArtifact;
const persistStore = zit.receive.persistStore;
const persistStoreOpen = zit.receive.persistStoreOpen;
const plural = @import("sync_dialogue.zig").plural;
const printForks = @import("sync_dialogue.zig").printForks;
const readPasswordNoEcho = @import("sync_dialogue.zig").readPasswordNoEcho;
const refreshClusterSidecar = zit.receive.refreshClusterSidecar;
const renderStorage = cmd_init.renderStorage;
const repeat = @import("zit").repeat;
const repo_filename = zit.repo_filename;
const reportPeerRefusal = @import("sync_dialogue.zig").reportPeerRefusal;
const reportTruncatedPayloads = @import("sync_dialogue.zig").reportTruncatedPayloads;
const reportUvSkipped = @import("sync_dialogue.zig").reportUvSkipped;
const repostate = zit.repostate;
const runInit = cmd_init.runInit;
const serve_mod = @import("../net/serve.zig");
const stateOom = diagnostics.stateOom;
const storage = @import("zit").storage;
const syncOpenFail = @import("sync_dialogue.zig").syncOpenFail;
const syncRunFail = @import("sync_dialogue.zig").syncRunFail;
const sync_client_version_info = zit.sync_session.sync_client_version_info;
const sync_driver = @import("../net/sync_driver.zig");
const sync_projectcode = zit.sync_session.sync_projectcode;
const transport = @import("../net/transport.zig");
const treeManifestBytes = @import("sync_test_support.zig").treeManifestBytes;
const updateConfig = zit.repo_config.updateConfig;
const updateConfigOpen = zit.repo_config.updateConfigOpen;
const urlUser = @import("sync_dialogue.zig").urlUser;
const writeSyncCursor = zit.sync_session.writeSyncCursor;
const zit = @import("zit");
/// Clone phase edges let benchmark code attribute the real command without
/// coupling clocks or counters into production behavior.
pub const ClonePhase = enum { transport, initialize, exchange, persist, config, warm_cache };
pub const ClonePhaseEvent = enum { begin, end };
pub const CloneObserver = struct {
context: *anyopaque,
callback: *const fn (context: *anyopaque, phase: ClonePhase, event: ClonePhaseEvent) void,
};
fn observeClonePhase(observer: ?*const CloneObserver, phase: ClonePhase, event: ClonePhaseEvent) void {
if (observer) |value| value.callback(value.context, phase, event);
}
// ---------------------------------------------------------------------------
// `zit clone URL DIR` — the interop milestone: a real client converging against
// a real `zit serve` over HTTP. Create a fresh repo at DIR, open an HTTP
// transport to URL, pump the sans-IO clone client (`sync.Client`) to §3.7
// convergence through `sync_driver.runSync`, then persist the received
// artifacts into DIR's on-disk log so the clone is durable. Anonymous only.
// ---------------------------------------------------------------------------
/// The canonical form of a clone URL for persistence: rebuilt from parsed
/// components — scheme://[user@]host[:port]/path[?query] — so a userinfo
/// password is excluded BY CONSTRUCTION: a component the rebuild never names
/// cannot survive into the stored value, even if a future parser change adds
/// fields (fossil url.c's canonical form works the same way). The username is
/// kept, matching fossil. The REQUESTED URL is what canonicalizes, never the
/// redirect-resolved one: an alias in front of a moving host is load-bearing,
/// and the redirect chain re-resolves on every sync anyway.
pub fn canonicalRemoteUrl(gpa: std.mem.Allocator, url: []const u8) error{ OutOfMemory, InvalidUrl }![]u8 {
const uri = std.Uri.parse(url) catch return error.InvalidUrl;
const bytesOf = struct {
fn f(c: std.Uri.Component) []const u8 {
return switch (c) {
.raw => |r| r,
.percent_encoded => |p| p,
};
}
}.f;
var out: std.ArrayList(u8) = .empty;
errdefer out.deinit(gpa);
try out.appendSlice(gpa, uri.scheme);
try out.appendSlice(gpa, "://");
if (uri.user) |u| {
try out.appendSlice(gpa, bytesOf(u));
try out.append(gpa, '@');
}
const host = uri.host orelse return error.InvalidUrl;
try out.appendSlice(gpa, bytesOf(host));
if (uri.port) |p| try out.print(gpa, ":{d}", .{p});
const path = bytesOf(uri.path);
if (path.len == 0) try out.append(gpa, '/') else try out.appendSlice(gpa, path);
if (uri.query) |q| {
try out.append(gpa, '?');
try out.appendSlice(gpa, bytesOf(q));
}
return out.toOwnedSlice(gpa);
}
test "canonicalRemoteUrl: password excluded by construction, username kept" {
const gpa = std.testing.allocator;
const cases = [_]struct { in: []const u8, want: []const u8 }{
.{ .in = "http://user:secret@host/repo", .want = "http://user@host/repo" },
.{ .in = "http://user@host:8080/repo", .want = "http://user@host:8080/repo" },
.{ .in = "https://host", .want = "https://host/" },
.{ .in = "https://host/a/b?x=1#frag", .want = "https://host/a/b?x=1" },
};
for (cases) |c| {
const got = try canonicalRemoteUrl(gpa, c.in);
defer gpa.free(got);
try std.testing.expectEqualStrings(c.want, got);
try std.testing.expect(std.mem.indexOf(u8, got, "secret") == null);
}
try std.testing.expectError(error.InvalidUrl, canonicalRemoteUrl(gpa, "not a url"));
}
/// `zit clone URL DIR` — thin wiring over three deep modules: `transport`
/// (HTTP/IO), `sync.Client` + `sync_driver.runSync` (sans-IO §3.7 loop), and
/// `LogStore` (persistence). The URL is used verbatim (sync §4 r1 — the client
/// never appends /xfer). All-or-nothing failures surface an honest exit-1
/// diagnostic; a bad URL scheme is rejected before any repo is created.
pub fn runClone(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
return runCloneObserved(gpa, io, arena, verb_args, streams, amb, null);
}
/// `runClone` with benchmark-owned phase attribution. Sync §8.2's project-code
/// adoption remains in the observed config phase; a null observer is the
/// ordinary CLI path, which reads no clock and owns no counters.
pub fn runCloneObserved(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
observer: ?*const CloneObserver,
) cli.Exit {
var made: CloneMade = .{};
const exit = cloneObserved(gpa, io, arena, verb_args, streams, amb, observer, &made);
if (exit != .ok) {
if (made.initialized and !made.committed) abandonClone(arena, io, &made);
return exit;
}
// A clone is followed by the `open` the operator would type next
// (Fossil 2.x), run inside DIR by this same binary so the two verbs stay
// one implementation; --no-open keeps the bare store. It runs only here,
// after cloneObserved returned and released its exclusive store lock —
// the child takes that same lock. The child opens the same cone the clone
// elided to: members from the flags and from --cone-file (read here,
// relative to this cwd) are passed explicitly.
const parsed = switch (cli.parseCloneArgs(verb_args)) {
.ok => |p| p,
else => return exit,
};
if (parsed.help or !parsed.open) return exit;
var members = parsed.cone_members;
var len = parsed.cone_len;
var not_members = parsed.cone_not_members;
var not_len = parsed.cone_not_len;
if (parsed.cone_file) |cf| {
if (cli.appendConeFile(arena, io, cf, &members, len, ¬_members, not_len)) |lens| {
len = lens.members;
not_len = lens.not_members;
}
}
return autoOpen(gpa, io, arena, streams, parsed.dir, members[0..len], not_members[0..not_len], parsed.cone_parents);
}
/// Spawn this same binary as `zit open` inside the freshly cloned DIR.
fn autoOpen(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
streams: cli.Streams,
dir: []const u8,
cone: []const []const u8,
cone_not: []const []const u8,
cone_parents: bool,
) cli.Exit {
streams.out.flush() catch {};
streams.err.flush() catch {};
const exe = std.process.executablePathAlloc(io, gpa) catch {
streams.err.writeAll("zit clone: cloned, but cannot locate zit to open the working tree; run 'zit open' inside the directory\n") catch {};
return .failure;
};
defer gpa.free(exe);
var argv = std.ArrayList([]const u8).empty;
argv.append(arena, exe) catch return .failure;
argv.append(arena, "open") catch return .failure;
for (cone) |m| argv.append(arena, std.fmt.allocPrint(arena, "--cone={s}", .{m}) catch return .failure) catch return .failure;
for (cone_not) |m| argv.append(arena, std.fmt.allocPrint(arena, "--cone-not={s}", .{m}) catch return .failure) catch return .failure;
if (cone_parents) argv.append(arena, "--cone-parents") catch return .failure;
var child = std.process.spawn(io, .{
.argv = argv.items,
.cwd = .{ .path = dir },
}) catch {
streams.err.print("zit clone: cloned, but 'zit open' could not be started; run it inside '{s}'\n", .{dir}) catch {};
return .failure;
};
const term = child.wait(io) catch return .failure;
return switch (term) {
.exited => |code| if (code == 0) .ok else .failure,
else => .failure,
};
}
fn cloneObserved(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
observer: ?*const CloneObserver,
made: *CloneMade,
) cli.Exit {
const parsed = switch (cli.parseCloneArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |tok| return cli.usageError(streams, "clone", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "clone", .extra_argument, tok),
.no_url => return cli.usageError(streams, "clone", .no_url, ""),
.no_dir => return cli.usageError(streams, "clone", .no_dir, ""),
.empty_value => |tok| return cli.usageError(streams, "clone", .empty_value, tok),
.too_many_cones => |tok| return cli.usageError(streams, "clone", .too_many_cones, tok),
.repeated_option => |tok| return cli.usageError(streams, "clone", .repeated_option, tok),
.bad_resolve => |tok| return cli.usageError(streams, "clone", .bad_resolve, tok),
.too_many_resolves => |tok| return cli.usageError(streams, "clone", .too_many_resolves, tok),
};
if (parsed.help) {
streams.out.writeAll(clone_help) catch {};
return .ok;
}
// Open the transport first: a malformed URL fails before any repo lands on
// disk (nothing half-made to roll back). `open` performs no network I/O.
observeClonePhase(observer, .transport, .begin);
// sync §6 — a private repository is unreadable anonymously. The login is
// --user or the URL's user@; the secret cannot be derived until the round-0
// reply names the project code, so the client holds name and password and
// signs from round 1, exactly as fossil does. The plaintext lives only in
// pw_buf/argv/env and is scrubbed on every exit.
var pw_buf: [max_password_len]u8 = undefined;
defer std.crypto.secureZero(u8, &pw_buf);
const login_user: ?[]const u8 = blk: {
const u = parsed.user orelse urlUser(parsed.url) orelse break :blk null;
if (u.len == 0) break :blk null;
if (zit.sync.containsFossilWhitespace(u)) {
streams.err.writeAll("zit clone: --user must not contain whitespace\n") catch {};
return .usage;
}
break :blk u;
};
const login: ?zit.sync.PendingLogin = if (login_user) |u| blk: {
const password = parsed.password orelse amb.push_password orelse (readPasswordNoEcho(streams, u, &pw_buf) orelse {
if (parsed.user != null) {
streams.err.print("zit clone: no password for '{s}' (give --password, set ZIT_PASSWORD, or run on a terminal)\n", .{u}) catch {};
return .failure;
}
// A user named only by the URL is a hint: clone anonymously, and say so.
streams.err.print("zit clone: no password for '{s}' from the URL (set ZIT_PASSWORD or pass --user/--password); cloning anonymously\n", .{u}) catch {};
break :blk null;
});
break :blk .{ .user = u, .password = password };
} else null;
var tr = transport.Transport.open(io, gpa, parsed.url) catch |e| {
observeClonePhase(observer, .transport, .end);
return syncOpenFail(streams, "clone", parsed.url, e);
};
observeClonePhase(observer, .transport, .end);
defer tr.close();
tr.setResolve(parsed.resolve.slice());
notePlaintext(streams, "clone", parsed.url);
if (login) |l| tr.setBasicAuth(l.user, l.password) catch return stateOom(streams, "clone");
// The cone and horizon are checked before DIR is made: a refused option
// leaves nothing behind.
var clone_members = parsed.cone_members;
var clone_len = parsed.cone_len;
var clone_not_members = parsed.cone_not_members;
var clone_not_len = parsed.cone_not_len;
if (parsed.cone_file) |cf| {
const lens = cli.appendConeFile(arena, io, cf, &clone_members, clone_len, &clone_not_members, clone_not_len) orelse {
streams.err.print("zit clone: cannot read --cone-file '{s}' (or it overflows the {d}-member cap)\n", .{ cf, zit.cone.max_members }) catch {};
return .failure;
};
clone_len = lens.members;
clone_not_len = lens.not_members;
}
if (parsed.cone_parents and clone_len == 0) {
streams.err.writeAll("zit clone: --cone-parents needs --cone=PATH: it is a property of the cone being set\n") catch {};
return .failure;
}
// Horizon draft section 3: the temporal member, validated here so the
// fault is named before DIR exists. The hash is stored and sent in its
// canonical lower-case form.
var clone_horizon: ?zit.cone.Horizon = null;
// Horizon draft section 6.2.7 -- a horizon given by name, or by a hash
// and a depth, is resolved by the origin (or here, against a peer that
// does not) and only the hash is ever recorded.
var horizon_name: ?zit.sync.HorizonName = null;
if (parsed.horizon) |h| {
const rule: zit.cone.HorizonRule = if (parsed.horizon_rule) |r|
zit.cone.HorizonRule.parse(r) orelse {
streams.err.print("zit clone: --horizon-rule must be 'descendants' or 'leaves', got '{s}'\n", .{r}) catch {};
return .failure;
}
else
.descendants;
const depth: u32 = if (parsed.depth) |d| zit.sync.parseHorizonDepth(d) orelse {
streams.err.print("zit clone: --depth needs a whole number of check-ins, got '{s}'\n", .{d}) catch {};
return .failure;
} else 0;
const level: zit.cone.HorizonDepth = if (parsed.structural) .structural else .content;
if (zit.cone.Horizon.validHash(h) and depth == 0) {
const lower = std.ascii.allocLowerString(arena, h) catch return stateOom(streams, "clone");
clone_horizon = .{ .hash = lower, .rule = rule, .depth = level };
} else if (h.len > zit.sync.max_horizon_name_len) {
streams.err.print("zit clone: --horizon names a check-in in at most {d} bytes\n", .{zit.sync.max_horizon_name_len}) catch {};
return .failure;
} else {
horizon_name = .{ .name = h, .depth = depth, .rule = rule, .level = level };
}
} else if (parsed.horizon_rule != null or parsed.structural or parsed.depth != null) {
streams.err.writeAll("zit clone: --horizon-rule, --depth and --structural need --horizon=HASH or --horizon=NAME\n") catch {};
return .failure;
}
// Create a fresh `.zit` at DIR by reusing the init sequence, quietly (its
// own success line is suppressed; clone prints its own). A pre-existing repo
// there is init's "already exists" error, propagated verbatim.
made.* = cloneMadeBefore(arena, io, parsed.dir) catch return stateOom(streams, "clone");
observeClonePhase(observer, .initialize, .begin);
const init_rc = runInit(gpa, io, arena, &.{ "-q", "--", parsed.dir }, streams);
observeClonePhase(observer, .initialize, .end);
if (init_rc != .ok) return init_rc;
made.initialized = true;
// Converge an in-memory store against the peer, then persist it into DIR.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
// §9.1 — the clone client's own instance identity, derived from the
// destination it is creating (informational; a fresh instance is exactly
// what a clone is).
const clone_code = blk: {
const cwd = (cli.currentPathAlloc(arena, io) catch return stateOom(streams, "clone")) orelse
break :blk zit.sync.deriveServerCode(&.{parsed.dir});
const abs = std.fs.path.resolve(arena, &.{ cwd, parsed.dir }) catch return stateOom(streams, "clone");
break :blk zit.sync.deriveServerCode(&.{abs});
};
// Live per-round progress, on stderr and ONLY when stderr is a
// terminal: an operator watching a slow clone sees each round land instead
// of silence, while piped stderr (scripts, tests) stays byte-identical to
// before. The line is \r-overwritten in place — never fossil's
// percent-flood — and finished with one accounting line when the exchange
// ends, either way it ends.
const stderr_tty = std.Io.File.stderr().isTty(io) catch false;
var progress: CloneProgress = .{ .err = streams.err };
const sink: ?sync_driver.RoundSink = if (stderr_tty)
.{ .ctx = &progress, .emit = CloneProgress.emit }
else
null;
observeClonePhase(observer, .exchange, .begin);
// The destination log, open BEFORE the exchange: a dense clone
// persists each round as it converges instead of holding the repository
// resident, so the log has to exist while rounds are still arriving. A
// clone that fails before the log commits is taken back by the caller
// (`abandonClone`), so no partial `.zit` is left for the operator.
var target = std.Io.Dir.cwd().openDir(io, parsed.dir, .{}) catch
return renderStorage(streams, parsed.dir, "cannot reopen the clone target directory");
defer target.close(io);
var store_append = cli.openStoreAppend(io, target) catch
return renderStorage(streams, parsed.dir, "cannot write the cloned repository");
defer store_append.close(io);
const file = store_append.file;
const st = file.stat(io) catch
return renderStorage(streams, parsed.dir, "cannot write the cloned repository");
var fs = storage.FileStorage.init(io, file, st.size);
// A horizon clone is made with the horizon stamp: every path that keeps
// a horizon commits one and every other fails before the commit, so the
// store's live header is never restamped after it holds frames.
const stamp = if (clone_horizon != null or horizon_name != null) logstore.horizon_version else logstore.version;
var log = logstore.LogStore.openStamped(gpa, fs.storageFile(), 0, stamp) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"),
};
defer log.deinit(gpa);
// Stream the persist unless a cone is requested. A cone-scoped
// clone must classify its keep set over the CONVERGED store (sparse-cone
// RFC section 6.3) unless its origin filtered for it, so it decides
// after the first reply (`ConeClonePersist`); every dense clone drains
// per round and releases what it wrote, bounding peak memory by the round
// batch plus the deltas whose basis has not arrived (git index-pack's
// shape, applied to fossil's reverse-delta wire).
const cone_scoped = clone_len + clone_not_len != 0 or clone_horizon != null or horizon_name != null;
// Every cluster this clone writes, for the index it creates.
var clone_clusters: std.ArrayListUnmanaged([]const u8) = .empty;
defer {
for (clone_clusters.items) |n| gpa.free(n);
clone_clusters.deinit(gpa);
}
// One pass whichever way the clone persists: per round, or once over
// the converged store with the cone's keep set.
var pass = PersistPass.init(gpa, &log, ms.store(), null, &ms, 0, &clone_clusters) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"),
};
defer pass.deinit();
var clone_dropped: std.ArrayListUnmanaged([]const u8) = .empty;
defer {
for (clone_dropped.items) |n| gpa.free(n);
clone_dropped.deinit(gpa);
}
pass.dropped_out = &clone_dropped;
var cone_persist: ConeClonePersist = .{
.ms = &ms,
.pass = &pass,
.horizon = if (clone_horizon) |h| h.depth else if (horizon_name) |n| n.level else null,
.members = clone_len,
.not_members = clone_not_len,
};
// The core's persist pass types its drain with PersistError because it
// cannot import the driver; the two sets must stay identical.
comptime std.debug.assert(sync_driver.DrainError == PersistError);
var drain: CloneIdentityDrain = .{
.gpa = gpa,
.log = &log,
.inner = if (cone_scoped)
.{ .ctx = &cone_persist, .run = ConeClonePersist.drainErased }
else
.{ .ctx = &pass, .run = PersistPass.drainErased },
};
var peer_error: PeerErrorCopy = .{};
var truncated: u64 = 0;
var outcome = cloneRun(gpa, &tr, &ms, &clone_code, login, sink, &drain, clone_members[0..clone_len], clone_not_members[0..clone_not_len], parsed.cone_parents, clone_horizon, horizon_name, if (cone_scoped) &cone_persist else null, &peer_error, &truncated) catch |e| {
progress.finishLine();
observeClonePhase(observer, .exchange, .end);
reportTruncatedPayloads(streams, "clone", truncated);
// A peer that stated its refusal is quoted; anything else keeps the
// generic mapping (sparse-cone RFC section 5.3 diagnostic path).
if (e == error.FatalProtocolError) {
if (peer_error.text()) |text| {
reportPeerRefusal(streams, "clone", text);
return .failure;
}
}
return syncRunFail(streams, "clone", parsed.url, e, &tr);
};
// §9.3 r28 — the latched shun rows are the one thing this outcome owns.
defer outcome.deinit(gpa);
progress.finishLine();
// Fossil fails a clone on any handler error. A cut-off reply also loses
// the clone_seqno card that would have asked for the next batch, so the
// exchange can end looking complete when it is not. It fails here, before
// anything is committed, so the caller removes what the clone made, as
// for any other failed exchange.
if (truncated > 0) {
observeClonePhase(observer, .exchange, .end);
reportTruncatedPayloads(streams, "clone", truncated);
streams.err.print("zit clone: the clone of {s} is incomplete; clone again\n", .{parsed.url}) catch {};
return .failure;
}
if (stderr_tty) {
streams.err.print(
"clone exchange: {d} rounds, {d} protocol bytes received, {d} sent\n",
.{ outcome.rounds, outcome.received_bytes, outcome.sent_bytes },
) catch {};
}
observeClonePhase(observer, .exchange, .end);
// A cone clone whose origin filtered for it streamed its rounds; what it
// received is read back from the log, not from `ms`, which released it.
const cone_streamed = cone_scoped and (cone_persist.streaming orelse false);
// Sparse-cone RFC section 6.3: a cone-scoped clone filters LOCALLY --
// correct against every peer, including cone-ignorant ones. The keep set
// is classified over the converged in-memory store before any frame is
// written; nothing is discarded silently (the count prints below).
var clone_cone: ?zit.cone.Cone = null;
defer if (clone_cone) |*c| c.deinit(gpa);
var keep_bits: ?SparseClassification = null;
defer if (keep_bits) |*k| k.deinit(gpa);
// The streamed log read back: the last round's stragglers and every
// delta whose basis never came are written first, so it holds the whole
// transfer.
var streamed_view: ?*LogIndexStore = null;
defer if (streamed_view) |v| v.destroy();
if (cone_streamed) {
const settled: PersistError!void = res: {
pass.drainNew() catch |e| break :res e;
break :res pass.settle();
};
settled catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"),
};
streamed_view = LogIndexStore.open(gpa, fs.storageFile()) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
else => return renderStorage(streams, parsed.dir, "cannot read back the cloned repository"),
};
}
// Horizon draft section 6.2.7 -- the named horizon, as the origin
// resolved it, or resolved here against a peer that ignored the name:
// that peer served densely, so the transfer holds what resolving needs.
if (horizon_name) |hn| {
if (outcome.horizon_resolved_fault) {
streams.err.print("zit clone: the origin's answer to --horizon={s} is malformed or out of place; nothing was cloned\n", .{hn.name}) catch {};
return .failure;
}
const resolved: ?zit.sync.HorizonResolved = if (cloneHorizonResolved(&outcome)) |r| r else blk: {
streams.err.print("zit clone: the origin did not resolve --horizon={s}; resolving it here from the transfer\n", .{hn.name}) catch {};
break :blk resolveHorizonLocally(gpa, arena, if (streamed_view) |v| v.store() else ms.store(), hn.name, hn.depth) catch return stateOom(streams, "clone");
};
const r = resolved orelse {
streams.err.print("zit clone: horizon name not found: {s}; nothing was cloned\n", .{hn.name}) catch {};
return .failure;
};
if (r.steps < hn.depth) {
streams.err.print("zit clone: --depth={d}: history ran out after {d} step{s}; the horizon is the root\n", .{ hn.depth, r.steps, plural(r.steps) }) catch {};
}
const lower = std.ascii.allocLowerString(arena, r.hash) catch return stateOom(streams, "clone");
clone_horizon = .{ .hash = lower, .rule = hn.rule, .depth = hn.level };
streams.err.print("zit clone: --horizon={s} is {s}\n", .{ hn.name, lower }) catch {};
}
// Horizon draft section 5.2.4 -- a seed not bound to this horizon is
// refused, as a wrongly signed one is: the clone keeps the content
// horizon below.
if (clone_horizon) |h| if (outcome.seed) |b| {
if (h.depth == .structural and (zit.sync.judgeSeed(gpa, b, h, null) catch return stateOom(streams, "clone")) != .ok) {
streams.err.writeAll("zit clone: the origin's seed is not bound to this horizon's check-in and rule; refusing it\n") catch {};
gpa.free(b);
outcome.seed = null;
}
};
// Horizon draft section 6.3 -- a structural request answered by an origin
// that did not say `horizon-served structural` has no seed and no labels:
// the transfer falls back to the content horizon and says so.
if (clone_horizon) |*h| {
if (h.depth == .structural and (outcome.horizon_served != .structural or outcome.seed == null)) {
// Horizon draft section 5.1 -- no seed, no structural cut: the
// derivation over K alone would carry no branch state at all.
streams.err.writeAll("zit clone: the origin did not serve the structural horizon with a seed; keeping the content horizon instead\n") catch {};
h.depth = .content;
}
}
// Horizon draft section 3.1 -- C is a check-in the origin holds. A hash it
// does not hold, or holds as something else, names no held set at all; a
// clone built on it would hold nothing, or everything, while claiming a
// horizon. Nothing is committed yet, so refusing here leaves no store.
if (clone_horizon) |h| {
if (!holdsCheckin(arena, if (streamed_view) |v| v.store() else ms.store(), h.hash)) {
streams.err.print("zit clone: --horizon names {s}, which the origin does not hold as a check-in; nothing was cloned\n", .{h.hash}) catch {};
return .failure;
}
}
if (cone_scoped) {
clone_cone = zit.cone.Cone.initWith(gpa, clone_members[0..clone_len], clone_not_members[0..clone_not_len], .{ .horizon = clone_horizon, .parents = parsed.cone_parents }) catch |e|
return cloneConeFail(streams, e);
if (streamed_view) |v| {
// What the origin withheld is only ever named, so the cited
// names are looked up, never interned: the session holds the
// log's names and no more. Nothing is cached from it -- a
// sidecar is a later client's, classified as that client
// classifies, interning what it cites.
// What the rounds past the clone sequence brought is not in the
// log yet; the view holds it beside the log so the cone judges
// it with everything else.
for (pass.held_back.items) |rid| {
const bytes = ms.store().content(rid) orelse continue;
_ = v.classifyStore().put(ms.store().hashOf(rid), bytes) catch return stateOom(streams, "clone");
}
keep_bits = zit.sparse.classifyKeepRidsOpts(gpa, v.classifyStore(), &clone_cone.?, .{ .intern_absent = false }) catch
return stateOom(streams, "clone");
pass.judgeHeldBack(v.store(), &keep_bits.?);
} else {
// One artifact inflated at a time, not the store.
ms.releasePlaintext();
keep_bits = classifyKeepRids(gpa, ms.transientStore(), &clone_cone.?) catch
return stateOom(streams, "clone");
}
}
if (streamed_view) |v| {
v.destroy();
streamed_view = null;
}
// Batched, this is what persists; streamed, it only tells an elided
// sub-manifest node from a gap.
if (keep_bits) |*k| pass.keep = k;
observeClonePhase(observer, .persist, .begin);
const persisted = blk: {
// Streaming, the rounds already drained; settle any straggler the
// last round interned. Batched, each artifact's resident bytes go
// once its frame is written, as on the streaming path.
if (!cone_streamed) pass.drainNew() catch |e| break :blk @as(PersistError!PersistSummary, e);
break :blk pass.finish();
} catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the peer sent a malformed artifact name"),
};
const written = persisted.written;
observeClonePhase(observer, .persist, .end);
// A cone clone holds every artifact it received until this persist (a
// dense one released them round by round); all of it is in the log now.
// Drop it and the classification before the cache derive below, which on
// a deep sub-manifest history needs gigabytes of its own: the two
// together overran a 4 GB client that either alone fits. The store's
// pool (`MemStore.pool`) is what makes the drop return memory.
if (cone_scoped) {
pass.keep = null;
if (keep_bits) |*k| k.deinit(gpa);
keep_bits = null;
ms.deinit();
ms = repostate.MemStore.init(gpa);
}
// §8.2 and submanifests §6.3/§8.1 — adopt every origin-owned setting in
// one field-preserving overlay. Missing fields leave `init`'s values
// untouched, and one combined frame avoids intermediate clone config state.
observeClonePhase(observer, .config, .begin);
const project_code = outcome.projectCode();
// Save the clone URL as the default remote (fossil's behavior), unless
// --once. The canonical rebuild strips a userinfo password by construction;
// `zit remote` remains the explicit override and the audit view. The
// InvalidUrl arm is unreachable in practice — `transport.open` parsed this
// same URL — and degrades to not-saving rather than failing a completed
// clone.
const saved_remote: ?[]const u8 = if (parsed.once) null else canonicalRemoteUrl(arena, parsed.url) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.InvalidUrl => null,
};
const cloned_patch: ConfigPatch = .{
// The round drain wrote it ahead of the first artifact.
.project_code = if (project_code.len > 0 and !drain.written) project_code else null,
// The login the peer accepted from round 1 on, stored the way a push
// stores its own, so the next pull or push authenticates unasked. A
// login is bound to the saved remote, so --once, which saves none,
// saves no login either.
.login = if (parsed.once) null else outcome.login,
.remote_url = saved_remote,
.enable_submanifests = outcome.submanifest.enable,
.submanifest_k = outcome.submanifest.k,
.submanifest_auto_retune = outcome.submanifest.auto_retune,
.git_email_domain = if (outcome.submanifest.git_email_domain) |*d| d.slice() else null,
};
if (cloned_patch.any()) {
updateConfigOpen(gpa, &log, cloned_patch) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
error.Storage => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
error.Corrupt => return renderStorage(streams, parsed.dir, "the cloned repository config is unreadable"),
};
}
// §9.3 r28 — adopt the origin's shun table. Written BEFORE the commit
// below, so a clone is durable with its tombstones or not at all: a
// repository that came up holding the content but not the record of its
// suppression is the exact state this carriage exists to prevent.
//
// The wire carries hex; the log keys by binary digest. A name that will not
// decode is skipped rather than fatal — the artifacts are already down, and
// failing a finished clone over one unparseable row helps nobody. The count
// is reported so a skip is never silent.
// The owner key the origin pins, adopted the way `interwiki:` rows are:
// trust on first use, from the origin the operator chose. Nothing but a
// clone (and the operator's `zit shun key pin`) ever writes it.
if (outcome.erasure_key) |key| {
const hex = zit.erasure.keyHex(key);
log.putSetting(.{ .name = zit.erasure.setting_name, .value = &hex, .mtime = amb.now_epoch }) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
else => return renderStorage(streams, parsed.dir, "cannot write the cloned repository"),
};
}
var shun_adopted: usize = 0;
var shun_skipped: usize = 0;
for (outcome.shun_rows) |row| {
var digest_buf: [32]u8 = undefined;
if (row.hash.len != 40 and row.hash.len != 64) {
shun_skipped += 1;
continue;
}
const digest = digest_buf[0 .. row.hash.len / 2];
_ = std.fmt.hexToBytes(digest, row.hash) catch {
shun_skipped += 1;
continue;
};
log.putShun(digest, row.reason, row.epoch) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "clone"),
else => {
shun_skipped += 1;
continue;
},
};
shun_adopted += 1;
}
if (shun_adopted != 0 or shun_skipped != 0) {
streams.out.print("adopted {d} shun row{s} from the origin", .{ shun_adopted, plural(shun_adopted) }) catch {};
if (shun_skipped != 0) streams.out.print("; {d} unreadable and skipped", .{shun_skipped}) catch {};
streams.out.writeAll("\n") catch {};
}
// Every persisted artifact and the optional combined config frame must be
// durable before the cache observes this log or clone reports success.
log.commit() catch return renderStorage(streams, parsed.dir, "cannot write the cloned repository");
made.committed = true;
zit.repocache.recordDroppedSeeds(gpa, io, target, clone_dropped.items) catch return stateOom(streams, "clone");
// A store carrying a credential is owner-only (best-effort, as for push).
if (outcome.login != null) file.setPermissions(io, File.Permissions.fromMode(0o600)) catch {};
refreshClusterSidecar(gpa, io, target, &log, clone_clusters.items, true) catch return stateOom(streams, "clone");
observeClonePhase(observer, .config, .end);
// The committed writable log is still open, so reuse its derivation path
// directly instead of reopening `.zit` solely to prime the disposable cache.
// Say so first: on a large store this phase is minutes of derive, and an
// operator watching a silent clone cannot tell transfer from derivation.
streams.out.print("clone from {s}: transferred {d} artifact{s} in {d} round{s}; deriving caches\n", .{ parsed.url, written, plural(written), outcome.rounds, plural(outcome.rounds) }) catch {};
observeClonePhase(observer, .warm_cache, .begin);
Repository.warmCacheFromLog(gpa, io, target, &log);
observeClonePhase(observer, .warm_cache, .end);
if (clone_cone) |*c| {
// Policy lands with the disposable tier (RFC section 5.2 loss rule).
zit.repocache.writeStoreCone(gpa, io, target, c);
// Horizon draft section 5.2 -- and the seed beside it, when the origin
// sent one and the cone kept its structural depth.
if (c.horizon != null and c.horizon.?.depth == .structural) {
if (outcome.seed) |b| zit.repocache.writeHorizonSeed(io, target, b);
}
if (c.horizon != null) zit.repocache.writeHorizonUrl(io, target, outcome.horizon_url.get());
// Horizon draft section 6.2.6 -- the clone was served through the
// origin's cursor; the first pull need not announce what it got.
if (c.horizon != null) writeSyncCursor(gpa, io, target, parsed.url, c, .{ .peer = outcome.peer_cursor, .local = @intCast(log.entryCount()) });
if (c.horizon != null and cone_streamed) {
// Horizon draft section 6.2.5 -- the origin withheld the elided
// set itself and the clone kept everything it was sent.
streams.out.print(
"cloned {d} artifacts into {s} in {d} rounds (horizon clone; origin served: {s}; nothing elided locally)\n",
.{ written, parsed.dir, outcome.rounds, @tagName(outcome.horizon_served.?) },
) catch {};
} else if (c.horizon != null) {
// Horizon draft section 6.2.5 -- say what the origin did: a
// horizon-aware origin withheld the elided set itself, so the
// local count is only what the local filter still had to drop.
streams.out.print(
"cloned {d} artifacts into {s} in {d} rounds (horizon clone; origin served: {s}; {d} elided locally)\n",
.{ written, parsed.dir, outcome.rounds, if (outcome.horizon_served) |l| @tagName(l) else "dense", persisted.elided },
) catch {};
} else {
// Streamed, nothing was filtered here but the transit the rounds
// past the clone sequence brought: a reply the peer did not
// acknowledge after the first was kept as it came.
const mode = if (!cone_streamed or persisted.filtered != 0)
outcome.cone_mode.phrase()
else if (outcome.cone_mode == .mixed)
"applied by the peer; it did not acknowledge every reply, and those were kept as sent"
else
"applied by the peer; nothing filtered locally";
streams.out.print(
"cloned {d} artifacts into {s} in {d} rounds ({d} elided by the cone; {s})\n",
.{ written, parsed.dir, outcome.rounds, persisted.elided, mode },
) catch {};
}
} else {
streams.out.print("cloned {d} artifact{s} into {s} in {d} round{s}\n", .{ written, plural(written), parsed.dir, outcome.rounds, plural(outcome.rounds) }) catch {};
}
if (project_code.len > 0) streams.out.print("project-code: {s}\n", .{project_code}) catch {};
reportUvSkipped(streams, "clone", outcome.uv_skipped);
// submanifests §7.1/§7.3 — a clone converges on the cfile cursor, so it
// copies the peer's holdings faithfully, INCLUDING an incomplete tree
// closure the peer itself was carrying. The phantoms the persist hook minted
// say so; report them, because a check-in whose file set cannot be produced
// (§5.2) is not a converged clone, and "exit 0 with nothing said" is exactly
// how a repository stays quietly broken. Reported AFTER the success line and
// AFTER the repository is fully written: a partial clone is still progress,
// and this is the same shape `pull` already uses for artifacts it did not
// obtain.
if (persisted.submanifest_gaps > 0) {
streams.err.print(
"zit clone: {d} sub-manifest references are missing; the cloned repository cannot enumerate every check-in\n",
.{persisted.submanifest_gaps},
) catch {};
return .failure;
}
// A clone brought every leaf, so any fork would count as new; it is the
// origin's, and nothing of this caller's was forked, so it is told and
// the clone still exits 0.
if (forksAfter(gpa, io, arena, target, streams, "clone", &.{}, null)) |forks| printForks(streams.err, forks);
return .ok;
}
/// §8.2 — persist the origin's project code into the freshly cloned repo at
/// `dir/.zit`, preserving every other field `init` wrote. A no-op when the peer
/// sent no code. Thin wrapper over `updateConfig` (the field-preserving RMW).
fn storeClonedProjectCode(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
project_code: []const u8,
) PersistLockedError!void {
if (project_code.len == 0) return; // peer sent none; keep the local init code
return updateConfig(gpa, io, dir, .{ .project_code = project_code });
}
/// draft-fossil-submanifests-00 §6.3/§8.1 (and §6.4) over sync §9.3
/// — persist the origin's three sub-manifest settings and its Git email domain
/// into the freshly cloned repo at `dir/.zit`, preserving every other field
/// `init` wrote. A no-op when the peer named none. Thin wrapper over
/// `updateConfig`, the same shape `storeClonedProjectCode` has and for the same
/// reason.
///
/// One-way and clone-only, deliberately. A fresh clone has nothing of its own
/// to conflict with, so adoption needs no merge; an EXISTING repository would,
/// and §9.3's channel resolves conflicts by MTIME, which zit's config frame does
/// not store (see `sync.submanifest_config_mtime`). Refreshing settings on a
/// later pull is therefore a real conflict-resolution feature with a config
/// schema change behind it, not plumbing this wrapper can be widened into —
/// `runPull`/`runPush` deliberately have no seam here.
fn storeClonedSubmanifestSettings(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
cfg: zit.sync.SubmanifestConfig,
) PersistLockedError!void {
if (!cfg.any()) return; // peer sent none; keep the local init defaults
return updateConfig(gpa, io, dir, .{
.enable_submanifests = cfg.enable,
.submanifest_k = cfg.k,
.submanifest_auto_retune = cfg.auto_retune,
.git_email_domain = if (cfg.git_email_domain) |*d| d.slice() else null,
});
}
/// Drives a modern-clone client (§3.8 CLONE_ROUND0 → EXCHANGE) to convergence
/// over `tr`, filling `ms` with the peer's artifacts; returns the round count.
/// Sans-IO on this side of `tr.exchange`: the state machine only moves
/// `[]const u8` bytes. Its closed error set is the driver's own causes unioned
/// with the transport's, so `runClone` switches the whole surface exhaustively.
/// The result of a clone run: the round count plus the origin's project code
/// (§8.2), copied out of the sans-IO client so the caller can persist the
/// shared identity after the client is torn down.
fn cloneHorizonResolved(o: *const CloneOutcome) ?zit.sync.HorizonResolved {
const steps = o.horizon_resolved_steps orelse return null;
return .{ .hash = o.horizon_resolved_buf[0..o.horizon_resolved_len], .steps = steps };
}
const CloneOutcome = struct {
rounds: usize,
/// Sparse-cone RFC section 6.3 -- how the peer answered a cone request.
cone_mode: ConeMode = .none,
/// The login the peer accepted from round 1 on: the derived secret, copied
/// out so the CLI can store it in the new repository once it exists. The
/// user borrows argv.
login: ?zit.sync.Credential = null,
uv_skipped: u64 = 0,
/// Uncompressed protocol bytes moved (`RunStats` totals), for the
/// end-of-clone accounting line.
sent_bytes: usize,
received_bytes: usize,
project_code_buf: [zit.sync.max_projectcode_len]u8,
project_code_len: usize,
/// Horizon draft section 6.2.6 -- the origin's cursor from the last round.
peer_cursor: ?u64 = null,
/// Horizon draft section 6.2.7 -- C as the origin resolved a horizon
/// name, the hash in `horizon_resolved_buf`; null steps when no name was
/// asked, or the peer did not answer it. Read through `cloneHorizonResolved`:
/// the outcome is returned by value, so it holds no slice into itself.
horizon_resolved_steps: ?u32 = null,
/// The client met a `horizon-resolved` it must not accept.
horizon_resolved_fault: bool = false,
horizon_resolved_len: usize = 0,
horizon_resolved_buf: [64]u8 = undefined,
/// Horizon draft section 6.2.5 -- what the origin said it served, for the
/// end-of-clone line; null against a horizon-ignorant peer.
horizon_served: ?zit.sync.HorizonLevel = null,
horizon_url: HorizonUrl = .{},
/// Horizon draft section 6.2.3 -- the seed the origin sent, owned by the
/// outcome (moved out of the client), for the sidecar beside the cone.
seed: ?[]u8 = null,
/// submanifests §6.3/§8.1 over sync §9.3 — the origin's three project-scope
/// settings, learned from the cleanup round's config card. Copied out of the
/// sans-IO client for the same reason the project code is: the client is
/// torn down before the caller persists anything. Owns no memory (three
/// optional scalars and a by-value Git email domain), so no buffer twin is
/// needed.
submanifest: zit.sync.SubmanifestConfig,
/// §9.3 r28 — the origin's shun table, learned from the same cleanup round.
/// Unlike the settings above this OWNS memory (a row count the wire
/// decides), so `deinit` is not optional: the client that parsed these rows
/// is torn down before the caller writes anything durable.
///
/// Carrying the shun table is not a nicety. fossil's shun table is its
/// takedown mechanism; a clone that drops it will, on its next sync with
/// any third party, re-accept exactly the content an operator was ordered
/// to remove.
shun_rows: []zit.sync.OwnedShunRow = &.{},
/// The origin's pinned `erasure-key`, adopted at clone and never by an
/// ordinary pull: trust on first use, from the origin the operator chose.
erasure_key: ?zit.erasure.PublicKey = null,
fn projectCode(self: *const CloneOutcome) []const u8 {
return self.project_code_buf[0..self.project_code_len];
}
fn deinit(self: *CloneOutcome, gpa: std.mem.Allocator) void {
if (self.seed) |b| gpa.free(b);
self.seed = null;
for (self.shun_rows) |row| {
gpa.free(row.hash);
gpa.free(row.reason);
}
gpa.free(self.shun_rows);
self.shun_rows = &.{};
}
};
/// Map a Cone.init refusal for clone. Same wording discipline as the other
/// verbs' mappers.
/// Whether `store` holds `hash` as a check-in manifest.
fn holdsCheckin(arena: std.mem.Allocator, store: repostate.ArtifactStore, hash: []const u8) bool {
const rid = store.lookup(hash) orelse return false;
if (!store.held(rid)) return false;
const plain = store.content(rid) orelse return false;
if (!zit.structgraph.looksStructural(plain)) return false;
const im = zit.artifact.parseForImport(arena, plain) catch return false;
return im.manifest.kind == .manifest;
}
/// Horizon draft section 6.2.7 -- a horizon name resolved over a transfer
/// that holds the whole repository, as a peer ignorant of the name serves it.
/// The derivation is built beside `store`, which is left unchanged; the hash
/// is allocated from `arena`. Null when NAME names no check-in.
fn resolveHorizonLocally(gpa: std.mem.Allocator, arena: std.mem.Allocator, store: repostate.ArtifactStore, name: []const u8, depth: u32) error{OutOfMemory}!?zit.sync.HorizonResolved {
var scratch = repostate.MemStore.init(gpa);
defer scratch.deinit();
var state = repostate.RepoState.init(gpa, scratch.store());
defer state.deinit();
var parents: std.ArrayListUnmanaged([]const u8) = .empty;
defer parents.deinit(gpa);
var tags: std.ArrayListUnmanaged(repostate.TagOp) = .empty;
defer tags.deinit(gpa);
const n = store.count();
var rid: repostate.Rid = 1;
while (rid <= n) : (rid += 1) {
if (!store.held(rid)) continue;
const plain = store.content(rid) orelse continue;
if (!zit.structgraph.looksStructural(plain)) continue;
var im = artifact.parseForImport(gpa, plain) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => continue,
};
defer im.deinit(gpa);
const m = &im.manifest;
if (repostate.sweepDisposition(m.kind) != .crosslink) continue;
if (zit.sync.isSeed(m)) continue; // never applied, as at every derive
const d = m.d orelse continue;
const mtime = repostate.parseMtime(d.timestamp) orelse continue;
tags.clearRetainingCapacity();
for (m.t) |tc| try tags.append(gpa, .{
.type = switch (tc.sign) {
.add => .singleton,
.cancel => .cancel,
.propagate => .propagate,
},
.name = tc.name,
.target = tc.target,
.value = tc.value,
});
const hash = store.hashOf(rid);
if (m.kind == .manifest) {
parents.clearRetainingCapacity();
for (m.p) |pc| {
var it = pc.hashes();
while (it.next()) |p| try parents.append(gpa, p);
}
_ = try state.crosslink(.{ .hash = hash, .mtime = mtime, .parents = parents.items, .tags = tags.items });
} else {
_ = try state.ingestControlTags(hash, mtime, tags.items);
}
}
const pick = state.horizonCheckin(name, depth) catch |e| switch (e) {
error.NotFound => return null,
error.OutOfMemory => return error.OutOfMemory,
};
return .{ .hash = try arena.dupe(u8, scratch.store().hashOf(pick.rid)), .steps = pick.steps };
}
/// What a clone made at DIR, so a clone that fails before its log commits
/// leaves DIR as it found it. A clone that committed keeps what it wrote,
/// even when it then reports a fault: that store is progress, not debris.
const CloneMade = struct {
dir: []const u8 = "",
dir_existed: bool = false,
/// The `.zit`-named entries DIR already held; never removed.
prior: []const []const u8 = &.{},
initialized: bool = false,
committed: bool = false,
/// DIR could not be read in full before the clone, so what it held is
/// unknown and a failure removes nothing.
unsure: bool = false,
};
/// Every name a clone writes into DIR is `.zit` or starts `.zit-`, so only
/// such entries are candidates for removal from a directory that already
/// existed; a user's `.zitignore` never is.
fn isCloneDebris(name: []const u8) bool {
return std.mem.eql(u8, name, ".zit") or std.mem.startsWith(u8, name, ".zit-");
}
fn cloneMadeBefore(arena: std.mem.Allocator, io: std.Io, dir: []const u8) error{OutOfMemory}!CloneMade {
// Only a DIR that is plainly absent counts as the clone's to remove; any
// other failure to read it leaves its contents unknown.
var d = std.Io.Dir.cwd().openDir(io, dir, .{ .iterate = true }) catch |e| return switch (e) {
error.FileNotFound => .{ .dir = dir },
else => .{ .dir = dir, .dir_existed = true, .unsure = true },
};
defer d.close(io);
var prior: std.ArrayListUnmanaged([]const u8) = .empty;
var it = d.iterate();
while (true) {
const e = (it.next(io) catch return .{ .dir = dir, .dir_existed = true, .unsure = true }) orelse break;
if (isCloneDebris(e.name)) try prior.append(arena, try arena.dupe(u8, e.name));
}
return .{ .dir = dir, .dir_existed = true, .prior = prior.items };
}
/// Removes what a failed clone made: DIR itself when the clone created it,
/// else only the `.zit`-named entries that were not there before.
fn abandonClone(arena: std.mem.Allocator, io: std.Io, made: *const CloneMade) void {
const cwd = std.Io.Dir.cwd();
if (made.unsure) return;
if (!made.dir_existed) {
cwd.deleteTree(io, made.dir) catch {};
return;
}
var d = cwd.openDir(io, made.dir, .{ .iterate = true }) catch return;
defer d.close(io);
// Collected first: removing entries while iterating a directory is
// unspecified on some filesystems.
var made_here: std.ArrayListUnmanaged([]const u8) = .empty;
var it = d.iterate();
scan: while (it.next(io) catch null) |e| {
if (!isCloneDebris(e.name)) continue;
for (made.prior) |p| if (std.mem.eql(u8, p, e.name)) continue :scan;
const name = arena.dupe(u8, e.name) catch break;
made_here.append(arena, name) catch break;
}
for (made_here.items) |name| d.deleteTree(io, name) catch {};
}
fn cloneConeFail(s: cli.Streams, err: zit.cone.Error) cli.Exit {
switch (err) {
error.OutOfMemory => s.err.writeAll("zit clone: out of memory\n") catch {},
error.InvalidMember => s.err.writeAll(
"zit clone: invalid --cone member: members are repository-relative paths using '/', with no '.', '..', or empty components\n",
) catch {},
error.ReservedMember => s.err.writeAll("zit clone: a --cone member names a reserved control path\n") catch {},
error.Bounds => s.err.writeAll("zit clone: --cone needs between 1 and 255 members (exclusions included), at least one inclusion\n") catch {},
error.DeadExclusion => s.err.writeAll("zit clone: a --cone-not member no --cone member covers is a dead rule\n") catch {},
error.Contradiction => s.err.writeAll("zit clone: one path given as both --cone and --cone-not\n") catch {},
}
return .failure;
}
/// Set from `$ZIT_CLONE_BATCHED` at the one environment seam in `main`: a
/// cone clone keeps the batched local-filter path even from an origin that
/// filtered for it. The differential test compares the two stores through it.
pub var clone_batched_persist: bool = false;
/// The `RoundDrain` every clone runs, around the one that persists its
/// rounds: before a round's artifacts reach the log it appends the origin's
/// project code, learned from the round-0 reply's `push` card (sync §8.2,
/// §9.1). The log is append-only, so a clone cut at any point holds content
/// only under the origin's code, and the `zit pull` that finishes it is not
/// refused as another project (sync §8.5 L1688-1689).
const CloneIdentityDrain = struct {
gpa: std.mem.Allocator,
log: *logstore.LogStore,
inner: sync_driver.RoundDrain,
/// Set by `cloneRun` for the length of the exchange.
client: ?*const zit.sync.Client = null,
written: bool = false,
fn roundDrain(self: *CloneIdentityDrain) sync_driver.RoundDrain {
return .{ .ctx = self, .run = run };
}
fn run(ctx: *anyopaque) sync_driver.DrainError!void {
const self: *CloneIdentityDrain = @ptrCast(@alignCast(ctx));
if (!self.written) {
const code = self.client.?.peerProjectCode();
if (code.len > 0) {
try updateConfigOpen(self.gpa, self.log, .{ .project_code = code });
self.written = true;
}
}
return self.inner.run(self.inner.ctx);
}
};
/// The `RoundDrain` of a cone-scoped clone. Sparse-cone RFC section 6.3 has
/// the clone filter locally, over the converged store, so against an origin
/// that did not filter every round is held (containers only; each round's
/// plaintext dropped) until the exchange ends. An origin whose first reply
/// says it filtered for this very request -- `horizon-served` at the depth
/// asked for, and `cone-served` when the request names paths -- sent what
/// the cone keeps and nothing else, so its rounds stream to the log as a
/// dense clone's do, and the classification runs over the log afterwards.
const ConeClonePersist = struct {
ms: *repostate.MemStore,
pass: *PersistPass,
horizon: ?zit.cone.HorizonDepth,
/// The request's cone members; `cone-served` acknowledges only a
/// request that names one.
members: usize,
not_members: usize,
/// Set by `cloneRun` for the length of the exchange.
client: ?*const zit.sync.Client = null,
/// Decided once the first reply is folded in; null until then.
streaming: ?bool = null,
/// Whether the round being drained was asked for past the clone
/// sequence: the client's cursor read zero when the last drain ended.
past_sequence: bool = false,
fn drainErased(ctx: *anyopaque) sync_driver.DrainError!void {
const self: *ConeClonePersist = @ptrCast(@alignCast(ctx));
if (self.streaming == null) {
self.streaming = self.filteredByOrigin();
// Held until the persist and then dropped whole: pooled, that
// memory goes back whole before the cache derive. A streamed
// clone releases each round's bytes as it writes them, which a
// pool would keep.
if (!self.streaming.?) try self.ms.poolDurable();
}
defer self.past_sequence = self.client.?.clone_seqno == 0;
if (self.streaming.?) {
self.pass.hold = self.past_sequence;
return self.pass.drainNew();
}
// A later read inflates again from the container.
self.ms.releasePlaintext();
}
fn filteredByOrigin(self: *const ConeClonePersist) bool {
if (clone_batched_persist) return false;
const client = self.client.?;
if (self.horizon) |depth| {
const served = client.horizon_served orelse return false;
// A structural request served at content falls back after the
// exchange (horizon draft section 6.3): the batched path decides.
if (!std.mem.eql(u8, @tagName(served), @tagName(depth))) return false;
} else if (self.members == 0) return false; // nothing to acknowledge
if (self.members + self.not_members == 0) return true;
// Path rules the origin may not have applied: section 6.1 asks it
// to, and only an acknowledged reply says it did.
return self.members != 0 and client.cone_replies_unacknowledged == 0 and client.cone_replies_served != 0;
}
};
fn cloneRun(
gpa: std.mem.Allocator,
tr: *transport.Transport,
ms: *repostate.MemStore,
self_servercode: []const u8,
login: ?zit.sync.PendingLogin,
sink: ?sync_driver.RoundSink,
drain: *CloneIdentityDrain,
cone_members: []const []const u8,
cone_not_members: []const []const u8,
cone_parents: bool,
horizon: ?zit.cone.Horizon,
horizon_name: ?zit.sync.HorizonName,
cone_persist: ?*ConeClonePersist,
peer_error_out: *PeerErrorCopy,
truncated_out: *u64,
) sync_driver.RunError(*transport.Transport)!CloneOutcome {
var client = zit.sync.Client{
.state = zit.sync.initialState(.clone),
.operation = .clone,
.self_servercode = self_servercode,
.project_code = sync_projectcode,
.version = sync_client_version_info,
.pending_login = login,
// Sparse-cone RFC section 6.1: request server-side filtering; the
// local persist filter stays as the section 6.3 belt regardless.
.cone_members = cone_members,
.cone_not_members = cone_not_members,
.cone_parents = cone_parents,
.horizon = horizon,
.horizon_name = horizon_name,
};
defer client.deinit(gpa);
// Read on every exit, the error ones included: the cut-off replies are
// reported whether or not the run went on to fail.
defer truncated_out.* = client.payloads_truncated;
if (cone_persist) |cp| cp.client = &client;
defer if (cone_persist) |cp| {
cp.client = null;
};
drain.client = &client;
defer drain.client = null;
const stats = sync_driver.runSyncProgress(gpa, &client, ms.store(), tr, max_stalled_sync_rounds, sink, drain.roundDrain()) catch |e| {
// A refusing peer's words outlive the aborted client (bounded copy)
// so the caller renders them instead of a generic protocol error.
peer_error_out.set(client.peerErrorText());
return e;
};
// §8.2 — the peer's project code, learned from the clone reply's `push`
// card, copied out before the client is deinitialized.
var outcome = CloneOutcome{
.rounds = stats.rounds,
.uv_skipped = client.uv_skipped,
.sent_bytes = stats.sent_bytes,
.received_bytes = stats.received_bytes,
.project_code_buf = undefined,
.project_code_len = 0,
// §9.3 — the origin's sub-manifest settings, latched from the config
// card the §8.4 cleanup round asked for.
.submanifest = client.peerSubmanifestConfig(),
.horizon_served = client.horizon_served,
.horizon_url = .of(&client),
.seed = client.seed_bytes,
.peer_cursor = client.peer_cursor,
.cone_mode = coneMode(&client),
};
outcome.horizon_resolved_fault = client.horizon_resolved_fault;
if (client.horizon_resolved) |h| {
@memcpy(outcome.horizon_resolved_buf[0..h.len], h);
outcome.horizon_resolved_len = h.len;
outcome.horizon_resolved_steps = client.horizon_steps.?;
}
client.seed_bytes = null; // moved to the outcome
const pc = client.peerProjectCode();
@memcpy(outcome.project_code_buf[0..pc.len], pc);
outcome.project_code_len = pc.len;
outcome.login = client.credential;
outcome.erasure_key = client.peer_erasure_key;
// §9.3 r28 — copy the latched shun rows out of the client, which the
// `defer` above tears down on return.
const latched = client.peerShunRows();
if (latched.len != 0) {
var rows = try gpa.alloc(zit.sync.OwnedShunRow, latched.len);
var filled: usize = 0;
errdefer {
for (rows[0..filled]) |r| {
gpa.free(r.hash);
gpa.free(r.reason);
}
gpa.free(rows);
}
for (latched, rows) |src_row, *dst| {
const h = try gpa.dupe(u8, src_row.hash);
errdefer gpa.free(h);
const r = try gpa.dupe(u8, src_row.reason);
dst.* = .{ .hash = h, .epoch = src_row.epoch, .reason = r };
filled += 1;
}
outcome.shun_rows = rows;
}
return outcome;
}
test "sync clone persistence: open-log artifacts and one combined config overlay preserve the wrapper result" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var wrapped_dir = std.testing.tmpDir(.{});
defer wrapped_dir.cleanup();
var open_dir = std.testing.tmpDir(.{});
defer open_dir.cleanup();
try d4SeedConfiguredRepo(gpa, io, wrapped_dir.dir);
try d4SeedConfiguredRepo(gpa, io, open_dir.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const content = "one content-bearing clone artifact";
var name_buf: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, content, &name_buf);
const container = try zit.container.encode(gpa, content, .default);
defer gpa.free(container);
const rid = try ms.store().put(name, content);
try ms.store().retain(rid, .{ .full = container });
const patch: ConfigPatch = .{
.project_code = "origin-code",
.enable_submanifests = true,
.submanifest_k = 5,
.submanifest_auto_retune = true,
};
const wrapped = try persistStore(gpa, io, wrapped_dir.dir, ms.store(), null, 0);
try updateConfig(gpa, io, wrapped_dir.dir, patch);
var open_fs: storage.FileStorage = undefined;
var open_log = try d4OpenLog(gpa, io, open_dir.dir, &open_fs);
defer open_fs.file.close(io);
defer open_log.deinit(gpa);
const opened = try persistStoreOpen(gpa, &open_log, ms.store(), null, 0, null, null);
try updateConfigOpen(gpa, &open_log, patch);
try open_log.commit();
try std.testing.expectEqual(wrapped.written, opened.written);
try std.testing.expectEqual(wrapped.submanifest_gaps, opened.submanifest_gaps);
try std.testing.expectEqual(@as(u64, 3), (try open_log.logKey()).frame_count);
var wrapped_fs: storage.FileStorage = undefined;
var wrapped_log = try d4OpenLog(gpa, io, wrapped_dir.dir, &wrapped_fs);
defer wrapped_fs.file.close(io);
defer wrapped_log.deinit(gpa);
var wrapped_cfg = (try wrapped_log.readConfig(gpa)).?;
defer wrapped_cfg.deinit(gpa);
var opened_cfg = (try open_log.readConfig(gpa)).?;
defer opened_cfg.deinit(gpa);
try std.testing.expectEqualDeep(wrapped_cfg.config, opened_cfg.config);
try std.testing.expectEqualStrings("origin-code", opened_cfg.config.project_code);
try std.testing.expectEqualStrings("operator", opened_cfg.config.login_user);
try std.testing.expectEqualStrings("https://example.test/remote", opened_cfg.config.remote_url);
try std.testing.expect(opened_cfg.config.seen_submanifest);
try std.testing.expect(opened_cfg.config.enable_submanifests);
try std.testing.expectEqual(@as(u8, 5), opened_cfg.config.submanifest_k);
try std.testing.expect(opened_cfg.config.submanifest_auto_retune);
try std.testing.expectEqualDeep(@as([32]u8, @splat(0x42)), opened_cfg.config.submanifest_settled_cost_shape.?);
var digest_buf: [digest_len]u8 = undefined;
const stored = (try open_log.content(gpa, open_log.lookup(decodeWireName(name, &digest_buf).?).?)).?;
defer gpa.free(stored);
try std.testing.expectEqualStrings(content, stored);
}
/// An origin that predates the crossing-delta rule (sparse-cone section 6.2),
/// as every release before horizons is: it acknowledges a path cone, serves a
/// kept blob as its stored delta against a base the cone elides, and answers
/// the clone's gimme for that base. The serve path itself would send the
/// blob in full, so the store hides the blob's plaintext from it (the
/// fallback to the verbatim delta) and the base is appended by hand.
const PreCrossingOrigin = struct {
server: zit.sync.Server,
inner: repostate.ArtifactStore,
/// The kept blob the serve path must not reconstruct.
hidden: repostate.Rid,
base_hash: []const u8,
base: []const u8,
fn peer(self: *PreCrossingOrigin) transport.InProcessPeer {
return .{ .ctx = self, .exchangeFn = exchange };
}
fn store(self: *PreCrossingOrigin) repostate.ArtifactStore {
return .{ .context = self, .vtable = &vtable };
}
fn exchange(ctx: *anyopaque, gpa: std.mem.Allocator, framed: []const u8) transport.TransportError!transport.InProcessReply {
const self: *PreCrossingOrigin = @ptrCast(@alignCast(ctx));
const request = transport.unframe(gpa, framed) catch return error.ProtocolError;
defer gpa.free(request);
var out: std.ArrayListUnmanaged(u8) = .empty;
errdefer out.deinit(gpa);
_ = self.server.respond(gpa, self.store(), request, &out) catch return error.ProtocolError;
var gimme_buf: [80]u8 = undefined;
const gimme = std.fmt.bufPrint(&gimme_buf, "gimme {s}\n", .{self.base_hash}) catch unreachable;
if (std.mem.indexOf(u8, request, gimme) != null) {
var head_buf: [96]u8 = undefined;
const head = std.fmt.bufPrint(&head_buf, "file {s} {d}\n", .{ self.base_hash, self.base.len }) catch unreachable;
try out.appendSlice(gpa, head);
try out.appendSlice(gpa, self.base);
}
return .{ .body = try out.toOwnedSlice(gpa), .content_type = "application/x-fossil-uncompressed" };
}
const vtable: repostate.ArtifactStore.VTable = .{
.intern = intern,
.put = put,
.content = content,
.hashOf = hashOf,
.lookup = lookup,
.count = count,
.wire = wire,
.retain = retain,
.held = held,
};
fn of(ctx: *anyopaque) *PreCrossingOrigin {
return @ptrCast(@alignCast(ctx));
}
fn intern(ctx: *anyopaque, hash: []const u8) error{OutOfMemory}!repostate.Rid {
return of(ctx).inner.intern(hash);
}
fn put(ctx: *anyopaque, hash: []const u8, bytes: []const u8) error{OutOfMemory}!repostate.Rid {
return of(ctx).inner.put(hash, bytes);
}
fn content(ctx: *anyopaque, rid: repostate.Rid) ?[]const u8 {
if (rid == of(ctx).hidden) return null;
return of(ctx).inner.content(rid);
}
fn hashOf(ctx: *anyopaque, rid: repostate.Rid) []const u8 {
return of(ctx).inner.hashOf(rid);
}
fn lookup(ctx: *anyopaque, hash: []const u8) ?repostate.Rid {
return of(ctx).inner.lookup(hash);
}
fn count(ctx: *anyopaque) usize {
return of(ctx).inner.count();
}
fn wire(ctx: *anyopaque, rid: repostate.Rid) ?repostate.WireFrame {
return of(ctx).inner.wire(rid);
}
fn retain(ctx: *anyopaque, rid: repostate.Rid, ingest: repostate.WireIngest) error{OutOfMemory}!void {
return of(ctx).inner.retain(rid, ingest);
}
fn held(ctx: *anyopaque, rid: repostate.Rid) bool {
return of(ctx).inner.held(rid);
}
};
/// The classification such an origin answers with: a fixed keep set.
const FixedConeClassifier = struct {
cls: *const SparseClassification,
fn classifier(self: *FixedConeClassifier) zit.sync.ConeClassifier {
return .{ .context = self, .vtable = &.{ .classify = classify } };
}
fn classify(context: *anyopaque, _: []const []const u8, _: []const []const u8, _: bool, _: ?zit.cone.Horizon) ?*const SparseClassification {
const self: *FixedConeClassifier = @ptrCast(@alignCast(context));
return self.cls;
}
};
test "clone --cone from an origin that ships a crossing delta keeps the delta's blob and drops its out-of-cone base" {
// Goal: the clone's own cone decides what it stores, whichever origin it
// cloned from (sparse-cone RFC section 6.3). An acknowledging origin from
// before the crossing-delta rule sends a kept blob as a delta against an
// elided base, and the clone has to fetch that base to materialize the
// blob; the base is transit, as the batched local filter has always
// treated it, and persisting it would hold an artifact the store's cone
// elides. Method: one check-in naming src/a (kept) and doc/b (elided),
// src/a stored as a delta against doc/b, cloned with --cone=src through
// an in-process origin; pin the log's holdings and the report.
const gpa = std.testing.allocator;
const io = std.testing.io;
const base = "doc/b: a shared run long enough that src/a is plainly a compact forward delta of it\n";
const target = base ++ "src/a's own tail\n";
var base_buf: [64]u8 = undefined;
const base_hash = zit.artifact.artifactName(.sha3_256, base, &base_buf);
var target_buf: [64]u8 = undefined;
const target_hash = zit.artifact.artifactName(.sha3_256, target, &target_buf);
var origin_ms = repostate.MemStore.init(gpa);
defer origin_ms.deinit();
const os = origin_ms.store();
const base_rid = try os.put(base_hash, base);
const base_container = try zit.container.encode(gpa, base, .default);
defer gpa.free(base_container);
try os.retain(base_rid, .{ .full = base_container });
const raw = try zit.delta.create(gpa, base, target);
defer gpa.free(raw);
const delta_container = try zit.container.encode(gpa, raw, .default);
defer gpa.free(delta_container);
const target_rid = try os.put(target_hash, target);
try os.retain(target_rid, .{ .delta = .{ .src = base_hash, .container = delta_container, .target_size = target.len } });
const files = [_]artifact.FCard{
.{ .filename = "doc/b", .hash = base_hash, .perm = null, .old_name = null, .mode = .regular },
.{ .filename = "src/a", .hash = target_hash, .perm = null, .old_name = null, .mode = .regular },
};
var m = artifact.Manifest{
.arena_state = std.heap.ArenaAllocator.init(gpa),
.kind = .manifest,
.a = null,
.b = null,
.c = .{ .comment = "one" },
.d = .{ .timestamp = "2026-10-23T12:00:00.000" },
.e = null,
.g = null,
.h = null,
.i = null,
.k = null,
.l = null,
.n = null,
.r = null,
.u = .{ .login = "u" },
.z = null,
.w = null,
.f = &files,
.j = &.{},
.m = &.{},
.p = &.{},
.q = &.{},
.t = &.{},
};
defer m.deinit();
const manifest = try artifact.create(&m, gpa);
defer gpa.free(manifest);
var manifest_buf: [64]u8 = undefined;
const manifest_hash = zit.artifact.artifactName(.sha3_256, manifest, &manifest_buf);
const manifest_rid = try os.put(manifest_hash, manifest);
const manifest_container = try zit.container.encode(gpa, manifest, .default);
defer gpa.free(manifest_container);
try os.retain(manifest_rid, .{ .full = manifest_container });
// The origin's cone keeps the check-in and src/a, never doc/b.
var keep = try std.DynamicBitSetUnmanaged.initEmpty(gpa, os.count() + 1);
keep.set(manifest_rid);
keep.set(target_rid);
var cls: SparseClassification = .{ .keep = keep, .out_only = try std.DynamicBitSetUnmanaged.initEmpty(gpa, keep.capacity()), .arena = std.heap.ArenaAllocator.init(gpa) };
defer cls.deinit(gpa);
var fixed: FixedConeClassifier = .{ .cls = &cls };
const serve_code = zit.sync.deriveServerCode(&.{"pre-crossing-origin"});
var origin: PreCrossingOrigin = .{
.server = .{
.version = serve_mod.server_version,
.self_servercode = &serve_code,
.project_code = "8f6c1d0e2b3a4f5e6d7c8b9a0f1e2d3c4b5a6978",
.cone_classifier = fixed.classifier(),
},
.inner = os,
.hidden = target_rid,
.base_hash = base_hash,
.base = base,
};
transport.in_process_peer = origin.peer();
defer transport.in_process_peer = null;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
const root = try tmp.dir.realPathFileAlloc(io, ".", arena);
const dir = try std.fs.path.join(arena, &.{ root, "clone" });
var out: std.Io.Writer.Allocating = .init(gpa);
defer out.deinit();
var err: std.Io.Writer.Allocating = .init(gpa);
defer err.deinit();
const streams: cli.Streams = .{ .out = &out.writer, .err = &err.writer };
const exit = runClone(gpa, io, arena, &.{ "--no-open", "--once", "--cone=src", "http://127.0.0.1:9/", dir }, streams, .{ .now_epoch = 1_800_000_000, .user_env = "test" });
if (exit != .ok) std.debug.print("clone stderr:\n{s}\n", .{err.written()});
try std.testing.expectEqual(cli.Exit.ok, exit);
const reported = std.mem.indexOf(u8, out.written(), "(1 elided by the cone; applied by the peer)") != null;
if (!reported) std.debug.print("clone stdout:\n{s}\n", .{out.written()});
try std.testing.expect(reported);
var clone_dir = try tmp.dir.openDir(io, "clone", .{});
defer clone_dir.close(io);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, clone_dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var digest_buf: [digest_len]u8 = undefined;
if (log.lookup(decodeWireName(base_hash, &digest_buf).?)) |rid| try std.testing.expect(log.isPhantom(rid));
const stored = (try log.content(gpa, log.lookup(decodeWireName(target_hash, &digest_buf).?).?)).?;
defer gpa.free(stored);
try std.testing.expectEqualStrings(target, stored);
}
/// An in-process origin answering every round from one `Server` over a
/// held store, as `zit serve` would.
const PlainOrigin = struct {
server: zit.sync.Server,
store: repostate.ArtifactStore,
fn peer(self: *PlainOrigin) transport.InProcessPeer {
return .{ .ctx = self, .exchangeFn = exchange };
}
fn exchange(ctx: *anyopaque, gpa: std.mem.Allocator, framed: []const u8) transport.TransportError!transport.InProcessReply {
const self: *PlainOrigin = @ptrCast(@alignCast(ctx));
const request = transport.unframe(gpa, framed) catch return error.ProtocolError;
defer gpa.free(request);
var out: std.ArrayListUnmanaged(u8) = .empty;
errdefer out.deinit(gpa);
_ = self.server.respond(gpa, self.store, request, &out) catch return error.ProtocolError;
return .{ .body = try out.toOwnedSlice(gpa), .content_type = "application/x-fossil-uncompressed" };
}
};
test "sync §8.2/§9.1: a clone cut after its first artifact already carries the origin's project code" {
// A clone writes its artifacts round by round, so a crash can leave a
// store holding some of them. Holding anything from the origin, it must
// hold the origin's project code too, or the `zit pull` that would finish
// it is refused as another project. Method: clone from an in-process
// origin, then read the store as a crash just after its first artifact
// frame would have left it.
const gpa = std.testing.allocator;
const io = std.testing.io;
const origin_code = "8f6c1d0e2b3a4f5e6d7c8b9a0f1e2d3c4b5a6978";
var origin_ms = repostate.MemStore.init(gpa);
defer origin_ms.deinit();
const os = origin_ms.store();
for ([_][]const u8{ "first origin artifact\n", "second origin artifact\n", "third origin artifact\n" }) |content| {
var name_buf: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, content, &name_buf);
const rid = try os.put(name, content);
const container = try zit.container.encode(gpa, content, .default);
defer gpa.free(container);
try os.retain(rid, .{ .full = container });
}
const serve_code = zit.sync.deriveServerCode(&.{"plain-origin"});
var origin: PlainOrigin = .{
.server = .{ .version = serve_mod.server_version, .self_servercode = &serve_code, .project_code = origin_code },
.store = os,
};
transport.in_process_peer = origin.peer();
defer transport.in_process_peer = null;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
const root = try tmp.dir.realPathFileAlloc(io, ".", arena);
const dir = try std.fs.path.join(arena, &.{ root, "clone" });
var out: std.Io.Writer.Allocating = .init(gpa);
defer out.deinit();
var err: std.Io.Writer.Allocating = .init(gpa);
defer err.deinit();
const streams: cli.Streams = .{ .out = &out.writer, .err = &err.writer };
const exit = runClone(gpa, io, arena, &.{ "--no-open", "--once", "http://127.0.0.1:9/", dir }, streams, .{ .now_epoch = 1_800_000_000, .user_env = "test" });
if (exit != .ok) std.debug.print("clone stderr:\n{s}\n", .{err.written()});
try std.testing.expectEqual(cli.Exit.ok, exit);
var clone_dir = try tmp.dir.openDir(io, "clone", .{});
defer clone_dir.close(io);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, clone_dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var first_end: ?u64 = null;
var rid: logstore.Rid = 1;
while (rid <= log.entryCount()) : (rid += 1) {
if (log.isPhantom(rid)) continue;
const end = log.offsetOf(rid) + log.frameSpanOf(rid);
if (first_end == null or end < first_end.?) first_end = end;
}
// The store as a crash right after that frame leaves it.
var cut_fs = storage.FileStorage.initReadOnly(io, fs.file, first_end.?);
var cut = try logstore.LogStore.openReadOnly(gpa, cut_fs.storageFile());
defer cut.deinit(gpa);
try std.testing.expect(cut.entryCount() >= 1);
var cfg = (try cut.readConfig(gpa)).?;
defer cfg.deinit(gpa);
try std.testing.expectEqualStrings(origin_code, cfg.config.project_code);
}
test "sync clone config: absent origin settings append no frame and preserve init values" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedConfiguredRepo(gpa, io, tmp.dir);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
const patch: ConfigPatch = .{};
try std.testing.expect(!patch.any());
if (patch.any()) try updateConfigOpen(gpa, &log, patch);
try log.commit();
try std.testing.expectEqual(@as(u64, 1), (try log.logKey()).frame_count);
var cfg = (try log.readConfig(gpa)).?;
defer cfg.deinit(gpa);
try std.testing.expectEqual(@as(u8, 9), cfg.config.hash_policy);
try std.testing.expectEqualStrings("stable", cfg.config.default_branch);
try std.testing.expectEqualStrings("project", cfg.config.project_id);
try std.testing.expectEqualStrings("init-code", cfg.config.project_code);
try std.testing.expectEqualStrings("operator", cfg.config.login_user);
try std.testing.expectEqualStrings("https://example.test/remote", cfg.config.remote_url);
try std.testing.expect(cfg.config.seen_submanifest);
try std.testing.expect(!cfg.config.enable_submanifests);
try std.testing.expectEqual(@as(u8, 3), cfg.config.submanifest_k);
try std.testing.expect(!cfg.config.submanifest_auto_retune);
try std.testing.expectEqualDeep(@as([32]u8, @splat(0x42)), cfg.config.submanifest_settled_cost_shape.?);
}
test "streaming persist writes the byte-identical log and releases every persisted entry" {
const gpa = std.testing.allocator;
const io = std.testing.io;
// Two stores fed the same reverse-delta chain (older-as-delta-of-newer,
// fossil's wire shape), one persisted batched and one streamed in
// increments with release. The logs must match byte for byte — streaming
// reorders nothing — and the streamed store must end empty of payload:
// every entry held, none resident.
const links = 6;
var bodies: [links][]u8 = undefined;
var made: usize = 0;
defer for (bodies[0..made]) |b| gpa.free(b);
var names: [links][64]u8 = undefined;
var nlen: [links]usize = undefined;
for (0..links) |i| {
bodies[i] = try std.fmt.allocPrint(gpa, "streamed chain body: one long shared prefix kept across every revision -- revision {d}", .{i});
made += 1;
nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len;
}
const seed = struct {
fn one(a: std.mem.Allocator, s: repostate.ArtifactStore, i: usize, bs: *const [links][]u8, ns: *const [links][64]u8, nl: *const [links]usize) !void {
const rid = try s.put(ns[i][0..nl[i]], bs[i]);
if (i == links - 1) {
const c = try zit.container.encode(a, bs[i], .default);
defer a.free(c);
try s.retain(rid, .{ .full = c });
} else {
const dr = try zit.delta.create(a, bs[i + 1], bs[i]);
defer a.free(dr);
const c = try zit.container.encode(a, dr, .default);
defer a.free(c);
try s.retain(rid, .{ .delta = .{ .src = ns[i + 1][0..nl[i + 1]], .container = c, .target_size = bs[i].len } });
}
}
}.one;
// Batched reference.
var tmp_a = std.testing.tmpDir(.{});
defer tmp_a.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp_a.dir);
var ms_a = repostate.MemStore.init(gpa);
defer ms_a.deinit();
for (0..links) |i| try seed(gpa, ms_a.store(), i, &bodies, &names, &nlen);
{
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
const sum = try persistStoreOpen(gpa, &log, ms_a.store(), null, 0, null, null);
try std.testing.expectEqual(@as(usize, links), sum.written);
try log.commit();
}
// Streamed: seed in two "rounds", draining after each, releasing as it goes.
var tmp_b = std.testing.tmpDir(.{});
defer tmp_b.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp_b.dir);
var ms_b = repostate.MemStore.init(gpa);
defer ms_b.deinit();
{
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp_b.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var pass = try PersistPass.init(gpa, &log, ms_b.store(), null, &ms_b, 0, null);
defer pass.deinit();
for (0..3) |i| try seed(gpa, ms_b.store(), i, &bodies, &names, &nlen);
try pass.drainNew(); // rids 1..3 are deltas of unseen bases: all defer
for (3..links) |i| try seed(gpa, ms_b.store(), i, &bodies, &names, &nlen);
try pass.drainNew(); // the full form lands and the whole chain drains
const sum = try pass.finish();
try std.testing.expectEqual(@as(usize, links), sum.written);
try std.testing.expectEqual(@as(usize, 0), sum.submanifest_gaps);
try log.commit();
}
// Every streamed entry is now frame-backed and released: held, no bytes.
for (1..links + 1) |rid_us| {
const rid: repostate.Rid = @intCast(rid_us);
try std.testing.expect(ms_b.store().held(rid));
try std.testing.expect(ms_b.store().wire(rid) == null);
try std.testing.expect(ms_b.store().content(rid) == null);
}
// The two logs are byte-identical: streaming changed when frames were
// written, never what or in what order.
const a_bytes = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 20));
defer gpa.free(a_bytes);
const b_bytes = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 20));
defer gpa.free(b_bytes);
try std.testing.expectEqualSlices(u8, a_bytes, b_bytes);
}
test "a basis interned as a phantom before the cursor passed it is revisited and persisted" {
const gpa = std.testing.allocator;
const io = std.testing.io;
// The fossil-peer streaming shape: a delta cfile arrives naming an unseen
// DELTASRC, which interns the basis as a PHANTOM at a low rid; the basis
// content lands at that same rid in a later round, behind the drain's
// cursor. The pass must come back for it — a drain that only moves forward
// leaves the basis unwritten and its waiters unresolvable.
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s_store = ms.store();
const base_body = "phantom-first basis body: the long shared prefix every revision keeps\n";
const delta_body = "phantom-first basis body: the long shared prefix every revision keeps, edited\n";
var base_name: [64]u8 = undefined;
const bn = zit.artifact.artifactName(.sha3_256, base_body, &base_name).len;
var delta_name: [64]u8 = undefined;
const dn = zit.artifact.artifactName(.sha3_256, delta_body, &delta_name).len;
// Round 1: the basis is interned as a phantom (rid 1), then the delta
// arrives as content (rid 2).
const base_rid = try s_store.intern(base_name[0..bn]);
const delta_rid = try s_store.put(delta_name[0..dn], delta_body);
{
const dr = try zit.delta.create(gpa, base_body, delta_body);
defer gpa.free(dr);
const c = try zit.container.encode(gpa, dr, .default);
defer gpa.free(c);
try s_store.retain(delta_rid, .{ .delta = .{ .src = base_name[0..bn], .container = c, .target_size = delta_body.len } });
}
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var pass = try PersistPass.init(gpa, &log, s_store, null, &ms, 0, null);
defer pass.deinit();
try pass.drainNew(); // cursor passes the phantom; the delta defers on it
// Round 2: the basis content lands at the SAME rid, behind the cursor.
{
const got = try s_store.put(base_name[0..bn], base_body);
try std.testing.expectEqual(base_rid, got);
const c = try zit.container.encode(gpa, base_body, .default);
defer gpa.free(c);
try s_store.retain(base_rid, .{ .full = c });
}
try pass.drainNew(); // the revisit sweep drives it, unblocking the delta
const sum = try pass.finish();
try std.testing.expectEqual(@as(usize, 2), sum.written);
try log.commit();
// Both artifacts are durable frames, and both entries were released.
var d: [digest_len]u8 = undefined;
try std.testing.expect(log.lookup(decodeWireName(base_name[0..bn], &d).?) != null);
try std.testing.expect(log.lookup(decodeWireName(delta_name[0..dn], &d).?) != null);
try std.testing.expect(s_store.wire(base_rid) == null and s_store.held(base_rid));
try std.testing.expect(s_store.wire(delta_rid) == null and s_store.held(delta_rid));
}
/// Six versions of one file, each a wire delta of the one before, the first
/// full, in `ms`; `lie` makes the last delta rebuild bytes its name does not
/// hash. Returns the versions' names (owned by `arena`).
fn deferredChainFixture(gpa: std.mem.Allocator, arena: std.mem.Allocator, ms: *repostate.MemStore, lie: bool) ![]const []const u8 {
const store = ms.store();
var names: std.ArrayList([]const u8) = .empty;
var prev: ?[]const u8 = null;
var k: usize = 0;
while (k < 6) : (k += 1) {
var body: std.ArrayList(u8) = .empty;
var line: usize = 0;
while (line < 200) : (line += 1) try body.print(arena, "line {d} of a long file{s}\n", .{ line, if (line == k * 7) " edited" else "" });
var name_buf: [64]u8 = undefined;
const name = try arena.dupe(u8, zit.artifact.artifactName(.sha3_256, body.items, &name_buf));
const rid = try store.put(name, body.items);
if (prev) |p| {
const target = if (lie and k == 5) "not what the name hashes\n" else body.items;
const dr = try zit.delta.create(gpa, p, target);
defer gpa.free(dr);
const c = try zit.container.encode(gpa, dr, .default);
defer gpa.free(c);
try store.retain(rid, .{ .delta = .{ .src = names.items[k - 1], .container = c, .target_size = target.len } });
} else {
const c = try zit.container.encode(gpa, body.items, .default);
defer gpa.free(c);
try store.retain(rid, .{ .full = c });
}
try names.append(arena, name);
prev = body.items;
}
return names.items;
}
test "sync persist: a delta whose basis is not memoised is stored as it arrived and checked before the commit" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
try d4SeedConfiguredRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const names = try deferredChainFixture(gpa, arena_state.allocator(), &ms, false);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var pass = try PersistPass.init(gpa, &log, ms.store(), null, null, 0, null);
defer pass.deinit();
pass.cache.max_bytes = 1; // nothing is memoised: every basis misses
try pass.drainNew();
try std.testing.expectEqual(@as(usize, 5), pass.unverified.items.len);
const sum = try pass.finish();
try std.testing.expectEqual(@as(usize, 6), sum.written);
try std.testing.expectEqual(@as(usize, 0), pass.unverified.items.len);
try log.commit();
// Every version reads back as itself, and each stayed a delta.
var d: [digest_len]u8 = undefined;
for (names, 0..) |name, i| {
const rid = log.lookup(decodeWireName(name, &d).?).?;
const got = (try log.content(gpa, rid)).?;
defer gpa.free(got);
try std.testing.expect(zit.artifact.verify(name, got));
try std.testing.expectEqual(@as(u8, @intCast(i)), log.depthOf(rid));
}
}
test "sync persist: a deferred delta that rebuilds the wrong bytes fails the pass before the commit" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
try d4SeedConfiguredRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try deferredChainFixture(gpa, arena_state.allocator(), &ms, true);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var pass = try PersistPass.init(gpa, &log, ms.store(), null, null, 0, null);
defer pass.deinit();
pass.cache.max_bytes = 1;
try pass.drainNew();
try std.testing.expectError(error.Corrupt, pass.finish());
}
test "sync persistStoreOpen: malformed artifact retains the wrapper's corrupt error and writes no frame" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const content = "content that the wire name does not actually name\n";
const lying_name = repeat("0", 64);
const rid = try ms.store().put(lying_name, content);
const container = try zit.container.encode(gpa, content, .default);
defer gpa.free(container);
try ms.store().retain(rid, .{ .full = container });
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
try std.testing.expectError(error.Corrupt, persistStoreOpen(gpa, &log, ms.store(), null, 0, null, null));
try log.commit();
var digest_buf: [digest_len]u8 = undefined;
try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null);
}
test "sync §7.7: persistStore recomputes the artifact hash and aborts on a lying container, writing no frame" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// A received full cfile whose wire NAME does not name its content (§7.7 —
// a clone skips the transfer-layer pre-storage check, so a lying container
// reaches the persist commit hook).
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const content = "content that the wire name does not actually name\n";
const lying_name = repeat("0", 64); // not SHA3-256(content)
const rid = try s.put(lying_name, content);
const container = try zit.container.encode(gpa, content, .default);
defer gpa.free(container);
try s.retain(rid, .{ .full = container });
// §7.7 L1496-1498 — recompute and abort before ANY frame is written.
try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, ms.store(), null, 0));
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var digest_buf: [digest_len]u8 = undefined;
const digest = decodeWireName(lying_name, &digest_buf).?;
try std.testing.expect(log.lookup(digest) == null); // no frame landed
}
test "submanifests §7.1: the persist commit hook phantomizes every absent S reference" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Two leaves that were never received — the state a clone from a peer whose
// OWN closure is incomplete lands in. Nothing on the receive path can see
// this: a clone stores wire-backed and inflates nothing.
const leaf_a = "F a.txt " ++ (repeat("1", 64)) ++ "\n";
const leaf_b = "F b.txt " ++ (repeat("2", 64)) ++ "\n";
var an: [64]u8 = undefined;
var bn: [64]u8 = undefined;
const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an);
const name_b = zit.artifact.artifactName(.sha3_256, leaf_b, &bn);
const tree = try treeManifestBytes(gpa, &.{ name_a, name_b });
defer gpa.free(tree);
var tn: [64]u8 = undefined;
const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const rid = try s.put(tree_name, tree);
const container = try zit.container.encode(gpa, tree, .default);
defer gpa.free(container);
try s.retain(rid, .{ .full = container });
// §7.1: "Storing a structural artifact that carries S cards whose referenced
// sub-manifests are absent MUST create a phantom for each missing reference."
const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), summary.written);
try std.testing.expectEqual(@as(usize, 2), summary.submanifest_gaps);
}
test "persist scoped past the holdings writes nothing and re-walks only received" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
// Five full-form blobs (the already-durable holdings), named by true hash.
inline for (.{ "alpha", "bravo", "charlie", "delta", "echo" }) |body| {
var nm: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, body, &nm);
const rid = try s.put(name, body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
}
const first = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 5), first.written);
// A pull that received nothing: scope past all five holdings. The walk
// never begins, so nothing is persisted and nothing is re-visited (this
// was O(store) — every holding read once from disk to be discarded).
const noop = try persistStore(gpa, io, tmp.dir, ms.store(), null, 5);
try std.testing.expectEqual(@as(usize, 0), noop.written);
// Three artifacts arrive (rids 6..8); scope past the original five. Only the
// new three are persisted — a mirror's incremental pull no longer re-walks
// the millions it already holds.
inline for (.{ "foxtrot", "golf", "hotel" }) |body| {
var nm: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, body, &nm);
const rid = try s.put(name, body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
}
const incr = try persistStore(gpa, io, tmp.dir, ms.store(), null, 5);
try std.testing.expectEqual(@as(usize, 3), incr.written);
// A full re-persist (start 0) early-returns every artifact and reaches the
// same eight-artifact total, so scoping removed work, never content.
const full = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 8), full.written);
}
test "persist appends through the store's scan index, current or a prefix, and every reader sees the result" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const bodies = [_][]const u8{ "alpha", "bravo", "charlie", "delta", "echo", "foxtrot" };
var names: [bodies.len][64]u8 = undefined;
for (bodies, &names) |body, *nm| _ = zit.artifact.artifactName(.sha3_256, body, nm);
for (bodies[0..2], names[0..2]) |body, *nm| {
const rid = try s.put(nm, body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
}
_ = try persistStore(gpa, io, tmp.dir, s, null, 0);
// A current scan index, as any read verb leaves behind.
{
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
const blob = try log.serializeIndex(gpa);
defer gpa.free(blob);
zit.repocache.storeLogIndex(io, tmp.dir, blob);
}
// Two received over the current index, then two more over what is now
// only a prefix of the log. Neither persist walks the log's frames.
const scans_before = zit.receive.persist_append_scans;
var start: repostate.Rid = 2;
for (0..2) |round| {
for (bodies[2 + 2 * round ..][0..2], names[2 + 2 * round ..][0..2]) |body, *nm| {
const rid = try s.put(nm, body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
}
const sum = try persistStore(gpa, io, tmp.dir, s, null, start);
try std.testing.expectEqual(@as(usize, 2), sum.written);
start += 2;
}
try std.testing.expectEqual(scans_before, zit.receive.persist_append_scans);
// A full scan and an index-adopting open both read every artifact back.
var fs: storage.FileStorage = undefined;
var scanned = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer scanned.deinit(gpa);
var adopted = try openLogAdoptingIndex(gpa, io, tmp.dir, fs.storageFile(), .read_only);
defer adopted.deinit(gpa);
for (bodies, &names) |body, *nm| {
var digest_buf: [digest_len]u8 = undefined;
const digest = decodeWireName(nm, &digest_buf).?;
for ([_]*logstore.LogStore{ &scanned, &adopted }) |log| {
const got = (try log.content(gpa, log.lookup(digest).?)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(body, got);
}
}
try std.testing.expectEqual(@as(usize, bodies.len), scanned.entryCount());
}
test "persist that received nothing never takes the writer lock" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
inline for (.{ "alpha", "bravo" }) |body| {
var nm: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, body, &nm);
const rid = try s.put(name, body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
}
_ = try persistStore(gpa, io, tmp.dir, s, null, 0);
// A phantom past the holdings is not something received.
const phantom_name: [64]u8 = @splat('0');
_ = try s.intern(&phantom_name);
// The lock is held, so a persist that opened the store for append would
// fail here: a no-op sync must not scan the log or wait on another writer.
var held = try cli.openStoreAppend(io, tmp.dir);
defer held.close(io);
const noop = try persistStore(gpa, io, tmp.dir, s, null, 2);
try std.testing.expectEqual(@as(usize, 0), noop.written);
try std.testing.expectEqual(@as(usize, 0), noop.submanifest_gaps);
}
test "submanifests §8.1: a repository predating this draft records no indicator and rescans nothing" {
// The compatibility half of the rebuild: the indicator is false, so the
// rescan is unreachable and a second persist over the same store is exactly
// the sweep it always was — no config frame appended, no gap invented.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
const flat = "C msg\nD 2026-07-27T12:00:00.000\nF a.txt " ++ (repeat("1", 64)) ++ "\nU u\n";
var fn_buf: [64]u8 = undefined;
const flat_name = zit.artifact.artifactName(.sha3_256, flat, &fn_buf);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(flat_name, flat);
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
const again = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 0), again.submanifest_gaps);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// No indicator was written at all: `d4SeedEmptyRepo` stores no config, and
// nothing above gave it a reason to start one.
try std.testing.expect((try log.readConfig(gpa)) == null);
}
test "submanifests §7.1: a persisted tree whose sub-manifests are held reports no gap" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
const leaf_a = "F a.txt " ++ (repeat("1", 64)) ++ "\n";
var an: [64]u8 = undefined;
const name_a = zit.artifact.artifactName(.sha3_256, leaf_a, &an);
const tree = try treeManifestBytes(gpa, &.{name_a});
defer gpa.free(tree);
var tn: [64]u8 = undefined;
const tree_name = zit.artifact.artifactName(.sha3_256, tree, &tn);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
// The sub-manifest is stored at a LOWER rid than the tree in one direction
// and a higher one in the other; the gap verdict must not depend on which,
// which is why it is taken once, after the whole sweep.
_ = try s.put(tree_name, tree);
_ = try s.put(name_a, leaf_a);
const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 2), summary.written);
try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps);
// The §7.1 phantom was minted for the leaf BEFORE the leaf's own frame was
// written (the tree persists first), so this asserts the phantom-upgrade
// path storage actually takes: the frame must read back byte-for-byte, not
// merely exist. A §7.1 duty that corrupted a frame would be worse than an
// unmet one.
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var ldig: [digest_len]u8 = undefined;
const leaf_digest = decodeWireName(name_a, &ldig).?;
const lrid = log.lookup(leaf_digest).?;
try std.testing.expect(!log.isPhantom(lrid));
const got = (try log.content(gpa, lrid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(leaf_a, got);
}
test "submanifests §7.1: an artifact predating this draft costs the persist hook no gap and no phantom" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// The backwards-compatibility guarantee as a test: a flat manifest carries
// no S card, so §7.1's duty over it is empty and the persisted bytes and
// frame count are exactly what they were before this rule existed.
const flat = "C msg\nD 2026-07-27T12:00:00.000\nF a.txt " ++ (repeat("1", 64)) ++ "\nU u\n";
var fn_buf: [64]u8 = undefined;
const flat_name = zit.artifact.artifactName(.sha3_256, flat, &fn_buf);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(flat_name, flat);
const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), summary.written);
try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
const rids = try log.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
var artifacts: usize = 0;
for (rids) |r| {
if (log.kindOf(r) != .config) artifacts += 1;
}
try std.testing.expectEqual(@as(usize, 1), artifacts);
}
test "submanifests §7.1: a malformed S run mints no phantom at the persist hook" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// §3.2: a third token on S is rejected. Validate-then-apply means the FIRST
// (well-formed) S card mints nothing either — a half-read artifact never
// leaves a half-minted phantom set behind.
const bad = "C tree\nS a.txt " ++ (repeat("1", 64)) ++ "\nS b.txt " ++ (repeat("2", 64)) ++ " extra\nU u\n";
var bn: [64]u8 = undefined;
const bad_name = zit.artifact.artifactName(.sha3_256, bad, &bn);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(bad_name, bad);
const summary = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), summary.written);
try std.testing.expectEqual(@as(usize, 0), summary.submanifest_gaps);
}
test "sync §3.3: persistStore falls back to full form when a retained delta's basis is absent, never a phantom-based delta" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Only the delta is present; its basis was never received/stored (a legal
// §3.3 outcome that must not persist a delta over a phantom base).
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const base = "an absent basis whose long shared run would make a compact delta if it were here at all";
const target = base ++ " mutated once";
var bn: [64]u8 = undefined;
var tn: [64]u8 = undefined;
const base_name = zit.artifact.artifactName(.sha3_256, base, &bn);
const target_name = zit.artifact.artifactName(.sha3_256, target, &tn);
const raw = try zit.delta.create(gpa, base, target);
defer gpa.free(raw);
const dc = try zit.container.encode(gpa, raw, .default);
defer gpa.free(dc);
const ts = try zit.delta.targetSize(raw);
const rid = try s.put(target_name, target);
try s.retain(rid, .{ .delta = .{ .src = base_name, .container = dc, .target_size = ts } });
const written = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try std.testing.expectEqual(@as(usize, 1), written.written);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// The target persisted as a FULL (.zlib) frame — the fallback, not a delta.
var tdig: [digest_len]u8 = undefined;
const target_digest = decodeWireName(target_name, &tdig).?;
const trid = log.lookup(target_digest).?;
try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(trid));
const got = (try log.content(gpa, trid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(target, got);
// No phantom basis frame was minted for the absent DELTASRC.
var bdig: [digest_len]u8 = undefined;
const base_digest = decodeWireName(base_name, &bdig).?;
try std.testing.expect(log.lookup(base_digest) == null);
}
test "sync §7.7/§3.3: persistStore stores a retained delta chain up to the read bound and falls back to full form beyond it" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const cap: usize = logstore.max_delta_chain; // the READ bound (32)
const n: usize = cap + 2; // links 0..cap+1: link cap+1's basis sits at depth cap
// A chain grown one byte per link so each delta is tiny. Link i's basis is
// link i-1 (§8.2 RID order); the persist path stores a `.delta` iff the base
// depth is under `max_delta_chain`, so link cap+1 falls back to full form.
var content: std.ArrayList(u8) = .empty;
defer content.deinit(gpa);
try content.appendSlice(gpa, "chain-root-artifact-with-a-long-shared-run-for-compact-forward-deltas");
var prev_name: [64]u8 = undefined;
var prev_name_len: usize = 0;
var deep_name: [64]u8 = undefined; // link cap (deepest stored delta)
var deep_len: usize = 0;
var last_name: [64]u8 = undefined; // link cap+1 (the fallback)
var last_len: usize = 0;
var i: usize = 0;
while (i < n) : (i += 1) {
try content.append(gpa, 'a' + @as(u8, @intCast(i % 26)));
const cur = content.items;
const prev = cur[0 .. cur.len - 1];
var nbuf: [64]u8 = undefined;
const name = zit.artifact.artifactName(.sha3_256, cur, &nbuf);
const rid = try s.put(name, cur);
if (i == 0) {
const c = try zit.container.encode(gpa, cur, .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
} else {
const raw = try zit.delta.create(gpa, prev, cur);
defer gpa.free(raw);
const c = try zit.container.encode(gpa, raw, .default);
defer gpa.free(c);
const tsz = try zit.delta.targetSize(raw);
try s.retain(rid, .{ .delta = .{ .src = prev_name[0..prev_name_len], .container = c, .target_size = tsz } });
}
@memcpy(prev_name[0..name.len], name);
prev_name_len = name.len;
if (i == cap) {
@memcpy(deep_name[0..name.len], name);
deep_len = name.len;
}
if (i == n - 1) {
@memcpy(last_name[0..name.len], name);
last_len = name.len;
}
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// Link cap is the deepest `.delta` the write bound allows; link cap+1 fell
// back to a full `.zlib` frame yet still reconstructs (never a Corrupt read).
var ddig: [digest_len]u8 = undefined;
const deep_rid = log.lookup(decodeWireName(deep_name[0..deep_len], &ddig).?).?;
try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(deep_rid));
var ldig: [digest_len]u8 = undefined;
const last_rid = log.lookup(decodeWireName(last_name[0..last_len], &ldig).?).?;
try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(last_rid));
const got = (try log.content(gpa, last_rid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(content.items, got);
}
test "sync §7.2/§3.3: persistStore stores a whole reverse-delta chain, ordering every basis frame before the delta that points at it" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// A reverse-delta chain in fossil's storage shape: revision i is stored as a
// delta of the NEWER revision i+1, the newest stored full. The store rids run
// oldest..newest (put order), so EVERY delta's basis sits at a HIGHER rid than
// the delta — the worst case for physical ordering. persist's worklist must
// still write each basis frame before its delta so `delta_of` points backward;
// naive rid-order would degrade all but the oldest link to full form.
const links = 6;
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
var bodies: [links][]u8 = undefined;
var made: usize = 0;
defer for (bodies[0..made]) |b| gpa.free(b);
var names: [links][64]u8 = undefined;
var nlen: [links]usize = undefined;
for (0..links) |i| {
bodies[i] = try std.fmt.allocPrint(gpa, "chain body: one long shared prefix kept across every revision for compact reverse deltas -- revision {d}", .{i});
made += 1;
nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len;
}
// put oldest..newest (store rids 1..links); older i is a reverse delta of newer i+1.
for (0..links) |i| {
const rid = try s.put(names[i][0..nlen[i]], bodies[i]);
if (i == links - 1) {
const c = try zit.container.encode(gpa, bodies[i], .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
} else {
const dr = try zit.delta.create(gpa, bodies[i + 1], bodies[i]); // older from newer
defer gpa.free(dr);
const c = try zit.container.encode(gpa, dr, .default);
defer gpa.free(c);
try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } });
}
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
// The newest link is full; every older link is a `.delta` whose basis frame is
// physically earlier (a LOWER log rid = a lower offset), and all reconstruct.
for (0..links) |i| {
var d: [digest_len]u8 = undefined;
const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?;
const rgot = (try log.content(gpa, rid)).?;
defer gpa.free(rgot);
try std.testing.expectEqualStrings(bodies[i], rgot);
if (i == links - 1) {
try std.testing.expectEqual(logstore.FrameKind.zlib, log.kindOf(rid));
} else {
try std.testing.expectEqual(logstore.FrameKind.delta, log.kindOf(rid));
var bd: [digest_len]u8 = undefined;
const brid = log.lookup(decodeWireName(names[i + 1][0..nlen[i + 1]], &bd).?).?;
try std.testing.expect(brid < rid); // basis frame precedes the reverse-delta
}
}
}
test "sync §7.7: persistStore verifies a retained delta's reconstruction and refuses a lying delta container, writing no frame" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// A retained delta whose stored plaintext does NOT hash to its wire name (a
// clone skips the transfer-layer pre-storage check, §7.7 r6, so a lying
// delta reaches the persist commit hook). The §7.7 recompute runs BEFORE the
// wire-form branch, so a delta-form entry is rejected exactly like a full one.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const content = "content that the wire name does not actually name, delivered as a delta\n";
const lying_name = repeat("0", 64); // not SHA3-256(content)
const some_basis = repeat("1", 64); // an absent basis; verify fails before it is even consulted
const raw = try zit.delta.create(gpa, "a plausible basis blob", content);
defer gpa.free(raw);
const dc = try zit.container.encode(gpa, raw, .default);
defer gpa.free(dc);
const rid = try s.put(lying_name, content);
try s.retain(rid, .{ .delta = .{ .src = some_basis, .container = dc, .target_size = content.len } });
try std.testing.expectError(error.Corrupt, persistStore(gpa, io, tmp.dir, ms.store(), null, 0));
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
var digest_buf: [digest_len]u8 = undefined;
try std.testing.expect(log.lookup(decodeWireName(lying_name, &digest_buf).?) == null); // no frame landed
}
fn persistReverseChainNoLeak(allocator: std.mem.Allocator) !void {
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(allocator, io, tmp.dir);
// A small reverse-delta chain built on the (failing) allocator so injected
// OOM is exercised in both the store build and the persist worklist.
var ms = repostate.MemStore.init(allocator);
defer ms.deinit();
const s = ms.store();
const links = 4;
var bodies: [links][]u8 = undefined;
var made: usize = 0;
defer for (bodies[0..made]) |b| allocator.free(b);
var names: [links][64]u8 = undefined;
var nlen: [links]usize = undefined;
for (0..links) |i| {
bodies[i] = try std.fmt.allocPrint(allocator, "reverse chain oom: one shared body across revisions -- revision {d}", .{i});
made += 1;
nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len;
}
for (0..links) |i| {
const rid = try s.put(names[i][0..nlen[i]], bodies[i]);
if (i == links - 1) {
const c = try zit.container.encode(allocator, bodies[i], .default);
defer allocator.free(c);
try s.retain(rid, .{ .full = c });
} else {
const dr = try zit.delta.create(allocator, bodies[i + 1], bodies[i]);
defer allocator.free(dr);
const c = try zit.container.encode(allocator, dr, .default);
defer allocator.free(c);
try s.retain(rid, .{ .delta = .{ .src = names[i + 1][0..nlen[i + 1]], .container = c, .target_size = bodies[i].len } });
}
}
_ = try persistStore(allocator, io, tmp.dir, ms.store(), null, 0);
}
/// A control artifact's body plus its Z card (§4.7.19).
fn sealedControl(gpa: std.mem.Allocator, body: []const u8) ![]u8 {
var digest: [16]u8 = undefined;
std.crypto.hash.Md5.hash(body, &digest, .{});
return std.fmt.allocPrint(gpa, "{s}Z {s}\n", .{ body, std.fmt.bytesToHex(digest, .lower) });
}
test "horizon §5.2.3: a received seed is never stored, as a raw put, a full container or a wire delta, and a delta on one lands in full" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
const c = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
const labelled = try sealedControl(gpa, "D 2026-10-24T00:00:00\nT +sym-labelled " ++ c ++ "\nU " ++ zit.sync.seed_user ++ "\n");
defer gpa.free(labelled);
const bound = try sealedControl(gpa, "D 2026-10-24T00:00:00\nT +horizon-seed " ++ c ++ " descendants\nT +sym-bound " ++ c ++ "\nU someone\n");
defer gpa.free(bound);
const bound_delta = try sealedControl(gpa, "D 2026-10-24T00:00:01\nT +horizon-seed " ++ c ++ " leaves\nT +sym-delta " ++ c ++ "\nU someone\n");
defer gpa.free(bound_delta);
const plain = try sealedControl(gpa, "D 2026-10-24T00:00:00\nT +sym-plain " ++ c ++ "\nU someone\n");
defer gpa.free(plain);
const base = "an ordinary blob that a seed is sent as a delta against\n";
const on_seed = "a blob sent as a delta against the labelled seed\n";
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
const Named = struct { buf: [64]u8 = undefined, len: usize = 0 };
var nm: [6]Named = .{ .{}, .{}, .{}, .{}, .{}, .{} };
const bodies = [_][]const u8{ on_seed, labelled, bound, base, bound_delta, plain };
for (bodies, 0..) |b, i| nm[i].len = zit.artifact.artifactName(.sha3_256, b, &nm[i].buf).len;
const name = struct {
fn of(n: *const Named) []const u8 {
return n.buf[0..n.len];
}
}.of;
// A blob whose wire form is a delta on the labelled seed, put first so
// it defers on a basis that is then dropped.
{
const rid = try s.put(name(&nm[0]), on_seed);
const d = try zit.delta.create(gpa, labelled, on_seed);
defer gpa.free(d);
const ct = try zit.container.encode(gpa, d, .default);
defer gpa.free(ct);
try s.retain(rid, .{ .delta = .{ .src = name(&nm[1]), .container = ct, .target_size = on_seed.len } });
}
_ = try s.put(name(&nm[1]), labelled); // a raw put
{
const rid = try s.put(name(&nm[2]), bound);
const ct = try zit.container.encode(gpa, bound, .default);
defer gpa.free(ct);
try s.retain(rid, .{ .full = ct });
}
{
const rid = try s.put(name(&nm[3]), base);
const ct = try zit.container.encode(gpa, base, .default);
defer gpa.free(ct);
try s.retain(rid, .{ .full = ct });
}
{
const rid = try s.put(name(&nm[4]), bound_delta);
const d = try zit.delta.create(gpa, base, bound_delta);
defer gpa.free(d);
const ct = try zit.container.encode(gpa, d, .default);
defer gpa.free(ct);
try s.retain(rid, .{ .delta = .{ .src = name(&nm[3]), .container = ct, .target_size = bound_delta.len } });
}
_ = try s.put(name(&nm[5]), plain);
const summary = try persistStore(gpa, io, tmp.dir, s, null, 0);
try std.testing.expectEqual(@as(usize, 3), summary.written);
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
const stored = [_]bool{ true, false, false, true, false, true };
for (nm, stored) |n, want| {
var buf: [digest_len]u8 = undefined;
const digest = decodeWireName(name(&n), &buf).?;
const held = if (log.lookup(digest)) |r| !log.isPhantom(r) else false;
try std.testing.expectEqual(want, held);
if (want) {
const got = (try log.content(gpa, log.lookup(digest).?)).?;
defer gpa.free(got);
try std.testing.expect(zit.artifact.verify(name(&n), got));
}
}
}
test "sync §7.2/§3.3: persistStore drains the reverse-delta worklist without leaking under allocation failure" {
try zit.NoGrowAllocator.checkAllAllocationFailures(std.testing.allocator, persistReverseChainNoLeak, .{});
}
/// Drives `persistArtifact` over a FORWARD delta chain (each link a delta of the
/// immediately older, already-persisted lower-rid link) in rid order, so a test
/// owns the shared basis memo and can read its hit counter. Every basis precedes
/// its delta, so `allow_defer=false` never defers and no worklist is needed —
/// mirrors `persistStore`'s open/commit around a plain rid-order drive.
fn persistForwardChain(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
ms: *repostate.MemStore,
cache: *logstore.PlaintextCache,
) !void {
var store_append = try cli.openStoreAppend(io, dir);
defer store_append.close(io);
const file = store_append.file;
const st = try file.stat(io);
var fs = storage.FileStorage.init(io, file, st.size);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0);
defer log.deinit(gpa);
const store = ms.store();
const n = store.count();
var srefs: SubManifestRefs = .{};
defer srefs.deinit(gpa);
var rid: repostate.Rid = 1;
while (rid <= n) : (rid += 1) {
switch (try persistArtifact(gpa, ms.store(), &log, rid, false, false, cache, &srefs, false, false, null, null)) {
.persisted, .dropped => {},
.deferred => unreachable, // forward chain: no basis is ever absent
}
}
try log.commit();
}
test "sync §7.2/§7.4: the basis memo reuses each basis (hits) and reconstructs a byte-identical store to the uncached path" {
const gpa = std.testing.allocator;
const io = std.testing.io;
const links = 8;
// A forward delta chain: link 0 is full, every later link is a delta that
// reconstructs its body FROM the older body (its basis a lower, already-
// persisted rid) — the case where the memo turns each basis reconstruction
// into a lookup for the delta built on it. Store owns its copies (retain
// dupes), but the bodies/names back the reconstructed-content assertions.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
const s = ms.store();
var bodies: [links][]u8 = undefined;
var made: usize = 0;
defer for (bodies[0..made]) |b| gpa.free(b);
var names: [links][64]u8 = undefined;
var nlen: [links]usize = undefined;
for (0..links) |i| {
bodies[i] = try std.fmt.allocPrint(gpa, "forward chain shared prefix across every revision -- revision {d}", .{i});
made += 1;
nlen[i] = zit.artifact.artifactName(.sha3_256, bodies[i], &names[i]).len;
}
for (0..links) |i| {
const rid = try s.put(names[i][0..nlen[i]], bodies[i]);
if (i == 0) {
const c = try zit.container.encode(gpa, bodies[i], .default);
defer gpa.free(c);
try s.retain(rid, .{ .full = c });
} else {
const dr = try zit.delta.create(gpa, bodies[i - 1], bodies[i]); // newer body FROM its older basis
defer gpa.free(dr);
const c = try zit.container.encode(gpa, dr, .default);
defer gpa.free(c);
try s.retain(rid, .{ .delta = .{ .src = names[i - 1][0..nlen[i - 1]], .container = c, .target_size = bodies[i].len } });
}
}
// (A) a normal memo; (B) a zero-budget memo that can hold nothing — the exact
// uncached reconstruction. Same store, two freshly-seeded repos.
var tmp_a = std.testing.tmpDir(.{});
defer tmp_a.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp_a.dir);
var cache_a = logstore.PlaintextCache.init();
defer cache_a.deinit(gpa);
try persistForwardChain(gpa, io, tmp_a.dir, &ms, &cache_a);
var tmp_b = std.testing.tmpDir(.{});
defer tmp_b.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp_b.dir);
var cache_b = logstore.PlaintextCache.init();
cache_b.max_bytes = 0; // never memoises: forces a fresh reconstruction each time
defer cache_b.deinit(gpa);
try persistForwardChain(gpa, io, tmp_b.dir, &ms, &cache_b);
// The memo reused every basis exactly once (one hit per delta link); the
// zero-budget run reused nothing — the two ways the same bytes get built.
try std.testing.expectEqual(@as(u64, links - 1), cache_a.hits);
try std.testing.expectEqual(@as(u64, 0), cache_b.hits);
// Byte-identical store: the memo changes only the work, never the bytes.
const bytes_a = try tmp_a.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(bytes_a);
const bytes_b = try tmp_b.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(bytes_b);
try std.testing.expectEqualSlices(u8, bytes_b, bytes_a);
// Every link reconstructs to its exact body (§7.7 chain correctness).
var fs: storage.FileStorage = undefined;
var log = try d4OpenLog(gpa, io, tmp_a.dir, &fs);
defer fs.file.close(io);
defer log.deinit(gpa);
for (0..links) |i| {
var d: [digest_len]u8 = undefined;
const rid = log.lookup(decodeWireName(names[i][0..nlen[i]], &d).?).?;
const got = (try log.content(gpa, rid)).?;
defer gpa.free(got);
try std.testing.expectEqualStrings(bodies[i], got);
}
}
test "clone observer forwards phase edges without owning measurement state" {
const Probe = struct {
count: usize = 0,
phase: ClonePhase = .transport,
event: ClonePhaseEvent = .begin,
fn callback(context: *anyopaque, phase: ClonePhase, event: ClonePhaseEvent) void {
const self: *@This() = @ptrCast(@alignCast(context));
self.count += 1;
self.phase = phase;
self.event = event;
}
};
var probe = Probe{};
const observer = CloneObserver{ .context = &probe, .callback = Probe.callback };
observeClonePhase(&observer, .persist, .end);
try std.testing.expectEqual(@as(usize, 1), probe.count);
try std.testing.expectEqual(ClonePhase.persist, probe.phase);
try std.testing.expectEqual(ClonePhaseEvent.end, probe.event);
}
test "the cluster sidecar is created by persist, adopted without a walk, rebuilt by one walk when lost, extended by a later persist" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Two blobs and a cluster naming one of them, persisted into the fresh log.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
var names: [2][64]u8 = undefined;
inline for (.{ "clustered blob", "loose blob" }, 0..) |body, i| {
_ = zit.artifact.artifactName(.sha3_256, body, &names[i]);
const rid = try ms.store().put(&names[i], body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try ms.store().retain(rid, .{ .full = c });
}
const cluster = try std.fmt.allocPrint(gpa, "M {s}\nZ 00000000000000000000000000000000\n", .{&names[0]});
defer gpa.free(cluster);
var cn: [64]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, cluster, &cn);
{
const rid = try ms.store().put(&cn, cluster);
const c = try zit.container.encode(gpa, cluster, .default);
defer gpa.free(c);
try ms.store().retain(rid, .{ .full = c });
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
// 1. Adopted: persist created the index, so a load walks nothing.
{
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const before = clusterindex.walk_rids;
var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store());
defer r.deinit();
try std.testing.expectEqual(before, clusterindex.walk_rids);
try std.testing.expectEqual(@as(usize, 1), r.cluster_names.len);
try std.testing.expectEqualStrings(&cn, r.cluster_names[0]);
try std.testing.expect(r.clustered.isSet(holdings.store().lookup(&names[0]).?));
try std.testing.expect(!r.clustered.isSet(holdings.store().lookup(&names[1]).?));
try std.testing.expect(!r.clustered.isSet(holdings.store().lookup(&cn).?));
}
// 2. Lost: one full walk rebuilds it; the load after that adopts again.
try tmp.dir.deleteFile(io, zit.repocache.clusters_filename);
{
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const before = clusterindex.walk_rids;
var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store());
defer r.deinit();
try std.testing.expectEqual(before + 3, clusterindex.walk_rids);
try std.testing.expectEqual(@as(usize, 1), r.cluster_names.len);
const again = clusterindex.walk_rids;
var r2 = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store());
defer r2.deinit();
try std.testing.expectEqual(again, clusterindex.walk_rids);
}
// 3. Extended: a later persist records the cluster it writes; no walk.
const cluster2 = try std.fmt.allocPrint(gpa, "M {s}\nZ 00000000000000000000000000000000\n", .{&names[1]});
defer gpa.free(cluster2);
var cn2: [64]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, cluster2, &cn2);
{
const rid = try ms.store().put(&cn2, cluster2);
const c = try zit.container.encode(gpa, cluster2, .default);
defer gpa.free(c);
try ms.store().retain(rid, .{ .full = c });
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 3);
{
var holdings = try ClientHoldings.open(gpa, io, tmp.dir);
defer holdings.deinit(io);
const before = clusterindex.walk_rids;
var r = try loadClusterIndex(gpa, io, tmp.dir, &holdings.index.log, holdings.store());
defer r.deinit();
try std.testing.expectEqual(before, clusterindex.walk_rids);
try std.testing.expectEqual(@as(usize, 2), r.cluster_names.len);
try std.testing.expect(r.clustered.isSet(holdings.store().lookup(&names[1]).?));
}
}
test "horizon §6.2.7: a name resolved from the transfer sees a branch tag that arrived before its check-ins" {
// The fallback derives over the transfer in rid order, which is arrival
// order: a control artifact that starts a branch may precede the
// check-in it tags, and the propagated tag must still reach the tip.
const gpa = std.testing.allocator;
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
const Sealed = struct {
fn of(a: std.mem.Allocator, body: []const u8) ![]const u8 {
var buf: [32]u8 = undefined;
return std.fmt.allocPrint(a, "{s}Z {s}\n", .{ body, artifact.selfChecksum(body, &buf) });
}
fn name(a: std.mem.Allocator, bytes: []const u8) ![]const u8 {
var buf: [64]u8 = undefined;
return a.dupe(u8, artifact.artifactName(.sha3_256, bytes, &buf));
}
};
const c1 = try Sealed.of(arena, "C one\nD 2026-01-01T00:00:01\nU u\n");
const h1 = try Sealed.name(arena, c1);
const c2 = try Sealed.of(arena, try std.fmt.allocPrint(arena, "C two\nD 2026-01-01T00:00:02\nP {s}\nU u\n", .{h1}));
const h2 = try Sealed.name(arena, c2);
const c3 = try Sealed.of(arena, try std.fmt.allocPrint(arena, "C three\nD 2026-01-01T00:00:03\nP {s}\nU u\n", .{h2}));
const h3 = try Sealed.name(arena, c3);
const tag = try Sealed.of(arena, try std.fmt.allocPrint(arena, "D 2026-01-01T00:00:04\nT *branch {s} dev\nT *sym-dev {s}\nU u\n", .{ h2, h2 }));
const ht = try Sealed.name(arena, tag);
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
_ = try ms.store().put(ht, tag); // the tag first
_ = try ms.store().put(h3, c3);
_ = try ms.store().put(h2, c2);
_ = try ms.store().put(h1, c1);
const r = (try resolveHorizonLocally(gpa, arena, ms.store(), "dev", 0)).?;
try std.testing.expectEqualStrings(h3, r.hash);
const back = (try resolveHorizonLocally(gpa, arena, ms.store(), "dev", 1)).?;
try std.testing.expectEqualStrings(h2, back.hash);
}