//! `zit pack`: rewrite the store into a compact new generation and swap it in
//! atomically; the rewrite is shared with `zit shun` and erasure reclaim.
const std = @import("std");
const ClientHoldings = zit.sync_session.ClientHoldings;
const LogIndexStore = zit.log_index_store.LogIndexStore;
const NameSet = @import("shun.zig").NameSet;
const SparseClassification = zit.sparse.SparseClassification;
const classifyKeepRids = zit.sparse.classifyKeepRids;
const cli = @import("zit").cli;
const clusterindex = @import("zit").clusterindex;
const d4SeedEmptyRepo = @import("sync_test_support.zig").d4SeedEmptyRepo;
const decodeWireName = zit.digest_name.decodeWireName;
const diagnostics = @import("../cli/diagnostics.zig");
const digestToWireName = zit.digest_name.digestToWireName;
const digest_len = zit.digest_name.digest_len;
const extras = @import("../cli/extras.zig");
const help_pages = @import("../cli/help.zig");
const hexToDigest = zit.digest_name.hexToDigest;
const loadClusterIndex = zit.receive.loadClusterIndex;
const logstore = @import("zit").logstore;
const packEvolvingBlob = @import("sync_test_support.zig").packEvolvingBlob;
const packOpenReadOnly = @import("sync_test_support.zig").packOpenReadOnly;
const packOpenWritable = @import("sync_test_support.zig").packOpenWritable;
const packSeedBlob = @import("sync_test_support.zig").packSeedBlob;
const packSeedCheckin = @import("sync_test_support.zig").packSeedCheckin;
const packSeedDeltaStore = @import("sync_test_support.zig").packSeedDeltaStore;
const packSeedFullStore = @import("sync_test_support.zig").packSeedFullStore;
const pack_help = help_pages.pack_help;
const persistStore = zit.receive.persistStore;
const reasonOf = diagnostics.reasonOf;
const refreshSettingLogIndex = zit.log_index_store.refreshSettingLogIndex;
const repeat = @import("zit").repeat;
const repoReadFail = diagnostics.repoReadFail;
const repo_filename = zit.repo_filename;
const repostate = zit.repostate;
const serveAppendFrame = @import("sync_test_support.zig").serveAppendFrame;
const stateOom = diagnostics.stateOom;
const storage = @import("zit").storage;
const storeReadFail = diagnostics.storeReadFail;
const zit = @import("zit");
// ---------------------------------------------------------------------------
// `zit pack` — repack the store into a compact new generation.
//
// On-disk storage is UNSPECIFIED by the RFCs (they fix the wire format and
// content-addressed naming, §4.1 L889-892/L918-919 and artifact §3.1 L318-326), so pack is free to reorganize
// the log as long as every artifact's exact content and name survive and the
// §7.2 verbatim wire bytes are still servable. It re-ingests every artifact,
// in the source's offset order, into a FRESH log through `putArtifact` with a
// lineage base hint — a check-in against its parent manifest, a file blob
// against the same path's blob in the parent check-in (the commit path's own
// §4.1/§7.2 compaction heuristic, applied offline). Offset order plus the
// strictly-backward `delta_of` invariant means every base is already written
// before the delta that names it, so no worklist is needed (unlike
// `persistStore`, whose input is rid-ordered).
//
// The new log is written to a temporary file beside `.zit`, fsynced, and then
// atomically renamed over `.zit`: a crash anywhere before the rename leaves the
// original byte-identical (the temp is discarded next run). The `.zit-cache`
// invalidates itself on the tail change and rebuilds on the next read verb.
// ---------------------------------------------------------------------------
/// pack's core store-to-store transform lives in the `zit` module (pack.zig)
/// so the DST/fuzz harnesses drive the REAL pass in-process; the CLI wiring
/// below (temp file, atomic rename, stats rendering) stays here. These aliases
/// keep every call site and test reading
/// `packLog`/`PackError`/`PackStats`/`Filter`/`shunFilter`.
const PackError = zit.pack.PackError;
pub const PackStats = zit.pack.PackStats;
const packLog = zit.pack.packLog;
const Filter = zit.pack.Filter;
const shunFilter = zit.pack.shunFilter;
/// The name of the transient new-generation file the rewriting verbs share,
/// beside `.zit`. A fixed name (never a random one) so a temp left by a crashed
/// run is reused rather than accumulating, and so `extras.zig` can name it in
/// the single never-tracked predicate. Because it IS fixed and TWO verbs now
/// reach for it, `rewriteAndSwap` holds an exclusive advisory lock on it for the
/// whole rewrite — see there.
pub const pack_tmp_filename = extras.pack_tmp_filename;
/// The generation-rewrite lock: `pack` and `shun --force` exclude each other
/// with it, never waiting, since a second rewrite of the same store only
/// repeats the first. Its busy notice is the caller's to render.
pub const rewrite_lock_name = pack_tmp_filename ++ ".lock";
pub fn lockRewrite(io: std.Io, dir: std.Io.Dir) error{ RewriteBusy, Storage }!zit.storelock.Lock {
var sink = std.Io.Writer.Discarding.init(&.{});
return zit.storelock.acquire(io, dir, rewrite_lock_name, 0, &sink.writer) catch |e| switch (e) {
error.Busy, error.Ambiguous => error.RewriteBusy,
error.HeldByThisProcess, error.Io => error.Storage,
};
}
/// `zit pack` — repack the current repository's store into a compact new
/// generation and atomically swap it in. Thin CLI wiring over `packLog`: parse
/// (no positional), discover the checkout root, open `.zit` read-only and
/// capture its tail key, stream the packed log into a temp file, fsync, re-check
/// the source tail (the no-repo-lock concurrency guard), then rename the temp
/// over `.zit`. On any failure the temp is deleted and the original store is
/// intact.
///
/// The integrity guarantee this rewrite depends on is UNCHANGED, but it is no
/// longer collected at open. `openReadOnly` used to whole-frame-CRC-verify every
/// frame while scanning, so a corrupt source aborted before a byte was written —
/// an eager pre-flight. It scans header-only now. The check that matters happens
/// on the reads instead: `packLog` pulls every artifact through `srcContent`,
/// which propagates `error.Corrupt` rather than skipping the frame, and
/// `readFrameAt` CRC-checks every frame it reads. So a rotted payload still
/// aborts the pack with the temp deleted and `.zit` untouched — it is caught in
/// the copy loop rather than at open, and the operator sees the same outcome.
/// This ordering is load-bearing for a verb that RENAMES OVER the original: if
/// `srcContent` were ever changed to answer null on a corrupt read, the rewrite
/// would silently omit the damaged artifact and then delete the only copy of it.
/// The packed-generation memo's byte shape: magic, then the four
/// `LogKey` fields little-endian — 32 bytes exactly. No self-checksum: a
/// damaged memo can only fail the equality below, and the cost of a miss is
/// one real pack.
const packed_key_magic = "zPK1";
const packed_key_len = 4 + 8 + 8 + 8 + 4;
fn serializePackedKey(buf: *[packed_key_len]u8, key: logstore.LogStore.LogKey) []const u8 {
@memcpy(buf[0..4], packed_key_magic);
std.mem.writeInt(u64, buf[4..12], key.end, .little);
std.mem.writeInt(u64, buf[12..20], key.frame_count, .little);
std.mem.writeInt(u64, buf[20..28], key.last_frame_offset, .little);
std.mem.writeInt(u32, buf[28..32], key.last_header_crc, .little);
return buf[0..packed_key_len];
}
fn parsePackedKey(blob: []const u8) ?logstore.LogStore.LogKey {
if (blob.len != packed_key_len or !std.mem.eql(u8, blob[0..4], packed_key_magic)) return null;
return .{
.end = std.mem.readInt(u64, blob[4..12], .little),
.frame_count = std.mem.readInt(u64, blob[12..20], .little),
.last_frame_offset = std.mem.readInt(u64, blob[20..28], .little),
.last_header_crc = std.mem.readInt(u32, blob[28..32], .little),
};
}
/// Record the just-swapped generation's key so the next identity-filter
/// pack over an unchanged log is a no-op, and refresh the scan-index sidecar
/// the rename invalidated — the open below pays the scan either way, so the
/// next read verb should not pay it again. Best-effort throughout: any failure
/// leaves the memo absent, which costs a real pack, never an answer.
fn recordPackedGeneration(gpa: std.mem.Allocator, io: std.Io, store_dir: std.Io.Dir) void {
var file = cli.openStoreRead(io, store_dir) catch return;
defer file.close(io);
const st = file.stat(io) catch return;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch return;
defer log.deinit(gpa);
const key = log.logKey() catch return;
var buf: [packed_key_len]u8 = undefined;
zit.repocache.storePackedKey(io, store_dir, serializePackedKey(&buf, key));
refreshSettingLogIndex(gpa, io, store_dir, &log);
}
pub fn runPack(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
var drop_elided = false;
var pack_args: std.ArrayListUnmanaged([]const u8) = .empty;
defer pack_args.deinit(gpa);
for (verb_args) |tok| {
if (std.mem.eql(u8, tok, "--drop-elided")) {
drop_elided = true;
} else {
pack_args.append(gpa, tok) catch return stateOom(streams, "pack");
}
}
const help = switch (cli.parseNoPositionalArgs(pack_args.items)) {
.ok => |a| a.help,
.unknown_option => |tok| return cli.usageError(streams, "pack", .unknown_option, tok),
.extra_argument => |tok| return cli.usageError(streams, "pack", .extra_argument, tok),
};
if (help) {
streams.out.writeAll(pack_help) catch {};
return .ok;
}
// `-R` names a bare store as readily as a checkout: nothing below reads
// a working tree.
const root = (cli.resolveRoot(arena, io, amb.repo_override) catch return stateOom(streams, "pack")) orelse {
diagnostics.reportNoRoot(streams, io, "pack", amb.repo_override);
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
// pack is store-scoped end to end: the reads, the pack-tmp lock
// and the final rename must all land beside the STORE, so a detached
// worktree resolves its pointer once here and the whole flow inherits it.
if (cli.storeDirOf(io, root_dir) catch return repoReadFail(streams, "pack")) |resolved| {
root_dir.close(io);
root_dir = resolved;
}
// Open the source read-only: `openReadOnly` scans and whole-frame-CRC-checks
// every frame, so a corrupt source aborts here (no temp written). The tail
// key pins the log we packed for the pre-rename concurrency re-check.
//
// LIMIT OF THE RECLAIM'S DAMAGE TOLERANCE, stated where it is imposed: this
// pre-flight runs before `packLog` sees the filter, so it covers the shunned
// frames too. The reclaim contract's filter-before-reconstruction ordering
// rescues a shunned artifact whose CONTENT no longer hashes to its name (and
// one that will not decode) — but NOT one whose frame CHECKSUM is damaged,
// which fails here with the reclaim never attempted. `pack_help` says so in
// those terms; narrowing the pre-flight to kept frames would trade a
// whole-store integrity gate for a rarer recovery path and is not this
// slice's call to make.
var src_file = cli.openStoreRead(io, root_dir) catch |e| return storeReadFail(streams, "pack", e);
defer src_file.close(io);
const src_st = src_file.stat(io) catch return repoReadFail(streams, "pack");
var src_fs = storage.FileStorage.init(io, src_file, src_st.size);
var src = logstore.LogStore.openReadOnly(gpa, src_fs.storageFile()) catch |e| {
switch (e) {
error.OutOfMemory => streams.err.writeAll("zit pack: out of memory\n") catch {},
else => streams.err.print("zit pack: cannot read repository: {s}\n", .{reasonOf(e)}) catch {},
}
return .failure;
};
defer src.deinit(gpa);
const src_key = src.logKey() catch return packFail(streams, error.Corrupt);
// The no-op gate: a log whose key equals the one the last pack
// recorded has had NO append since (a commit, a shun row, and a pull all
// grow the log), so there is nothing to compact and nothing to reclaim
// that the last pack did not already handle. Gated to the identity
// shape: a `--drop-elided` run composes a cone filter the memo knows
// nothing about, so it always runs.
if (!drop_elided) {
if (zit.repocache.tryLoadPackedKey(gpa, io, root_dir)) |blob| {
defer gpa.free(blob);
if (parsePackedKey(blob)) |recorded| {
if (std.meta.eql(recorded, src_key)) {
streams.out.writeAll("already packed; nothing to do\n") catch {};
return .ok;
}
}
}
}
// `zit pack` is where the repository's deferred shuns are RECLAIMED: every
// artifact the source's shun table names is dropped from the new generation
// instead of relocated into it (the pack filter). The tombstones themselves
// carry forward, so the name keeps reading as absent and a later sync will
// not re-fetch what the operator removed. pack names no NEW tombstone — that
// is `zit shun`'s job — so `extra_shuns` is empty.
// Sparse-cone RFC section 5.4 -- narrow-then-compact, the spec's ONLY
// deletion: an explicit `pack --drop-elided` drops what the store cone
// elides (blobs referenced exclusively out-of-cone), composed with the
// ordinary shun reclaim. Refused without a store cone: with no policy on
// record there is nothing "elided" to drop, and a bare pack must never
// guess one.
var elided_dropper: ?ElidedDropper = null;
defer if (elided_dropper) |*d| d.deinit(gpa);
if (drop_elided) {
var cone = zit.repocache.readStoreCone(gpa, io, root_dir) orelse {
streams.err.writeAll(
"zit pack: --drop-elided needs a store cone and this store has none\n" ++
"zit pack: narrow first ('zit update --cone=...'), or drop the flag\n",
) catch {};
return .failure;
};
elided_dropper = ElidedDropper.build(gpa, src_fs.storageFile(), &cone) orelse {
cone.deinit(gpa);
streams.err.writeAll("zit pack: cannot classify the store for --drop-elided\n") catch {};
return .failure;
};
cone.deinit(gpa);
}
const pack_filter: Filter = if (elided_dropper) |*d|
d.composed(shunFilter(&src))
else
shunFilter(&src);
const stats = rewriteAndSwap(
gpa,
io,
root_dir,
&src,
src_key,
pack_filter,
&.{},
) catch |e| switch (e) {
error.TailChanged => return tailChangedFail(streams, "pack"),
error.RewriteBusy => return rewriteBusyFail(streams, "pack"),
else => |pe| return packFail(streams, pe),
};
renderPackStats(streams.out, stats);
// Only the identity-filter shape records the memo: a --drop-elided pack's
// outcome depends on a cone the memo does not capture, so the next plain
// pack must run for real.
if (!drop_elided) recordPackedGeneration(gpa, io, root_dir);
return .ok;
}
/// `rewriteAndSwap`'s causes: everything `packLog` can fail with, plus the two
/// aborts that are not failures of the rewrite at all and must not be reported as
/// if they were. `TailChanged` means the source log is intact and NEWER than the
/// snapshot we packed; `RewriteBusy` means another rewrite already owns the temp.
/// In both the original store is untouched and the operator's next move is
/// obvious, whereas every `PackError` arm means the new generation could not be
/// built.
pub const RewriteError = PackError || error{ TailChanged, RewriteBusy };
/// The new-generation rewrite `pack` and `shun` share: create the fixed-name
/// readable temp beside `.zit`, stream `packLog(src → temp)` through `filter`
/// (recording `extra_shuns` in the new generation only), fsync, re-check the
/// source tail against `src_key`, then atomically rename the temp over `.zit`.
/// Returns the pack's own stats; on every failing path the temp is deleted and
/// the original store is byte-identical.
///
/// ONE durable transition — the rename — and that is why the two verbs share
/// this body rather than each having its own: the property is a property of this
/// sequence, so a second copy would be a second place for it to stop holding.
/// `src` is already open (both callers must read it before they can decide what
/// to filter); the pre-rename check deliberately reads NEITHER it nor the
/// caller's file handle.
///
/// THAT IS THE GUARD, and it takes a fresh HANDLE, not merely a fresh stat.
/// `storage.FileStorage` caches the append watermark it was constructed with and
/// `size()` returns that cached value without a stat (src/store/storage.zig
/// `sizeImpl`), so re-checking the tail through the caller's existing view
/// compares `key.end` against the very number the key was derived from — it
/// cannot fail, and the concurrency guard is then a no-op that reads as a guard.
/// The caller's HANDLE fails the same way against the other concurrent writer:
/// once a competing rewrite has renamed its generation over `.zit`, that handle
/// names an ORPHAN inode whose size and tail are frozen, so it agrees with the
/// key forever — and this run would then rename a snapshot taken before the
/// winner existed over the winner, silently un-shunning what it just shunned.
/// Reopening `.zit` BY NAME under the lock is what makes both an append (`zit
/// commit`) and a rename (a racing `pack`/`shun --force`) detectable, and — for
/// `shun` — is what makes "an aborted run leaves NO tombstone" something more
/// than an assertion about code that never runs. Pinned by the two "a run the
/// concurrency guard aborts leaves .zit untouched and records NO tombstone"
/// tests, one per concurrent-writer shape. Same argument, same shape, as
/// `shunAppendRows`.
///
/// MUTUAL EXCLUSION, and why it lives here rather than in either verb. The temp
/// has a FIXED name, so two overlapping rewrites would each truncate the other's
/// in-flight new generation and then fail the §7.7 reconstruct-and-rehash pass on
/// frames that had become holes — reported as `error.Corrupt`, i.e. two healthy
/// processes telling the operator their repository is corrupt. A cron'd `zit pack`
/// overlapping an operator's `zit shun --force` is an entirely ordinary way to hit
/// that, and the shun is the run that must not silently not happen. So the temp is
/// written only under the rewrite lock, `.zit-tmp-pack.lock`, taken by exclusive
/// create without waiting: the second arrival gets `error.RewriteBusy` and says
/// so, instead of a corruption claim. A crashed run's lock names a process that
/// no longer exists and is broken by the next one (`storelock`).
///
/// The final check and the rename run under the store's WRITER lock too. An
/// append that landed between the check and the rename would go into the
/// generation the rename discards; holding the writer lock across both makes the
/// check final.
///
/// Parameterised by DATA only — a drop predicate and a tombstone list — never by
/// a policy callback. Which names those are, what to tell the operator, and
/// whether to refuse at all stay entirely in the callers; nothing here knows
/// which verb it is running, which is why it needs no verb label.
///
/// `pub` for one reason: `src/sim/shun_sweep.zig` drives THIS FUNCTION under a
/// fault-injecting `std.Io`. The property the sweep exists to pin — one durable
/// transition, so a crash leaves either the old store or the new one — is a
/// property of this sequence and of nothing else, so the sweep has to reach the
/// sequence. It deliberately does NOT drive `runPack`/`runShun`: both begin at
/// `cli.discoverRoot`, which reads the process's current directory, and a sweep
/// that depends on ambient cwd is a sweep whose failures do not replay.
pub fn rewriteAndSwap(
gpa: std.mem.Allocator,
io: std.Io,
root_dir: std.Io.Dir,
src: *logstore.LogStore,
src_key: logstore.LogStore.LogKey,
filter: Filter,
extra_shuns: []const logstore.ShunRecord,
) RewriteError!PackStats {
// The rewrite lock first; the busy path must NOT touch the temp, which
// belongs to the run that holds it.
var rewrite_lock = try lockRewrite(io, root_dir);
defer rewrite_lock.release(io);
// New-generation file: a readable temp beside `.zit` (createFileAtomic opens
// write-only, but pack reads its own delta bases back, so we roll our own
// readable temp and rename it). Truncated on open: whatever a prior crashed
// run left is ours to discard now that the lock is held, and `LogStore.open`
// below writes from offset 0, so trailing bytes of a longer stale temp would
// otherwise survive past the new generation's end. Removed on every path but
// the successful rename, before the lock is released.
var tmp_file = root_dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = true }) catch
return error.Storage;
var renamed = false;
defer {
if (!renamed) root_dir.deleteFile(io, pack_tmp_filename) catch {};
tmp_file.close(io);
}
var dst_fs = storage.FileStorage.init(io, tmp_file, 0);
var dst = try logstore.LogStore.open(gpa, dst_fs.storageFile(), src.hash_kind);
const stats = packLog(gpa, src, &dst, filter, extra_shuns) catch |e| {
dst.deinit(gpa);
return e;
};
dst.deinit(gpa); // `packLog` already fsynced; deinit only frees memory
// The new generation keeps the source's header version: a packed horizon
// clone must stay one a release predating horizons refuses
// (`logstore.horizon_version`).
var src_header: [logstore.header_len]u8 = undefined;
if ((src.file.pread(0, &src_header) catch return error.Storage) != src_header.len) return error.Storage;
if (logstore.headerVersion(&src_header) == logstore.horizon_version) {
var dst_header: [logstore.header_len]u8 = undefined;
if ((tmp_file.readPositionalAll(io, &dst_header, 0) catch return error.Storage) != dst_header.len) return error.Storage;
const stamped = logstore.restampHeader(dst_header, logstore.horizon_version) orelse return error.Storage;
tmp_file.writePositionalAll(io, &stamped, 0) catch return error.Storage;
tmp_file.sync(io) catch return error.Storage;
}
// No repo lock exists: if the source log changed under us since we scanned
// it, abort rather than clobber a concurrent write with our stale snapshot.
// O(1) (one stat + one header read); the original stays intact. For `shun`
// this is also what makes an aborted run leave NO tombstone behind: the rows
// were written into the temp this path is about to delete.
//
// A FRESH HANDLE, a fresh stat and a fresh storage view, for the reason given
// in the doc above: the caller's view answers `size()` from a cached
// watermark, so asking it would always agree with the key — and the caller's
// HANDLE would agree just as unconditionally once a competing rewrite has
// renamed its generation over `.zit`, since that leaves the handle on an
// orphan inode whose size and tail can never move again. Reopening by NAME is
// what makes that case an abort; the caller's handle only ever catches an
// append to the same inode.
var writer_lock = cli.lockStoreWriter(io, root_dir) catch return error.Storage;
defer writer_lock.release(io);
var check_file = root_dir.openFile(io, repo_filename, .{}) catch return error.Storage;
defer check_file.close(io);
const now_st = check_file.stat(io) catch return error.Storage;
var check_fs = storage.FileStorage.init(io, check_file, now_st.size);
if (!logstore.LogStore.validateTail(check_fs.storageFile(), src_key)) return error.TailChanged;
// The single durable transition: atomically replace `.zit` with the packed
// log. A crash before this leaves the original untouched; after it, the
// complete new log. There is no window where `.zit` is partial.
root_dir.rename(pack_tmp_filename, root_dir, repo_filename, io) catch return error.Storage;
renamed = true;
// The rename is durable only once the directory is. Without this a power
// loss can undo it after the verb has reported success, and a shunned
// artifact comes back.
storage.syncDirectory(io, root_dir) catch return error.Storage;
// A name the new generation masks, purge's graveyard must not keep
// either: a `purge undo` would otherwise restore a shunned or erased
// artifact. Only shunned names: a purge's own rewrite drops the names it
// has just buried there. The graveyard is a recovery aid, so a failure
// here does not undo the swap.
_ = zit.graveyard.dropWhere(gpa, io, root_dir, ShunnedAfter{ .src = src, .extra = extra_shuns }, ShunnedAfter.has) catch {};
return stats;
}
/// The names a rewrite's new generation masks: the source's active shuns and
/// the rows the rewrite records.
const ShunnedAfter = struct {
src: *const logstore.LogStore,
extra: []const logstore.ShunRecord,
fn has(self: ShunnedAfter, digest: []const u8) bool {
if (!logstore.isCompleteDigest(digest.len)) return false;
if (self.src.isShunned(digest)) return true;
for (self.extra) |row| if (std.mem.eql(u8, row.hash, digest)) return true;
return false;
}
};
/// The no-repo-lock concurrency guard's diagnostic, general over `verb` because
/// both rewriting verbs abort on it identically. Not a crash and not corruption:
/// the wording leads with what the operator most needs to know.
pub fn tailChangedFail(s: cli.Streams, verb: []const u8) cli.Exit {
s.err.print(
"zit {s}: repository changed during {s}; aborted (original is intact)\n",
.{ verb, verb },
) catch {};
return .failure;
}
/// The temp-lock diagnostic, general over `verb` for the same reason: both
/// rewriting verbs contend for one fixed-name temp. Says what is happening and
/// what to do, and — critically — does NOT say "corrupt", which is what an
/// unguarded collision used to produce.
pub fn rewriteBusyFail(s: cli.Streams, verb: []const u8) cli.Exit {
s.err.print(
"zit {s}: another repository rewrite is in progress; try again when it finishes\n",
.{verb},
) catch {};
return .failure;
}
/// Render pack's failure `err` as `zit pack: <reason>` on stderr, exit 1. The
/// original store is always intact when this runs (every write went to the
/// temp, which the caller's `defer` deletes).
fn packFail(s: cli.Streams, err: PackError) cli.Exit {
s.err.print("zit pack: {s}\n", .{reasonOf(err)}) catch {};
return .failure;
}
/// One human line summarizing a repack. "already packed" only when the run
/// changed NOTHING an operator would care about — the log did not shrink AND
/// nothing was reclaimed (the store was already compact; pack still verified every
/// frame, its integrity-scrub value). A reclaim is never described as "already
/// packed" even when the log grew, which it legitimately can: dropping an artifact
/// that others were deltified against moves its content into one of them, so a
/// byte count is exactly the wrong signal for whether work happened.
///
/// The reclaim clause appears only when the run actually dropped something: a
/// repository with no deferred shuns is the common case and "0 shunned reclaimed"
/// on every pack would be noise, while a run that DID remove bytes must say so —
/// dropping content silently is the one outcome an operator must never have to
/// infer from a byte count. `pack_help` states the clause the same conditional way.
///
/// `artifacts` is labelled "walked" rather than "packed" because `PackStats`
/// counts SOURCE frames: on a reclaiming run `artifacts - dropped` is what the new
/// generation holds, and an unlabelled first number would read as the packed
/// count. Split from `runPack` so the exact wording is unit tested without a
/// filesystem.
fn renderPackStats(out: *std.Io.Writer, stats: PackStats) void {
const unchanged = stats.bytes_after >= stats.bytes_before and stats.dropped == 0;
const verb = if (unchanged) "already packed" else "packed";
out.print("{s}: {d} artifacts walked, {d} re-deltaed, ", .{ verb, stats.artifacts, stats.deltas }) catch {};
if (stats.dropped > 0) out.print("{d} shunned reclaimed, ", .{stats.dropped}) catch {};
out.print("{d} -> {d} bytes\n", .{ stats.bytes_before, stats.bytes_after }) catch {};
}
/// sync §3.3 L553-556 / §8.8 L1839-1841 — this repository's shun table as the
/// `zit.sync.ShunSet` the phantom scan asks. Owns a copy of every shunned
/// DIGEST, taken while the caller already has the log open, so the set outlives
/// the handle it was read through.
///
/// FRESHNESS, stated exactly. This is a SNAPSHOT, not a live view: `load` reads
/// `shunList` once and the set answers from that copy thereafter. On the SERVE
/// path that is per-round freshness, because `LogIndexStore` rebuilds — and so
/// re-loads this — whenever the stat gate sees the log grow, which a `zit shun`
/// append does; a shun written between two served rounds is honoured by the next
/// one. On the CLIENT path it is a PROCESS snapshot: `ClientHoldings` builds
/// its index once at run start and never rebuilds mid-run, so a shun applied
/// mid-run lands on the next process, not the next round. That is a
/// deliberate limit, recorded here rather than implied away — the protocol layer
/// re-probes every scan (`sync.ShunSet`), so nothing else has to change if this
/// ever grows a re-read.
///
/// This is the ONE place a wire NAME becomes a log DIGEST for shun purposes —
/// the boundary `LogIndexStore.names` already declares is held to one place. The
/// width/hex validation is a CHECKED rejection rather than an assert, because an
/// assert is nothing in ReleaseFast: `logstore.LogStore.isShunned`'s caller
/// contract (see its doc) binds this boundary to reject a non-complete name
/// BEFORE decoding it, so the width is gated on the NAME, ahead of the decode.
/// The sync-facing elided-set: answers RFC section 5.2 rule 3 over a
/// classification computed once at run start. Same total-function contract
/// as `RepoShunSet`; borrows both the classification and the store, so both
/// must outlive the run.
/// `pack --drop-elided`'s filter half (sparse-cone RFC section 5.4): built
/// once over the SOURCE log, drops exactly the classification's elide set.
/// A digest that does not resolve in the index is kept -- unknown provenance
/// is never deleted.
const ElidedDropper = struct {
index: *LogIndexStore,
classification: SparseClassification,
inner: zit.pack.Filter,
fn build(gpa: std.mem.Allocator, src: storage.StorageFile, cone: *const zit.cone.Cone) ?ElidedDropper {
const index = LogIndexStore.open(gpa, src) catch return null;
const classification = classifyKeepRids(gpa, index.classifyStore(), cone) catch {
index.deinit();
return null;
};
return .{ .index = index, .classification = classification, .inner = zit.pack.Filter.keep_all };
}
fn drop(ctx: *anyopaque, digest: []const u8) bool {
const self: *ElidedDropper = @ptrCast(@alignCast(ctx));
if (self.inner.dropFn(self.inner.ctx, digest)) return true; // shun reclaim first
var name_buf: [2 * logstore.max_hash_len]u8 = undefined;
const name = digestToWireName(digest[0..digest.len], &name_buf);
const rid = self.index.store().lookup(name) orelse return false;
return self.classification.elided(rid);
}
fn composed(self: *ElidedDropper, inner: zit.pack.Filter) zit.pack.Filter {
self.inner = inner;
return .{ .ctx = @ptrCast(self), .dropFn = drop };
}
fn deinit(self: *ElidedDropper, gpa: std.mem.Allocator) void {
self.classification.deinit(gpa);
self.index.destroy();
self.* = undefined;
}
};
/// Seed a store holding a wire delta that pack CANNOT beat by re-deltification,
/// so ONLY the §6.1 monotonic-shrink guard keeps the store from regressing. `B`
/// is a one-line edit of `A`, stored as a COMPACT zit delta against `A` — the
/// smallest form pack could ever reproduce. But `max_base_candidates` unrelated
/// filler blobs sit between `A` and `B`, so pack's bounded backward candidate
/// window (§5.2) saturates on the fillers and never reaches `A`; with no lineage
/// hint either (bare blobs, no F-card path), pack's best re-encoding of `B` is a
/// full zlib frame STRICTLY LARGER than the stored A-based delta. The pre-guard
/// pack replaced the delta with that larger full frame (the +33% clone
/// regression in miniature); the guard keeps the existing smaller frame. Fills
/// A's and B's 64-hex names.
fn packSeedUnbeatableWireDelta(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
a_name: *[64]u8,
b_name: *[64]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
// A: B's ideal base, stored full (zlib).
const a = try packEvolvingBlob(gpa, 1);
defer gpa.free(a);
var a_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(a, &a_digest, .{});
const a_rid = try log.putZlib(&a_digest, a);
_ = zit.artifact.artifactName(.sha3_256, a, a_name);
// `max_base_candidates` unrelated fillers between A and B: they saturate B's
// backward candidate window so A is out of reach when pack re-encodes B.
var f: usize = 0;
while (f < logstore.max_base_candidates) : (f += 1) {
var fbuf: std.ArrayList(u8) = .empty;
defer fbuf.deinit(gpa);
var i: usize = 0;
while (i < 200) : (i += 1) try fbuf.print(gpa, "filler {d} unrelated row {d}\n", .{ f, i });
var fdig: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(fbuf.items, &fdig, .{});
_ = try log.putZlib(&fdig, fbuf.items);
}
// B: a one-line edit of A, stored as a COMPACT zit delta against A (the
// smallest form pack could reproduce — and only against A, which pack's
// window cannot reach here).
const b = try packEvolvingBlob(gpa, 2);
defer gpa.free(b);
var b_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{});
_ = zit.artifact.artifactName(.sha3_256, b, b_name);
const raw = try zit.delta.create(gpa, a, b);
defer gpa.free(raw);
const container = try zit.container.encode(gpa, raw, .default);
defer gpa.free(container);
_ = try log.putDeltaContainer(&b_digest, a_rid, container);
try log.commit();
}
/// Assert the §6.1 per-frame monotonicity: NO artifact's packed frame is larger
/// on disk than its source frame. Pack keeps the smaller of {existing frame, new
/// encoding} for every artifact, so a repack is non-worsening frame by frame.
/// This holds universally over THESE fixtures because their chains stay below the
/// read cap; the one path that can enlarge a frame — `copySrcFrame`'s §3.3
/// deep-chain fallback re-storing a kept delta as full zlib — needs a base at
/// `max_delta_chain` depth, which the seeds never build.
fn packExpectNoFrameLarger(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void {
const src_rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(src_rids);
for (src_rids) |sr| {
const dr = dst.lookup(src.hashOf(sr)) orelse return error.PackedArtifactMissing;
try std.testing.expect(dst.frameSpanOf(dr) <= src.frameSpanOf(sr));
}
}
/// Count the `.delta` frames among a store's persisted artifacts.
fn packDeltaFrameCount(gpa: std.mem.Allocator, log: *logstore.LogStore) !usize {
const rids = try log.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
var n: usize = 0;
for (rids) |r| {
if (log.kindOf(r) == .delta) n += 1;
}
return n;
}
/// Assert that every artifact in `src` reconstructs byte-identically in `dst`
/// and the two artifact SETS are equal (no drop, no addition).
fn packExpectContentEqual(gpa: std.mem.Allocator, src: *logstore.LogStore, dst: *logstore.LogStore) !void {
const src_rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(src_rids);
const dst_rids = try dst.ridsInPersistedOrder(gpa);
defer gpa.free(dst_rids);
try std.testing.expectEqual(src_rids.len, dst_rids.len);
for (src_rids) |sr| {
const digest = src.hashOf(sr);
const dr = dst.lookup(digest) orelse return error.PackedArtifactMissing;
const a = (try src.content(gpa, sr)).?;
defer gpa.free(a);
const b = (try dst.content(gpa, dr)).?;
defer gpa.free(b);
try std.testing.expectEqualSlices(u8, a, b);
}
}
test "pack: the packed store's artifact set + every reconstructed content byte-equal the original (config preserved)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// No drop: the packed content-frame count equals the source's, and every
// artifact reconstructs byte-for-byte.
try std.testing.expectEqual(src.entries.items.len, stats.artifacts);
try packExpectContentEqual(gpa, &src, &dst);
// The live config survives as exactly one frame with identical fields.
var soc = (try src.readConfig(gpa)).?;
defer soc.deinit(gpa);
var doc = (try dst.readConfig(gpa)).?;
defer doc.deinit(gpa);
try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy);
try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch);
try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id);
try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code);
}
test "pack re-deltas full-form frames (delta count rises, .zit shrinks) with byte-identical content" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// The source is all full-form: not one delta frame.
{
const rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
for (rids) |r| try std.testing.expect(src.kindOf(r) != .delta);
}
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// The two child blobs (v2 against v1, v3 against v2) re-delta against the
// same path's parent blob, so the delta count rises and the store shrinks —
// with content preserved exactly.
try std.testing.expect(stats.deltas >= 2);
try std.testing.expect(stats.bytes_after < stats.bytes_before);
try packExpectContentEqual(gpa, &src, &dst);
}
test "pack preserves wire deltas — a store with delta frames packs to <= source size (regression for the 8.65x inflation bug)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedDeltaStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// The source genuinely holds retained wire deltas (the compact form pack
// must not discard) whose bases pack's lineage hint cannot re-derive.
const src_deltas = try packDeltaFrameCount(gpa, &src);
try std.testing.expect(src_deltas >= 2);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// MONOTONIC: the packed store is never larger than the source. The old pack
// reconstructed each delta to a full zlib frame and inflated the store; a
// correct pack preserves the compact deltas verbatim.
try std.testing.expect(stats.bytes_after <= stats.bytes_before);
// Every source delta survives AS a delta frame — none reconstructed to full
// (the optional re-delta may add further deltas, so the total only grows).
{
const rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
for (rids) |sr| {
if (src.kindOf(sr) != .delta) continue;
const dr = dst.lookup(src.hashOf(sr)).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr));
}
}
try std.testing.expect(try packDeltaFrameCount(gpa, &dst) >= src_deltas);
// No drop, every reconstruction byte-exact.
try packExpectContentEqual(gpa, &src, &dst);
}
test "pack copies a delta frame verbatim re-linked to its base's new rid, content byte-identical" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedDeltaStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
_ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// For every delta frame in the source, the packed store holds it as a delta
// whose stored container is byte-identical (verbatim re-serve) and whose base
// names the SAME artifact (re-linked to the base's new rid). Content matches.
const rids = try src.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
var checked: usize = 0;
for (rids) |sr| {
if (src.kindOf(sr) != .delta) continue;
const digest = src.hashOf(sr);
const dr = dst.lookup(digest).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dr));
var sw = (try src.wire(gpa, sr)).?;
defer sw.deinit(gpa);
var dw = (try dst.wire(gpa, dr)).?;
defer dw.deinit(gpa);
// Verbatim container bytes and the same base hash (re-link preserves the
// logical base while its rid/offset differs in the packed generation).
try std.testing.expectEqualSlices(u8, sw.delta.container, dw.delta.container);
try std.testing.expectEqualSlices(u8, sw.delta.baseHash(), dw.delta.baseHash());
const a = (try src.content(gpa, sr)).?;
defer gpa.free(a);
const b = (try dst.content(gpa, dr)).?;
defer gpa.free(b);
try std.testing.expectEqualSlices(u8, a, b);
checked += 1;
}
try std.testing.expect(checked >= 2);
}
/// The rid naming the 64-hex `name` in `log`, or null when absent (test-only
/// name→digest→rid convenience over `lookup`).
fn packRidByName(log: *logstore.LogStore, name: *const [64]u8) ?logstore.Rid {
var d_buf: [digest_len]u8 = undefined;
const d = hexToDigest(name, &d_buf) orelse return null;
return log.lookup(d);
}
/// Seed a store modeling a peer-supplied (verbatim wire) delta whose sender
/// chose a POOR base. `A` (a large file) and `C` (an unrelated large blob) are
/// stored full; `B` (a one-line edit of `A`) is stored as a hand-crafted §4.1
/// delta container against `C` — a large, suboptimal delta the sender picked.
/// `A` precedes `B` chronologically and is B's ideal base, so pack's best-of-N
/// re-deltification (§6.1) must re-base `B` onto `A` rather than trust the
/// sender's `C`. Fills the 64-hex names of A/B/C.
fn packSeedSuboptimalWireDelta(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
a_name: *[64]u8,
b_name: *[64]u8,
c_name: *[64]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
// A: the ideal base for B (near-identical predecessor), stored full.
const a = try packEvolvingBlob(gpa, 1);
defer gpa.free(a);
a_name.* = try packSeedBlob(log, a);
// C: an unrelated large blob (a poor base for B), stored full.
var cbuf: std.ArrayList(u8) = .empty;
defer cbuf.deinit(gpa);
var i: usize = 0;
while (i < 200) : (i += 1) try cbuf.print(gpa, "unrelated content row {d}\n", .{i});
const c = cbuf.items;
c_name.* = try packSeedBlob(log, c);
const c_rid = packRidByName(log, c_name).?;
// B: a one-line edit of A. delta(A,B) is tiny; delta(C,B) is large.
const b = try packEvolvingBlob(gpa, 2);
defer gpa.free(b);
_ = zit.artifact.artifactName(.sha3_256, b, b_name);
var b_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(b, &b_digest, .{});
// Hand-craft the verbatim wire delta against the POOR base C — models a peer
// that chose C as base, retained at rest by `putDeltaContainer`.
const raw = try zit.delta.create(gpa, c, b);
defer gpa.free(raw);
const container = try zit.container.encode(gpa, raw, .default);
defer gpa.free(container);
_ = try log.putDeltaContainer(&b_digest, c_rid, container);
try log.commit();
}
test "pack §6.1: a verbatim wire delta is re-deltified against the size-minimizing base, not the sender's" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var a_name: [64]u8 = undefined;
var b_name: [64]u8 = undefined;
var c_name: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedSuboptimalWireDelta(gpa, &wlog, &a_name, &b_name, &c_name);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// Precondition: the source really holds B as a delta whose base is C (the
// sender's suboptimal choice) — the exact frame the old pack copied verbatim.
{
const sb = packRidByName(&src, &b_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb));
var sw = (try src.wire(gpa, sb)).?;
defer sw.deinit(gpa);
var cbin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&cbin, &c_name);
try std.testing.expectEqualSlices(u8, &cbin, sw.delta.baseHash());
}
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// After pack, B is still a delta, but its base is now A (the size-minimizing
// candidate) — the sender's base C is no longer trusted at rest (§6.1/§3).
const db = packRidByName(&dst, &b_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db));
var dw = (try dst.wire(gpa, db)).?;
defer dw.deinit(gpa);
var abin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&abin, &a_name);
try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash());
// Every artifact still reconstructs byte-for-byte and the store shrank (a
// tiny A-based delta replaced the large C-based one).
try packExpectContentEqual(gpa, &src, &dst);
try std.testing.expect(stats.bytes_after < stats.bytes_before);
try std.testing.expectEqual(dst.size(), stats.bytes_after);
try std.testing.expectEqual(src.size(), stats.bytes_before);
}
/// Seed the §6.2 same-path scenario: two versions of ONE tracked file whose ideal
/// base (the earlier version) sits OUTSIDE pack's bounded chronological window.
/// `v1` of "tracked.txt" is stored full, then `max_base_candidates` unrelated
/// filler blobs saturate the backward candidate window, then `v2` (a one-line
/// edit of `v1`) is stored full. With ONLY the chronological window (step §6.1)
/// pack cannot reach `v1` from `v2` — the fillers fill every slot — so `v2` stays
/// a full frame and the store does not shrink. The mlink-analog same-path walk
/// (§6.2) reads both check-in manifests, learns tracked.txt: [v1, v2], and offers
/// `v1` as `v2`'s priority candidate, so pack re-deltifies `v2` onto `v1`. Both
/// check-ins are FULL frames (pack's re-delta target); fills v1/v2's 64-hex names.
fn packSeedSamePathOutOfWindow(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
v1_name: *[64]u8,
v2_name: *[64]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
// v1 of tracked.txt (full), and the root check-in that names it.
const v1 = try packEvolvingBlob(gpa, 1);
defer gpa.free(v1);
v1_name.* = try packSeedBlob(log, v1);
var root_name: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &root_name);
// `max_base_candidates` unrelated fillers between v1 and v2: they saturate
// v2's bounded backward window so v1 is out of chronological reach.
var f: usize = 0;
while (f < logstore.max_base_candidates) : (f += 1) {
var fbuf: std.ArrayList(u8) = .empty;
defer fbuf.deinit(gpa);
var i: usize = 0;
while (i < 200) : (i += 1) try fbuf.print(gpa, "samepath filler {d} unrelated row {d}\n", .{ f, i });
_ = try packSeedBlob(log, fbuf.items);
}
// v2 of tracked.txt (a one-line edit of v1), stored FULL, and its check-in.
const v2 = try packEvolvingBlob(gpa, 2);
defer gpa.free(v2);
v2_name.* = try packSeedBlob(log, v2);
var tip_name: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &root_name, "tracked.txt", v2_name, &tip_name);
try log.commit();
}
test "pack §6.2: a same-path predecessor beyond the chronological window becomes v2's base (store shrinks)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var v1_name: [64]u8 = undefined;
var v2_name: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedSamePathOutOfWindow(gpa, &wlog, &v1_name, &v2_name);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// Precondition: the source holds v2 as a NON-delta (full) frame — the case
// step §6.1's chronological window alone cannot improve (v1 is out of reach).
{
const sv2 = packRidByName(&src, &v2_name).?;
try std.testing.expect(src.kindOf(sv2) != .delta);
}
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// After pack, v2 is a delta whose base is the same-path predecessor v1 — the
// §6.2 mlink analog reached a base the chronological window could not.
const dv2 = packRidByName(&dst, &v2_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(dv2));
var dw = (try dst.wire(gpa, dv2)).?;
defer dw.deinit(gpa);
var v1bin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&v1bin, &v1_name);
try std.testing.expectEqualSlices(u8, &v1bin, dw.delta.baseHash());
// The store shrank (a tiny v1-based delta replaced the full v2 frame), every
// artifact still reconstructs byte-for-byte, and no frame grew.
try std.testing.expect(stats.bytes_after < stats.bytes_before);
try packExpectContentEqual(gpa, &src, &dst);
try packExpectNoFrameLarger(gpa, &src, &dst);
}
/// Seed the §6.2 same-path REVERT scenario: one tracked file whose content goes
/// A→B→A across three check-ins — the third restores a byte-identical earlier
/// blob. Because the store is content-addressed, the reverted version resolves to
/// the SAME blob rid as v1, so the pack manifest walk records that rid twice,
/// non-consecutively, for one path. Exercises `deriveCandidateHints`'s
/// distinct-rid compaction: a blob must never be offered as its own same-path
/// predecessor. Fills v1/v2's 64-hex names.
fn packSeedSamePathRevert(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
v1_name: *[64]u8,
v2_name: *[64]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
const v1 = try packEvolvingBlob(gpa, 1);
defer gpa.free(v1);
const v2 = try packEvolvingBlob(gpa, 2);
defer gpa.free(v2);
// v1 of tracked.txt (full) + its root check-in.
v1_name.* = try packSeedBlob(log, v1);
var c1: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-01T00:00:00", null, "tracked.txt", v1_name, &c1);
// v2 (a one-line edit) + its check-in.
v2_name.* = try packSeedBlob(log, v2);
var c2: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-02T00:00:00", &c1, "tracked.txt", v2_name, &c2);
// Revert: tracked.txt returns to v1's byte-identical content. The blob is
// already stored (content-addressed), so this check-in re-references v1's rid
// — the non-consecutive duplicate that recordPathBlob's consecutive-only dedup
// lets through.
var c3: [64]u8 = undefined;
try packSeedCheckin(gpa, log, "2020-01-03T00:00:00", &c2, "tracked.txt", v1_name, &c3);
try log.commit();
}
test "pack §6.2: a same-path revert (A→B→A) re-references one blob rid twice and packs cleanly (no self-predecessor panic)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var v1_name: [64]u8 = undefined;
var v2_name: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedSamePathRevert(gpa, &wlog, &v1_name, &v2_name);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
// Before the distinct-rid compaction in deriveCandidateHints this panicked on
// a self-predecessor (v1 sorted adjacent to its own duplicate ⇒ equal offsets
// fail the strict offset-below assert). Now the pack completes and every
// artifact reconstructs byte-for-byte with no frame enlarged.
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
try packExpectContentEqual(gpa, &src, &dst);
try packExpectNoFrameLarger(gpa, &src, &dst);
try std.testing.expectEqual(dst.size(), stats.bytes_after);
try std.testing.expectEqual(src.size(), stats.bytes_before);
}
test "pack §6.1 monotonic guard: a wire delta pack cannot beat is KEPT, not replaced by a larger frame (no regression)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var a_name: [64]u8 = undefined;
var b_name: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedUnbeatableWireDelta(gpa, &wlog, &a_name, &b_name);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
// Precondition: the source holds B as a compact delta whose base is A.
{
const sb = packRidByName(&src, &b_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, src.kindOf(sb));
var sw = (try src.wire(gpa, sb)).?;
defer sw.deinit(gpa);
var abin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&abin, &a_name);
try std.testing.expectEqualSlices(u8, &abin, sw.delta.baseHash());
}
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
// (a) decode-all byte-identical pre/post.
try packExpectContentEqual(gpa, &src, &dst);
// (b) NO frame is larger post-pack than its pre-pack encoding.
try packExpectNoFrameLarger(gpa, &src, &dst);
// (c) the whole store did not regress on this wire-delta-heavy fixture.
try std.testing.expect(stats.bytes_after <= stats.bytes_before);
// Accounting exact: reported sizes are the real store sizes.
try std.testing.expectEqual(src.size(), stats.bytes_before);
try std.testing.expectEqual(dst.size(), stats.bytes_after);
// B stayed a delta (the compact A-based frame was kept, re-linked to A's new
// rid) — the guard refused to replace it with the larger full zlib pack's
// window-only re-encoding would have produced.
const db = packRidByName(&dst, &b_name).?;
try std.testing.expectEqual(logstore.FrameKind.delta, dst.kindOf(db));
var dw = (try dst.wire(gpa, db)).?;
defer dw.deinit(gpa);
var abin: [digest_len]u8 = undefined;
_ = try std.fmt.hexToBytes(&abin, &a_name);
try std.testing.expectEqualSlices(u8, &abin, dw.delta.baseHash());
// The kept frame is no larger than the source's (equal — copied verbatim).
try std.testing.expect(dst.frameSpanOf(db) <= src.frameSpanOf(packRidByName(&src, &b_name).?));
}
/// Seed a store exercising EVERY frame class pack must handle in one place: a
/// raw frame, a zlib frame, a locally-created delta, a verbatim wire delta, an
/// interned phantom (no frame on disk), and the config singleton. Fills the
/// content-bearing artifacts' 64-hex names plus the phantom's binary digest.
fn packSeedMixedStore(
gpa: std.mem.Allocator,
log: *logstore.LogStore,
raw_name: *[64]u8,
zlib_name: *[64]u8,
delta_name: *[64]u8,
wire_name: *[64]u8,
phantom_digest: *[digest_len]u8,
) !void {
try log.putConfig(.{ .hash_policy = 2, .default_branch = "trunk", .project_id = "pid", .project_code = "code-abc" });
// raw: a unique blob large enough to be a real (if unrelated) base.
var rbuf: std.ArrayList(u8) = .empty;
defer rbuf.deinit(gpa);
var i: usize = 0;
while (i < 120) : (i += 1) try rbuf.print(gpa, "raw-only content row {d}\n", .{i});
const rbytes = rbuf.items;
raw_name.* = try packSeedBlob(log, rbytes); // `put` => .raw
// zlib: a large evolving blob stored as a full zlib container.
const z = try packEvolvingBlob(gpa, 1);
defer gpa.free(z);
var z_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(z, &z_digest, .{});
const z_rid = try log.putZlib(&z_digest, z);
_ = zit.artifact.artifactName(.sha3_256, z, zlib_name);
// local delta: an edit of the zlib blob, delta-encoded against it.
const d = try packEvolvingBlob(gpa, 2);
defer gpa.free(d);
var d_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(d, &d_digest, .{});
_ = try log.putArtifact(&d_digest, d, z_rid);
_ = zit.artifact.artifactName(.sha3_256, d, delta_name);
// wire delta: a hand-crafted §4.1 delta container against the raw blob (a
// poor base) — the verbatim-retained peer form.
const w = try packEvolvingBlob(gpa, 3);
defer gpa.free(w);
var w_digest: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(w, &w_digest, .{});
_ = zit.artifact.artifactName(.sha3_256, w, wire_name);
const r_rid = packRidByName(log, raw_name).?;
const raw_delta = try zit.delta.create(gpa, rbytes, w);
defer gpa.free(raw_delta);
const container = try zit.container.encode(gpa, raw_delta, .default);
defer gpa.free(container);
_ = try log.putDeltaContainer(&w_digest, r_rid, container);
// phantom: an interned forward reference that never reaches disk.
std.crypto.hash.sha3.Sha3_256.hash("absent forward-referenced artifact", phantom_digest, .{});
_ = try log.intern(phantom_digest);
try log.commit();
}
test "pack §6.1/§3: a mixed raw/zlib/delta/wire-delta/phantom/config store round-trips, config+phantom preserved, accounting exact" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var raw_name: [64]u8 = undefined;
var zlib_name: [64]u8 = undefined;
var delta_name: [64]u8 = undefined;
var wire_name: [64]u8 = undefined;
var phantom_digest: [digest_len]u8 = undefined;
// A phantom never reaches disk, so we pack the in-memory writable store
// directly (a reopen would drop it) — the case pack must tolerate without
// choking or emitting a garbage frame.
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedMixedStore(gpa, &wlog, &raw_name, &zlib_name, &delta_name, &wire_name, &phantom_digest);
// The seeded store really holds one frame of each class plus the phantom.
try std.testing.expectEqual(logstore.FrameKind.raw, wlog.kindOf(packRidByName(&wlog, &raw_name).?));
try std.testing.expectEqual(logstore.FrameKind.zlib, wlog.kindOf(packRidByName(&wlog, &zlib_name).?));
try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &delta_name).?));
try std.testing.expectEqual(logstore.FrameKind.delta, wlog.kindOf(packRidByName(&wlog, &wire_name).?));
try std.testing.expect(wlog.isPhantom(wlog.lookup(&phantom_digest).?));
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), wlog.hash_kind);
defer dst.deinit(gpa);
const stats = try packLog(gpa, &wlog, &dst, Filter.keep_all, &.{});
// Decode-all byte-identical pre/post, and the artifact SETS match (the
// phantom is excluded from both, so counts still agree).
try packExpectContentEqual(gpa, &wlog, &dst);
// §6.1 per-frame monotonicity: no frame grew in the repack.
try packExpectNoFrameLarger(gpa, &wlog, &dst);
// Config preserved verbatim in fields.
var soc = (try wlog.readConfig(gpa)).?;
defer soc.deinit(gpa);
var doc = (try dst.readConfig(gpa)).?;
defer doc.deinit(gpa);
try std.testing.expectEqual(soc.config.hash_policy, doc.config.hash_policy);
try std.testing.expectEqualStrings(soc.config.default_branch, doc.config.default_branch);
try std.testing.expectEqualStrings(soc.config.project_id, doc.config.project_id);
try std.testing.expectEqualStrings(soc.config.project_code, doc.config.project_code);
// Phantom preserved as a contentless forward reference: it carries no bytes,
// so pack writes no frame for it (a phantom never reaches disk) and it is not
// resurrected as a garbage artifact in the packed store.
try std.testing.expect(dst.lookup(&phantom_digest) == null);
// Accounting exact: the reported sizes are the actual store sizes and the
// artifact tally is the source's real (non-phantom) frame count.
try std.testing.expectEqual(wlog.size(), stats.bytes_before);
try std.testing.expectEqual(dst.size(), stats.bytes_after);
const src_rids = try wlog.ridsInPersistedOrder(gpa);
defer gpa.free(src_rids);
try std.testing.expectEqual(src_rids.len, stats.artifacts);
}
test "pack §7.7: packLog aborts on a source frame whose content does not name it (no packed frame written)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// A frame keyed by a name that is NOT sha3-256 of its bytes: `put` stores
// bytes verbatim under any key, so this models a corrupt/lying source frame
// that survives `openReadOnly` (its CRC + embedded-hash label are intact).
const lying_name = @as([digest_len]u8, @splat(0xab));
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
_ = try wlog.put(&lying_name, "content these bytes do not name");
try wlog.commit();
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var dfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer dfile.close(io);
var dfs = storage.FileStorage.init(io, dfile, 0);
var dst = try logstore.LogStore.open(gpa, dfs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
try std.testing.expectError(error.Corrupt, packLog(gpa, &src, &dst, Filter.keep_all, &.{}));
try std.testing.expect(dst.lookup(&lying_name) == null); // no frame landed on the abort
}
test "pack is idempotent: pack∘pack is byte-identical (0 additional re-deltas)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
// First pack: source (full) → L1.
{
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var f1 = try tmp.dir.createFile(io, "pack1", .{ .read = true, .truncate = true });
defer f1.close(io);
var f1fs = storage.FileStorage.init(io, f1, 0);
var dst = try logstore.LogStore.open(gpa, f1fs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
_ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
}
// Second pack: L1 → L2, byte-for-byte equal, and no further re-deltaing.
var l1fs: storage.FileStorage = undefined;
var l1 = try packOpenReadOnly2(gpa, io, tmp.dir, "pack1", &l1fs);
defer l1fs.file.close(io);
defer l1.deinit(gpa);
var f2 = try tmp.dir.createFile(io, "pack2", .{ .read = true, .truncate = true });
defer f2.close(io);
var f2fs = storage.FileStorage.init(io, f2, 0);
var l2 = try logstore.LogStore.open(gpa, f2fs.storageFile(), l1.hash_kind);
defer l2.deinit(gpa);
const stats2 = try packLog(gpa, &l1, &l2, Filter.keep_all, &.{});
try std.testing.expectEqual(l1.size(), l2.size());
const a = try tmp.dir.readFileAlloc(io, "pack1", gpa, .limited(1 << 24));
defer gpa.free(a);
const b = try tmp.dir.readFileAlloc(io, "pack2", gpa, .limited(1 << 24));
defer gpa.free(b);
try std.testing.expectEqualSlices(u8, a, b); // byte-identical: pack∘pack == pack
// An already-packed store still reports its deltas (they were re-created
// identically), and packing it does not shrink further.
try std.testing.expectEqual(l1.size(), stats2.bytes_after);
}
/// Reopen a named log file in `dir` read-only (a sibling to `packOpenReadOnly`
/// that takes an arbitrary name, for the multi-generation idempotence test).
fn packOpenReadOnly2(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, name: []const u8, fs_out: *storage.FileStorage) !logstore.LogStore {
const f = try dir.openFile(io, name, .{});
const st = try f.stat(io);
fs_out.* = storage.FileStorage.init(io, f, st.size);
return logstore.LogStore.openReadOnly(gpa, fs_out.storageFile());
}
test "pack: packLog never writes the source (a crash before the rename leaves the original byte-identical)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
const before = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(before);
// Pack into a separate temp (the new generation). packLog only ever reads
// the source; the atomic swap in `runPack` is the sole mutation of `.zit`.
{
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var f = try tmp.dir.createFile(io, pack_tmp_filename, .{ .read = true, .truncate = true });
defer f.close(io);
var ffs = storage.FileStorage.init(io, f, 0);
var dst = try logstore.LogStore.open(gpa, ffs.storageFile(), src.hash_kind);
defer dst.deinit(gpa);
_ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
}
// No rename happened (the "crash"): the original `.zit` is byte-for-byte
// unchanged and still opens.
const after = try tmp.dir.readFileAlloc(io, repo_filename, gpa, .limited(1 << 24));
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, before, after);
}
/// The allocation-failure probe: pack a prebuilt source `.zit` into a fresh temp
/// destination, using the failing allocator throughout. Every OOM point must
/// unwind cleanly (no leak) and a fully-funded run must succeed.
fn packOomProbe(alloc: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !void {
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(alloc, io, dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(alloc);
var f = try dir.createFile(io, "oomdst", .{ .read = true, .truncate = true });
defer f.close(io);
var ffs = storage.FileStorage.init(io, f, 0);
var dst = try logstore.LogStore.open(alloc, ffs.storageFile(), src.hash_kind);
defer dst.deinit(alloc);
_ = try packLog(alloc, &src, &dst, Filter.keep_all, &.{});
}
test "pack: checkAllAllocationFailures on packLog (every OOM path unwinds cleanly)" {
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(std.testing.allocator, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(std.testing.allocator);
try packSeedFullStore(std.testing.allocator, &wlog, &tip);
}
// Zig's allocator grows a buffer in place when the heap allows, so the
// count of allocations depends on heap state; refuse in-place growth.
try zit.NoGrowAllocator.checkAllAllocationFailures(std.testing.allocator, packOomProbe, .{ io, tmp.dir });
}
test "pack∘derive == from-scratch derive: the packed store derives the identical repo view" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var tip: [64]u8 = undefined;
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
try packSeedFullStore(gpa, &wlog, &tip);
}
var sfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &sfs);
defer sfs.file.close(io);
defer src.deinit(gpa);
var src_view: zit.repoview.RepoView = undefined;
try src_view.load(gpa, &src);
defer src_view.deinit(gpa);
var pfile = try tmp.dir.createFile(io, "packed", .{ .read = true, .truncate = true });
defer pfile.close(io);
var pfs = storage.FileStorage.init(io, pfile, 0);
var dst = try logstore.LogStore.open(gpa, pfs.storageFile(), src.hash_kind);
_ = try packLog(gpa, &src, &dst, Filter.keep_all, &.{});
dst.deinit(gpa);
var rfs: storage.FileStorage = undefined;
var packed_log = try packOpenReadOnly2(gpa, io, tmp.dir, "packed", &rfs);
defer rfs.file.close(io);
defer packed_log.deinit(gpa);
var packed_view: zit.repoview.RepoView = undefined;
try packed_view.load(gpa, &packed_log);
defer packed_view.deinit(gpa);
// Same derived view: main branch, artifact count, and the leaf set + order
// (§7.5) are identical — pack changed only the storage layout.
try std.testing.expectEqualStrings(src_view.main_branch, packed_view.main_branch);
try std.testing.expectEqual(src_view.store.store().count(), packed_view.store.store().count());
var la = src_view.state.leaves();
var lb = packed_view.state.leaves();
while (la.next()) |ra| {
const rb = lb.next() orelse return error.LeafCountMismatch;
try std.testing.expectEqualStrings(src_view.hashOf(ra), packed_view.hashOf(rb));
}
try std.testing.expect(lb.next() == null);
}
test "renderPackStats: 'packed' when the store shrank, 'already packed' otherwise" {
var buf: [128]u8 = undefined;
{
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 1695, .deltas = 1373, .dropped = 0, .shuns_recorded = 0, .bytes_before = 9_680_000, .bytes_after = 920_000 });
try std.testing.expectEqualStrings(
"packed: 1695 artifacts walked, 1373 re-deltaed, 9680000 -> 920000 bytes\n",
w.buffered(),
);
}
{
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 3, .deltas = 2, .dropped = 0, .shuns_recorded = 0, .bytes_before = 4096, .bytes_after = 4096 });
try std.testing.expectEqualStrings(
"already packed: 3 artifacts walked, 2 re-deltaed, 4096 -> 4096 bytes\n",
w.buffered(),
);
}
}
test "renderPackStats: a reclaiming run names the dropped count; a run that drops nothing stays silent" {
// Reclamation is the one pack outcome an operator must not have to infer
// from a shrinking byte count: bytes shrink on an ordinary re-deltify too.
// So the clause appears exactly when `dropped > 0`, and nowhere else.
var buf: [128]u8 = undefined;
{
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 12, .deltas = 4, .dropped = 3, .shuns_recorded = 0, .bytes_before = 8192, .bytes_after = 4096 });
try std.testing.expectEqualStrings(
"packed: 12 artifacts walked, 4 re-deltaed, 3 shunned reclaimed, 8192 -> 4096 bytes\n",
w.buffered(),
);
}
{
// `artifacts` counts the source frames WALKED, so a run that dropped
// every one of them still reports 12 walked and 12 reclaimed.
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 12, .deltas = 0, .dropped = 12, .shuns_recorded = 0, .bytes_before = 8192, .bytes_after = 64 });
try std.testing.expectEqualStrings(
"packed: 12 artifacts walked, 0 re-deltaed, 12 shunned reclaimed, 8192 -> 64 bytes\n",
w.buffered(),
);
}
{
// A reclaim that GREW the log is still "packed". Reclaiming an artifact
// other artifacts were deltified against moves its content into one of
// them, so the log can legitimately end up larger — and "already packed"
// on a run that just irreversibly deleted content would be the most
// misleading line this verb could print.
var w: std.Io.Writer = .fixed(&buf);
renderPackStats(&w, .{ .artifacts = 7, .deltas = 5, .dropped = 1, .shuns_recorded = 0, .bytes_before = 5760, .bytes_after = 6248 });
try std.testing.expectEqualStrings(
"packed: 7 artifacts walked, 5 re-deltaed, 1 shunned reclaimed, 5760 -> 6248 bytes\n",
w.buffered(),
);
}
}
test "rewriteAndSwap: a run the concurrency guard aborts leaves .zit untouched and records NO tombstone" {
// THE "ONE DURABLE TRANSITION" INVARIANT, made observable. `zit shun --force`
// must never leave a tombstone behind without also having removed the content
// it names; the mechanism is that the rows are written into the TEMP and only
// the rename makes anything durable. This drives the abort seam that is a live
// guard rather than an injected fault — the no-repo-lock concurrency guard: a
// concurrent writer extends `.zit` between our scan and the rename, so `validateTail`
// fails after `packLog` has already written every row into the temp. If the
// rows had been appended to the live log first, they would still be here.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const doomed_body = repeat("content the operator named in this invocation ", 4);
var doomed: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(doomed_body, &doomed, .{});
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
_ = try packSeedBlob(&wlog, doomed_body);
_ = try packSeedBlob(&wlog, repeat("content with nothing against it ", 4));
try wlog.commit();
}
// The scan the rewrite will be built from, and the tail key that pins it.
var rfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &rfs);
defer src.deinit(gpa);
const src_key = try src.logKey();
// The concurrent writer. One appended frame is exactly what the guard exists
// to detect; the snapshot in `src` is now stale.
{
var f = try tmp.dir.openFile(io, repo_filename, .{ .mode = .read_write });
defer f.close(io);
const st = try f.stat(io);
var wfs = storage.FileStorage.init(io, f, st.size);
var wlog = try logstore.LogStore.open(gpa, wfs.storageFile(), 0);
defer wlog.deinit(gpa);
_ = try packSeedBlob(&wlog, repeat("a frame a concurrent writer appended ", 3));
try wlog.commit();
}
const extended = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(extended);
var set: NameSet = .{};
defer set.deinit(gpa);
try set.add(gpa, &doomed);
const rows = [_]logstore.ShunRecord{
.{ .hash = &doomed, .epoch = 1753000077, .reason = "takedown notice 4417" },
};
try std.testing.expectError(error.TailChanged, rewriteAndSwap(
gpa,
io,
tmp.dir,
&src,
src_key,
set.filter(),
&rows,
));
// THE PINS, tombstone FIRST because it is the one this test exists for: the
// aborted run left no row, `.zit` is the concurrent writer's version byte for
// byte (our stale rewrite never landed), the content the run intended to
// remove is intact, and the temp is gone.
var vfs: storage.FileStorage = undefined;
var after_log = try packOpenReadOnly(gpa, io, tmp.dir, &vfs);
defer after_log.deinit(gpa);
try std.testing.expectEqual(@as(usize, 0), after_log.shunList().len);
try std.testing.expect(!after_log.isShunned(&doomed));
try std.testing.expect(after_log.lookup(&doomed) != null);
{
const after = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, extended, after);
}
if (tmp.dir.openFile(io, pack_tmp_filename, .{})) |f| {
var mf = f;
mf.close(io);
return error.TempSurvivedTheAbort;
} else |_| {}
}
test "rewriteAndSwap: a run the concurrency guard aborts leaves .zit untouched and records NO tombstone (competing RENAME)" {
// THE SIBLING SHAPE, and the one the caller's own file handle cannot see. The
// test above simulates a concurrent writer that APPENDS to `.zit`; here a
// competing rewrite (`zit pack`, or another `zit shun --force`) renames a whole
// new generation over the name while our snapshot's handle stays open. That
// handle then names an ORPHAN inode whose size and tail can never move again,
// so a guard that re-stats it agrees with its own key forever and this run
// renames a pre-competitor snapshot over the winner — silently restoring
// content the winner shunned, with both runs exiting 0. Reopening `.zit` by
// NAME is what turns it into `error.TailChanged`.
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const doomed_body = repeat("content the operator named in this invocation ", 4);
var doomed: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(doomed_body, &doomed, .{});
{
var wfs: storage.FileStorage = undefined;
var wlog = try packOpenWritable(gpa, io, tmp.dir, &wfs);
defer wfs.file.close(io);
defer wlog.deinit(gpa);
_ = try packSeedBlob(&wlog, doomed_body);
_ = try packSeedBlob(&wlog, repeat("content with nothing against it ", 4));
try wlog.commit();
}
// The scan the rewrite will be built from, and the tail key that pins it. This
// handle is the one the competing rename is about to orphan.
var rfs: storage.FileStorage = undefined;
var src = try packOpenReadOnly(gpa, io, tmp.dir, &rfs);
defer src.deinit(gpa);
const src_key = try src.logKey();
// The competing rewrite: a NEW generation built beside `.zit` and renamed over
// it. Nothing touches the inode `src` is reading, which is the whole point.
{
var f = try tmp.dir.createFile(io, "gen2", .{ .read = true, .truncate = true });
defer f.close(io);
var gfs = storage.FileStorage.init(io, f, 0);
var glog = try logstore.LogStore.open(gpa, gfs.storageFile(), src.hash_kind);
defer glog.deinit(gpa);
_ = try packSeedBlob(&glog, repeat("the generation the competing rewrite produced ", 3));
try glog.commit();
}
try tmp.dir.rename("gen2", tmp.dir, repo_filename, io);
const winner = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(winner);
var set: NameSet = .{};
defer set.deinit(gpa);
try set.add(gpa, &doomed);
const rows = [_]logstore.ShunRecord{
.{ .hash = &doomed, .epoch = 1753000077, .reason = "takedown notice 4417" },
};
try std.testing.expectError(error.TailChanged, rewriteAndSwap(
gpa,
io,
tmp.dir,
&src,
src_key,
set.filter(),
&rows,
));
// The winner's generation survives byte for byte, and the aborted run recorded
// nothing — the loser did not resurrect its own snapshot's contents.
{
const after = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, winner, after);
}
var vfs: storage.FileStorage = undefined;
var after_log = try packOpenReadOnly(gpa, io, tmp.dir, &vfs);
defer after_log.deinit(gpa);
try std.testing.expectEqual(@as(usize, 0), after_log.shunList().len);
try std.testing.expect(after_log.lookup(&doomed) == null);
if (tmp.dir.openFile(io, pack_tmp_filename, .{})) |f| {
var mf = f;
mf.close(io);
return error.TempSurvivedTheAbort;
} else |_| {}
}
/// Read `dir/.zit` whole (caller frees) — the byte-identical oracle for a run
/// that must have changed nothing.
fn storeSnapshot(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) ![]u8 {
var file = try dir.openFile(io, repo_filename, .{});
defer file.close(io);
const st = try file.stat(io);
const buf = try gpa.alloc(u8, @intCast(st.size));
errdefer gpa.free(buf);
const n = try file.readPositionalAll(io, buf, 0);
try std.testing.expectEqual(buf.len, n);
return buf;
}
test "a rewrite staged before an import cannot publish over pending state" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const body = "held content\n";
var hash: [digest_len]u8 = undefined;
std.crypto.hash.sha3.Sha3_256.hash(body, &hash, .{});
try serveAppendFrame(gpa, io, tmp.dir, &hash, body);
const snapshot = try cli.openStoreRead(io, tmp.dir);
defer snapshot.close(io);
var fs = storage.FileStorage.init(io, snapshot, snapshot.size);
var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile());
defer log.deinit(gpa);
const key = try log.logKey();
const before = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(before);
try tmp.dir.createDir(io, zit.import_transaction.pending_name, .default_dir);
try std.testing.expectError(error.Storage, rewriteAndSwap(gpa, io, tmp.dir, &log, key, Filter.keep_all, &.{}));
const after = try storeSnapshot(gpa, io, tmp.dir);
defer gpa.free(after);
try std.testing.expectEqualSlices(u8, before, after);
}
/// What `loadClusterIndex` answers for `tmp`'s store now, beside what a
/// fresh resolve over the same store answers: equal when no stale sidecar was
/// trusted.
fn expectResolvedFresh(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, clustered_name: []const u8, want_clustered: bool) !void {
var holdings = try ClientHoldings.open(gpa, io, dir);
defer holdings.deinit(io);
var got = try loadClusterIndex(gpa, io, dir, &holdings.index.log, holdings.store());
defer got.deinit();
var names: std.ArrayListUnmanaged([]const u8) = .empty;
defer names.deinit(gpa);
try names.appendSlice(gpa, got.cluster_names);
var fresh = try clusterindex.resolve(gpa, holdings.store(), names.items);
defer fresh.deinit();
const s = holdings.store();
var rid: repostate.Rid = 1;
while (rid <= s.count()) : (rid += 1) {
try std.testing.expectEqual(fresh.clustered.isSet(rid), got.clustered.isSet(rid));
}
try std.testing.expectEqual(fresh.missing.len, got.missing.len);
const held = if (s.lookup(clustered_name)) |r| s.held(r) else false;
try std.testing.expectEqual(want_clustered, held and got.clustered.isSet(s.lookup(clustered_name).?));
var listed_missing = false;
for (got.missing) |m| {
if (std.mem.eql(u8, m, clustered_name)) listed_missing = true;
}
try std.testing.expectEqual(!want_clustered, listed_missing);
}
test "the resolved cluster sidecar is rebuilt, not trusted, after a pack, a shun and a purge" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try d4SeedEmptyRepo(gpa, io, tmp.dir);
// Three blobs; one cluster names the first two.
var ms = repostate.MemStore.init(gpa);
defer ms.deinit();
var names: [3][64]u8 = undefined;
inline for (.{ "shunned member", "purged member", "loose blob" }, 0..) |body, i| {
_ = zit.artifact.artifactName(.sha3_256, body, &names[i]);
const rid = try ms.store().put(&names[i], body);
const c = try zit.container.encode(gpa, body, .default);
defer gpa.free(c);
try ms.store().retain(rid, .{ .full = c });
}
const cluster = try std.fmt.allocPrint(gpa, "M {s}\nM {s}\nZ 00000000000000000000000000000000\n", .{ &names[0], &names[1] });
defer gpa.free(cluster);
var cn: [64]u8 = undefined;
_ = zit.artifact.artifactName(.sha3_256, cluster, &cn);
{
const rid = try ms.store().put(&cn, cluster);
const c = try zit.container.encode(gpa, cluster, .default);
defer gpa.free(c);
try ms.store().retain(rid, .{ .full = c });
}
_ = try persistStore(gpa, io, tmp.dir, ms.store(), null, 0);
try expectResolvedFresh(gpa, io, tmp.dir, &names[1], true);
const Rewrite = struct {
fn run(dir: std.Io.Dir, filter: Filter) !void {
const a = std.testing.allocator;
const sio = std.testing.io;
const snapshot = try cli.openStoreRead(sio, dir);
defer snapshot.close(sio);
var fs = storage.FileStorage.init(sio, snapshot, snapshot.size);
var log = try logstore.LogStore.openReadOnly(a, fs.storageFile());
defer log.deinit(a);
_ = try rewriteAndSwap(a, sio, dir, &log, try log.logKey(), filter, &.{});
}
};
// Pack: a new generation of the same artifacts.
try Rewrite.run(tmp.dir, Filter.keep_all);
try expectResolvedFresh(gpa, io, tmp.dir, &names[1], true);
// Shun: the first member is masked, so it is a promise no longer held.
{
var sa = try cli.openStoreAppend(io, tmp.dir);
defer sa.close(io);
const st = try sa.file.stat(io);
var fs = storage.FileStorage.init(io, sa.file, st.size);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0);
defer log.deinit(gpa);
var digest_buf: [digest_len]u8 = undefined;
try log.putShun(decodeWireName(&names[0], &digest_buf).?, "", 0);
try log.commit();
}
try expectResolvedFresh(gpa, io, tmp.dir, &names[0], false);
// Purge: the second member's frame is gone from the new generation.
const Drop = struct {
var digest: [digest_len]u8 = undefined;
fn drop(_: *anyopaque, d: []const u8) bool {
return std.mem.eql(u8, d, &digest);
}
};
_ = decodeWireName(&names[1], &Drop.digest).?;
var drop_ctx: u8 = 0;
try Rewrite.run(tmp.dir, .{ .ctx = &drop_ctx, .dropFn = Drop.drop });
try expectResolvedFresh(gpa, io, tmp.dir, &names[1], false);
}