//! The submanifest conversion pass: re-mint a fossil corpus as a new
//! project with S-card tree manifests from the first commit.
//!
//! The load-bearing observation is the RE-MINT CASCADE RULE: an artifact's
//! name changes only when its BYTES change, and (with content-blob names
//! kept, the phase-1 posture) bytes change only for an artifact that either
//! gains the S-tree structure (every check-in manifest) or references a
//! re-minted artifact in a hash-bearing card. So the remap surface is:
//!
//! check-in manifests all re-mint (S-trees + remapped P/Q/B; D/C/U and
//! the R card survive -- R is over the file SET, which
//! is preserved exactly)
//! control artifacts re-mint iff a T card targets a remapped check-in
//! events (technotes) re-mint iff their P chain or T target remaps
//! wiki / forum P/I/G reference their own kind's chain -- verbatim
//! unless an ancestor re-minted (transitively closed)
//! attachments target a wiki NAME or ticket uuid + a CONTENT hash
//! -- verbatim in phase 1
//! tickets K ids are uuids, not artifact hashes -- verbatim
//! clusters dropped (sync housekeeping, not history)
//! content blobs verbatim (phase 1 keeps their names)
//! PGP signatures cannot survive rehashing: stripped and counted
//!
//! The census pass below is the converter's front door: it classifies the
//! whole corpus, counts the remap surface and the casualties, and is the
//! baseline the count-reconciliation acceptance check audits against.
const std = @import("std");
const repeat = @import("zit").repeat;
const zit = @import("zit");
const cli = zit.cli;
const artifact = zit.artifact;
const logstore = zit.logstore;
const Repository = zit.Repository;
const repostate = zit.repostate;
const editor = zit.submanifest_editor;
const LogTreeStore = zit.log_tree_store.LogTreeStore;
/// One corpus walk: every stored artifact classified by parsed type, with
/// the phase-1 remap surface and casualty counts. Phantoms hold no bytes
/// and are skipped (they are the sync layer's business, not history's).
pub const Census = struct {
total: u64 = 0,
phantoms: u64 = 0,
blobs: u64 = 0,
checkins: u64 = 0,
submanifests: u64 = 0,
controls: u64 = 0,
wikis: u64 = 0,
tickets: u64 = 0,
attachments: u64 = 0,
events: u64 = 0,
forums: u64 = 0,
clusters: u64 = 0,
/// Check-in manifests wrapped in a PGP signature: the signature is a
/// stated casualty of any conversion (stripped and counted).
signed: u64 = 0,
pub fn structural(self: *const Census) u64 {
return self.checkins + self.submanifests + self.controls + self.wikis +
self.tickets + self.attachments + self.events + self.forums + self.clusters;
}
};
/// Count the phantoms in the source store without reading any content —
/// the cheap pre-flight that lets an incomplete corpus refuse before a
/// staging directory exists. The classifying walk itself happens once, in
/// `remint`'s first pass (it used to happen twice).
pub fn phantomCount(gpa: std.mem.Allocator, repo: *Repository) error{ OutOfMemory, Storage }!u64 {
const log = repo.logStore(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
};
var phantoms: u64 = 0;
const n: u64 = log.artifactCount();
var rid: u64 = 1;
while (rid <= n) : (rid += 1) {
if (log.isPhantom(@intCast(rid))) phantoms += 1;
}
return phantoms;
}
/// Render the census for the operator: what converts, what carries, what
/// dies. The converter's own accounting reconciles against these numbers.
pub fn renderCensus(c: *const Census, streams: cli.Streams) void {
streams.out.print(
"census: {d} artifacts — {d} check-ins to re-mint ({d} signed: signatures strip)," ++
" {d} content blobs carried verbatim\n",
.{ c.total, c.checkins, c.signed, c.blobs },
) catch {};
if (c.controls + c.wikis + c.tickets + c.attachments + c.events + c.forums != 0) {
streams.out.print(
"census: {d} control, {d} wiki, {d} ticket, {d} attachment, {d} technote, {d} forum" ++
" — carried, re-minting only where a reference remaps\n",
.{ c.controls, c.wikis, c.tickets, c.attachments, c.events, c.forums },
) catch {};
}
if (c.clusters != 0) {
streams.out.print("census: {d} cluster(s) dropped (sync housekeeping, not history)\n", .{c.clusters}) catch {};
}
if (c.submanifests != 0) {
streams.out.print("census: {d} sub-manifest slice(s) already present in the source\n", .{c.submanifests}) catch {};
}
if (c.phantoms != 0) {
streams.out.print("census: {d} phantom/masked name(s) skipped — clone the origin to completeness first\n", .{c.phantoms}) catch {};
}
}
/// One F card over a static filename/hash pair, for the mint tests below.
fn testFCard(name: []const u8, hash: []const u8) artifact.FCard {
return .{ .filename = name, .hash = hash, .perm = null, .old_name = null, .mode = .regular };
}
test "convert §6.2 step 2: a check-in whose names do not partition mints FLAT, not empty" {
// The regression: `buildTree` reports `isFlat()` for a name set that
// yields fewer than two groups, and `rewriteManifest` dropped the source's
// F cards while emitting an empty S run — publishing a check-in with NO
// file set. The differential verify caught it and aborted the whole
// conversion, which is how it presented (an ordinary libspatialite
// trunk baseline, 38 files, at k=5).
const gpa = std.testing.allocator;
const entries = [_]artifact.FCard{
testFCard("Makefile", repeat("0", 40)),
testFCard("src/a.c", repeat("1", 40)),
testFCard("src/b.c", repeat("2", 40)),
};
// max_k makes the span enormous relative to three names, so the boundary
// test never fires and the whole set stays one group — §6.2 step 2.
var build = try artifact.buildTree(gpa, &entries, artifact.max_k, .sha3_256);
defer build.deinit();
try std.testing.expect(build.isFlat());
const source =
"C comment\n" ++
"D 2009-04-16T15:57:11\n" ++
"F Makefile " ++ repeat("0", 40) ++ "\n" ++
"F src/a.c " ++ repeat("1", 40) ++ "\n" ++
"F src/b.c " ++ repeat("2", 40) ++ "\n" ++
"P " ++ repeat("3", 40) ++ "\n" ++
"U someone\n" ++
"Z " ++ repeat("4", 32) ++ "\n";
var map: std.StringHashMapUnmanaged([]const u8) = .empty;
defer map.deinit(gpa);
const minted = try rewriteManifest(gpa, source, &entries, build.s, &map);
defer gpa.free(minted);
// The mint must carry the file set as an F run and no S run at all, and it
// must still parse as a check-in manifest.
var m = try artifact.parse(gpa, minted);
defer m.deinit();
try std.testing.expectEqual(@as(usize, 0), m.s.len);
try std.testing.expectEqual(entries.len, m.f.len);
for (&entries, m.f) |src_card, dst_card| {
try std.testing.expectEqualStrings(src_card.filename, dst_card.filename);
try std.testing.expectEqualStrings(src_card.hash.?, dst_card.hash.?);
}
}
test "convert §6.2 step 3: a partitioning name set still mints as an S tree with no F run" {
// The other arm, so the fix above cannot be "flatten everything": at k=1
// the same shape of input does partition, and the mint must be a tree.
const gpa = std.testing.allocator;
var entries: std.ArrayList(artifact.FCard) = .empty;
defer entries.deinit(gpa);
var names: [64][8]u8 = undefined;
var hashes: [64][40]u8 = undefined;
for (0..names.len) |i| {
names[i] = [_]u8{ 'f', 'i', 'l', 'e', '0' + @as(u8, @intCast(i / 10)), '0' + @as(u8, @intCast(i % 10)), 0, 0 };
hashes[i] = @as([40]u8, @splat('a'));
hashes[i][0] = '0' + @as(u8, @intCast(i % 10));
try entries.append(gpa, testFCard(names[i][0..6], hashes[i][0..]));
}
var build = try artifact.buildTree(gpa, entries.items, artifact.min_k, .sha3_256);
defer build.deinit();
try std.testing.expect(!build.isFlat());
const source = "C c\nD 2009-04-16T15:57:11\nU someone\nZ " ++ repeat("4", 32) ++ "\n";
var map: std.StringHashMapUnmanaged([]const u8) = .empty;
defer map.deinit(gpa);
const minted = try rewriteManifest(gpa, source, entries.items, build.s, &map);
defer gpa.free(minted);
var m = try artifact.parse(gpa, minted);
defer m.deinit();
try std.testing.expectEqual(@as(usize, 0), m.f.len);
try std.testing.expect(m.s.len >= 2); // §6.2: a tree partitions at least in two
}
test "census: an empty-ish census sums structurally" {
// Goal: the structural() partition is total over ArtifactType — a new
// member without a census bucket fails the exhaustive switch in census()
// at compile time, and this pin keeps the arithmetic honest.
var c: Census = .{ .checkins = 2, .controls = 1, .clusters = 3 };
try std.testing.expectEqual(@as(u64, 6), c.structural());
}
// ---------------------------------------------------------------------------
// The re-mint pass: check-ins become S-tree manifests under new names,
// every hash-bearing reference chases the old->new map, everything else
// carries verbatim. Phase 1 keeps content-blob names; new structural
// artifacts mint under SHA3-256 (the new project's native policy).
// ---------------------------------------------------------------------------
pub const RemintError = error{
OutOfMemory,
Storage,
Corrupt,
/// A parent reference left the corpus (the census phantom refusal should
/// have caught this; a hole here is a source-store defect).
BrokenAncestry,
/// The differential verify found a (old,new) pair whose file sets differ
/// -- the one bug class this converter must never publish.
Diverged,
};
pub const Remint = struct {
provenance_notes: u64 = 0,
checkins: u64 = 0,
blobs: u64 = 0,
carried: u64 = 0,
controls_remapped: u64 = 0,
signatures_stripped: u64 = 0,
clusters_dropped: u64 = 0,
verified_pairs: u64 = 0,
/// What the first pass saw, rendered for the operator and reconciled
/// against the converter's own accounting.
census: Census = .{},
/// Wall time inside the tree builder and inside the per-pair proof,
/// summed over every check-in — the two costs the builder separates.
build_ns: u64 = 0,
verify_ns: u64 = 0,
enum_ns: u64 = 0,
/// The verb the operator typed, for diagnostics.
verb: []const u8 = "import-fossil",
};
/// Re-mint the whole corpus from `repo` into `dest`. `dest` is a fresh
/// writable LogStore (the caller owns staging and publication). On success
/// every source check-in has a converted counterpart whose EFFECTIVE FILE
/// SET is byte-equivalent, proven pair by pair before returning.
pub fn remint(
gpa: std.mem.Allocator,
io: std.Io,
repo: *Repository,
dest: *logstore.LogStore,
k: u4,
project_code: []const u8,
verb: []const u8,
streams: cli.Streams,
) RemintError!Remint {
var clock = Phase.start(io, streams);
const log = repo.logStore(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
};
var out: Remint = .{ .verb = verb };
// old full-hex -> new full-hex, both owned. Sized by the census's
// structural count in spirit; grown on demand in practice.
var map: std.StringHashMapUnmanaged([]const u8) = .empty;
defer {
var it = map.iterator();
while (it.next()) |e| {
gpa.free(e.key_ptr.*);
gpa.free(e.value_ptr.*);
}
map.deinit(gpa);
}
// Pass 1: content blobs carry verbatim (phase 1 keeps their names), and
// every artifact classifies once into the work lists.
var carry_rids: std.ArrayList(logstore.Rid) = .empty;
defer carry_rids.deinit(gpa);
{
var c: Census = .{};
const n: u64 = log.artifactCount();
var rid: u64 = 1;
while (rid <= n) : (rid += 1) {
c.total += 1;
const r: logstore.Rid = @intCast(rid);
if (log.isPhantom(r)) return error.BrokenAncestry;
const bytes = (log.content(gpa, r) catch |e| return contentErr(e)) orelse return error.BrokenAncestry;
defer gpa.free(bytes);
const signed = artifact.isSignatureWrapped(bytes);
var m = artifact.parseWindow(gpa, bytes) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => {
// Delta-encode at write time. The flat source's own
// chain names the best base; when it is already in DEST that
// hint leads, and the recent window covers the rest.
var base_buf: [logstore.max_hash_len]u8 = undefined;
const hint: ?logstore.Rid = if (log.deltaBaseHashOf(r, &base_buf)) |bh| dest.lookup(bh) else null;
_ = dest.putArtifactPreferring(log.hashOf(r), bytes, hint) catch return error.Storage;
out.blobs += 1;
c.blobs += 1;
continue;
},
};
defer m.deinit();
switch (m.kind) {
.manifest => {
// Collected via the DERIVED state below (topo order needs
// the P DAG, which repostate already holds).
c.checkins += 1;
if (signed) c.signed += 1;
},
.cluster => {
out.clusters_dropped += 1;
c.clusters += 1;
},
else => {
switch (m.kind) {
.submanifest => c.submanifests += 1,
.control => c.controls += 1,
.wiki => c.wikis += 1,
.ticket => c.tickets += 1,
.attachment => c.attachments += 1,
.event => c.events += 1,
.forum => c.forums += 1,
.manifest, .cluster => unreachable,
}
carry_rids.append(gpa, r) catch return error.OutOfMemory;
},
}
}
renderCensus(&c, streams);
out.census = c;
}
clock.done("classified and carried content blobs");
// Pass 2: check-ins in parents-first order. `events` holds every
// check-in; Kahn over `parentsOf` gives the order without recursion.
// Each tree is built by the sub-manifest editor over its primary
// parent's root, the same builder the commit verb uses, so a
// check-in costs its change set, not its file set.
{
var tree_store = LogTreeStore{ .gpa = gpa, .log = dest };
defer tree_store.deinitCache();
var ed = editor.TreeEditor.init(gpa, tree_store.treeStore(), .{
.hash_algo = .sha3_256,
.shape = .{ .tree = .{ .k = k, .height_max = @intCast(artifact.max_construct_height) } },
}, 64 << 20) catch |e| return treeErr(streams, verb, "-", e);
defer ed.deinit();
var parents: ParentCache = .empty;
defer {
var pit = parents.valueIterator();
while (pit.next()) |pt| pt.*.deinit(gpa);
parents.deinit(gpa);
}
// How many children build from each check-in as their PRIMARY
// parent: the cache entry is released when the last has.
var primary_kids: std.AutoHashMapUnmanaged(repostate.Rid, u32) = .empty;
defer primary_kids.deinit(gpa);
// The proof's flatten memo: consecutive check-ins share nearly every
// node, so one memo across the run keeps the walk at its
// distinct-artifact cost.
var sub_memo: ?repostate.SubManifestMemo = null;
defer if (sub_memo) |*m| m.deinit();
var intern = Intern.init(gpa);
defer intern.deinit(gpa);
// The source side's baseline memo: consecutive delta
// check-ins share a baseline, so it is read and parsed once per
// distinct baseline, not once per check-in.
var baseline_memo: ?*Repository.BaselineMemo = null;
defer if (baseline_memo) |m| m.deinit(gpa);
const state = repo.state();
const all = state.events.keys();
var indegree: std.AutoHashMapUnmanaged(repostate.Rid, u32) = .empty;
defer indegree.deinit(gpa);
var children: std.AutoHashMapUnmanaged(repostate.Rid, std.ArrayListUnmanaged(repostate.Rid)) = .empty;
defer {
var cit = children.valueIterator();
while (cit.next()) |l| l.deinit(gpa);
children.deinit(gpa);
}
for (all) |rid| {
var deg: u32 = 0;
var pit = state.parentsOf(rid);
while (pit.next()) |p| {
if (state.mtimeOf(p) == null) continue; // not a check-in here
if (deg == 0) {
const pk = primary_kids.getOrPut(gpa, p) catch return error.OutOfMemory;
if (!pk.found_existing) pk.value_ptr.* = 0;
pk.value_ptr.* += 1;
}
deg += 1;
const gop = children.getOrPut(gpa, p) catch return error.OutOfMemory;
if (!gop.found_existing) gop.value_ptr.* = .empty;
gop.value_ptr.append(gpa, rid) catch return error.OutOfMemory;
}
indegree.put(gpa, rid, deg) catch return error.OutOfMemory;
}
var queue: std.ArrayList(repostate.Rid) = .empty;
defer queue.deinit(gpa);
for (all) |rid| {
if ((indegree.get(rid) orelse 0) == 0) queue.append(gpa, rid) catch return error.OutOfMemory;
}
var taken: usize = 0;
while (taken < queue.items.len) {
const rid = queue.items[taken];
taken += 1;
try remintCheckin(gpa, repo, dest, &ed, io, &parents, &sub_memo, &intern, &baseline_memo, primary_kids.get(rid) orelse 0, rid, &map, &out, streams);
if (children.getPtr(rid)) |kids| {
for (kids.items) |kid| {
const d = indegree.getPtr(kid).?;
d.* -= 1;
if (d.* == 0) queue.append(gpa, kid) catch return error.OutOfMemory;
}
}
}
if (taken != all.len) return error.BrokenAncestry; // a cycle is corrupt history
out.checkins = @intCast(taken);
}
streams.err.print(
"import-fossil: of which source enumeration {d}s, tree builds {d}s, pair proofs {d}s\n",
.{ out.enum_ns / std.time.ns_per_s, out.build_ns / std.time.ns_per_s, out.verify_ns / std.time.ns_per_s },
) catch {};
clock.done("re-minted and verified check-ins");
// Pass 3: the carried structural artifacts. Controls target check-ins
// and re-mint through the map; wiki/forum/ticket/attachment/event chains
// reference only their own kind or uuids and carry verbatim (the
// cascade rule in this module's doc).
for (carry_rids.items) |r| {
const bytes = (log.content(gpa, r) catch |e| return contentErr(e)) orelse return error.BrokenAncestry;
defer gpa.free(bytes);
const rewritten = remapReferences(gpa, bytes, &map) catch return error.OutOfMemory;
if (rewritten) |new_bytes| {
defer gpa.free(new_bytes);
var name_buf: [64]u8 = undefined;
const new_hex = artifact.artifactName(.sha3_256, new_bytes, &name_buf);
var raw: [32]u8 = undefined;
_ = std.fmt.hexToBytes(raw[0 .. new_hex.len / 2], new_hex) catch unreachable;
_ = dest.putArtifact(raw[0 .. new_hex.len / 2], new_bytes, null) catch return error.Storage;
out.controls_remapped += 1;
} else {
var base_buf: [logstore.max_hash_len]u8 = undefined;
const hint: ?logstore.Rid = if (log.deltaBaseHashOf(r, &base_buf)) |bh| dest.lookup(bh) else null;
_ = dest.putArtifactPreferring(log.hashOf(r), bytes, hint) catch return error.Storage;
out.carried += 1;
}
}
// The provenance technote (non-negotiable): the old->new map as an
// event artifact INSIDE the converted repository -- it syncs with the
// repo, touches no check-in's file set, and keeps every external hash
// citation resolvable by lookup. Deterministic: the tip's own timestamp
// and an id derived from the project code, so one source converts to
// byte-identical output every time.
clock.done("carried structural artifacts");
try mintProvenance(gpa, repo, dest, &map, project_code, &out);
clock.done("minted provenance");
return out;
}
/// Wall-clock phase reporter: one stderr line per completed phase, so a
/// multi-hour conversion can be watched from outside.
pub const Phase = struct {
io: std.Io,
streams: cli.Streams,
t0: i64,
pub fn start(io: std.Io, streams: cli.Streams) Phase {
return .{ .io = io, .streams = streams, .t0 = std.Io.Timestamp.now(io, .real).toSeconds() };
}
pub fn done(self: *Phase, label: []const u8) void {
const now = std.Io.Timestamp.now(self.io, .real).toSeconds();
self.streams.err.print("import-fossil: {s} ({d}s)\n", .{ label, now - self.t0 }) catch {};
self.t0 = now;
}
};
fn contentErr(e: logstore.ContentError) RemintError {
return switch (e) {
error.OutOfMemory => error.OutOfMemory,
error.Storage => error.Storage,
error.Corrupt => error.Corrupt,
};
}
/// One re-minted check-in's file set and root, kept only while a child still
/// has to build from it: the editor applies a child's change set over
/// this root, so unchanged slices are reused by name instead of rebuilt.
const ParentTree = struct {
arena_state: std.heap.ArenaAllocator,
files: []const artifact.FCard,
/// Empty when the parent's tree came out flat (§6.2 step 2).
s: []const artifact.SCard,
/// Children that will build from this parent and have not yet.
pending: u32,
fn view(self: *const ParentTree) editor.RootView {
if (self.s.len != 0) return .{ .tree = self.s };
return .{ .flat = self.files };
}
fn deinit(self: *ParentTree, gpa: std.mem.Allocator) void {
self.arena_state.deinit();
gpa.destroy(self);
}
};
const ParentCache = std.AutoHashMapUnmanaged(repostate.Rid, *ParentTree);
/// One copy of every distinct name, hash and perm token the parent cache
/// holds, for the run. Consecutive check-ins share nearly every card, so
/// retaining a check-in's file set costs a lookup per card rather than an
/// allocation per card; memory is bounded by the distinct strings in the
/// history, not by history length times width.
const Intern = struct {
arena_state: std.heap.ArenaAllocator,
table: std.StringHashMapUnmanaged(void) = .empty,
fn init(gpa: std.mem.Allocator) Intern {
return .{ .arena_state = std.heap.ArenaAllocator.init(gpa) };
}
fn deinit(self: *Intern, gpa: std.mem.Allocator) void {
self.table.deinit(gpa);
self.arena_state.deinit();
}
fn get(self: *Intern, gpa: std.mem.Allocator, s: []const u8) error{OutOfMemory}![]const u8 {
const gop = try self.table.getOrPut(gpa, s);
if (!gop.found_existing) {
const owned = try self.arena_state.allocator().dupe(u8, s);
gop.key_ptr.* = owned;
}
return gop.key_ptr.*;
}
fn getOpt(self: *Intern, gpa: std.mem.Allocator, s: ?[]const u8) error{OutOfMemory}!?[]const u8 {
return if (s) |v| try self.get(gpa, v) else null;
}
};
fn internCard(intern: *Intern, gpa: std.mem.Allocator, f: artifact.FCard) error{OutOfMemory}!artifact.FCard {
return .{
.filename = try intern.get(gpa, f.filename),
.hash = try intern.getOpt(gpa, f.hash),
.perm = try intern.getOpt(gpa, f.perm),
// Kept: a rename annotation is a fact of ONE check-in, so a child
// that inherits the file without it is a change the editor must see.
.old_name = try intern.getOpt(gpa, f.old_name),
.mode = f.mode,
};
}
fn sameCard(a: artifact.FCard, b: artifact.FCard) bool {
return sameOpt(a.hash, b.hash) and sameOpt(a.perm, b.perm) and sameOpt(a.old_name, b.old_name);
}
fn sameOpt(a: ?[]const u8, b: ?[]const u8) bool {
if (a == null or b == null) return a == null and b == null;
return std.mem.eql(u8, a.?, b.?);
}
/// The commands that turn `parent`'s file set into `child`'s: both are in
/// canonical filename order (§8.1), so one merge walk finds every added,
/// changed and removed name.
fn diffCommands(
gpa: std.mem.Allocator,
parent: []const artifact.FCard,
child: []const artifact.FCard,
) error{OutOfMemory}!std.ArrayList(editor.TreeCommand) {
var cmds: std.ArrayList(editor.TreeCommand) = .empty;
errdefer cmds.deinit(gpa);
var i: usize = 0;
var j: usize = 0;
while (i < parent.len or j < child.len) {
if (j >= child.len) {
try cmds.append(gpa, .{ .delete = parent[i].filename });
i += 1;
continue;
}
if (i >= parent.len) {
try cmds.append(gpa, .{ .upsert = child[j] });
j += 1;
continue;
}
switch (std.mem.order(u8, parent[i].filename, child[j].filename)) {
.lt => {
try cmds.append(gpa, .{ .delete = parent[i].filename });
i += 1;
},
.gt => {
try cmds.append(gpa, .{ .upsert = child[j] });
j += 1;
},
.eq => {
if (!sameCard(parent[i], child[j])) try cmds.append(gpa, .{ .upsert = child[j] });
i += 1;
j += 1;
},
}
}
return cmds;
}
fn treeErr(streams: cli.Streams, verb: []const u8, old_hex: []const u8, e: editor.Error) RemintError {
switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.StorageFailure => return error.Storage,
else => {
streams.err.print("zit {s}: tree build failed at source check-in {s}: {s}\n", .{ verb, old_hex, @errorName(e) }) catch {};
return error.Corrupt;
},
}
}
/// Re-mint ONE check-in: effective file set through the enumeration seam,
/// S-tree build, text-level card rewrite (F run out, S run in at its sorted
/// position, B dropped, P/Q/T hash tokens chased through the map, PGP
/// armor stripped, Z resealed), new name recorded in the map.
fn remintCheckin(
gpa: std.mem.Allocator,
repo: *Repository,
dest: *logstore.LogStore,
ed: *editor.TreeEditor,
ed_io: std.Io,
parents: *ParentCache,
sub_memo: *?repostate.SubManifestMemo,
intern: *Intern,
baseline_memo: *?*Repository.BaselineMemo,
primary_kids: u32,
rid: repostate.Rid,
map: *std.StringHashMapUnmanaged([]const u8),
out: *Remint,
streams: cli.Streams,
) RemintError!void {
const old_hex = repo.hashOf(rid);
const t_enum = std.Io.Timestamp.now(ed_io, .real).nanoseconds;
var fl = (repo.filesMemo(gpa, rid, baseline_memo) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
}) orelse return error.BrokenAncestry;
defer fl.deinit(gpa);
var entries: std.ArrayList(artifact.FCard) = .empty;
defer entries.deinit(gpa);
var it = fl.iterator();
while (it.next()) |f| entries.append(gpa, f) catch return error.OutOfMemory;
out.enum_ns += @intCast(std.Io.Timestamp.now(ed_io, .real).nanoseconds - t_enum);
// The primary parent is the first P card that names a check-in; its
// re-minted root is the base the editor applies this check-in's change
// set over. A root check-in (or a parent whose entry was released)
// applies over the empty flat root, the editor's base case.
const state = repo.state();
var primary: ?repostate.Rid = null;
{
var pit = state.parentsOf(rid);
while (pit.next()) |p| {
if (state.mtimeOf(p) == null) continue;
primary = p;
break;
}
}
const parent_tree: ?*ParentTree = if (primary) |p| parents.get(p) else null;
var cmds = if (parent_tree) |pt|
diffCommands(gpa, pt.files, entries.items) catch return error.OutOfMemory
else blk: {
var all: std.ArrayList(editor.TreeCommand) = .empty;
for (entries.items) |f| all.append(gpa, .{ .upsert = f }) catch return error.OutOfMemory;
break :blk all;
};
defer cmds.deinit(gpa);
const parent_view: editor.RootView = if (parent_tree) |pt| pt.view() else .{ .flat = &.{} };
const t_build = std.Io.Timestamp.now(ed_io, .real).nanoseconds;
var built = ed.apply(parent_view, cmds.items) catch |e| return treeErr(streams, out.verb, old_hex, e);
out.build_ns += @intCast(std.Io.Timestamp.now(ed_io, .real).nanoseconds - t_build);
defer built.deinit();
const s_cards: []const artifact.SCard = switch (built.root) {
.tree => |cards| cards,
.flat => &.{},
};
// The file list retains the parsed manifest, so its bytes are the source
// text: one parse per check-in, not two.
const source_text = stripClearsign(fl.target.bytes, out);
const new_bytes = rewriteManifest(gpa, source_text, entries.items, s_cards, map) catch return error.OutOfMemory;
defer gpa.free(new_bytes);
var name_buf: [64]u8 = undefined;
const new_hex = artifact.artifactName(.sha3_256, new_bytes, &name_buf);
var raw: [32]u8 = undefined;
_ = std.fmt.hexToBytes(raw[0..32], new_hex) catch unreachable;
_ = dest.put(raw[0..32], new_bytes) catch return error.Storage;
// The map owns both strings once `put` succeeds; until then they are
// ours to free. No errdefer may outlive the hand-off, or a later failure
// in this function frees a key the map frees again at deinit.
const old_key = gpa.dupe(u8, old_hex) catch return error.OutOfMemory;
const new_val = gpa.dupe(u8, new_hex) catch {
gpa.free(old_key);
return error.OutOfMemory;
};
map.put(gpa, old_key, new_val) catch {
gpa.free(old_key);
gpa.free(new_val);
return error.OutOfMemory;
};
// The differential proof for THIS pair, while the source file set is
// still in hand: the tree just written must read back through the same
// resolver every verb uses and flatten to exactly `entries`. Proving
// each pair here instead of in a later pass spares a second
// whole-history enumeration of the source; publication still
// waits on every pair.
const t_verify = std.Io.Timestamp.now(ed_io, .real).nanoseconds;
try verifyMinted(gpa, dest, sub_memo, out.verb, old_key, new_val, entries.items, streams);
out.verified_pairs += 1;
out.verify_ns += @intCast(std.Io.Timestamp.now(ed_io, .real).nanoseconds - t_verify);
// Keep this root for the children that build from it; release the
// parent's once its last such child (this one) has consumed it.
if (primary_kids > 0) {
const pt = gpa.create(ParentTree) catch return error.OutOfMemory;
errdefer gpa.destroy(pt);
pt.* = .{ .arena_state = std.heap.ArenaAllocator.init(gpa), .files = &.{}, .s = &.{}, .pending = primary_kids };
errdefer pt.arena_state.deinit();
const arena = pt.arena_state.allocator();
const files = arena.alloc(artifact.FCard, entries.items.len) catch return error.OutOfMemory;
for (entries.items, 0..) |f, i| files[i] = internCard(intern, gpa, f) catch return error.OutOfMemory;
const s_copy = arena.alloc(artifact.SCard, s_cards.len) catch return error.OutOfMemory;
for (s_cards, 0..) |c, i| s_copy[i] = .{
.filename = intern.get(gpa, c.filename) catch return error.OutOfMemory,
.hash = intern.get(gpa, c.hash) catch return error.OutOfMemory,
};
pt.files = files;
pt.s = s_copy;
parents.put(gpa, rid, pt) catch return error.OutOfMemory;
}
if (parent_tree) |pt| {
pt.pending -= 1;
if (pt.pending == 0) {
_ = parents.remove(primary.?);
pt.deinit(gpa);
}
}
}
/// Unwrap a clearsign envelope (PGP or SSH — the same §4.1.1 wrapper the
/// parser strips), counting the casualty. The payload is exactly the lines
/// between the armor's blank separator and the signature block.
fn stripClearsign(bytes: []const u8, out: *Remint) []const u8 {
const body = artifact.stripSignatureWrapper(bytes);
if (body.len != bytes.len) out.signatures_stripped += 1;
return body;
}
fn rewriteManifest(
gpa: std.mem.Allocator,
text: []const u8,
entries: []const artifact.FCard,
s_cards: []const artifact.SCard,
map: *const std.StringHashMapUnmanaged([]const u8),
) error{OutOfMemory}![]u8 {
var outb: std.ArrayList(u8) = .empty;
errdefer outb.deinit(gpa);
// submanifests §6.2 step 2: a name set that partitions into fewer than two
// groups yields no tree, and the writer "MUST emit a flat manifest and no
// sub-manifests". The run this manifest carries is therefore an F run, not
// an S run — and it sorts where F sorts (before P), not where S does (after
// R). Dropping the source's F cards and emitting `build.s` regardless is
// what published a check-in with no file set at all.
const flat = s_cards.len == 0;
const run_card: u8 = if (flat) 'F' else 'S';
var emitted_s = false;
var lines = std.mem.splitScalar(u8, text, '\n');
while (lines.next()) |line| {
if (line.len == 0) continue;
const card = line[0];
if (!emitted_s and card > run_card and card != 'Z') {
try appendRun(gpa, &outb, s_cards, entries, flat);
emitted_s = true;
}
switch (card) {
// The source's own file run is replaced by the one minted above,
// whichever letter it used: a source that already carries
// sub-manifests must not keep its S cards beside the new run.
'F', 'B', 'S' => continue,
'Z' => {
if (!emitted_s) {
try appendRun(gpa, &outb, s_cards, entries, flat);
emitted_s = true;
}
try appendZ(gpa, &outb);
},
'P', 'Q', 'T' => try appendRemapped(gpa, &outb, line, map),
else => {
try outb.appendSlice(gpa, line);
try outb.append(gpa, '\n');
},
}
}
if (!emitted_s) try appendRun(gpa, &outb, s_cards, entries, flat);
return outb.toOwnedSlice(gpa);
}
/// The file-set run this minted manifest carries: the tree's S run, or — when
/// §6.2 construction declined to build a tree — the effective set as flat F
/// cards. The B card is dropped either way, so the flat arm materializes the
/// whole effective set (`entries`), not the source's own F lines, which for a
/// delta manifest name only what that check-in changed.
fn appendRun(
gpa: std.mem.Allocator,
outb: *std.ArrayList(u8),
s_cards: []const artifact.SCard,
entries: []const artifact.FCard,
flat: bool,
) error{OutOfMemory}!void {
if (!flat) return appendSRun(gpa, outb, s_cards);
for (entries) |f| try artifact.appendFlatFCard(outb, gpa, f);
}
fn appendSRun(gpa: std.mem.Allocator, outb: *std.ArrayList(u8), s_cards: []const artifact.SCard) error{OutOfMemory}!void {
for (s_cards) |s_card| {
var enc_buf: [2 * 4096]u8 = undefined;
const encoded = artifact.encodeEscapes(s_card.filename, &enc_buf);
try outb.appendSlice(gpa, "S ");
try outb.appendSlice(gpa, encoded);
try outb.append(gpa, ' ');
try outb.appendSlice(gpa, s_card.hash);
try outb.append(gpa, '\n');
}
}
/// One card line with every 40/64-hex token replaced by its mapped
/// counterpart when one exists. Tokens outside the map (content hashes,
/// '*', tag names) pass through untouched.
fn appendRemapped(
gpa: std.mem.Allocator,
outb: *std.ArrayList(u8),
line: []const u8,
map: *const std.StringHashMapUnmanaged([]const u8),
) error{OutOfMemory}!void {
var toks = std.mem.splitScalar(u8, line, ' ');
var first = true;
while (toks.next()) |tok| {
if (!first) try outb.append(gpa, ' ');
first = false;
if (map.get(tok)) |new| {
try outb.appendSlice(gpa, new);
} else if (tok.len > 1 and (tok[0] == '+' or tok[0] == '-' or tok[0] == '*')) {
// Q cards carry a sign prefix on the hash token.
if (map.get(tok[1..])) |new| {
try outb.append(gpa, tok[0]);
try outb.appendSlice(gpa, new);
} else {
try outb.appendSlice(gpa, tok);
}
} else {
try outb.appendSlice(gpa, tok);
}
}
try outb.append(gpa, '\n');
}
/// Seal the §6.2 Z card: md5 over every byte already emitted.
fn appendZ(gpa: std.mem.Allocator, outb: *std.ArrayList(u8)) error{OutOfMemory}!void {
var digest: [16]u8 = undefined;
std.crypto.hash.Md5.hash(outb.items, &digest, .{});
const hex = std.fmt.bytesToHex(digest, .lower);
try outb.appendSlice(gpa, "Z ");
try outb.appendSlice(gpa, &hex);
try outb.append(gpa, '\n');
}
/// Rewrite a carried structural artifact's references. Returns the new
/// bytes when anything changed (caller re-mints), null when nothing did
/// (caller carries verbatim). Only T lines can target a check-in here;
/// the map lookup is what decides, not the card letter.
fn remapReferences(
gpa: std.mem.Allocator,
bytes: []const u8,
map: *const std.StringHashMapUnmanaged([]const u8),
) error{OutOfMemory}!?[]u8 {
var outb: std.ArrayList(u8) = .empty;
errdefer outb.deinit(gpa);
var changed = false;
var lines = std.mem.splitScalar(u8, bytes, '\n');
while (lines.next()) |line| {
if (line.len == 0) continue;
if (line[0] == 'Z') {
try appendZ(gpa, &outb);
continue;
}
if (line[0] == 'T') {
const before = outb.items.len;
try appendRemapped(gpa, &outb, line, map);
const slice = outb.items[before..];
if (!std.mem.eql(u8, slice[0 .. slice.len - 1], line)) changed = true;
continue;
}
try outb.appendSlice(gpa, line);
try outb.append(gpa, '\n');
}
if (!changed) {
outb.deinit(gpa);
return null;
}
return try outb.toOwnedSlice(gpa);
}
/// The differential proof for one re-minted check-in: the converted
/// manifest, read back from `dest` and flattened through the same resolver
/// every verb uses, must carry exactly `src_cards` — the source's effective
/// file set, name for name, hash for hash. Any other outcome is the one bug
/// class this converter must never publish, and `divergedAt`/`divergedWhy`
/// make that class loud.
fn verifyMinted(
gpa: std.mem.Allocator,
dest: *logstore.LogStore,
sub_memo: *?repostate.SubManifestMemo,
verb: []const u8,
old_hex: []const u8,
new_hex: []const u8,
src_cards: []const artifact.FCard,
streams: cli.Streams,
) RemintError!void {
var raw: [32]u8 = undefined;
_ = std.fmt.hexToBytes(raw[0 .. new_hex.len / 2], new_hex) catch return error.Corrupt;
const new_rid = dest.lookup(raw[0 .. new_hex.len / 2]) orelse return divergedWhy(streams, verb, old_hex, "the converted manifest is not in the destination store");
const new_bytes = (dest.content(gpa, new_rid) catch |e| return contentErr(e)) orelse return divergedWhy(streams, verb, old_hex, "the converted manifest has no content");
defer gpa.free(new_bytes);
var m = artifact.parse(gpa, new_bytes) catch |e| return divergedWhy(streams, verb, old_hex, @errorName(e));
defer m.deinit();
Repository.flattenTreeFromLogMemo(gpa, dest, &m, sub_memo) catch |e| return divergedWhy(streams, verb, old_hex, @errorName(e));
// `flattened` is null for a manifest that carries no S run at all, which
// since the §6.2 step-2 flat arm above is an ordinary outcome, not a fault:
// the file set is then the manifest's own F run. Same substitution
// `repository.manifestFilesOf` makes, for the same reason — a consumer must
// not distinguish a walked enumeration from a flat one.
const new_files = m.flattened orelse m.f;
for (src_cards, 0..) |src_card, i| {
if (i >= new_files.len) return divergedAt(streams, verb, old_hex, src_card.filename);
const dst_card = new_files[i];
if (!std.mem.eql(u8, src_card.filename, dst_card.filename)) return divergedAt(streams, verb, old_hex, src_card.filename);
const src_hash = src_card.hash orelse return divergedAt(streams, verb, old_hex, src_card.filename);
const dst_hash = dst_card.hash orelse return divergedAt(streams, verb, old_hex, src_card.filename);
if (!std.mem.eql(u8, src_hash, dst_hash)) return divergedAt(streams, verb, old_hex, src_card.filename);
}
if (src_cards.len != new_files.len) return divergedAt(streams, verb, old_hex, "(extra converted entries)");
}
fn divergedWhy(streams: cli.Streams, verb: []const u8, old_hex: []const u8, why: []const u8) RemintError {
streams.err.print(
"zit {s}: VERIFY FAILURE at source check-in {s}: {s} — nothing was published\n",
.{ verb, old_hex, why },
) catch {};
return error.Diverged;
}
fn divergedAt(streams: cli.Streams, verb: []const u8, old_hex: []const u8, path: []const u8) RemintError {
streams.err.print(
"zit {s}: VERIFY DIVERGENCE at source check-in {s}, path '{s}' — nothing was published\n",
.{ verb, old_hex, path },
) catch {};
return error.Diverged;
}
/// Compose and store the provenance technote(s). Pairs are emitted in
/// sorted-old-hash order (map iteration order is not deterministic; the
/// output must be). One note holds up to `pairs_per_note` pairs; a corpus
/// larger than that chains notes, each E-carded under an id derived from
/// the project code and its chunk index.
const pairs_per_note: usize = 16 * 1024;
fn mintProvenance(
gpa: std.mem.Allocator,
repo: *Repository,
dest: *logstore.LogStore,
map: *const std.StringHashMapUnmanaged([]const u8),
project_code: []const u8,
out: *Remint,
) RemintError!void {
if (map.count() == 0) return;
// The tip's D timestamp: the conversion's clock, without ambient time.
const tip = repo.resolve("tip", .checkin) catch return error.BrokenAncestry;
var tip_owned = (repo.manifest(gpa, tip) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.Storage,
}) orelse return error.BrokenAncestry;
defer tip_owned.deinit(gpa);
const timestamp = if (tip_owned.manifest.d) |d| d.timestamp else "1970-01-01T00:00:00";
var olds: std.ArrayList([]const u8) = .empty;
defer olds.deinit(gpa);
var kit = map.keyIterator();
while (kit.next()) |k| olds.append(gpa, k.*) catch return error.OutOfMemory;
std.mem.sort([]const u8, olds.items, {}, sliceLess);
var chunk: usize = 0;
var offset: usize = 0;
while (offset < olds.items.len) : (chunk += 1) {
const end = @min(offset + pairs_per_note, olds.items.len);
var body: std.ArrayList(u8) = .empty;
defer body.deinit(gpa);
body.appendSlice(gpa, "import-fossil provenance: OLD-check-in-hash NEW-check-in-hash, one pair per line.\n") catch return error.OutOfMemory;
for (olds.items[offset..end]) |old_hex| {
body.appendSlice(gpa, old_hex) catch return error.OutOfMemory;
body.append(gpa, ' ') catch return error.OutOfMemory;
body.appendSlice(gpa, map.get(old_hex).?) catch return error.OutOfMemory;
body.append(gpa, '\n') catch return error.OutOfMemory;
}
offset = end;
var id_material: std.ArrayList(u8) = .empty;
defer id_material.deinit(gpa);
id_material.appendSlice(gpa, "import-fossil-provenance:") catch return error.OutOfMemory;
id_material.appendSlice(gpa, project_code) catch return error.OutOfMemory;
id_material.append(gpa, ':') catch return error.OutOfMemory;
id_material.append(gpa, @intCast('0' + (chunk % 10))) catch return error.OutOfMemory;
var id_buf: [64]u8 = undefined;
const note_id = artifact.artifactName(.sha3_256, id_material.items, &id_buf);
var note: std.ArrayList(u8) = .empty;
defer note.deinit(gpa);
const head = std.fmt.allocPrint(
gpa,
"D {s}\nE {s} {s}\nU import-fossil\nW {d}\n{s}\n",
.{ timestamp, timestamp, note_id, body.items.len, body.items },
) catch return error.OutOfMemory;
defer gpa.free(head);
note.appendSlice(gpa, head) catch return error.OutOfMemory;
appendZ(gpa, ¬e) catch return error.OutOfMemory;
var name_buf: [64]u8 = undefined;
const note_hex = artifact.artifactName(.sha3_256, note.items, &name_buf);
var raw: [32]u8 = undefined;
_ = std.fmt.hexToBytes(raw[0..32], note_hex) catch unreachable;
_ = dest.put(raw[0..32], note.items) catch return error.Storage;
out.provenance_notes += 1;
}
}
fn sliceLess(_: void, a: []const u8, b: []const u8) bool {
return std.mem.order(u8, a, b) == .lt;
}