Artifact 496e350c7220a1974f3562ba05456da9f2ced3405cbfb74e0c9c7f3d6cdbbb9c:
- File
src/fossil/fossil_convert.zig
— part of check-in
[38b9743ff8]
at
2026-08-29 12:04:37
on branch trunk
— import-fossil --submanifest: delta-encode the converted store as it is written (zit#169)
Blobs take the flat source's own delta base as the hint, manifests their parent, carried artifacts the recent window. A 28x full-form gap on a toy corpus became 1.3x of packed, with no second pass and no double-disk peak. (user: dmestas, size: 45368) [annotate] [blame] [check-ins using]
//! The submanifest conversion pass (#61): re-mint a fossil corpus as a new //! project with S-card tree manifests from the first commit. //! //! The load-bearing observation is the RE-MINT CASCADE RULE: an artifact's //! name changes only when its BYTES change, and (with content-blob names //! kept, the phase-1 posture) bytes change only for an artifact that either //! gains the S-tree structure (every check-in manifest) or references a //! re-minted artifact in a hash-bearing card. So the remap surface is: //! //! check-in manifests all re-mint (S-trees + remapped P/Q/B; D/C/U and //! the R card survive -- R is over the file SET, which //! is preserved exactly) //! control artifacts re-mint iff a T card targets a remapped check-in //! events (technotes) re-mint iff their P chain or T target remaps //! wiki / forum P/I/G reference their own kind's chain -- verbatim //! unless an ancestor re-minted (transitively closed) //! attachments target a wiki NAME or ticket uuid + a CONTENT hash //! -- verbatim in phase 1 //! tickets K ids are uuids, not artifact hashes -- verbatim //! clusters dropped (sync housekeeping, not history) //! content blobs verbatim (phase 1 keeps their names) //! PGP signatures cannot survive rehashing: stripped and counted //! //! The census pass below is the converter's front door: it classifies the //! whole corpus, counts the remap surface and the casualties, and is the //! baseline the count-reconciliation acceptance check audits against. const std = @import("std"); const zit = @import("zit"); const cli = zit.cli; const artifact = zit.artifact; const logstore = zit.logstore; const Repository = zit.Repository; const repostate = zit.repostate; const editor = zit.submanifest_editor; const LogTreeStore = zit.log_tree_store.LogTreeStore; /// One corpus walk: every stored artifact classified by parsed type, with /// the phase-1 remap surface and casualty counts. Phantoms hold no bytes /// and are skipped (they are the sync layer's business, not history's). pub const Census = struct { total: u64 = 0, phantoms: u64 = 0, blobs: u64 = 0, checkins: u64 = 0, submanifests: u64 = 0, controls: u64 = 0, wikis: u64 = 0, tickets: u64 = 0, attachments: u64 = 0, events: u64 = 0, forums: u64 = 0, clusters: u64 = 0, /// Check-in manifests wrapped in a PGP signature: the signature is a /// stated casualty of any conversion (stripped and counted). signed: u64 = 0, pub fn structural(self: *const Census) u64 { return self.checkins + self.submanifests + self.controls + self.wikis + self.tickets + self.attachments + self.events + self.forums + self.clusters; } }; /// Count the phantoms in the source store without reading any content — /// the cheap pre-flight that lets an incomplete corpus refuse before a /// staging directory exists. The classifying walk itself happens once, in /// `remint`'s first pass (#155: it used to happen twice). pub fn phantomCount(gpa: std.mem.Allocator, repo: *Repository) error{ OutOfMemory, Storage }!u64 { const log = repo.logStore(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; var phantoms: u64 = 0; const n: u64 = log.artifactCount(); var rid: u64 = 1; while (rid <= n) : (rid += 1) { if (log.isPhantom(@intCast(rid))) phantoms += 1; } return phantoms; } /// Render the census for the operator: what converts, what carries, what /// dies. The converter's own accounting reconciles against these numbers. pub fn renderCensus(c: *const Census, streams: cli.Streams) void { streams.out.print( "census: {d} artifacts — {d} check-ins to re-mint ({d} signed: signatures strip)," ++ " {d} content blobs carried verbatim\n", .{ c.total, c.checkins, c.signed, c.blobs }, ) catch {}; if (c.controls + c.wikis + c.tickets + c.attachments + c.events + c.forums != 0) { streams.out.print( "census: {d} control, {d} wiki, {d} ticket, {d} attachment, {d} technote, {d} forum" ++ " — carried, re-minting only where a reference remaps\n", .{ c.controls, c.wikis, c.tickets, c.attachments, c.events, c.forums }, ) catch {}; } if (c.clusters != 0) { streams.out.print("census: {d} cluster(s) dropped (sync housekeeping, not history)\n", .{c.clusters}) catch {}; } if (c.submanifests != 0) { streams.out.print("census: {d} sub-manifest slice(s) already present in the source\n", .{c.submanifests}) catch {}; } if (c.phantoms != 0) { streams.out.print("census: {d} phantom/masked name(s) skipped — clone the origin to completeness first\n", .{c.phantoms}) catch {}; } } /// One F card over a static filename/hash pair, for the mint tests below. fn testFCard(name: []const u8, hash: []const u8) artifact.FCard { return .{ .filename = name, .hash = hash, .perm = null, .old_name = null, .mode = .regular }; } test "convert §6.2 step 2 (zit#126): a check-in whose names do not partition mints FLAT, not empty" { // The regression: `buildTree` reports `isFlat()` for a name set that // yields fewer than two groups, and `rewriteManifest` dropped the source's // F cards while emitting an empty S run — publishing a check-in with NO // file set. The differential verify caught it and aborted the whole // conversion, which is how zit#126 presented (an ordinary libspatialite // trunk baseline, 38 files, at k=5). const gpa = std.testing.allocator; const entries = [_]artifact.FCard{ testFCard("Makefile", "0" ** 40), testFCard("src/a.c", "1" ** 40), testFCard("src/b.c", "2" ** 40), }; // max_k makes the span enormous relative to three names, so the boundary // test never fires and the whole set stays one group — §6.2 step 2. var build = try artifact.buildTree(gpa, &entries, artifact.max_k, .sha3_256); defer build.deinit(); try std.testing.expect(build.isFlat()); const source = "C comment\n" ++ "D 2009-04-16T15:57:11\n" ++ "F Makefile " ++ "0" ** 40 ++ "\n" ++ "F src/a.c " ++ "1" ** 40 ++ "\n" ++ "F src/b.c " ++ "2" ** 40 ++ "\n" ++ "P " ++ "3" ** 40 ++ "\n" ++ "U someone\n" ++ "Z " ++ "4" ** 32 ++ "\n"; var map: std.StringHashMapUnmanaged([]const u8) = .empty; defer map.deinit(gpa); const minted = try rewriteManifest(gpa, source, &entries, build.s, &map); defer gpa.free(minted); // The mint must carry the file set as an F run and no S run at all, and it // must still parse as a check-in manifest. var m = try artifact.parse(gpa, minted); defer m.deinit(); try std.testing.expectEqual(@as(usize, 0), m.s.len); try std.testing.expectEqual(entries.len, m.f.len); for (&entries, m.f) |src_card, dst_card| { try std.testing.expectEqualStrings(src_card.filename, dst_card.filename); try std.testing.expectEqualStrings(src_card.hash.?, dst_card.hash.?); } } test "convert §6.2 step 3: a partitioning name set still mints as an S tree with no F run" { // The other arm, so the fix above cannot be "flatten everything": at k=1 // the same shape of input does partition, and the mint must be a tree. const gpa = std.testing.allocator; var entries: std.ArrayList(artifact.FCard) = .empty; defer entries.deinit(gpa); var names: [64][8]u8 = undefined; var hashes: [64][40]u8 = undefined; for (0..names.len) |i| { names[i] = [_]u8{ 'f', 'i', 'l', 'e', '0' + @as(u8, @intCast(i / 10)), '0' + @as(u8, @intCast(i % 10)), 0, 0 }; hashes[i] = [_]u8{'a'} ** 40; hashes[i][0] = '0' + @as(u8, @intCast(i % 10)); try entries.append(gpa, testFCard(names[i][0..6], hashes[i][0..])); } var build = try artifact.buildTree(gpa, entries.items, artifact.min_k, .sha3_256); defer build.deinit(); try std.testing.expect(!build.isFlat()); const source = "C c\nD 2009-04-16T15:57:11\nU someone\nZ " ++ "4" ** 32 ++ "\n"; var map: std.StringHashMapUnmanaged([]const u8) = .empty; defer map.deinit(gpa); const minted = try rewriteManifest(gpa, source, entries.items, build.s, &map); defer gpa.free(minted); var m = try artifact.parse(gpa, minted); defer m.deinit(); try std.testing.expectEqual(@as(usize, 0), m.f.len); try std.testing.expect(m.s.len >= 2); // §6.2: a tree partitions at least in two } test "census: an empty-ish census sums structurally" { // Goal: the structural() partition is total over ArtifactType — a new // member without a census bucket fails the exhaustive switch in census() // at compile time, and this pin keeps the arithmetic honest. var c: Census = .{ .checkins = 2, .controls = 1, .clusters = 3 }; try std.testing.expectEqual(@as(u64, 6), c.structural()); } // --------------------------------------------------------------------------- // The re-mint pass (#61): check-ins become S-tree manifests under new names, // every hash-bearing reference chases the old->new map, everything else // carries verbatim. Phase 1 keeps content-blob names; new structural // artifacts mint under SHA3-256 (the new project's native policy). // --------------------------------------------------------------------------- pub const RemintError = error{ OutOfMemory, Storage, Corrupt, /// A parent reference left the corpus (the census phantom refusal should /// have caught this; a hole here is a source-store defect). BrokenAncestry, /// The differential verify found a (old,new) pair whose file sets differ /// -- the one bug class this converter must never publish. Diverged, }; pub const Remint = struct { provenance_notes: u64 = 0, checkins: u64 = 0, blobs: u64 = 0, carried: u64 = 0, controls_remapped: u64 = 0, signatures_stripped: u64 = 0, clusters_dropped: u64 = 0, verified_pairs: u64 = 0, /// What the first pass saw, rendered for the operator and reconciled /// against the converter's own accounting. census: Census = .{}, /// Wall time inside the tree builder and inside the per-pair proof, /// summed over every check-in — the two costs #167 separates. build_ns: u64 = 0, verify_ns: u64 = 0, enum_ns: u64 = 0, /// The verb the operator typed, for diagnostics. verb: []const u8 = "import-fossil", }; /// Re-mint the whole corpus from `repo` into `dest`. `dest` is a fresh /// writable LogStore (the caller owns staging and publication). On success /// every source check-in has a converted counterpart whose EFFECTIVE FILE /// SET is byte-equivalent, proven pair by pair before returning. pub fn remint( gpa: std.mem.Allocator, io: std.Io, repo: *Repository, dest: *logstore.LogStore, k: u4, project_code: []const u8, verb: []const u8, streams: cli.Streams, ) RemintError!Remint { var clock = Phase.start(io, streams); const log = repo.logStore(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; var out: Remint = .{ .verb = verb }; // old full-hex -> new full-hex, both owned. Sized by the census's // structural count in spirit; grown on demand in practice. var map: std.StringHashMapUnmanaged([]const u8) = .empty; defer { var it = map.iterator(); while (it.next()) |e| { gpa.free(e.key_ptr.*); gpa.free(e.value_ptr.*); } map.deinit(gpa); } // Pass 1: content blobs carry verbatim (phase 1 keeps their names), and // every artifact classifies once into the work lists. var carry_rids: std.ArrayList(logstore.Rid) = .empty; defer carry_rids.deinit(gpa); { var c: Census = .{}; const n: u64 = log.artifactCount(); var rid: u64 = 1; while (rid <= n) : (rid += 1) { c.total += 1; const r: logstore.Rid = @intCast(rid); if (log.isPhantom(r)) return error.BrokenAncestry; const bytes = (log.content(gpa, r) catch |e| return contentErr(e)) orelse return error.BrokenAncestry; defer gpa.free(bytes); const signed = artifact.isSignatureWrapped(bytes); var m = artifact.parseWindow(gpa, bytes) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => { // zit#169: delta-encode at write time. The flat source's own // chain names the best base; when it is already in DEST that // hint leads, and the recent window covers the rest. var base_buf: [logstore.max_hash_len]u8 = undefined; const hint: ?logstore.Rid = if (log.deltaBaseHashOf(r, &base_buf)) |bh| dest.lookup(bh) else null; _ = dest.putArtifact(log.hashOf(r), bytes, hint) catch return error.Storage; out.blobs += 1; c.blobs += 1; continue; }, }; defer m.deinit(); switch (m.kind) { .manifest => { // Collected via the DERIVED state below (topo order needs // the P DAG, which repostate already holds). c.checkins += 1; if (signed) c.signed += 1; }, .cluster => { out.clusters_dropped += 1; c.clusters += 1; }, else => { switch (m.kind) { .submanifest => c.submanifests += 1, .control => c.controls += 1, .wiki => c.wikis += 1, .ticket => c.tickets += 1, .attachment => c.attachments += 1, .event => c.events += 1, .forum => c.forums += 1, .manifest, .cluster => unreachable, } carry_rids.append(gpa, r) catch return error.OutOfMemory; }, } } renderCensus(&c, streams); out.census = c; } clock.done("classified and carried content blobs"); // Pass 2: check-ins in parents-first order. `events` holds every // check-in; Kahn over `parentsOf` gives the order without recursion. // Each tree is built by the sub-manifest editor over its primary // parent's root (#167), the same builder the commit verb uses, so a // check-in costs its change set, not its file set. { var tree_store = LogTreeStore{ .gpa = gpa, .log = dest }; defer tree_store.deinitCache(); var ed = editor.TreeEditor.init(gpa, tree_store.treeStore(), .{ .hash_algo = .sha3_256, .shape = .{ .tree = .{ .k = k, .height_max = @intCast(artifact.max_construct_height) } }, }, 64 << 20) catch |e| return treeErr(streams, verb, "-", e); defer ed.deinit(); var parents: ParentCache = .empty; defer { var pit = parents.valueIterator(); while (pit.next()) |pt| pt.*.deinit(gpa); parents.deinit(gpa); } // How many children build from each check-in as their PRIMARY // parent: the cache entry is released when the last has. var primary_kids: std.AutoHashMapUnmanaged(repostate.Rid, u32) = .empty; defer primary_kids.deinit(gpa); // The proof's flatten memo: consecutive check-ins share nearly every // node, so one memo across the run keeps the walk at its // distinct-artifact cost (#167). var sub_memo: ?repostate.SubManifestMemo = null; defer if (sub_memo) |*m| m.deinit(); var intern = Intern.init(gpa); defer intern.deinit(gpa); // The source side's baseline memo (zit#167): consecutive delta // check-ins share a baseline, so it is read and parsed once per // distinct baseline, not once per check-in. var baseline_memo: ?*Repository.BaselineMemo = null; defer if (baseline_memo) |m| m.deinit(gpa); const state = repo.state(); const all = state.events.keys(); var indegree: std.AutoHashMapUnmanaged(repostate.Rid, u32) = .empty; defer indegree.deinit(gpa); var children: std.AutoHashMapUnmanaged(repostate.Rid, std.ArrayListUnmanaged(repostate.Rid)) = .empty; defer { var cit = children.valueIterator(); while (cit.next()) |l| l.deinit(gpa); children.deinit(gpa); } for (all) |rid| { var deg: u32 = 0; var pit = state.parentsOf(rid); while (pit.next()) |p| { if (state.mtimeOf(p) == null) continue; // not a check-in here if (deg == 0) { const pk = primary_kids.getOrPut(gpa, p) catch return error.OutOfMemory; if (!pk.found_existing) pk.value_ptr.* = 0; pk.value_ptr.* += 1; } deg += 1; const gop = children.getOrPut(gpa, p) catch return error.OutOfMemory; if (!gop.found_existing) gop.value_ptr.* = .empty; gop.value_ptr.append(gpa, rid) catch return error.OutOfMemory; } indegree.put(gpa, rid, deg) catch return error.OutOfMemory; } var queue: std.ArrayList(repostate.Rid) = .empty; defer queue.deinit(gpa); for (all) |rid| { if ((indegree.get(rid) orelse 0) == 0) queue.append(gpa, rid) catch return error.OutOfMemory; } var taken: usize = 0; while (taken < queue.items.len) { const rid = queue.items[taken]; taken += 1; try remintCheckin(gpa, repo, dest, &ed, io, &parents, &sub_memo, &intern, &baseline_memo, primary_kids.get(rid) orelse 0, rid, &map, &out, streams); if (children.getPtr(rid)) |kids| { for (kids.items) |kid| { const d = indegree.getPtr(kid).?; d.* -= 1; if (d.* == 0) queue.append(gpa, kid) catch return error.OutOfMemory; } } } if (taken != all.len) return error.BrokenAncestry; // a cycle is corrupt history out.checkins = @intCast(taken); } streams.err.print( "import-fossil: of which source enumeration {d}s, tree builds {d}s, pair proofs {d}s\n", .{ out.enum_ns / std.time.ns_per_s, out.build_ns / std.time.ns_per_s, out.verify_ns / std.time.ns_per_s }, ) catch {}; clock.done("re-minted and verified check-ins"); // Pass 3: the carried structural artifacts. Controls target check-ins // and re-mint through the map; wiki/forum/ticket/attachment/event chains // reference only their own kind or uuids and carry verbatim (the // cascade rule in this module's doc). for (carry_rids.items) |r| { const bytes = (log.content(gpa, r) catch |e| return contentErr(e)) orelse return error.BrokenAncestry; defer gpa.free(bytes); const rewritten = remapReferences(gpa, bytes, &map) catch return error.OutOfMemory; if (rewritten) |new_bytes| { defer gpa.free(new_bytes); var name_buf: [64]u8 = undefined; const new_hex = artifact.artifactName(.sha3_256, new_bytes, &name_buf); var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0 .. new_hex.len / 2], new_hex) catch unreachable; _ = dest.putArtifact(raw[0 .. new_hex.len / 2], new_bytes, null) catch return error.Storage; out.controls_remapped += 1; } else { var base_buf: [logstore.max_hash_len]u8 = undefined; const hint: ?logstore.Rid = if (log.deltaBaseHashOf(r, &base_buf)) |bh| dest.lookup(bh) else null; _ = dest.putArtifact(log.hashOf(r), bytes, hint) catch return error.Storage; out.carried += 1; } } // The provenance technote (#61 non-negotiable): the old->new map as an // event artifact INSIDE the converted repository -- it syncs with the // repo, touches no check-in's file set, and keeps every external hash // citation resolvable by lookup. Deterministic: the tip's own timestamp // and an id derived from the project code, so one source converts to // byte-identical output every time. clock.done("carried structural artifacts"); try mintProvenance(gpa, repo, dest, &map, project_code, &out); clock.done("minted provenance"); return out; } /// Wall-clock phase reporter: one stderr line per completed phase, so a /// multi-hour conversion can be watched from outside (#155). pub const Phase = struct { io: std.Io, streams: cli.Streams, t0: i64, pub fn start(io: std.Io, streams: cli.Streams) Phase { return .{ .io = io, .streams = streams, .t0 = std.Io.Timestamp.now(io, .real).toSeconds() }; } pub fn done(self: *Phase, label: []const u8) void { const now = std.Io.Timestamp.now(self.io, .real).toSeconds(); self.streams.err.print("import-fossil: {s} ({d}s)\n", .{ label, now - self.t0 }) catch {}; self.t0 = now; } }; fn contentErr(e: logstore.ContentError) RemintError { return switch (e) { error.OutOfMemory => error.OutOfMemory, error.Storage => error.Storage, error.Corrupt => error.Corrupt, }; } /// One re-minted check-in's file set and root, kept only while a child still /// has to build from it (#167): the editor applies a child's change set over /// this root, so unchanged slices are reused by name instead of rebuilt. const ParentTree = struct { arena_state: std.heap.ArenaAllocator, files: []const artifact.FCard, /// Empty when the parent's tree came out flat (§6.2 step 2). s: []const artifact.SCard, /// Children that will build from this parent and have not yet. pending: u32, fn view(self: *const ParentTree) editor.RootView { if (self.s.len != 0) return .{ .tree = self.s }; return .{ .flat = self.files }; } fn deinit(self: *ParentTree, gpa: std.mem.Allocator) void { self.arena_state.deinit(); gpa.destroy(self); } }; const ParentCache = std.AutoHashMapUnmanaged(repostate.Rid, *ParentTree); /// One copy of every distinct name, hash and perm token the parent cache /// holds, for the run. Consecutive check-ins share nearly every card, so /// retaining a check-in's file set costs a lookup per card rather than an /// allocation per card; memory is bounded by the distinct strings in the /// history, not by history length times width. const Intern = struct { arena_state: std.heap.ArenaAllocator, table: std.StringHashMapUnmanaged(void) = .empty, fn init(gpa: std.mem.Allocator) Intern { return .{ .arena_state = std.heap.ArenaAllocator.init(gpa) }; } fn deinit(self: *Intern, gpa: std.mem.Allocator) void { self.table.deinit(gpa); self.arena_state.deinit(); } fn get(self: *Intern, gpa: std.mem.Allocator, s: []const u8) error{OutOfMemory}![]const u8 { const gop = try self.table.getOrPut(gpa, s); if (!gop.found_existing) { const owned = try self.arena_state.allocator().dupe(u8, s); gop.key_ptr.* = owned; } return gop.key_ptr.*; } fn getOpt(self: *Intern, gpa: std.mem.Allocator, s: ?[]const u8) error{OutOfMemory}!?[]const u8 { return if (s) |v| try self.get(gpa, v) else null; } }; fn internCard(intern: *Intern, gpa: std.mem.Allocator, f: artifact.FCard) error{OutOfMemory}!artifact.FCard { return .{ .filename = try intern.get(gpa, f.filename), .hash = try intern.getOpt(gpa, f.hash), .perm = try intern.getOpt(gpa, f.perm), // Kept: a rename annotation is a fact of ONE check-in, so a child // that inherits the file without it is a change the editor must see. .old_name = try intern.getOpt(gpa, f.old_name), .mode = f.mode, }; } fn sameCard(a: artifact.FCard, b: artifact.FCard) bool { return sameOpt(a.hash, b.hash) and sameOpt(a.perm, b.perm) and sameOpt(a.old_name, b.old_name); } fn sameOpt(a: ?[]const u8, b: ?[]const u8) bool { if (a == null or b == null) return a == null and b == null; return std.mem.eql(u8, a.?, b.?); } /// The commands that turn `parent`'s file set into `child`'s: both are in /// canonical filename order (§8.1), so one merge walk finds every added, /// changed and removed name. fn diffCommands( gpa: std.mem.Allocator, parent: []const artifact.FCard, child: []const artifact.FCard, ) error{OutOfMemory}!std.ArrayList(editor.TreeCommand) { var cmds: std.ArrayList(editor.TreeCommand) = .empty; errdefer cmds.deinit(gpa); var i: usize = 0; var j: usize = 0; while (i < parent.len or j < child.len) { if (j >= child.len) { try cmds.append(gpa, .{ .delete = parent[i].filename }); i += 1; continue; } if (i >= parent.len) { try cmds.append(gpa, .{ .upsert = child[j] }); j += 1; continue; } switch (std.mem.order(u8, parent[i].filename, child[j].filename)) { .lt => { try cmds.append(gpa, .{ .delete = parent[i].filename }); i += 1; }, .gt => { try cmds.append(gpa, .{ .upsert = child[j] }); j += 1; }, .eq => { if (!sameCard(parent[i], child[j])) try cmds.append(gpa, .{ .upsert = child[j] }); i += 1; j += 1; }, } } return cmds; } fn treeErr(streams: cli.Streams, verb: []const u8, old_hex: []const u8, e: editor.Error) RemintError { switch (e) { error.OutOfMemory => return error.OutOfMemory, error.StorageFailure => return error.Storage, else => { streams.err.print("zit {s}: tree build failed at source check-in {s}: {s}\n", .{ verb, old_hex, @errorName(e) }) catch {}; return error.Corrupt; }, } } /// Re-mint ONE check-in: effective file set through the enumeration seam, /// S-tree build, text-level card rewrite (F run out, S run in at its sorted /// position, B dropped, P/Q/T hash tokens chased through the map, PGP /// armor stripped, Z resealed), new name recorded in the map. fn remintCheckin( gpa: std.mem.Allocator, repo: *Repository, dest: *logstore.LogStore, ed: *editor.TreeEditor, ed_io: std.Io, parents: *ParentCache, sub_memo: *?repostate.SubManifestMemo, intern: *Intern, baseline_memo: *?*Repository.BaselineMemo, primary_kids: u32, rid: repostate.Rid, map: *std.StringHashMapUnmanaged([]const u8), out: *Remint, streams: cli.Streams, ) RemintError!void { const old_hex = repo.hashOf(rid); const t_enum = std.Io.Timestamp.now(ed_io, .real).nanoseconds; var fl = (repo.filesMemo(gpa, rid, baseline_memo) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.BrokenAncestry; defer fl.deinit(gpa); var entries: std.ArrayList(artifact.FCard) = .empty; defer entries.deinit(gpa); var it = fl.iterator(); while (it.next()) |f| entries.append(gpa, f) catch return error.OutOfMemory; out.enum_ns += @intCast(std.Io.Timestamp.now(ed_io, .real).nanoseconds - t_enum); // The primary parent is the first P card that names a check-in; its // re-minted root is the base the editor applies this check-in's change // set over. A root check-in (or a parent whose entry was released) // applies over the empty flat root, the editor's base case. const state = repo.state(); var primary: ?repostate.Rid = null; { var pit = state.parentsOf(rid); while (pit.next()) |p| { if (state.mtimeOf(p) == null) continue; primary = p; break; } } const parent_tree: ?*ParentTree = if (primary) |p| parents.get(p) else null; var cmds = if (parent_tree) |pt| diffCommands(gpa, pt.files, entries.items) catch return error.OutOfMemory else blk: { var all: std.ArrayList(editor.TreeCommand) = .empty; for (entries.items) |f| all.append(gpa, .{ .upsert = f }) catch return error.OutOfMemory; break :blk all; }; defer cmds.deinit(gpa); const parent_view: editor.RootView = if (parent_tree) |pt| pt.view() else .{ .flat = &.{} }; const t_build = std.Io.Timestamp.now(ed_io, .real).nanoseconds; var built = ed.apply(parent_view, cmds.items) catch |e| return treeErr(streams, out.verb, old_hex, e); out.build_ns += @intCast(std.Io.Timestamp.now(ed_io, .real).nanoseconds - t_build); defer built.deinit(); const s_cards: []const artifact.SCard = switch (built.root) { .tree => |cards| cards, .flat => &.{}, }; // The file list retains the parsed manifest, so its bytes are the source // text: one parse per check-in, not two. const source_text = stripClearsign(fl.target.bytes, out); const new_bytes = rewriteManifest(gpa, source_text, entries.items, s_cards, map) catch return error.OutOfMemory; defer gpa.free(new_bytes); var name_buf: [64]u8 = undefined; const new_hex = artifact.artifactName(.sha3_256, new_bytes, &name_buf); var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0..32], new_hex) catch unreachable; _ = dest.put(raw[0..32], new_bytes) catch return error.Storage; const old_key = gpa.dupe(u8, old_hex) catch return error.OutOfMemory; errdefer gpa.free(old_key); const new_val = gpa.dupe(u8, new_hex) catch return error.OutOfMemory; map.put(gpa, old_key, new_val) catch return error.OutOfMemory; // The differential proof for THIS pair, while the source file set is // still in hand: the tree just written must read back through the same // resolver every verb uses and flatten to exactly `entries`. Proving // each pair here instead of in a later pass spares a second // whole-history enumeration of the source (#155); publication still // waits on every pair. const t_verify = std.Io.Timestamp.now(ed_io, .real).nanoseconds; try verifyMinted(gpa, dest, sub_memo, out.verb, old_key, new_val, entries.items, streams); out.verified_pairs += 1; out.verify_ns += @intCast(std.Io.Timestamp.now(ed_io, .real).nanoseconds - t_verify); // Keep this root for the children that build from it; release the // parent's once its last such child (this one) has consumed it. if (primary_kids > 0) { const pt = gpa.create(ParentTree) catch return error.OutOfMemory; errdefer gpa.destroy(pt); pt.* = .{ .arena_state = std.heap.ArenaAllocator.init(gpa), .files = &.{}, .s = &.{}, .pending = primary_kids }; errdefer pt.arena_state.deinit(); const arena = pt.arena_state.allocator(); const files = arena.alloc(artifact.FCard, entries.items.len) catch return error.OutOfMemory; for (entries.items, 0..) |f, i| files[i] = internCard(intern, gpa, f) catch return error.OutOfMemory; const s_copy = arena.alloc(artifact.SCard, s_cards.len) catch return error.OutOfMemory; for (s_cards, 0..) |c, i| s_copy[i] = .{ .filename = intern.get(gpa, c.filename) catch return error.OutOfMemory, .hash = intern.get(gpa, c.hash) catch return error.OutOfMemory, }; pt.files = files; pt.s = s_copy; parents.put(gpa, rid, pt) catch return error.OutOfMemory; } if (parent_tree) |pt| { pt.pending -= 1; if (pt.pending == 0) { _ = parents.remove(primary.?); pt.deinit(gpa); } } } /// Unwrap a clearsign envelope (PGP or SSH — the same §4.1.1 wrapper the /// parser strips), counting the casualty. The payload is exactly the lines /// between the armor's blank separator and the signature block. fn stripClearsign(bytes: []const u8, out: *Remint) []const u8 { const body = artifact.stripSignatureWrapper(bytes); if (body.len != bytes.len) out.signatures_stripped += 1; return body; } fn rewriteManifest( gpa: std.mem.Allocator, text: []const u8, entries: []const artifact.FCard, s_cards: []const artifact.SCard, map: *const std.StringHashMapUnmanaged([]const u8), ) error{OutOfMemory}![]u8 { var outb: std.ArrayList(u8) = .empty; errdefer outb.deinit(gpa); // submanifests §6.2 step 2: a name set that partitions into fewer than two // groups yields no tree, and the writer "MUST emit a flat manifest and no // sub-manifests". The run this manifest carries is therefore an F run, not // an S run — and it sorts where F sorts (before P), not where S does (after // R). Dropping the source's F cards and emitting `build.s` regardless is // what published a check-in with no file set at all. const flat = s_cards.len == 0; const run_card: u8 = if (flat) 'F' else 'S'; var emitted_s = false; var lines = std.mem.splitScalar(u8, text, '\n'); while (lines.next()) |line| { if (line.len == 0) continue; const card = line[0]; if (!emitted_s and card > run_card and card != 'Z') { try appendRun(gpa, &outb, s_cards, entries, flat); emitted_s = true; } switch (card) { 'F' => continue, 'B' => continue, 'Z' => { if (!emitted_s) { try appendRun(gpa, &outb, s_cards, entries, flat); emitted_s = true; } try appendZ(gpa, &outb); }, 'P', 'Q', 'T' => try appendRemapped(gpa, &outb, line, map), else => { try outb.appendSlice(gpa, line); try outb.append(gpa, '\n'); }, } } if (!emitted_s) try appendRun(gpa, &outb, s_cards, entries, flat); return outb.toOwnedSlice(gpa); } /// The file-set run this minted manifest carries: the tree's S run, or — when /// §6.2 construction declined to build a tree — the effective set as flat F /// cards. The B card is dropped either way, so the flat arm materializes the /// whole effective set (`entries`), not the source's own F lines, which for a /// delta manifest name only what that check-in changed. fn appendRun( gpa: std.mem.Allocator, outb: *std.ArrayList(u8), s_cards: []const artifact.SCard, entries: []const artifact.FCard, flat: bool, ) error{OutOfMemory}!void { if (!flat) return appendSRun(gpa, outb, s_cards); for (entries) |f| try artifact.appendFlatFCard(outb, gpa, f); } fn appendSRun(gpa: std.mem.Allocator, outb: *std.ArrayList(u8), s_cards: []const artifact.SCard) error{OutOfMemory}!void { for (s_cards) |s_card| { var enc_buf: [2 * 4096]u8 = undefined; const encoded = artifact.encodeEscapes(s_card.filename, &enc_buf); try outb.appendSlice(gpa, "S "); try outb.appendSlice(gpa, encoded); try outb.append(gpa, ' '); try outb.appendSlice(gpa, s_card.hash); try outb.append(gpa, '\n'); } } /// One card line with every 40/64-hex token replaced by its mapped /// counterpart when one exists. Tokens outside the map (content hashes, /// '*', tag names) pass through untouched. fn appendRemapped( gpa: std.mem.Allocator, outb: *std.ArrayList(u8), line: []const u8, map: *const std.StringHashMapUnmanaged([]const u8), ) error{OutOfMemory}!void { var toks = std.mem.splitScalar(u8, line, ' '); var first = true; while (toks.next()) |tok| { if (!first) try outb.append(gpa, ' '); first = false; if (map.get(tok)) |new| { try outb.appendSlice(gpa, new); } else if (tok.len > 1 and (tok[0] == '+' or tok[0] == '-' or tok[0] == '*')) { // Q cards carry a sign prefix on the hash token. if (map.get(tok[1..])) |new| { try outb.append(gpa, tok[0]); try outb.appendSlice(gpa, new); } else { try outb.appendSlice(gpa, tok); } } else { try outb.appendSlice(gpa, tok); } } try outb.append(gpa, '\n'); } /// Seal the §6.2 Z card: md5 over every byte already emitted. fn appendZ(gpa: std.mem.Allocator, outb: *std.ArrayList(u8)) error{OutOfMemory}!void { var digest: [16]u8 = undefined; std.crypto.hash.Md5.hash(outb.items, &digest, .{}); const hex = std.fmt.bytesToHex(digest, .lower); try outb.appendSlice(gpa, "Z "); try outb.appendSlice(gpa, &hex); try outb.append(gpa, '\n'); } /// Rewrite a carried structural artifact's references. Returns the new /// bytes when anything changed (caller re-mints), null when nothing did /// (caller carries verbatim). Only T lines can target a check-in here; /// the map lookup is what decides, not the card letter. fn remapReferences( gpa: std.mem.Allocator, bytes: []const u8, map: *const std.StringHashMapUnmanaged([]const u8), ) error{OutOfMemory}!?[]u8 { var outb: std.ArrayList(u8) = .empty; errdefer outb.deinit(gpa); var changed = false; var lines = std.mem.splitScalar(u8, bytes, '\n'); while (lines.next()) |line| { if (line.len == 0) continue; if (line[0] == 'Z') { try appendZ(gpa, &outb); continue; } if (line[0] == 'T') { const before = outb.items.len; try appendRemapped(gpa, &outb, line, map); const slice = outb.items[before..]; if (!std.mem.eql(u8, slice[0 .. slice.len - 1], line)) changed = true; continue; } try outb.appendSlice(gpa, line); try outb.append(gpa, '\n'); } if (!changed) { outb.deinit(gpa); return null; } return try outb.toOwnedSlice(gpa); } /// The differential proof for one re-minted check-in: the converted /// manifest, read back from `dest` and flattened through the same resolver /// every verb uses, must carry exactly `src_cards` — the source's effective /// file set, name for name, hash for hash. Any other outcome is the one bug /// class this converter must never publish, and `divergedAt`/`divergedWhy` /// make that class loud. fn verifyMinted( gpa: std.mem.Allocator, dest: *logstore.LogStore, sub_memo: *?repostate.SubManifestMemo, verb: []const u8, old_hex: []const u8, new_hex: []const u8, src_cards: []const artifact.FCard, streams: cli.Streams, ) RemintError!void { var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0 .. new_hex.len / 2], new_hex) catch return error.Corrupt; const new_rid = dest.lookup(raw[0 .. new_hex.len / 2]) orelse return divergedWhy(streams, verb, old_hex, "the converted manifest is not in the destination store"); const new_bytes = (dest.content(gpa, new_rid) catch |e| return contentErr(e)) orelse return divergedWhy(streams, verb, old_hex, "the converted manifest has no content"); defer gpa.free(new_bytes); var m = artifact.parse(gpa, new_bytes) catch |e| return divergedWhy(streams, verb, old_hex, @errorName(e)); defer m.deinit(); Repository.flattenTreeFromLogMemo(gpa, dest, &m, sub_memo) catch |e| return divergedWhy(streams, verb, old_hex, @errorName(e)); // `flattened` is null for a manifest that carries no S run at all, which // since the §6.2 step-2 flat arm above is an ordinary outcome, not a fault: // the file set is then the manifest's own F run. Same substitution // `repository.manifestFilesOf` makes, for the same reason — a consumer must // not distinguish a walked enumeration from a flat one. const new_files = m.flattened orelse m.f; for (src_cards, 0..) |src_card, i| { if (i >= new_files.len) return divergedAt(streams, verb, old_hex, src_card.filename); const dst_card = new_files[i]; if (!std.mem.eql(u8, src_card.filename, dst_card.filename)) return divergedAt(streams, verb, old_hex, src_card.filename); const src_hash = src_card.hash orelse return divergedAt(streams, verb, old_hex, src_card.filename); const dst_hash = dst_card.hash orelse return divergedAt(streams, verb, old_hex, src_card.filename); if (!std.mem.eql(u8, src_hash, dst_hash)) return divergedAt(streams, verb, old_hex, src_card.filename); } if (src_cards.len != new_files.len) return divergedAt(streams, verb, old_hex, "(extra converted entries)"); } fn divergedWhy(streams: cli.Streams, verb: []const u8, old_hex: []const u8, why: []const u8) RemintError { streams.err.print( "zit {s}: VERIFY FAILURE at source check-in {s}: {s} — nothing was published\n", .{ verb, old_hex, why }, ) catch {}; return error.Diverged; } fn divergedAt(streams: cli.Streams, verb: []const u8, old_hex: []const u8, path: []const u8) RemintError { streams.err.print( "zit {s}: VERIFY DIVERGENCE at source check-in {s}, path '{s}' — nothing was published\n", .{ verb, old_hex, path }, ) catch {}; return error.Diverged; } /// Compose and store the provenance technote(s). Pairs are emitted in /// sorted-old-hash order (map iteration order is not deterministic; the /// output must be). One note holds up to `pairs_per_note` pairs; a corpus /// larger than that chains notes, each E-carded under an id derived from /// the project code and its chunk index. const pairs_per_note: usize = 16 * 1024; fn mintProvenance( gpa: std.mem.Allocator, repo: *Repository, dest: *logstore.LogStore, map: *const std.StringHashMapUnmanaged([]const u8), project_code: []const u8, out: *Remint, ) RemintError!void { if (map.count() == 0) return; // The tip's D timestamp: the conversion's clock, without ambient time. const tip = repo.resolve("tip", .checkin) catch return error.BrokenAncestry; var tip_owned = (repo.manifest(gpa, tip) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.BrokenAncestry; defer tip_owned.deinit(gpa); const timestamp = if (tip_owned.manifest.d) |d| d.timestamp else "1970-01-01T00:00:00"; var olds: std.ArrayList([]const u8) = .empty; defer olds.deinit(gpa); var kit = map.keyIterator(); while (kit.next()) |k| olds.append(gpa, k.*) catch return error.OutOfMemory; std.mem.sort([]const u8, olds.items, {}, sliceLess); var chunk: usize = 0; var offset: usize = 0; while (offset < olds.items.len) : (chunk += 1) { const end = @min(offset + pairs_per_note, olds.items.len); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); body.appendSlice(gpa, "import-fossil provenance: OLD-check-in-hash NEW-check-in-hash, one pair per line.\n") catch return error.OutOfMemory; for (olds.items[offset..end]) |old_hex| { body.appendSlice(gpa, old_hex) catch return error.OutOfMemory; body.append(gpa, ' ') catch return error.OutOfMemory; body.appendSlice(gpa, map.get(old_hex).?) catch return error.OutOfMemory; body.append(gpa, '\n') catch return error.OutOfMemory; } offset = end; var id_material: std.ArrayList(u8) = .empty; defer id_material.deinit(gpa); id_material.appendSlice(gpa, "import-fossil-provenance:") catch return error.OutOfMemory; id_material.appendSlice(gpa, project_code) catch return error.OutOfMemory; id_material.append(gpa, ':') catch return error.OutOfMemory; id_material.append(gpa, @intCast('0' + (chunk % 10))) catch return error.OutOfMemory; var id_buf: [64]u8 = undefined; const note_id = artifact.artifactName(.sha3_256, id_material.items, &id_buf); var note: std.ArrayList(u8) = .empty; defer note.deinit(gpa); const head = std.fmt.allocPrint( gpa, "D {s}\nE {s} {s}\nU import-fossil\nW {d}\n{s}\n", .{ timestamp, timestamp, note_id, body.items.len, body.items }, ) catch return error.OutOfMemory; defer gpa.free(head); note.appendSlice(gpa, head) catch return error.OutOfMemory; appendZ(gpa, ¬e) catch return error.OutOfMemory; var name_buf: [64]u8 = undefined; const note_hex = artifact.artifactName(.sha3_256, note.items, &name_buf); var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0..32], note_hex) catch unreachable; _ = dest.put(raw[0..32], note.items) catch return error.Storage; out.provenance_notes += 1; } } fn sliceLess(_: void, a: []const u8, b: []const u8) bool { return std.mem.order(u8, a, b) == .lt; }