Artifact 62688851dbee43e72f6227952b5fdcefeb8cd68b8c3cd860fd83a7e2a60e610a:
- File
src/fossil/fossil_convert.zig
— part of check-in
[98ddd81b75]
at
2026-08-23 23:47:53
on branch trunk
— fossil_convert: mint FLAT when the names do not partition (#126)
Committed under a wrong description, corrected here. The message claimed this silenced an unused parameter and left two tests failing; it does neither. The content is the finished #126 fix -- appendRun honours flat, materializing the whole effective entry set rather than falling through to the S run -- and the suite is green.
The stub I described was a transient reversion in the shared checkout, made to prove the new regression test fails without the fix. It was restored before I committed. fossil commit auto-syncs, so my commit captured the restored tree while my message described the tree I had read minutes earlier.
Authored by the #126 work, not by this commit's message. (user: dmestas, size: 34218) [annotate] [blame] [check-ins using]
//! The submanifest conversion pass (#61): re-mint a fossil corpus as a new //! project with S-card tree manifests from the first commit. //! //! The load-bearing observation is the RE-MINT CASCADE RULE: an artifact's //! name changes only when its BYTES change, and (with content-blob names //! kept, the phase-1 posture) bytes change only for an artifact that either //! gains the S-tree structure (every check-in manifest) or references a //! re-minted artifact in a hash-bearing card. So the remap surface is: //! //! check-in manifests all re-mint (S-trees + remapped P/Q/B; D/C/U and //! the R card survive -- R is over the file SET, which //! is preserved exactly) //! control artifacts re-mint iff a T card targets a remapped check-in //! events (technotes) re-mint iff their P chain or T target remaps //! wiki / forum P/I/G reference their own kind's chain -- verbatim //! unless an ancestor re-minted (transitively closed) //! attachments target a wiki NAME or ticket uuid + a CONTENT hash //! -- verbatim in phase 1 //! tickets K ids are uuids, not artifact hashes -- verbatim //! clusters dropped (sync housekeeping, not history) //! content blobs verbatim (phase 1 keeps their names) //! PGP signatures cannot survive rehashing: stripped and counted //! //! The census pass below is the converter's front door: it classifies the //! whole corpus, counts the remap surface and the casualties, and is the //! baseline the count-reconciliation acceptance check audits against. const std = @import("std"); const zit = @import("zit"); const cli = zit.cli; const artifact = zit.artifact; const logstore = zit.logstore; const Repository = zit.Repository; const repostate = zit.repostate; /// One corpus walk: every stored artifact classified by parsed type, with /// the phase-1 remap surface and casualty counts. Phantoms hold no bytes /// and are skipped (they are the sync layer's business, not history's). pub const Census = struct { total: u64 = 0, phantoms: u64 = 0, blobs: u64 = 0, checkins: u64 = 0, submanifests: u64 = 0, controls: u64 = 0, wikis: u64 = 0, tickets: u64 = 0, attachments: u64 = 0, events: u64 = 0, forums: u64 = 0, clusters: u64 = 0, /// Check-in manifests wrapped in a PGP signature: the signature is a /// stated casualty of any conversion (stripped and counted). signed: u64 = 0, pub fn structural(self: *const Census) u64 { return self.checkins + self.submanifests + self.controls + self.wikis + self.tickets + self.attachments + self.events + self.forums + self.clusters; } }; const pgp_prefix = "-----BEGIN PGP SIGNED MESSAGE-----"; /// Walk the source store and classify every artifact. Bounded by the /// store's artifact count; each artifact is parsed once with the Z-tolerant /// window parser (a parse failure is a content blob, by construction -- /// fossil's artifact grammar is what separates structure from content). pub fn census( gpa: std.mem.Allocator, repo: *Repository, ) error{ OutOfMemory, Storage, Corrupt }!Census { const log = repo.logStore(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; var out: Census = .{}; const n: u64 = log.artifactCount(); var rid: u64 = 1; while (rid <= n) : (rid += 1) { out.total += 1; const r: logstore.Rid = @intCast(rid); if (log.isPhantom(r)) { out.phantoms += 1; continue; } const bytes = (log.content(gpa, r) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Storage => return error.Storage, error.Corrupt => return error.Corrupt, }) orelse { out.phantoms += 1; // masked (shunned) reads as absent: not history continue; }; defer gpa.free(bytes); const signed = std.mem.startsWith(u8, bytes, pgp_prefix); var m = artifact.parseWindow(gpa, bytes) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => { out.blobs += 1; continue; }, }; defer m.deinit(); switch (m.kind) { .manifest => { out.checkins += 1; if (signed) out.signed += 1; }, .submanifest => out.submanifests += 1, .cluster => out.clusters += 1, .control => out.controls += 1, .wiki => out.wikis += 1, .ticket => out.tickets += 1, .attachment => out.attachments += 1, .event => out.events += 1, .forum => out.forums += 1, } } return out; } /// Render the census for the operator: what converts, what carries, what /// dies. The converter's own accounting reconciles against these numbers. pub fn renderCensus(c: *const Census, streams: cli.Streams) void { streams.out.print( "census: {d} artifacts — {d} check-ins to re-mint ({d} PGP-signed: signatures strip)," ++ " {d} content blobs carried verbatim\n", .{ c.total, c.checkins, c.signed, c.blobs }, ) catch {}; if (c.controls + c.wikis + c.tickets + c.attachments + c.events + c.forums != 0) { streams.out.print( "census: {d} control, {d} wiki, {d} ticket, {d} attachment, {d} technote, {d} forum" ++ " — carried, re-minting only where a reference remaps\n", .{ c.controls, c.wikis, c.tickets, c.attachments, c.events, c.forums }, ) catch {}; } if (c.clusters != 0) { streams.out.print("census: {d} cluster(s) dropped (sync housekeeping, not history)\n", .{c.clusters}) catch {}; } if (c.submanifests != 0) { streams.out.print("census: {d} sub-manifest slice(s) already present in the source\n", .{c.submanifests}) catch {}; } if (c.phantoms != 0) { streams.out.print("census: {d} phantom/masked name(s) skipped — clone the origin to completeness first\n", .{c.phantoms}) catch {}; } } /// One F card over a static filename/hash pair, for the mint tests below. fn testFCard(name: []const u8, hash: []const u8) artifact.FCard { return .{ .filename = name, .hash = hash, .perm = null, .old_name = null, .mode = .regular }; } test "convert §6.2 step 2 (zit#126): a check-in whose names do not partition mints FLAT, not empty" { // The regression: `buildTree` reports `isFlat()` for a name set that // yields fewer than two groups, and `rewriteManifest` dropped the source's // F cards while emitting an empty S run — publishing a check-in with NO // file set. The differential verify caught it and aborted the whole // conversion, which is how zit#126 presented (an ordinary libspatialite // trunk baseline, 38 files, at k=5). const gpa = std.testing.allocator; const entries = [_]artifact.FCard{ testFCard("Makefile", "0" ** 40), testFCard("src/a.c", "1" ** 40), testFCard("src/b.c", "2" ** 40), }; // max_k makes the span enormous relative to three names, so the boundary // test never fires and the whole set stays one group — §6.2 step 2. var build = try artifact.buildTree(gpa, &entries, artifact.max_k, .sha3_256); defer build.deinit(); try std.testing.expect(build.isFlat()); const source = "C comment\n" ++ "D 2009-04-16T15:57:11\n" ++ "F Makefile " ++ "0" ** 40 ++ "\n" ++ "F src/a.c " ++ "1" ** 40 ++ "\n" ++ "F src/b.c " ++ "2" ** 40 ++ "\n" ++ "P " ++ "3" ** 40 ++ "\n" ++ "U someone\n" ++ "Z " ++ "4" ** 32 ++ "\n"; var map: std.StringHashMapUnmanaged([]const u8) = .empty; defer map.deinit(gpa); const minted = try rewriteManifest(gpa, source, &entries, &build, &map); defer gpa.free(minted); // The mint must carry the file set as an F run and no S run at all, and it // must still parse as a check-in manifest. var m = try artifact.parse(gpa, minted); defer m.deinit(); try std.testing.expectEqual(@as(usize, 0), m.s.len); try std.testing.expectEqual(entries.len, m.f.len); for (&entries, m.f) |src_card, dst_card| { try std.testing.expectEqualStrings(src_card.filename, dst_card.filename); try std.testing.expectEqualStrings(src_card.hash.?, dst_card.hash.?); } } test "convert §6.2 step 3: a partitioning name set still mints as an S tree with no F run" { // The other arm, so the fix above cannot be "flatten everything": at k=1 // the same shape of input does partition, and the mint must be a tree. const gpa = std.testing.allocator; var entries: std.ArrayList(artifact.FCard) = .empty; defer entries.deinit(gpa); var names: [64][8]u8 = undefined; var hashes: [64][40]u8 = undefined; for (0..names.len) |i| { names[i] = [_]u8{ 'f', 'i', 'l', 'e', '0' + @as(u8, @intCast(i / 10)), '0' + @as(u8, @intCast(i % 10)), 0, 0 }; hashes[i] = [_]u8{'a'} ** 40; hashes[i][0] = '0' + @as(u8, @intCast(i % 10)); try entries.append(gpa, testFCard(names[i][0..6], hashes[i][0..])); } var build = try artifact.buildTree(gpa, entries.items, artifact.min_k, .sha3_256); defer build.deinit(); try std.testing.expect(!build.isFlat()); const source = "C c\nD 2009-04-16T15:57:11\nU someone\nZ " ++ "4" ** 32 ++ "\n"; var map: std.StringHashMapUnmanaged([]const u8) = .empty; defer map.deinit(gpa); const minted = try rewriteManifest(gpa, source, entries.items, &build, &map); defer gpa.free(minted); var m = try artifact.parse(gpa, minted); defer m.deinit(); try std.testing.expectEqual(@as(usize, 0), m.f.len); try std.testing.expect(m.s.len >= 2); // §6.2: a tree partitions at least in two } test "census: an empty-ish census sums structurally" { // Goal: the structural() partition is total over ArtifactType — a new // member without a census bucket fails the exhaustive switch in census() // at compile time, and this pin keeps the arithmetic honest. var c: Census = .{ .checkins = 2, .controls = 1, .clusters = 3 }; try std.testing.expectEqual(@as(u64, 6), c.structural()); } // --------------------------------------------------------------------------- // The re-mint pass (#61): check-ins become S-tree manifests under new names, // every hash-bearing reference chases the old->new map, everything else // carries verbatim. Phase 1 keeps content-blob names; new structural // artifacts mint under SHA3-256 (the new project's native policy). // --------------------------------------------------------------------------- pub const RemintError = error{ OutOfMemory, Storage, Corrupt, /// A parent reference left the corpus (the census phantom refusal should /// have caught this; a hole here is a source-store defect). BrokenAncestry, /// The differential verify found a (old,new) pair whose file sets differ /// -- the one bug class this converter must never publish. Diverged, }; pub const Remint = struct { provenance_notes: u64 = 0, checkins: u64 = 0, blobs: u64 = 0, carried: u64 = 0, controls_remapped: u64 = 0, signatures_stripped: u64 = 0, clusters_dropped: u64 = 0, verified_pairs: u64 = 0, }; /// Re-mint the whole corpus from `repo` into `dest`. `dest` is a fresh /// writable LogStore (the caller owns staging and publication). On success /// every source check-in has a converted counterpart whose EFFECTIVE FILE /// SET is byte-equivalent, proven pair by pair before returning. pub fn remint( gpa: std.mem.Allocator, repo: *Repository, dest: *logstore.LogStore, k: u4, project_code: []const u8, streams: cli.Streams, ) RemintError!Remint { const log = repo.logStore(gpa) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }; var out: Remint = .{}; // old full-hex -> new full-hex, both owned. Sized by the census's // structural count in spirit; grown on demand in practice. var map: std.StringHashMapUnmanaged([]const u8) = .empty; defer { var it = map.iterator(); while (it.next()) |e| { gpa.free(e.key_ptr.*); gpa.free(e.value_ptr.*); } map.deinit(gpa); } // Pass 1: content blobs carry verbatim (phase 1 keeps their names), and // every artifact classifies once into the work lists. var checkin_rids: std.ArrayList(repostate.Rid) = .empty; defer checkin_rids.deinit(gpa); var carry_rids: std.ArrayList(logstore.Rid) = .empty; defer carry_rids.deinit(gpa); { const n: u64 = log.artifactCount(); var rid: u64 = 1; while (rid <= n) : (rid += 1) { const r: logstore.Rid = @intCast(rid); if (log.isPhantom(r)) return error.BrokenAncestry; const bytes = (log.content(gpa, r) catch |e| return contentErr(e)) orelse return error.BrokenAncestry; defer gpa.free(bytes); var m = artifact.parseWindow(gpa, bytes) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => { _ = dest.put(log.hashOf(r), bytes) catch return error.Storage; out.blobs += 1; continue; }, }; defer m.deinit(); switch (m.kind) { .manifest => { // Collected via the DERIVED state below (topo order needs // the P DAG, which repostate already holds). }, .cluster => out.clusters_dropped += 1, else => carry_rids.append(gpa, r) catch return error.OutOfMemory, } } } // Pass 2: check-ins in parents-first order. `events` holds every // check-in; Kahn over `parentsOf` gives the order without recursion. { const state = repo.state(); const all = state.events.keys(); var indegree: std.AutoHashMapUnmanaged(repostate.Rid, u32) = .empty; defer indegree.deinit(gpa); var children: std.AutoHashMapUnmanaged(repostate.Rid, std.ArrayListUnmanaged(repostate.Rid)) = .empty; defer { var cit = children.valueIterator(); while (cit.next()) |l| l.deinit(gpa); children.deinit(gpa); } for (all) |rid| { var deg: u32 = 0; var pit = state.parentsOf(rid); while (pit.next()) |p| { if (state.mtimeOf(p) == null) continue; // not a check-in here deg += 1; const gop = children.getOrPut(gpa, p) catch return error.OutOfMemory; if (!gop.found_existing) gop.value_ptr.* = .empty; gop.value_ptr.append(gpa, rid) catch return error.OutOfMemory; } indegree.put(gpa, rid, deg) catch return error.OutOfMemory; } var queue: std.ArrayList(repostate.Rid) = .empty; defer queue.deinit(gpa); for (all) |rid| { if ((indegree.get(rid) orelse 0) == 0) queue.append(gpa, rid) catch return error.OutOfMemory; } var taken: usize = 0; while (taken < queue.items.len) { const rid = queue.items[taken]; taken += 1; try remintCheckin(gpa, repo, log, dest, rid, k, &map, &out); if (children.getPtr(rid)) |kids| { for (kids.items) |kid| { const d = indegree.getPtr(kid).?; d.* -= 1; if (d.* == 0) queue.append(gpa, kid) catch return error.OutOfMemory; } } } if (taken != all.len) return error.BrokenAncestry; // a cycle is corrupt history out.checkins = @intCast(taken); checkin_rids.appendSlice(gpa, queue.items) catch return error.OutOfMemory; } // Pass 3: the carried structural artifacts. Controls target check-ins // and re-mint through the map; wiki/forum/ticket/attachment/event chains // reference only their own kind or uuids and carry verbatim (the // cascade rule in this module's doc). for (carry_rids.items) |r| { const bytes = (log.content(gpa, r) catch |e| return contentErr(e)) orelse return error.BrokenAncestry; defer gpa.free(bytes); const rewritten = remapReferences(gpa, bytes, &map) catch return error.OutOfMemory; if (rewritten) |new_bytes| { defer gpa.free(new_bytes); var name_buf: [64]u8 = undefined; const new_hex = artifact.artifactName(.sha3_256, new_bytes, &name_buf); var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0 .. new_hex.len / 2], new_hex) catch unreachable; _ = dest.put(raw[0 .. new_hex.len / 2], new_bytes) catch return error.Storage; out.controls_remapped += 1; } else { _ = dest.put(log.hashOf(r), bytes) catch return error.Storage; out.carried += 1; } } // The provenance technote (#61 non-negotiable): the old->new map as an // event artifact INSIDE the converted repository -- it syncs with the // repo, touches no check-in's file set, and keeps every external hash // citation resolvable by lookup. Deterministic: the tip's own timestamp // and an id derived from the project code, so one source converts to // byte-identical output every time. try mintProvenance(gpa, repo, dest, &map, project_code, &out); // Pass 4: the differential proof, pair by pair, before anything is // published: the converted manifest's flattened file set must equal the // source's effective set exactly. The source side re-reads through the // same seam every verb uses; the destination side flattens the new tree // through a resolver over `dest`. for (checkin_rids.items) |rid| { try verifyPair(gpa, repo, dest, rid, &map, streams); out.verified_pairs += 1; } return out; } fn contentErr(e: logstore.ContentError) RemintError { return switch (e) { error.OutOfMemory => error.OutOfMemory, error.Storage => error.Storage, error.Corrupt => error.Corrupt, }; } /// Re-mint ONE check-in: effective file set through the enumeration seam, /// S-tree build, text-level card rewrite (F run out, S run in at its sorted /// position, B dropped, P/Q/T hash tokens chased through the map, PGP /// armor stripped, Z resealed), new name recorded in the map. fn remintCheckin( gpa: std.mem.Allocator, repo: *Repository, log: *logstore.LogStore, dest: *logstore.LogStore, rid: repostate.Rid, k: u4, map: *std.StringHashMapUnmanaged([]const u8), out: *Remint, ) RemintError!void { _ = log; var owned = (repo.manifest(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.BrokenAncestry; defer owned.deinit(gpa); var fl = (repo.files(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.BrokenAncestry; defer fl.deinit(gpa); var entries: std.ArrayList(artifact.FCard) = .empty; defer entries.deinit(gpa); var it = fl.iterator(); while (it.next()) |f| entries.append(gpa, f) catch return error.OutOfMemory; var build = artifact.buildTree(gpa, entries.items, k, .sha3_256) catch return error.OutOfMemory; defer build.deinit(); for (build.subs) |sub| { var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0 .. sub.name.len / 2], sub.name) catch return error.Corrupt; _ = dest.put(raw[0 .. sub.name.len / 2], sub.bytes) catch return error.Storage; } const source_text = stripClearsign(owned.bytes, out); const new_bytes = rewriteManifest(gpa, source_text, entries.items, &build, map) catch return error.OutOfMemory; defer gpa.free(new_bytes); var name_buf: [64]u8 = undefined; const new_hex = artifact.artifactName(.sha3_256, new_bytes, &name_buf); var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0..32], new_hex) catch unreachable; _ = dest.put(raw[0..32], new_bytes) catch return error.Storage; const old_key = gpa.dupe(u8, repo.hashOf(rid)) catch return error.OutOfMemory; errdefer gpa.free(old_key); const new_val = gpa.dupe(u8, new_hex) catch return error.OutOfMemory; map.put(gpa, old_key, new_val) catch return error.OutOfMemory; } /// Unwrap a PGP clearsign envelope, counting the casualty. Manifest card /// lines never begin with '-', so fossil's dash-escaping never fires and /// the payload is exactly the lines between the armor's blank separator and /// the signature block. fn stripClearsign(bytes: []const u8, out: *Remint) []const u8 { if (!std.mem.startsWith(u8, bytes, "-----BEGIN PGP SIGNED MESSAGE-----")) return bytes; const body_start = (std.mem.indexOf(u8, bytes, "\n\n") orelse return bytes) + 2; const sig = std.mem.indexOfPos(u8, bytes, body_start, "-----BEGIN PGP SIGNATURE-----") orelse return bytes; out.signatures_stripped += 1; return bytes[body_start..sig]; } /// The text-level manifest rewrite. Cards stay in their §4 sorted order: /// F lines are replaced by the S run (which sorts exactly where F did not: /// after R, before T), B lines drop (the effective set is fully /// materialized in the tree), P/Q/T lines have every hash token chased /// through the map, everything else carries byte-verbatim, and Z reseals /// over the result. fn rewriteManifest( gpa: std.mem.Allocator, text: []const u8, entries: []const artifact.FCard, build: *const artifact.TreeBuild, map: *const std.StringHashMapUnmanaged([]const u8), ) error{OutOfMemory}![]u8 { var outb: std.ArrayList(u8) = .empty; errdefer outb.deinit(gpa); // submanifests §6.2 step 2: a name set that partitions into fewer than two // groups yields no tree, and the writer "MUST emit a flat manifest and no // sub-manifests". The run this manifest carries is therefore an F run, not // an S run — and it sorts where F sorts (before P), not where S does (after // R). Dropping the source's F cards and emitting `build.s` regardless is // what published a check-in with no file set at all. const flat = build.isFlat(); const run_card: u8 = if (flat) 'F' else 'S'; var emitted_s = false; var lines = std.mem.splitScalar(u8, text, '\n'); while (lines.next()) |line| { if (line.len == 0) continue; const card = line[0]; if (!emitted_s and card > run_card and card != 'Z') { try appendRun(gpa, &outb, build, entries, flat); emitted_s = true; } switch (card) { 'F' => continue, 'B' => continue, 'Z' => { if (!emitted_s) { try appendRun(gpa, &outb, build, entries, flat); emitted_s = true; } try appendZ(gpa, &outb); }, 'P', 'Q', 'T' => try appendRemapped(gpa, &outb, line, map), else => { try outb.appendSlice(gpa, line); try outb.append(gpa, '\n'); }, } } if (!emitted_s) try appendRun(gpa, &outb, build, entries, flat); return outb.toOwnedSlice(gpa); } /// The file-set run this minted manifest carries: the tree's S run, or — when /// §6.2 construction declined to build a tree — the effective set as flat F /// cards. The B card is dropped either way, so the flat arm materializes the /// whole effective set (`entries`), not the source's own F lines, which for a /// delta manifest name only what that check-in changed. fn appendRun( gpa: std.mem.Allocator, outb: *std.ArrayList(u8), build: *const artifact.TreeBuild, entries: []const artifact.FCard, flat: bool, ) error{OutOfMemory}!void { if (!flat) return appendSRun(gpa, outb, build); for (entries) |f| try artifact.appendFlatFCard(outb, gpa, f); } fn appendSRun(gpa: std.mem.Allocator, outb: *std.ArrayList(u8), build: *const artifact.TreeBuild) error{OutOfMemory}!void { for (build.s) |s_card| { var enc_buf: [2 * 4096]u8 = undefined; const encoded = artifact.encodeEscapes(s_card.filename, &enc_buf); try outb.appendSlice(gpa, "S "); try outb.appendSlice(gpa, encoded); try outb.append(gpa, ' '); try outb.appendSlice(gpa, s_card.hash); try outb.append(gpa, '\n'); } } /// One card line with every 40/64-hex token replaced by its mapped /// counterpart when one exists. Tokens outside the map (content hashes, /// '*', tag names) pass through untouched. fn appendRemapped( gpa: std.mem.Allocator, outb: *std.ArrayList(u8), line: []const u8, map: *const std.StringHashMapUnmanaged([]const u8), ) error{OutOfMemory}!void { var toks = std.mem.splitScalar(u8, line, ' '); var first = true; while (toks.next()) |tok| { if (!first) try outb.append(gpa, ' '); first = false; if (map.get(tok)) |new| { try outb.appendSlice(gpa, new); } else if (tok.len > 1 and (tok[0] == '+' or tok[0] == '-' or tok[0] == '*')) { // Q cards carry a sign prefix on the hash token. if (map.get(tok[1..])) |new| { try outb.append(gpa, tok[0]); try outb.appendSlice(gpa, new); } else { try outb.appendSlice(gpa, tok); } } else { try outb.appendSlice(gpa, tok); } } try outb.append(gpa, '\n'); } /// Seal the §6.2 Z card: md5 over every byte already emitted. fn appendZ(gpa: std.mem.Allocator, outb: *std.ArrayList(u8)) error{OutOfMemory}!void { var digest: [16]u8 = undefined; std.crypto.hash.Md5.hash(outb.items, &digest, .{}); const hex = std.fmt.bytesToHex(digest, .lower); try outb.appendSlice(gpa, "Z "); try outb.appendSlice(gpa, &hex); try outb.append(gpa, '\n'); } /// Rewrite a carried structural artifact's references. Returns the new /// bytes when anything changed (caller re-mints), null when nothing did /// (caller carries verbatim). Only T lines can target a check-in here; /// the map lookup is what decides, not the card letter. fn remapReferences( gpa: std.mem.Allocator, bytes: []const u8, map: *const std.StringHashMapUnmanaged([]const u8), ) error{OutOfMemory}!?[]u8 { var outb: std.ArrayList(u8) = .empty; errdefer outb.deinit(gpa); var changed = false; var lines = std.mem.splitScalar(u8, bytes, '\n'); while (lines.next()) |line| { if (line.len == 0) continue; if (line[0] == 'Z') { try appendZ(gpa, &outb); continue; } if (line[0] == 'T') { const before = outb.items.len; try appendRemapped(gpa, &outb, line, map); const slice = outb.items[before..]; if (!std.mem.eql(u8, slice[0 .. slice.len - 1], line)) changed = true; continue; } try outb.appendSlice(gpa, line); try outb.append(gpa, '\n'); } if (!changed) { outb.deinit(gpa); return null; } return try outb.toOwnedSlice(gpa); } /// The pair proof: flatten the converted tree manifest over the destination /// store and compare its file set to the source's effective set, entry by /// entry. The one bug class this converter must never publish is a /// valid-looking manifest with the wrong content — this is the check that /// makes that class loud. fn verifyPair( gpa: std.mem.Allocator, repo: *Repository, dest: *logstore.LogStore, rid: repostate.Rid, map: *const std.StringHashMapUnmanaged([]const u8), streams: cli.Streams, ) RemintError!void { const old_hex = repo.hashOf(rid); const new_hex = map.get(old_hex) orelse return error.Diverged; var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0 .. new_hex.len / 2], new_hex) catch return error.Corrupt; const new_rid = dest.lookup(raw[0 .. new_hex.len / 2]) orelse return error.Diverged; const new_bytes = (dest.content(gpa, new_rid) catch |e| return contentErr(e)) orelse return error.Diverged; defer gpa.free(new_bytes); var m = artifact.parse(gpa, new_bytes) catch return error.Diverged; defer m.deinit(); Repository.flattenTreeFromLog(gpa, dest, &m) catch return error.Diverged; var fl = (repo.files(gpa, rid) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.BrokenAncestry; defer fl.deinit(gpa); var it = fl.iterator(); var i: usize = 0; // `flattened` is null for a manifest that carries no S run at all, which // since the §6.2 step-2 flat arm above is an ordinary outcome, not a fault: // the file set is then the manifest's own F run. Same substitution // `repository.manifestFilesOf` makes, for the same reason — a consumer must // not distinguish a walked enumeration from a flat one. const new_files = m.flattened orelse m.f; while (it.next()) |src_card| : (i += 1) { if (i >= new_files.len) return divergedAt(streams, old_hex, src_card.filename); const dst_card = new_files[i]; if (!std.mem.eql(u8, src_card.filename, dst_card.filename)) return divergedAt(streams, old_hex, src_card.filename); const src_hash = src_card.hash orelse return divergedAt(streams, old_hex, src_card.filename); const dst_hash = dst_card.hash orelse return divergedAt(streams, old_hex, src_card.filename); if (!std.mem.eql(u8, src_hash, dst_hash)) return divergedAt(streams, old_hex, src_card.filename); } if (i != new_files.len) return divergedAt(streams, old_hex, "(extra converted entries)"); } fn divergedAt(streams: cli.Streams, old_hex: []const u8, path: []const u8) RemintError { streams.err.print( "zit import-fossil: VERIFY DIVERGENCE at source check-in {s}, path '{s}' — nothing was published\n", .{ old_hex, path }, ) catch {}; return error.Diverged; } /// Compose and store the provenance technote(s). Pairs are emitted in /// sorted-old-hash order (map iteration order is not deterministic; the /// output must be). One note holds up to `pairs_per_note` pairs; a corpus /// larger than that chains notes, each E-carded under an id derived from /// the project code and its chunk index. const pairs_per_note: usize = 16 * 1024; fn mintProvenance( gpa: std.mem.Allocator, repo: *Repository, dest: *logstore.LogStore, map: *const std.StringHashMapUnmanaged([]const u8), project_code: []const u8, out: *Remint, ) RemintError!void { if (map.count() == 0) return; // The tip's D timestamp: the conversion's clock, without ambient time. const tip = repo.resolve("tip", .checkin) catch return error.BrokenAncestry; var tip_owned = (repo.manifest(gpa, tip) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, else => return error.Storage, }) orelse return error.BrokenAncestry; defer tip_owned.deinit(gpa); const timestamp = if (tip_owned.manifest.d) |d| d.timestamp else "1970-01-01T00:00:00"; var olds: std.ArrayList([]const u8) = .empty; defer olds.deinit(gpa); var kit = map.keyIterator(); while (kit.next()) |k| olds.append(gpa, k.*) catch return error.OutOfMemory; std.mem.sort([]const u8, olds.items, {}, sliceLess); var chunk: usize = 0; var offset: usize = 0; while (offset < olds.items.len) : (chunk += 1) { const end = @min(offset + pairs_per_note, olds.items.len); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); body.appendSlice(gpa, "import-fossil provenance: OLD-check-in-hash NEW-check-in-hash, one pair per line.\n") catch return error.OutOfMemory; for (olds.items[offset..end]) |old_hex| { body.appendSlice(gpa, old_hex) catch return error.OutOfMemory; body.append(gpa, ' ') catch return error.OutOfMemory; body.appendSlice(gpa, map.get(old_hex).?) catch return error.OutOfMemory; body.append(gpa, '\n') catch return error.OutOfMemory; } offset = end; var id_material: std.ArrayList(u8) = .empty; defer id_material.deinit(gpa); id_material.appendSlice(gpa, "import-fossil-provenance:") catch return error.OutOfMemory; id_material.appendSlice(gpa, project_code) catch return error.OutOfMemory; id_material.append(gpa, ':') catch return error.OutOfMemory; id_material.append(gpa, @intCast('0' + (chunk % 10))) catch return error.OutOfMemory; var id_buf: [64]u8 = undefined; const note_id = artifact.artifactName(.sha3_256, id_material.items, &id_buf); var note: std.ArrayList(u8) = .empty; defer note.deinit(gpa); const head = std.fmt.allocPrint( gpa, "D {s}\nE {s} {s}\nU import-fossil\nW {d}\n{s}\n", .{ timestamp, timestamp, note_id, body.items.len, body.items }, ) catch return error.OutOfMemory; defer gpa.free(head); note.appendSlice(gpa, head) catch return error.OutOfMemory; appendZ(gpa, ¬e) catch return error.OutOfMemory; var name_buf: [64]u8 = undefined; const note_hex = artifact.artifactName(.sha3_256, note.items, &name_buf); var raw: [32]u8 = undefined; _ = std.fmt.hexToBytes(raw[0..32], note_hex) catch unreachable; _ = dest.put(raw[0..32], note.items) catch return error.Storage; out.provenance_notes += 1; } } fn sliceLess(_: void, a: []const u8, b: []const u8) bool { return std.mem.order(u8, a, b) == .lt; }