Artifact 9de6bd483f6d8e461cad6bc573189469de28b24fdad846b456525ab148ef829b:
- File
src/git_import_impl.zig
— part of check-in
[8ae7041c2c]
at
2026-07-31 20:55:36
on branch sub-manifest
— import-git: pack the refs, and split the budget they shared with payload
zit now imports sqlite at defaults: 36,416 commits, 2,109 refs, 44m35s, no flags. Two independent walls stood in front of that.
Refs (issue 21). One file per ref made a ref's name a path, so the filesystem got a vote on which repositories are representable. sqlite has refs/remotes/origin/EQP-improvements and .../eqp-improvements at different commits -- legal, distinct git refs -- and on APFS the second landed on the first as a bare PathAlreadyExists, raised during publication after the whole import had run. Case was only the reachable instance; path length, reserved device names and each platform's own Unicode fold table are the same bug in other clothes.
They are one sorted file now, "refs", whose lines pair a canonical name with its target. Nothing reads it back -- "refs" is a reserved root the working-tree scan skips, and branches resolve through repostate's sym- tags -- so it is interop output and its job is to be legible. Sorting makes it byte-identical for a given ref set, so two imports of one repository can be compared.
This supersedes the ASCII-fold refusal from the previous commit. That was the right interim while a ref was a filename; with one file the collision is representable, so refusing it would be gratuitous. What survives is the genuine collision -- distinct sources normalizing to a byte-identical canonical name, refs/heads/./release against refs/heads/release -- which no layout can hold, and which now names both sources instead of only failing.
Budget (issue 20). The 4 GiB payload default refused sqlite's 12.2 GiB stream, and could not simply be raised: fileCommandCeiling divided the same number, so a larger allowance for a whole history was also a larger allowance for one commit's retained command list. Those bound different things and only one is memory -- payload streams and is released per event (sqlite's import held 21 MiB), while a commit's command list is held whole. The arena has max_command_arena_bytes of its own now and keeps the old 4 GiB; the cumulative total goes to 1 TiB.
The file-command ceiling is therefore structural: about 104 million commands in one commit, and no flag raises it. Its diagnostic says so rather than pointing at a knob that no longer moves it. The help text claimed the payload flag fixed both; it only ever fixed one. (user: dmestas, size: 71069) [annotate] [blame] [check-ins using]
const std = @import("std"); const builtin = @import("builtin"); const artifact = @import("artifact.zig"); const fast_export = @import("git_fast_export.zig"); const logstore = @import("logstore.zig"); const editor = @import("submanifest_editor.zig"); const control_paths = @import("control_paths.zig"); const submanifest = @import("repository.zig").submanifest; /// Tree types at the importer seam. These alias canonical editor types; they /// do not introduce a selectable runtime policy. pub const TreeStore = editor.TreeStore; pub const TreeFormat = editor.TreeFormat; pub const RootView = editor.RootView; pub const TreeCommand = editor.TreeCommand; pub const BuildResult = editor.BuildResult; pub const TreeError = editor.Error; pub const LoadedBytes = editor.LoadedBytes; pub const TreeStoreError = editor.StoreError; /// Import policy deliberately has one repository-wide partition choice. The /// translator resolves it once at initialization and never reconsiders it per /// event or commit. pub const Attribution = enum { committer, author, }; pub const UserOptions = struct { submanifest_k: ?u4 = null, attribution: Attribution = .committer, tree_cache_bytes: usize = 1 << 20, }; fn validateImportedPath(path: []const u8) !void { return control_paths.validateWorktreePath(path); } /// Import work observed at the parser/translator boundary. These counters are /// deliberately independent of a tree implementation so opt-in benchmark /// policies exercise exactly the same fast-export and ref finalization path. pub const CpuClock = enum { thread_rusage, process_rusage, }; pub const WorkCounters = struct { blobs: u64 = 0, commits: u64 = 0, resets: u64 = 0, tags: u64 = 0, changed_paths: u64 = 0, emitted_nodes: u64 = 0, reused_nodes: u64 = 0, max_tree_height: u8 = 0, cache_high_water_bytes: usize = 0, inherited_full_tree_materializations: u64 = 0, /// Tree entries omitted because the source names a Git submodule (Git mode /// 160000). Zit has no gitlink entry, so the import drops them; counting /// them here keeps that silent omission observable to every caller that /// already reads `WorkCounters`. skipped_submodule_entries: u64 = 0, tree_apply_count: u64 = 0, tree_apply_cpu_ns: u64 = 0, tree_apply_cpu_clock: CpuClock = treeApplyCpuClock(), }; pub const ResolvedOptions = struct { submanifest_k: u4, attribution: Attribution, }; pub fn resolveOptions(options: UserOptions) !ResolvedOptions { const k = options.submanifest_k orelse 5; if (k < artifact.min_k or k > artifact.max_k) return error.InvalidSubmanifestK; return .{ .submanifest_k = k, .attribution = options.attribution }; } /// draft-fossil-submanifests-00 §6.3's analysis pass run over a NAME SET /// alone, so an import can pin the analyzed optimum before it has read a /// single object. /// /// This is sound, not an approximation of the real scan: §6.1's partition /// predicate counts leading zero bits of PH(name) — "the SHA3-256 digest of a /// decoded filename" — so which names group together is fully determined by /// the names. The §6.3 objective's other input is byte counts, and a hash /// token's width is fixed by `algo`, not by what it hashes. Naming each card /// by the digest of its own filename therefore yields the same grouping and /// the same widths the real F-set would, with distinct tokens throughout. /// /// `names` need not be sorted or unique — `buildTree`'s strict-canonical-order /// precondition is established here. Returns null when the set cannot /// partition at all (§6.2 step 2's flat outcome), which is the caller's signal /// to keep its own default rather than invent a pin from nothing. pub fn analyzeSubmanifestK( gpa: std.mem.Allocator, names: []const []const u8, algo: artifact.HashAlgo, ) std.mem.Allocator.Error!?u4 { if (names.len < 2) return null; const sorted = try gpa.dupe([]const u8, names); defer gpa.free(sorted); std.mem.sort([]const u8, sorted, {}, struct { fn less(_: void, a: []const u8, b: []const u8) bool { return artifact.compareBytes(a, b) == .lt; } }.less); // One arena for the whole scan: the synthetic cards and their hash tokens // live exactly as long as the twelve `cost` calls that read them. var arena_state = std.heap.ArenaAllocator.init(gpa); defer arena_state.deinit(); const arena = arena_state.allocator(); var entries: std.ArrayList(artifact.FCard) = .empty; try entries.ensureTotalCapacity(arena, sorted.len); var previous: ?[]const u8 = null; for (sorted) |name| { if (previous) |last| if (artifact.compareBytes(last, name) != .lt) continue; previous = name; var hash_buf: [64]u8 = undefined; const hash = artifact.artifactName(algo, name, &hash_buf); entries.appendAssumeCapacity(.{ .filename = name, .hash = try arena.dupe(u8, hash), .perm = null, .old_name = null, .mode = .regular, }); } if (entries.items.len < 2) return null; var costs: [artifact.max_k - artifact.min_k + 1]submanifest.Cost = undefined; var n: usize = 0; var k: u4 = artifact.min_k; while (true) : (k += 1) { costs[n] = try submanifest.cost(gpa, entries.items, k, algo); n += 1; if (k == artifact.max_k) break; } const winner = submanifest.best(costs[0..n]) orelse return null; return winner.k; } /// Classifies a raw Git tree-entry mode. Mode 160000 is reported as `.gitlink` /// rather than refused: whether a submodule stops an import is import policy, /// which lives in the translator, not in mode classification. pub fn modeFromGit(mode: u32) !fast_export.FileMode { return switch (mode) { 0o100644 => .regular, 0o100755 => .executable, 0o120000 => .symlink, 0o160000 => .gitlink, else => error.UnknownMode, }; } const RusageSelector = enum(i32) { self = 0, thread = 1, }; /// Linux supports `RUSAGE_THREAD`; all other targets use the portable /// process selector so a benchmark never passes an unsupported rusage value. pub fn treeApplyCpuClock() CpuClock { return if (builtin.os.tag == .linux) .thread_rusage else .process_rusage; } fn treeApplyCpuNs() u64 { return rusageCpuNs(treeApplyCpuClock()); } fn rusageCpuNs(clock: CpuClock) u64 { const selector: RusageSelector = switch (clock) { .thread_rusage => .thread, .process_rusage => .self, }; const usage = std.posix.getrusage(@intFromEnum(selector)); const seconds: u64 = @intCast(usage.utime.sec + usage.stime.sec); const micros: u64 = @intCast(usage.utime.usec + usage.stime.usec); return seconds * std.time.ns_per_s + micros * std.time.ns_per_us; } /// Commit references retain their exact immutable importer object position, /// rather than inferring identity from a deduplicated Zit check-in hash. pub const CommitMarkState = struct { checkin_hash: []u8, object_index: usize, }; const ObjectKind = enum { blob, commit }; const MarkState = union(enum) { blob: []u8, commit: CommitMarkState, tagged: []u8, fn hash(self: *const MarkState) []const u8 { return switch (self.*) { .blob, .tagged => |artifact_hash| artifact_hash, .commit => |state| state.checkin_hash, }; } }; const OidState = struct { kind: ObjectKind, hash: []u8, object_index: ?usize = null, }; const ParentRoot = struct { bytes: []u8, manifest: artifact.Manifest, fn deinit(self: *ParentRoot, gpa: std.mem.Allocator) void { self.manifest.deinit(); gpa.free(self.bytes); self.* = undefined; } fn view(self: *const ParentRoot) editor.RootView { if (self.manifest.s.len != 0) return .{ .tree = self.manifest.s }; return .{ .flat = self.manifest.f }; } }; const CommitFlushObservation = struct { serialized_f_cards: u64, staged_artifact_bytes: u64, }; const StagedNode = struct { name: []u8, bytes: []u8, }; const StagingStore = struct { gpa: std.mem.Allocator, base: editor.TreeStore, nodes: std.ArrayList(StagedNode) = .empty, fn init(gpa: std.mem.Allocator, base: editor.TreeStore) StagingStore { return .{ .gpa = gpa, .base = base }; } fn deinit(self: *StagingStore) void { for (self.nodes.items) |node| { self.gpa.free(node.name); self.gpa.free(node.bytes); } self.nodes.deinit(self.gpa); self.* = undefined; } fn treeStore(self: *StagingStore) editor.TreeStore { return .{ .ctx = self, .load_fn = load, .put_fn = put, .put_batch_fn = putBatch }; } fn load(ctx: *anyopaque, allocator: std.mem.Allocator, name: []const u8) editor.StoreError!editor.LoadedBytes { const self: *StagingStore = @ptrCast(@alignCast(ctx)); for (self.nodes.items) |node| { if (!std.mem.eql(u8, node.name, name)) continue; const bytes = allocator.dupe(u8, node.bytes) catch return error.OutOfMemory; return .{ .bytes = bytes, .allocator = allocator }; } return self.base.load_fn(self.base.ctx, allocator, name); } fn put(ctx: *anyopaque, name: []const u8, bytes: []const u8) editor.StoreError!void { const self: *StagingStore = @ptrCast(@alignCast(ctx)); for (self.nodes.items) |node| { if (!std.mem.eql(u8, node.name, name)) continue; if (!std.mem.eql(u8, node.bytes, bytes)) return error.StorageFailure; return; } var name_buf: [64]u8 = undefined; if (!std.mem.eql(u8, name, artifact.artifactName(.sha1, bytes, &name_buf))) return error.StorageFailure; try self.nodes.ensureUnusedCapacity(self.gpa, 1); const owned_name = self.gpa.dupe(u8, name) catch return error.OutOfMemory; errdefer self.gpa.free(owned_name); const owned_bytes = self.gpa.dupe(u8, bytes) catch return error.OutOfMemory; self.nodes.appendAssumeCapacity(.{ .name = owned_name, .bytes = owned_bytes }); } fn putBatch(ctx: *anyopaque, nodes: []const editor.StoreBatchNode) editor.StoreError!void { const self: *StagingStore = @ptrCast(@alignCast(ctx)); for (nodes) |node| try put(self, node.name, node.bytes); } fn observation(self: *const StagingStore, checkin_bytes: []const u8) !CommitFlushObservation { var flush_observation = CommitFlushObservation{ .serialized_f_cards = try serializedFCardCount(self.gpa, checkin_bytes), .staged_artifact_bytes = @intCast(checkin_bytes.len), }; for (self.nodes.items) |node| { var existing = self.base.load_fn(self.base.ctx, self.gpa, node.name) catch |err| switch (err) { error.MissingNode => { flush_observation.serialized_f_cards = std.math.add(u64, flush_observation.serialized_f_cards, try serializedFCardCount(self.gpa, node.bytes)) catch return error.Overflow; flush_observation.staged_artifact_bytes = std.math.add(u64, flush_observation.staged_artifact_bytes, @intCast(node.bytes.len)) catch return error.Overflow; continue; }, else => return err, }; defer existing.deinit(); if (!std.mem.eql(u8, existing.bytes, node.bytes)) return error.StorageFailure; } return flush_observation; } fn flush( self: *const StagingStore, checkin_name: []const u8, checkin_bytes: []const u8, ) editor.StoreError!void { var batch: std.ArrayList(editor.StoreBatchNode) = .empty; defer batch.deinit(self.gpa); try batch.ensureTotalCapacity(self.gpa, self.nodes.items.len + 1); for (self.nodes.items) |node| { batch.appendAssumeCapacity(.{ .name = node.name, .bytes = node.bytes }); } batch.appendAssumeCapacity(.{ .name = checkin_name, .bytes = checkin_bytes }); try self.base.put_batch_fn(self.base.ctx, batch.items); } }; fn serializedFCardCount(gpa: std.mem.Allocator, bytes: []const u8) !u64 { var manifest = try artifact.parse(gpa, bytes); defer manifest.deinit(); return @intCast(manifest.f.len); } const OwnedIdentity = struct { name: []u8, email: []u8, timestamp: i64, timezone_offset_minutes: i16, fn deinit(self: *OwnedIdentity, gpa: std.mem.Allocator) void { gpa.free(self.name); gpa.free(self.email); self.* = undefined; } }; const ImportedObject = struct { oid: ?[]u8, hash: []u8, source_ref: []u8, first_parent_index: ?usize, tree_height: u8, is_merge: bool, author: OwnedIdentity, committer: OwnedIdentity, fn deinit(self: *ImportedObject, gpa: std.mem.Allocator) void { if (self.oid) |oid| gpa.free(oid); gpa.free(self.hash); gpa.free(self.source_ref); self.author.deinit(gpa); self.committer.deinit(gpa); self.* = undefined; } }; const ImportedBlob = struct { oid: ?[]u8, hash: []u8, fn deinit(self: *ImportedBlob, gpa: std.mem.Allocator) void { if (self.oid) |oid| gpa.free(oid); gpa.free(self.hash); self.* = undefined; } }; const CanonicalRefState = struct { active_count: usize, }; /// The two source refs a `CanonicalRefCollision` must name, borrowed from the /// importer's own ref states and valid until it is deinitialized. Retained /// rather than formatted for the same reason `FileCommandOverrun` is: the /// importer has no output channel, and the error name alone leaves an operator /// with a repository it cannot import and no idea which two refs to look at. /// /// The surviving collision is a genuine one: two distinct source refs that /// normalize to a byte-identical canonical name, such as `refs/heads/./release` /// and `refs/heads/release`. Names that merely differ by letter case are not a /// collision — `refs` is one packed file, so it holds both. pub const RefCollision = struct { first_source: []const u8, second_source: []const u8, canonical: []const u8, }; const RefOrigin = enum { commit, reset, annotated_tag }; const RefState = struct { source: []u8, canonical: []u8, target: ?[]u8, origin: RefOrigin, fn deinit(self: *RefState, gpa: std.mem.Allocator) void { gpa.free(self.source); gpa.free(self.canonical); if (self.target) |target| gpa.free(target); self.* = undefined; } }; const PendingRef = union(enum) { existing: struct { index: usize, target: ?[]u8, origin: RefOrigin, }, new: RefState, fn deinit(self: *PendingRef, gpa: std.mem.Allocator) void { switch (self.*) { .existing => |*state| if (state.target) |target| gpa.free(target), .new => |*state| state.deinit(gpa), } self.* = undefined; } fn publish(self: *PendingRef, importer: anytype) void { switch (self.*) { .existing => |state| { const current = &importer.ref_states.items[state.index]; const canonical = importer.canonical_ref_index.getPtr(current.canonical).?; if (current.target == null and state.target != null) canonical.active_count += 1; if (current.target != null and state.target == null) canonical.active_count -= 1; const old = current.target; current.target = state.target; current.origin = state.origin; if (old) |target| importer.gpa.free(target); }, .new => |state| { const index = importer.ref_states.items.len; importer.ref_states.appendAssumeCapacity(state); importer.ref_index.putAssumeCapacity(state.source, index); if (importer.canonical_ref_index.getPtr(state.canonical)) |canonical| { if (state.target != null) canonical.active_count += 1; } else { importer.canonical_ref_index.putAssumeCapacity( state.canonical, .{ .active_count = if (state.target == null) 0 else 1 }, ); } }, } self.* = undefined; } }; const ImportedTag = struct { source: []u8, name: []u8, oid: ?[]u8, target: []u8, tagger: OwnedIdentity, message: []u8, fn deinit(self: *ImportedTag, gpa: std.mem.Allocator) void { gpa.free(self.source); gpa.free(self.name); if (self.oid) |oid| gpa.free(oid); gpa.free(self.target); self.tagger.deinit(gpa); gpa.free(self.message); self.* = undefined; } }; pub const FinalizeOptions = struct { default_branch: ?[]const u8 = null, }; pub const FinalizedRef = struct { source: []u8, canonical: []u8, target: []u8, }; pub const FinalizedRefs = struct { controls: [][]u8, provenance_jsonl: []u8, refs: []FinalizedRef, primary_branch: []u8, pub fn deinit(self: *FinalizedRefs, gpa: std.mem.Allocator) void { for (self.controls) |bytes| gpa.free(bytes); gpa.free(self.controls); gpa.free(self.provenance_jsonl); for (self.refs) |mapping| { gpa.free(mapping.source); gpa.free(mapping.canonical); gpa.free(mapping.target); } gpa.free(self.refs); gpa.free(self.primary_branch); self.* = undefined; } }; pub const IncrementalTreePolicy = struct { pub fn apply( tree_editor: *editor.TreeEditor, _: std.mem.Allocator, tree_store: editor.TreeStore, parent: editor.RootView, commands: []const editor.TreeCommand, result: *editor.BuildResult, cache_high_water: *usize, ) editor.Error!void { tree_editor.rebindStore(tree_store); result.* = try tree_editor.apply(parent, commands); cache_high_water.* = tree_editor.cacheHighWater(); } pub fn inheritedFullTreeMaterialization(_: RootView) bool { return false; } }; pub const NoopObserver = struct { pub fn beforeEvent(_: *NoopObserver, _: *const WorkCounters, _: fast_export.Event) !void {} pub fn beforeCommitFlush(_: *NoopObserver, _: *const WorkCounters, _: anytype) !void {} pub fn afterCommitFlush(_: *NoopObserver, _: *const WorkCounters, _: anytype) void {} pub fn afterEvent(_: *NoopObserver, _: *const WorkCounters, _: fast_export.Event) !void {} }; /// Translation boundary between fast-export events and Zit artifacts. Both /// tables own every byte they retain; callers receive borrowed object hashes. pub fn Importer(comptime Policy: type, comptime Observer: type) type { return struct { const Self = @This(); gpa: std.mem.Allocator, log: *logstore.LogStore, format: editor.TreeFormat, attribution: Attribution, tree_store: editor.TreeStore, tree_editor: editor.TreeEditor, work: WorkCounters, marks: std.AutoHashMapUnmanaged(u64, MarkState) = .empty, oids: std.StringHashMapUnmanaged(OidState) = .empty, ref_states: std.ArrayList(RefState) = .empty, ref_index: std.StringHashMapUnmanaged(usize) = .empty, canonical_ref_index: std.StringHashMapUnmanaged(CanonicalRefState) = .empty, objects: std.ArrayList(ImportedObject) = .empty, observer: Observer, blobs: std.ArrayList(ImportedBlob) = .empty, tags: std.ArrayList(ImportedTag) = .empty, /// Paths the source stream has declared a Git submodule (mode 160000) /// at least once. Owned keys; grows only, because a lineage the /// importer has not reached yet may still delete or rename one of them. gitlink_paths: std.StringHashMapUnmanaged(void) = .empty, /// Retained by `refuseRefCollision`. Null until `finalizeRefs` returns /// `CanonicalRefCollision`; borrows from `ref_states`, so it is read /// before the importer is deinitialized or not at all. ref_collision: ?RefCollision = null, /// The production initializer fixes the incremental writer at compile time. /// No request, CLI argument, or persisted configuration can change it. pub fn init( gpa: std.mem.Allocator, log: *logstore.LogStore, tree_store: editor.TreeStore, options: UserOptions, ) !Self { return initObserved(gpa, log, tree_store, options, .{}); } pub fn initObserved( gpa: std.mem.Allocator, log: *logstore.LogStore, tree_store: editor.TreeStore, options: UserOptions, observer: Observer, ) !Self { const resolved = try resolveOptions(options); const format: editor.TreeFormat = .{ .k = resolved.submanifest_k, .hash_algo = .sha1, .height_max = @intCast(artifact.max_construct_height), }; var tree_editor = try editor.TreeEditor.init(gpa, tree_store, format, options.tree_cache_bytes); errdefer tree_editor.deinit(); return .{ .gpa = gpa, .log = log, .format = format, .attribution = resolved.attribution, .tree_store = tree_store, .tree_editor = tree_editor, .work = .{}, .observer = observer, }; } pub fn workCounters(self: *const Self) WorkCounters { return self.work; } pub fn retainedStateEntries(self: *const Self) usize { return self.marks.count() + self.oids.count() + self.objects.items.len + self.ref_states.items.len + self.blobs.items.len + self.tags.items.len; } pub fn deinit(self: *Self) void { self.tree_editor.deinit(); var marks_it = self.marks.iterator(); while (marks_it.next()) |entry| self.gpa.free(entry.value_ptr.hash()); self.marks.deinit(self.gpa); var oids_it = self.oids.iterator(); while (oids_it.next()) |entry| { self.gpa.free(entry.key_ptr.*); self.gpa.free(entry.value_ptr.hash); } self.oids.deinit(self.gpa); for (self.ref_states.items) |*state| state.deinit(self.gpa); self.ref_states.deinit(self.gpa); self.ref_index.deinit(self.gpa); self.canonical_ref_index.deinit(self.gpa); for (self.objects.items) |*object| object.deinit(self.gpa); for (self.blobs.items) |*blob| blob.deinit(self.gpa); self.blobs.deinit(self.gpa); self.objects.deinit(self.gpa); for (self.tags.items) |*tag| tag.deinit(self.gpa); self.tags.deinit(self.gpa); var gitlink_it = self.gitlink_paths.keyIterator(); while (gitlink_it.next()) |key| self.gpa.free(key.*); self.gitlink_paths.deinit(self.gpa); // The shared editor's cache owns its decoded immutable nodes. self.* = undefined; } pub fn ingest(self: *Self, event: fast_export.Event) !void { try self.observer.beforeEvent(&self.work, event); switch (event) { .blob => |blob| { try self.ingestBlob(blob); self.work.blobs += 1; }, .commit => |commit| { try self.ingestCommit(commit); self.work.commits += 1; }, .tag => |tag| { try self.ingestTag(tag); self.work.tags += 1; }, .reset => |reset| { try self.ingestReset(reset); self.work.resets += 1; }, .done => {}, } try self.observer.afterEvent(&self.work, event); } /// Drives the canonical fast-export parser into the importer. Callers own /// only the bytes and limits; parser leases, translation, and ref state /// stay on this shared path. pub fn ingestFastExport(self: *Self, bytes: []const u8, limits: fast_export.Limits) !void { const input = std.Io.Reader.fixed(bytes); var reader = fast_export.Reader.init(self.gpa, input, limits); defer reader.deinit(); while (try reader.next()) |lease| { var event = lease; defer event.deinit(); self.ingest(event.value) catch |err| { if (comptime @hasDecl(Observer, "stopOnError")) if (self.observer.stopOnError(err)) break; return err; }; } } pub fn objectForMark(self: *const Self, mark: u64) ![]const u8 { const state = self.marks.getPtr(mark) orelse return error.UnresolvedMark; return state.hash(); } pub fn objectForOid(self: *const Self, oid: []const u8) ![]const u8 { var normalized_buf: [64]u8 = undefined; const normalized = try canonicalOidBytes(oid, &normalized_buf); const state = self.oids.getPtr(normalized) orelse return error.UnresolvedMark; return state.hash; } fn ingestBlob(self: *Self, blob: fast_export.Blob) !void { if (self.marks.contains(blob.mark)) return error.DuplicateMark; var name_buf: [64]u8 = undefined; const hash = artifact.artifactName(self.format.hash_algo, blob.data, &name_buf); var digest_buf: [32]u8 = undefined; const digest = try decodeArtifactName(hash, &digest_buf); var oid_buf: [64]u8 = undefined; const oid = try canonicalOid(blob.original_oid, &oid_buf); const needs_oid = try self.canBindOid(oid, .blob, hash); try self.marks.ensureUnusedCapacity(self.gpa, 1); if (needs_oid) try self.oids.ensureUnusedCapacity(self.gpa, 1); try self.blobs.ensureUnusedCapacity(self.gpa, 1); const mark_hash = try self.gpa.dupe(u8, hash); var published = false; errdefer if (!published) self.gpa.free(mark_hash); const oid_owned = if (needs_oid) try self.gpa.dupe(u8, oid.?) else null; errdefer if (!published) if (oid_owned) |owned_oid| self.gpa.free(owned_oid); const oid_hash = if (needs_oid) try self.gpa.dupe(u8, hash) else null; errdefer if (!published) if (oid_hash) |value| self.gpa.free(value); const record_oid = if (blob.original_oid) |value| try self.gpa.dupe(u8, value) else null; errdefer if (record_oid) |value| self.gpa.free(value); const record_hash = try self.gpa.dupe(u8, hash); errdefer self.gpa.free(record_hash); const record = ImportedBlob{ .oid = record_oid, .hash = record_hash }; _ = try self.log.putArtifact(digest, blob.data, null); self.marks.putAssumeCapacity(blob.mark, .{ .blob = mark_hash }); if (oid_owned) |owned_oid| self.oids.putAssumeCapacity(owned_oid, .{ .kind = .blob, .hash = oid_hash.? }); published = true; self.blobs.appendAssumeCapacity(record); } fn ingestCommit(self: *Self, commit: fast_export.Commit) !void { if (self.marks.contains(commit.mark)) return error.DuplicateMark; var oid_buf: [64]u8 = undefined; const oid = try canonicalOid(commit.original_oid, &oid_buf); try self.preflightCommitOid(oid); var parent_hashes: std.ArrayList([]const u8) = .empty; defer parent_hashes.deinit(self.gpa); try parent_hashes.ensureTotalCapacity(self.gpa, commit.merges.len + @as(usize, if (commit.from == null) 0 else 1)); var parent_root: ?ParentRoot = null; defer if (parent_root) |*root| root.deinit(self.gpa); var root_view: editor.RootView = .{ .flat = &.{} }; var first_parent_index: ?usize = null; if (commit.from) |from| { const parent_index = try self.resolveCommitObjectIndex(from); const primary_hash = self.objects.items[parent_index].hash; first_parent_index = parent_index; parent_hashes.appendAssumeCapacity(primary_hash); parent_root = try self.loadParentRoot(primary_hash); root_view = parent_root.?.view(); } for (commit.merges) |merge| parent_hashes.appendAssumeCapacity(try self.resolveCommit(merge)); // Translation runs after the parent root is resolved: a Git submodule // entry is only resolvable against the tree it is applied to, since // whether the path currently holds a real file decides whether // omitting it needs a delete. var tree_commands: std.ArrayList(editor.TreeCommand) = .empty; defer tree_commands.deinit(self.gpa); try tree_commands.ensureTotalCapacity(self.gpa, commit.commands.len); if (self.needsSubmoduleLookups(commit.commands)) { // `Policy.apply` leaves the editor bound to the previous commit's // stack-local staging store, which no longer exists. Parent-tree // lookups below must read the durable store instead. self.tree_editor.rebindStore(self.tree_store); } var effective_parent = root_view; for (commit.commands) |command| { if (try self.treeCommand(command, effective_parent)) |translated| { tree_commands.appendAssumeCapacity(translated); } // Everything after a `deleteall` sees an empty tree, so a // submodule re-listed by a full-tree export needs no delete. if (std.meta.activeTag(command) == .delete_all) effective_parent = .{ .flat = &.{} }; } var staged = StagingStore.init(self.gpa, self.tree_store); defer staged.deinit(); var result: editor.BuildResult = undefined; const measure_tree_apply = self.work.commits != 0; const tree_apply_cpu_start = if (measure_tree_apply) treeApplyCpuNs() else 0; var cache_high_water: usize = 0; try Policy.apply( &self.tree_editor, self.gpa, staged.treeStore(), root_view, tree_commands.items, &result, &cache_high_water, ); if (measure_tree_apply) { self.work.tree_apply_count += 1; self.work.tree_apply_cpu_ns += treeApplyCpuNs() - tree_apply_cpu_start; } if (Policy.inheritedFullTreeMaterialization(root_view)) self.work.inherited_full_tree_materializations += 1; const tree_height = result.tree_height orelse if (first_parent_index) |parent_index| self.objects.items[parent_index].tree_height else return error.MalformedNode; defer result.deinit(); const parent_text = if (parent_hashes.items.len == 0) null else try std.mem.join(self.gpa, " ", parent_hashes.items); defer if (parent_text) |text| self.gpa.free(text); var parent_card: artifact.PCard = undefined; const parent_cards: []const artifact.PCard = if (parent_text) |text| blk: { parent_card = .{ .parents = text, .count = parent_hashes.items.len }; break :blk &.{parent_card}; } else &.{}; const attributed = switch (self.attribution) { .committer => commit.committer, .author => commit.author, }; var timestamp_buf: [19]u8 = undefined; const timestamp = try formatWriterTimestamp(attributed.timestamp, ×tamp_buf); const comment = if (commit.message.len == 0) "git import" else commit.message; var manifest: artifact.Manifest = .{ .arena_state = std.heap.ArenaAllocator.init(self.gpa), .kind = .manifest, .a = null, .b = null, .c = .{ .comment = comment }, .d = .{ .timestamp = timestamp }, .e = null, .g = null, .h = null, .i = null, .k = null, .l = null, .n = null, .r = null, .u = .{ .login = attributed.name }, .z = null, .w = null, .f = switch (result.root) { .flat => |cards| cards, .tree => &.{}, }, .j = &.{}, .m = &.{}, .p = parent_cards, .q = &.{}, .s = switch (result.root) { .flat => &.{}, .tree => |cards| cards, }, .t = &.{}, }; defer manifest.deinit(); const bytes = try artifact.create(&manifest, self.gpa); defer self.gpa.free(bytes); var name_buf: [64]u8 = undefined; const hash = artifact.artifactName(self.format.hash_algo, bytes, &name_buf); const needs_oid = try self.canBindOid(oid, .commit, hash); // The preflight above makes an extant commit OID fail before TreeEditor // can emit a candidate node. This post-serialization check sees only a // newly bound identity and cannot append before map capacity is reserved. try self.marks.ensureUnusedCapacity(self.gpa, 1); if (needs_oid) try self.oids.ensureUnusedCapacity(self.gpa, 1); const mark_hash = try self.gpa.dupe(u8, hash); var published = false; errdefer { if (!published) self.gpa.free(mark_hash); } const oid_owned = if (needs_oid) try self.gpa.dupe(u8, oid.?) else null; errdefer { if (!published) if (oid_owned) |owned_oid| self.gpa.free(owned_oid); } const oid_hash = if (needs_oid) try self.gpa.dupe(u8, hash) else null; errdefer { if (!published) if (oid_hash) |value| self.gpa.free(value); } const object_index = self.objects.items.len; var commit_record = try self.prepareCommitRecord(commit, hash, first_parent_index, tree_height); errdefer commit_record.deinit(self.gpa); var pending_ref = try self.prepareRef(commit.ref, hash, .commit); errdefer pending_ref.deinit(self.gpa); // All TreeEditor writes remain private until every importer allocation // needed for this check-in's mark/OID state has succeeded. const observation = try staged.observation(bytes); try self.observer.beforeCommitFlush(&self.work, observation); try staged.flush(hash, bytes); self.observer.afterCommitFlush(&self.work, observation); self.objects.appendAssumeCapacity(commit_record); self.marks.putAssumeCapacity(commit.mark, .{ .commit = .{ .checkin_hash = mark_hash, .object_index = object_index, } }); if (oid_owned) |owned_oid| { self.oids.putAssumeCapacity(owned_oid, .{ .kind = .commit, .hash = oid_hash.?, .object_index = object_index, }); } published = true; pending_ref.publish(self); self.work.changed_paths += result.changed_paths; self.work.emitted_nodes += result.emitted_nodes; self.work.reused_nodes += result.reused_nodes; self.work.cache_high_water_bytes = @max(self.work.cache_high_water_bytes, cache_high_water); self.work.max_tree_height = @max(self.work.max_tree_height, tree_height); } /// Translates one fast-export file command against `parent`, the tree the /// command is applied to. Returns null when the command contributes /// nothing to the Zit tree, which happens only for Git submodules: Zit has /// no gitlink entry, so a submodule path is absent from every tree this /// import writes. fn treeCommand( self: *Self, command: fast_export.FileCommand, parent: editor.RootView, ) !?editor.TreeCommand { return switch (command) { .modify => |modify| blk: { try validateImportedPath(modify.path); const mode: artifact.FileMode = switch (modify.mode) { .regular => .regular, .executable => .executable, .symlink => .symlink, .gitlink => break :blk try self.omitSubmodule(parent, modify.path), }; break :blk .{ .upsert = .{ .filename = modify.path, .hash = try self.resolveBlob(modify.data), .perm = switch (mode) { .regular => null, .executable => "x", .symlink => "l", }, .old_name = null, .mode = mode, } }; }, .delete => |path| blk: { try validateImportedPath(path); if (try self.omittedSubmodule(parent, path)) break :blk null; break :blk .{ .delete = path }; }, .rename => |rename| blk: { try validateImportedPath(rename.old); try validateImportedPath(rename.new); // Moving an omitted submodule moves the omission: the // destination becomes the submodule path, and a real file // sitting there is replaced exactly as Git replaces it. if (try self.omittedSubmodule(parent, rename.old)) { break :blk try self.omitSubmodule(parent, rename.new); } break :blk .{ .rename = .{ .old = rename.old, .new = rename.new } }; }, .copy => |copy| blk: { try validateImportedPath(copy.old); try validateImportedPath(copy.new); if (try self.omittedSubmodule(parent, copy.old)) { break :blk try self.omitSubmodule(parent, copy.new); } break :blk .{ .copy = .{ .old = copy.old, .new = copy.new } }; }, .delete_all => editor.TreeCommand.delete_all, }; } /// Records `path` as a submodule path and returns the tree command that /// keeps it out of the imported tree: a delete when `parent` still holds a /// real file there (a file replaced by a submodule), otherwise nothing. fn omitSubmodule(self: *Self, parent: editor.RootView, path: []const u8) !?editor.TreeCommand { try self.recordGitlinkPath(path); self.work.skipped_submodule_entries += 1; if ((try self.tree_editor.lookup(parent, path)) == null) return null; return .{ .delete = path }; } /// Whether `path` is absent from `parent` *because* this import omitted a /// submodule there, which is the only case where a delete, or a rename or /// copy source, may name a path the tree does not have. /// /// Both halves of the test do real work, and together they are why this /// cannot hide a genuine missing-path bug. Eligibility is restricted to /// paths the source itself declared mode 160000, so a command naming a /// path the importer should have written still reaches the tree editor and /// still fails with `MissingPath`. And `parent` is consulted per commit /// rather than trusting the recorded set alone, so a path that is a /// submodule on one branch and a real file on another is deleted normally /// on the branch where it is a file. fn omittedSubmodule(self: *Self, parent: editor.RootView, path: []const u8) !bool { if (!self.gitlink_paths.contains(path)) return false; return (try self.tree_editor.lookup(parent, path)) == null; } fn recordGitlinkPath(self: *Self, path: []const u8) !void { if (self.gitlink_paths.contains(path)) return; try self.gitlink_paths.ensureUnusedCapacity(self.gpa, 1); const owned = try self.gpa.dupe(u8, path); self.gitlink_paths.putAssumeCapacity(owned, {}); } /// Whether any command in `commands` can need a parent-tree lookup. A /// commit that touches no submodule path skips the rebind and every /// lookup, so the common case pays nothing for this policy. fn needsSubmoduleLookups(self: *const Self, commands: []const fast_export.FileCommand) bool { for (commands) |command| switch (command) { .modify => |modify| if (modify.mode == .gitlink) return true, .delete => |path| if (self.gitlink_paths.contains(path)) return true, .rename => |rename| if (self.gitlink_paths.contains(rename.old)) return true, .copy => |copy| if (self.gitlink_paths.contains(copy.old)) return true, .delete_all => {}, }; return false; } fn resolveBlob(self: *const Self, reference: fast_export.ObjectRef) ![]const u8 { return self.resolveKind(reference, .blob); } fn resolveCommit(self: *const Self, reference: fast_export.ObjectRef) ![]const u8 { return switch (reference) { .mark => |mark| { const state = self.marks.getPtr(mark) orelse return error.UnresolvedMark; return switch (state.*) { .commit => |commit| commit.checkin_hash, .tagged => |target| target, else => error.UnresolvedMark, }; }, .oid => |raw_oid| { var oid_buf: [64]u8 = undefined; const oid = try canonicalOidBytes(raw_oid, &oid_buf); const state = self.oids.getPtr(oid) orelse return error.UnresolvedMark; if (state.kind != .commit) return error.UnresolvedMark; return state.hash; }, }; } fn resolveCommitObjectIndex(self: *const Self, reference: fast_export.ObjectRef) !usize { return switch (reference) { .mark => |mark| { const state = self.marks.getPtr(mark) orelse return error.UnresolvedMark; return switch (state.*) { .commit => |commit| commit.object_index, else => error.UnresolvedMark, }; }, .oid => |raw_oid| { var oid_buf: [64]u8 = undefined; const oid = try canonicalOidBytes(raw_oid, &oid_buf); const state = self.oids.getPtr(oid) orelse return error.UnresolvedMark; if (state.kind != .commit) return error.UnresolvedMark; return state.object_index orelse error.UnresolvedMark; }, }; } fn resolveKind(self: *const Self, reference: fast_export.ObjectRef, expected: ObjectKind) ![]const u8 { return switch (reference) { .mark => |mark| { const state = self.marks.getPtr(mark) orelse return error.UnresolvedMark; return switch (state.*) { .blob => |hash| if (expected == .blob) hash else error.UnresolvedMark, else => error.UnresolvedMark, }; }, .oid => |raw_oid| { var oid_buf: [64]u8 = undefined; const oid = try canonicalOidBytes(raw_oid, &oid_buf); const state = self.oids.getPtr(oid) orelse return error.UnresolvedMark; if (state.kind != expected or expected != .blob) return error.UnresolvedMark; return state.hash; }, }; } fn canBindOid(self: *const Self, oid: ?[]const u8, kind: ObjectKind, hash: []const u8) !bool { const name = oid orelse return false; const existing = self.oids.getPtr(name) orelse return true; if (existing.kind != kind or !std.mem.eql(u8, existing.hash, hash)) return error.ConflictingOid; return false; } fn preflightCommitOid(self: *const Self, oid: ?[]const u8) !void { const name = oid orelse return; if (self.oids.contains(name)) return error.ConflictingOid; } fn loadParentRoot(self: *Self, hash: []const u8) !ParentRoot { var digest_buf: [32]u8 = undefined; const digest = try decodeArtifactName(hash, &digest_buf); const rid = self.log.lookup(digest) orelse return error.UnresolvedMark; const bytes = (try self.log.content(self.gpa, rid)) orelse return error.UnresolvedMark; errdefer self.gpa.free(bytes); const manifest = try artifact.parse(self.gpa, bytes); return .{ .bytes = bytes, .manifest = manifest }; } fn prepareCommitRecord( self: *Self, commit: fast_export.Commit, hash: []const u8, first_parent_index: ?usize, tree_height: u8, ) !ImportedObject { try self.objects.ensureUnusedCapacity(self.gpa, 1); const oid = if (commit.original_oid) |value| try self.gpa.dupe(u8, value) else null; errdefer if (oid) |value| self.gpa.free(value); const owned_hash = try self.gpa.dupe(u8, hash); errdefer self.gpa.free(owned_hash); const source_ref = try self.gpa.dupe(u8, commit.ref); errdefer self.gpa.free(source_ref); var author = try cloneIdentity(self.gpa, commit.author); errdefer author.deinit(self.gpa); var committer = try cloneIdentity(self.gpa, commit.committer); errdefer committer.deinit(self.gpa); return .{ .oid = oid, .hash = owned_hash, .source_ref = source_ref, .first_parent_index = first_parent_index, .tree_height = tree_height, .author = author, .committer = committer, .is_merge = commit.merges.len != 0, }; } fn prepareRef(self: *Self, source: []const u8, target: ?[]const u8, origin: RefOrigin) !PendingRef { if (self.ref_index.get(source)) |index| { return .{ .existing = .{ .index = index, .target = if (target) |value| try self.gpa.dupe(u8, value) else null, .origin = origin, } }; } try self.ref_states.ensureUnusedCapacity(self.gpa, 1); try self.ref_index.ensureUnusedCapacity(self.gpa, 1); try self.canonical_ref_index.ensureUnusedCapacity(self.gpa, 1); const owned_source = try self.gpa.dupe(u8, source); errdefer self.gpa.free(owned_source); const canonical = try canonicalRef(self.gpa, source); errdefer self.gpa.free(canonical); const owned_target = if (target) |value| try self.gpa.dupe(u8, value) else null; errdefer if (owned_target) |value| self.gpa.free(value); return .{ .new = .{ .source = owned_source, .canonical = canonical, .target = owned_target, .origin = origin, } }; } fn setRef(self: *Self, source: []const u8, target: ?[]const u8, origin: RefOrigin) !void { var pending = try self.prepareRef(source, target, origin); pending.publish(self); } /// Names the partner `state` collided with before refusing. The scan is /// linear over ref states, which is affordable exactly once on a path /// that is about to abandon the whole import. fn refuseRefCollision(self: *Self, state: RefState) error{CanonicalRefCollision} { for (self.ref_states.items) |other| { if (other.target == null) continue; if (std.mem.eql(u8, other.source, state.source)) continue; if (!std.mem.eql(u8, other.canonical, state.canonical)) continue; // `state` is the outer loop's first colliding ref state, so it // is the earlier of the pair; `other` is the partner the scan // found. Reporting them in that order matches the order the // source repository declares them in. self.ref_collision = .{ .first_source = state.source, .second_source = other.source, .canonical = state.canonical, }; break; } return error.CanonicalRefCollision; } pub fn finalizeRefs(self: *Self, options: FinalizeOptions) !FinalizedRefs { var refs: std.ArrayList(FinalizedRef) = .empty; errdefer { for (refs.items) |mapping| { self.gpa.free(mapping.source); self.gpa.free(mapping.canonical); self.gpa.free(mapping.target); } refs.deinit(self.gpa); } for (self.ref_states.items) |state| { const target = state.target orelse continue; const canonical_state = self.canonical_ref_index.get(state.canonical).?; if (canonical_state.active_count > 1) return self.refuseRefCollision(state); const canonical = try self.gpa.dupe(u8, state.canonical); var canonical_owned = true; errdefer if (canonical_owned) self.gpa.free(canonical); const source = try self.gpa.dupe(u8, state.source); var source_owned = true; errdefer if (source_owned) self.gpa.free(source); const owned_target = try self.gpa.dupe(u8, target); var target_owned = true; errdefer if (target_owned) self.gpa.free(owned_target); try refs.append(self.gpa, .{ .source = source, .canonical = canonical, .target = owned_target, }); canonical_owned = false; source_owned = false; target_owned = false; } const primary_source = blk: { if (options.default_branch) |requested| { if (requested.len == 0) return error.InvalidPrimaryBranch; for (refs.items) |mapping| { if (std.mem.startsWith(u8, mapping.source, "refs/heads/") and std.mem.eql(u8, mapping.source["refs/heads/".len..], requested)) break :blk requested; } return error.InvalidPrimaryBranch; } for (refs.items) |mapping| { if (std.mem.eql(u8, mapping.source, "refs/heads/main")) break :blk "main"; } for (refs.items) |mapping| { if (std.mem.startsWith(u8, mapping.source, "refs/heads/")) break :blk mapping.source["refs/heads/".len..]; } return error.InvalidPrimaryBranch; }; const primary_branch = try self.gpa.dupe(u8, primary_source); errdefer self.gpa.free(primary_branch); var controls: std.ArrayList([]u8) = .empty; errdefer { for (controls.items) |bytes| self.gpa.free(bytes); controls.deinit(self.gpa); } const heads = "refs/heads/"; for (self.objects.items) |object| { if (!std.mem.startsWith(u8, object.source_ref, heads)) continue; const branch = object.source_ref[heads.len..]; const introduces_branch = if (object.first_parent_index) |parent_index| !std.mem.eql(u8, self.objects.items[parent_index].source_ref, object.source_ref) else !std.mem.eql(u8, branch, primary_branch); if (!introduces_branch) continue; var target_manifest = try self.loadParentRoot(object.hash); defer target_manifest.deinit(self.gpa); const timestamp = if (target_manifest.manifest.d) |card| card.timestamp else "1970-01-01T00:00:00"; const login = if (target_manifest.manifest.u) |card| card.login else "git-import"; const bytes = try makeBranchControl(self.gpa, object.hash, branch, timestamp, login); errdefer self.gpa.free(bytes); try controls.append(self.gpa, bytes); } for (refs.items) |mapping| { var target_manifest = try self.loadParentRoot(mapping.target); defer target_manifest.deinit(self.gpa); const timestamp = if (target_manifest.manifest.d) |card| card.timestamp else "1970-01-01T00:00:00"; const login = if (target_manifest.manifest.u) |card| card.login else "git-import"; { const canonical_bytes = try makeSymbolicControl(self.gpa, mapping.target, mapping.canonical, timestamp, login); errdefer self.gpa.free(canonical_bytes); try controls.append(self.gpa, canonical_bytes); } if (std.mem.startsWith(u8, mapping.source, heads) and std.mem.eql(u8, mapping.source[heads.len..], primary_branch)) { const primary_alias = try std.fmt.allocPrint(self.gpa, "sym-{s}", .{primary_branch}); defer self.gpa.free(primary_alias); const alias_bytes = try makeSymbolicControl(self.gpa, mapping.target, primary_alias, timestamp, login); errdefer self.gpa.free(alias_bytes); try controls.append(self.gpa, alias_bytes); } } const provenance_jsonl = try self.buildProvenance(refs.items); errdefer self.gpa.free(provenance_jsonl); const owned_controls = try controls.toOwnedSlice(self.gpa); errdefer { for (owned_controls) |bytes| self.gpa.free(bytes); self.gpa.free(owned_controls); } const owned_refs = try refs.toOwnedSlice(self.gpa); errdefer { for (owned_refs) |mapping| { self.gpa.free(mapping.source); self.gpa.free(mapping.canonical); self.gpa.free(mapping.target); } self.gpa.free(owned_refs); } return .{ .controls = owned_controls, .provenance_jsonl = provenance_jsonl, .refs = owned_refs, .primary_branch = primary_branch, }; } fn hasFirstParent(self: *Self, target: []const u8) !bool { var root = try self.loadParentRoot(target); defer root.deinit(self.gpa); return root.manifest.p.len != 0 and root.manifest.p[0].count != 0; } fn buildProvenance(self: *const Self, refs: []const FinalizedRef) ![]u8 { var out: std.ArrayList(u8) = .empty; errdefer out.deinit(self.gpa); for (self.objects.items) |object| { const oid = object.oid orelse continue; try out.appendSlice(self.gpa, "{\"kind\":\"commit\",\"oid\":"); try appendJsonString(&out, self.gpa, oid); try out.appendSlice(self.gpa, ",\"zit\":"); try appendJsonString(&out, self.gpa, object.hash); try out.appendSlice(self.gpa, ",\"author\":"); try appendIdentityJson(&out, self.gpa, object.author); try out.appendSlice(self.gpa, ",\"committer\":"); try appendIdentityJson(&out, self.gpa, object.committer); try out.appendSlice(self.gpa, ",\"warning\":null}\n"); } for (self.blobs.items) |blob| { const oid = blob.oid orelse continue; try out.appendSlice(self.gpa, "{\"kind\":\"blob\",\"oid\":"); try appendJsonString(&out, self.gpa, oid); try out.appendSlice(self.gpa, ",\"zit\":"); try appendJsonString(&out, self.gpa, blob.hash); try out.appendSlice(self.gpa, ",\"warning\":null}\n"); } for (refs) |mapping| { const state_index = self.ref_index.get(mapping.source) orelse continue; const state = self.ref_states.items[state_index]; if (state.origin == .annotated_tag) continue; try out.appendSlice(self.gpa, "{\"kind\":\"ref\",\"source\":"); try appendJsonString(&out, self.gpa, mapping.source); try out.appendSlice(self.gpa, ",\"canonical\":"); try appendJsonString(&out, self.gpa, mapping.canonical); try out.appendSlice(self.gpa, ",\"zit\":"); try appendJsonString(&out, self.gpa, mapping.target); if (std.mem.startsWith(u8, mapping.source, "refs/tags/")) { try out.appendSlice(self.gpa, ",\"warning\":\"lightweight tag has no annotated metadata\"}\n"); } else try out.appendSlice(self.gpa, ",\"warning\":null}\n"); } for (self.tags.items) |tag| { try out.appendSlice(self.gpa, "{\"kind\":\"tag\",\"source\":"); try appendJsonString(&out, self.gpa, tag.source); const canonical = try canonicalRef(self.gpa, tag.source); defer self.gpa.free(canonical); try out.appendSlice(self.gpa, ",\"canonical\":"); try appendJsonString(&out, self.gpa, canonical); try out.appendSlice(self.gpa, ",\"name\":"); try appendJsonString(&out, self.gpa, tag.name); try out.appendSlice(self.gpa, ",\"oid\":"); if (tag.oid) |oid| try appendJsonString(&out, self.gpa, oid) else try out.appendSlice(self.gpa, "null"); try out.appendSlice(self.gpa, ",\"target_zit\":"); try appendJsonString(&out, self.gpa, tag.target); try out.appendSlice(self.gpa, ",\"tagger\":"); try appendIdentityJson(&out, self.gpa, tag.tagger); try out.appendSlice(self.gpa, ",\"message\":"); try appendJsonString(&out, self.gpa, tag.message); try out.appendSlice(self.gpa, ",\"warning\":null}\n"); } return out.toOwnedSlice(self.gpa); } fn ingestReset(self: *Self, reset: fast_export.Reset) !void { const target = if (reset.target) |reference| try self.resolveCommit(reference) else null; try self.setRef(reset.ref, target, .reset); } fn ingestTag(self: *Self, tag: fast_export.Tag) !void { if (tag.mark) |mark| { if (self.marks.contains(mark)) return error.DuplicateMark; try self.marks.ensureUnusedCapacity(self.gpa, 1); } const target = try self.resolveCommit(tag.target); const source = try std.fmt.allocPrint(self.gpa, "refs/tags/{s}", .{tag.name}); defer self.gpa.free(source); try self.tags.ensureUnusedCapacity(self.gpa, 1); const owned_source = try self.gpa.dupe(u8, source); errdefer self.gpa.free(owned_source); const name = try self.gpa.dupe(u8, tag.name); errdefer self.gpa.free(name); const oid = if (tag.original_oid) |value| try self.gpa.dupe(u8, value) else null; errdefer if (oid) |value| self.gpa.free(value); const owned_target = try self.gpa.dupe(u8, target); errdefer self.gpa.free(owned_target); const marked_target = if (tag.mark != null) try self.gpa.dupe(u8, target) else null; errdefer if (marked_target) |value| self.gpa.free(value); var tagger = try cloneIdentity(self.gpa, tag.tagger); errdefer tagger.deinit(self.gpa); const message = try self.gpa.dupe(u8, tag.message); errdefer self.gpa.free(message); var pending_ref = try self.prepareRef(source, target, .annotated_tag); errdefer pending_ref.deinit(self.gpa); self.tags.appendAssumeCapacity(.{ .source = owned_source, .name = name, .oid = oid, .target = owned_target, .tagger = tagger, .message = message, }); pending_ref.publish(self); if (tag.mark) |mark| self.marks.putAssumeCapacity(mark, .{ .tagged = marked_target.? }); } }; } fn cloneIdentity(gpa: std.mem.Allocator, identity: fast_export.Identity) !OwnedIdentity { const name = try gpa.dupe(u8, identity.name); errdefer gpa.free(name); const email = try gpa.dupe(u8, identity.email); return .{ .name = name, .email = email, .timestamp = identity.timestamp, .timezone_offset_minutes = identity.timezone_offset_minutes, }; } fn canonicalRef(gpa: std.mem.Allocator, source: []const u8) ![]u8 { const heads = "refs/heads/"; const tags = "refs/tags/"; const refs = "refs/"; const pair = if (std.mem.startsWith(u8, source, heads)) .{ @as([]const u8, "sym-heads/"), source[heads.len..] } else if (std.mem.startsWith(u8, source, tags)) .{ @as([]const u8, "sym-tags/"), source[tags.len..] } else if (std.mem.startsWith(u8, source, refs)) .{ @as([]const u8, "sym-refs/"), source[refs.len..] } else return error.InvalidRef; const prefix: []const u8 = pair[0]; const raw: []const u8 = pair[1]; var out: std.ArrayList(u8) = .empty; errdefer out.deinit(gpa); try out.appendSlice(gpa, prefix); var wrote_component = false; var components = std.mem.splitScalar(u8, raw, '/'); while (components.next()) |component| { if (component.len == 0 or std.mem.eql(u8, component, ".")) continue; if (std.mem.eql(u8, component, "..")) return error.InvalidRef; if (wrote_component) try out.append(gpa, '/'); try out.appendSlice(gpa, component); wrote_component = true; } if (!wrote_component) return error.InvalidRef; return out.toOwnedSlice(gpa); } fn makeBranchControl( gpa: std.mem.Allocator, target: []const u8, branch: []const u8, timestamp: []const u8, login: []const u8, ) ![]u8 { const cards = [_]artifact.TCard{.{ .sign = .propagate, .name = "branch", .target = target, .value = branch, }}; var manifest: artifact.Manifest = .{ .arena_state = std.heap.ArenaAllocator.init(gpa), .kind = .control, .a = null, .b = null, .c = null, .d = .{ .timestamp = timestamp }, .e = null, .g = null, .h = null, .i = null, .k = null, .l = null, .n = null, .r = null, .u = .{ .login = login }, .z = null, .w = null, .f = &.{}, .j = &.{}, .m = &.{}, .p = &.{}, .q = &.{}, .s = &.{}, .t = &cards, }; defer manifest.deinit(); return artifact.create(&manifest, gpa); } fn makeSymbolicControl( gpa: std.mem.Allocator, target: []const u8, name: []const u8, timestamp: []const u8, login: []const u8, ) ![]u8 { const cards = [_]artifact.TCard{.{ .sign = .add, .name = name, .target = target, .value = null, }}; var manifest: artifact.Manifest = .{ .arena_state = std.heap.ArenaAllocator.init(gpa), .kind = .control, .a = null, .b = null, .c = null, .d = .{ .timestamp = timestamp }, .e = null, .g = null, .h = null, .i = null, .k = null, .l = null, .n = null, .r = null, .u = .{ .login = login }, .z = null, .w = null, .f = &.{}, .j = &.{}, .m = &.{}, .p = &.{}, .q = &.{}, .s = &.{}, .t = &cards, }; defer manifest.deinit(); return artifact.create(&manifest, gpa); } fn appendJsonString(out: *std.ArrayList(u8), gpa: std.mem.Allocator, value: []const u8) !void { if (!std.unicode.utf8ValidateSlice(value)) return error.InvalidUtf8; try out.append(gpa, '"'); const hex = "0123456789abcdef"; for (value) |byte| switch (byte) { '"' => try out.appendSlice(gpa, "\\\""), '\\' => try out.appendSlice(gpa, "\\\\"), '\n' => try out.appendSlice(gpa, "\\n"), '\r' => try out.appendSlice(gpa, "\\r"), '\t' => try out.appendSlice(gpa, "\\t"), 0...8, 11...12, 14...0x1f => { try out.appendSlice(gpa, "\\u00"); try out.append(gpa, hex[byte >> 4]); try out.append(gpa, hex[byte & 0x0f]); }, else => try out.append(gpa, byte), }; try out.append(gpa, '"'); } fn appendNumber(out: *std.ArrayList(u8), gpa: std.mem.Allocator, value: anytype) !void { var buffer: [32]u8 = undefined; const text = try std.fmt.bufPrint(&buffer, "{d}", .{value}); try out.appendSlice(gpa, text); } fn appendIdentityJson(out: *std.ArrayList(u8), gpa: std.mem.Allocator, identity: OwnedIdentity) !void { try out.appendSlice(gpa, "{\"name\":"); try appendJsonString(out, gpa, identity.name); try out.appendSlice(gpa, ",\"email\":"); try appendJsonString(out, gpa, identity.email); try out.appendSlice(gpa, ",\"timestamp\":"); try appendNumber(out, gpa, identity.timestamp); try out.appendSlice(gpa, ",\"timezone_offset_minutes\":"); try appendNumber(out, gpa, identity.timezone_offset_minutes); try out.append(gpa, '}'); } fn formatWriterTimestamp(epoch: i64, out: *[19]u8) ![]const u8 { // 0000-01-01T00:00:00 through 9999-12-31T23:59:59 are the complete // four-digit writer range. Fast-export timestamps outside it have no // canonical Zit D-card representation. const min_writer_epoch: i64 = -62167219200; const max_writer_epoch: i64 = 253402300799; if (epoch < min_writer_epoch or epoch > max_writer_epoch) return error.InvalidTimestamp; const days = @divFloor(epoch, @as(i64, 86400)); const rem = @mod(epoch, @as(i64, 86400)); const z = days + 719468; const era = @divFloor(z, @as(i64, 146097)); const doe = z - era * 146097; const yoe = @divFloor(doe - @divFloor(doe, @as(i64, 1460)) + @divFloor(doe, @as(i64, 36524)) - @divFloor(doe, @as(i64, 146096)), @as(i64, 365)); const doy = doe - (365 * yoe + @divFloor(yoe, @as(i64, 4)) - @divFloor(yoe, @as(i64, 100))); const mp = @divFloor(5 * doy + 2, @as(i64, 153)); const day = doy - @divFloor(153 * mp + 2, @as(i64, 5)) + 1; const month = if (mp < 10) mp + 3 else mp - 9; const year = yoe + era * 400 + @as(i64, if (month <= 2) 1 else 0); if (year < 0 or year > 9999) return error.InvalidTimestamp; put2(out[0..2], @intCast(@divFloor(year, @as(i64, 100)))); put2(out[2..4], @intCast(@mod(year, @as(i64, 100)))); out[4] = '-'; put2(out[5..7], @intCast(month)); out[7] = '-'; put2(out[8..10], @intCast(day)); out[10] = 'T'; put2(out[11..13], @intCast(@divFloor(rem, @as(i64, 3600)))); out[13] = ':'; put2(out[14..16], @intCast(@divFloor(@mod(rem, @as(i64, 3600)), @as(i64, 60)))); out[16] = ':'; put2(out[17..19], @intCast(@mod(rem, @as(i64, 60)))); return out[0..19]; } fn canonicalOid(oid: ?[]const u8, out: *[64]u8) !?[]const u8 { const raw = oid orelse return null; return try canonicalOidBytes(raw, out); } fn canonicalOidBytes(oid: []const u8, out: *[64]u8) ![]const u8 { if (oid.len == 0 or oid.len > out.len) return error.InvalidOid; for (oid, 0..) |byte, index| { out[index] = switch (byte) { '0'...'9', 'a'...'f' => byte, 'A'...'F' => byte + ('a' - 'A'), else => return error.InvalidOid, }; } return out[0..oid.len]; } fn decodeArtifactName(name: []const u8, out: *[32]u8) ![]const u8 { const digest_len: usize = switch (name.len) { 40 => 20, 64 => 32, else => return error.InvalidArtifactName, }; for (out[0..digest_len], 0..) |*byte, index| { const high = name[index * 2]; const low = name[index * 2 + 1]; if (!isLowerHex(high) or !isLowerHex(low)) return error.InvalidArtifactName; byte.* = (std.fmt.charToDigit(high, 16) catch return error.InvalidArtifactName) << 4 | (std.fmt.charToDigit(low, 16) catch return error.InvalidArtifactName); } return out[0..digest_len]; } fn isLowerHex(byte: u8) bool { return (byte >= '0' and byte <= '9') or (byte >= 'a' and byte <= 'f'); } fn put2(out: []u8, value: u32) void { std.debug.assert(out.len >= 2 and value < 100); out[0] = '0' + @as(u8, @intCast((value / 10) % 10)); out[1] = '0' + @as(u8, @intCast(value % 10)); }