Artifact 2c5a121e5aba5c5f8d303f01c9288657e9a04c1e2de2c5143f67ecf752e48ad7:
- File
src/git/git_import_cli.zig
— part of check-in
[ce3c87e25c]
at
2026-08-10 23:48:29
on branch sub-manifest
— Move the git-interop implementation into src/git/
Seven files relocate with unchanged basenames: git_fast_export(+test), git_import, git_import_impl, git_import_cli, git_import_verify, git_import_benchmark_support. The six standalone module roots (test, refs_test, cli_test, e2e, benchmark, pipe_helper) stay flat because a Zig module cannot import above its root file's directory — the same rule that keeps main/e2e/dst/smoke at src/. Future export-git files follow the same split: roots flat, implementation in src/git/. Path-string edits only; build.zig unchanged. (user: dmestas, size: 143378) [annotate] [blame] [check-ins using]
const std = @import("std"); const builtin = @import("builtin"); pub const cli = @import("../cli.zig"); const zit = @import("zit"); const artifact = zit.artifact; const checkout = zit.checkout; const delta = zit.delta; const fast_export = zit.git_fast_export; const git_import = zit.git_import; const logstore = zit.logstore; const pack = zit.pack; const storage = zit.storage; const editor = zit.submanifest_editor; const verifier = @import("git_import_verify.zig"); extern "c" fn renameatx_np( from_fd: c_int, from: [*:0]const u8, to_fd: c_int, to: [*:0]const u8, flags: c_uint, ) c_int; pub const ChildLifecycle = struct { launched: usize = 0, terminated: usize = 0, stderr_drained: usize = 0, reaped: usize = 0, }; /// The child-facing half of the import transaction. The caller owns the /// implementation and its backing buffers for the full call to `run`. pub const ChildFactory = struct { context: *anyopaque, vtable: *const VTable, pub const VTable = struct { start_export: *const fn (context: *anyopaque, source: []const u8) anyerror!ExportPipes, terminate: *const fn (context: *anyopaque) anyerror!void, drain_stderr: *const fn (context: *anyopaque) anyerror!void, reap: *const fn (context: *anyopaque) anyerror!void, startup_diagnostics: ?*const fn (context: *anyopaque, err: *std.Io.Writer) void = null, }; }; pub const ExportPipes = struct { stdout: std.Io.Reader, stderr: std.Io.Reader, /// Scripted sessions supply their terminal status here. The OS binding gets /// its status during reap; an import succeeds only after the stream reaches /// `done` and EOF, so a nonzero scripted result is still an import failure. exit_code: u8 = 0, /// OS-backed sessions publish the wait status here during `reap`; scripted /// sessions use `exit_code` directly. exit_code_ref: ?*const u8 = null, owner: ?*ProductionGit = null, diagnostics: ?*const fn (pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void = null, detected_default_branch: ?[]const u8 = null, /// draft-fossil-submanifests-00 §6.3's optimum for the source's own name /// set, measured during preflight. Null when the source could not be /// listed or cannot partition; the caller then keeps its own default. analyzed_submanifest_k: ?u4 = null, }; /// The store's §7.2 default policy for a fresh import, derived from the /// algorithm the import actually names artifacts under rather than from a /// guess about what it produced. Both call sites read it, so the header and /// the persisted setting cannot drift apart. fn importDefaultPolicy() artifact.HashPolicy { const algo = git_import.import_hash_algo; return artifact.defaultPolicy(algo == .sha1, algo == .sha3_256); } /// The `git fast-export` command line the importer runs, as a value. /// /// Pure and public so the one policy decision it encodes — what happens to a /// signature Fossil has no card for — is testable without spawning a process. /// `refuse_signed` moves exactly the two signature modes and nothing else. pub fn exportArgv(source: []const u8, refuse_signed: bool) [10][]const u8 { return .{ "git", "-C", source, "fast-export", "--all", "--show-original-ids", "--mark-tags", if (refuse_signed) "--signed-commits=abort" else "--signed-commits=strip", if (refuse_signed) "--signed-tags=abort" else "--signed-tags=strip", "--use-done-feature", }; } /// OS-backed production dependency. It owns both pipe buffers through the /// import call; the `ExportPipes` readers retain a pointer to this value. pub const ProductionGit = struct { pub const StartupFailure = enum { after_spawn, after_stderr_handoff, after_stderr_thread }; /// Keep room in fixed CLI writers for the authoritative import diagnostic. const cli_diagnostic_reserve_bytes = 256; /// Bytes of output one preflight check may produce. Every check asks a /// bounded question — one ref, one boolean, one config value, or (for /// `fsck --connectivity-only --no-dangling`) nothing at all when the /// repository is intact — so an overrun means a check's shape was /// misjudged, never that the source is merely large. const preflight_output_limit_bytes = 8 * 1024; gpa: std.mem.Allocator, io: std.Io, child: ?std.process.Child = null, exit_code: u8 = 1, stdout_buffer: [8192]u8 = undefined, stderr_buffer: [8192]u8 = undefined, reaped: bool = false, reap_calls: usize = 0, /// Captured by the stdout reader before any parser-error cleanup can /// terminate the child. This distinguishes an independently completed /// truncated export from cancellation closing the pipe. stdout_reached_eof: bool = false, default_branch: ?[]const u8 = null, owns_default_branch: bool = false, analyzed_submanifest_k: ?u4 = null, /// The operator's explicit opt-out from the default signature policy. /// False (the default) strips; true restores the refusal. refuse_signed: bool = false, stderr_thread: ?std.Thread = null, /// Moved out of `child` before its reader thread starts, so draining never /// races cleanup or a platform-specific process handle implementation. stderr_file: ?std.Io.File = null, stderr_ring: [4096]u8 = undefined, stderr_ring_start: usize = 0, stderr_ring_len: usize = 0, stderr_bytes_drained: usize = 0, stderr_lock: std.Io.Mutex = .init, test_argv: ?[]const []const u8 = null, test_startup_failure: ?StartupFailure = null, pub fn init(gpa: std.mem.Allocator, io: std.Io) ProductionGit { return .{ .gpa = gpa, .io = io }; } /// Test-only process constructor: uses the identical spawn, stream, /// concurrent-diagnostic, termination, and reap path as `git fast-export`. pub fn initForTest(gpa: std.mem.Allocator, io: std.Io, argv: []const []const u8) ProductionGit { return .{ .gpa = gpa, .io = io, .test_argv = argv }; } fn childEnvironment(self: *ProductionGit) !std.process.Environ.Map { var count: usize = 0; while (std.c.environ[count] != null) : (count += 1) {} const inherited: [:null]const ?[*:0]const u8 = @ptrCast(std.c.environ[0..count :null]); var env = try std.process.Environ.createMap(.{ .block = .{ .slice = inherited } }, self.gpa); errdefer env.deinit(); try env.put("GIT_TERMINAL_PROMPT", "0"); try env.put("GIT_NO_LAZY_FETCH", "1"); return env; } pub fn setDefaultBranch(self: *ProductionGit, default_branch: ?[]const u8) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.default_branch = default_branch; self.owns_default_branch = false; } /// Set before `run`, alongside `setDefaultBranch`: the export argv is /// built inside `start_export`, which the factory vtable does not let the /// caller parameterize. pub fn setRefuseSigned(self: *ProductionGit, refuse: bool) void { self.refuse_signed = refuse; } pub fn deinit(self: *ProductionGit) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.* = undefined; } pub fn dependencies(self: *ProductionGit, publication: PublicationHooks) Dependencies { return .{ .git = .{ .context = self, .vtable = &vtable }, .publication = publication, }; } const vtable: ChildFactory.VTable = .{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, .startup_diagnostics = writeStartupDiagnostics, }; fn startExport(context: *anyopaque, source: []const u8) !ExportPipes { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.test_argv) |argv| return self.startSession(argv); try self.preflight(source); // Fossil has no signature card, so a signature cannot survive this // import in any representation. `strip` accepts that loss and is the // default — refusing by default only withheld the import, never the // signature. `abort` is the operator's explicit opt-out. There is no // honest third option: a verbatim pass would have to invent somewhere // to keep bytes the format cannot hold. const argv = exportArgv(source, self.refuse_signed); return self.startSession(&argv); } fn startSession(self: *ProductionGit, argv: []const []const u8) !ExportPipes { var env = try self.childEnvironment(); defer env.deinit(); self.reaped = false; self.exit_code = 1; self.reap_calls = 0; self.stdout_reached_eof = false; self.stderr_file = null; self.child = try std.process.spawn(self.io, .{ .argv = argv, .environ_map = &env, .stdin = .ignore, .stdout = .pipe, .stderr = .pipe, }); errdefer self.cleanupFailedStart(); try self.failStartForTest(.after_spawn); const child = &(self.child orelse unreachable); const stderr = child.stderr orelse return error.MissingStderr; child.stderr = null; self.stderr_file = stderr; try self.failStartForTest(.after_stderr_handoff); self.stderr_ring_start = 0; self.stderr_bytes_drained = 0; self.stderr_ring_len = 0; self.stderr_thread = try std.Thread.spawn(.{}, drainStderrThread, .{self}); try self.failStartForTest(.after_stderr_thread); return .{ .exit_code_ref = &self.exit_code, .detected_default_branch = self.default_branch, .analyzed_submanifest_k = self.analyzed_submanifest_k, .owner = self, .stdout = .{ .vtable = &stdout_reader_vtable, .buffer = &self.stdout_buffer, .seek = 0, .end = 0, }, .stderr = .{ .vtable = &stderr_reader_vtable, .buffer = &self.stderr_buffer, .seek = 0, .end = 0, }, .diagnostics = writeDiagnostics, }; } fn failStartForTest(self: *const ProductionGit, point: StartupFailure) !void { if (self.test_startup_failure == point) return error.InjectedStartFailure; } fn cleanupFailedStart(self: *ProductionGit) void { if (self.stderr_thread == null) { if (self.stderr_file) |file| file.close(self.io); self.stderr_file = null; } if (self.child) |*child| { if (child.stderr) |file| file.close(self.io); child.stderr = null; } terminate(@ptrCast(self)) catch {}; reap(@ptrCast(self)) catch {}; } fn preflight(self: *ProductionGit, source: []const u8) !void { const inside = try self.query("rev-parse --is-inside-work-tree", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-inside-work-tree" }); defer self.gpa.free(inside); if (!std.mem.eql(u8, std.mem.trim(u8, inside, " \t\r\n"), "true")) return error.NotGitRepository; const head = try self.query("rev-parse HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "HEAD" }); defer self.gpa.free(head); if (std.mem.trim(u8, head, " \t\r\n").len == 0) return error.MissingHead; const object_format = try self.query("rev-parse --show-object-format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" }); defer self.gpa.free(object_format); if (!std.mem.eql(u8, std.mem.trim(u8, object_format, " \t\r\n"), "sha1")) return error.UnsupportedObjectFormat; const shallow = try self.query("rev-parse --is-shallow-repository", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-shallow-repository" }); defer self.gpa.free(shallow); if (std.mem.eql(u8, std.mem.trim(u8, shallow, " \t\r\n"), "true")) return error.ShallowRepository; // `--no-dangling` is required, not tidiness. A dangling object is // unreachable but intact, so it says nothing about the connectivity // question this check exists to answer, yet git reports one // informational line per object on stdout: penpot emitted 14,763 bytes // and a healthy repository failed its import on the output limit. // A missing object is still reported, and still makes this check exit // non-zero. const connectivity = self.query("fsck --connectivity-only", &.{ "git", "-C", source, "--no-lazy-fetch", "fsck", "--connectivity-only", "--no-dangling", "--no-reflogs" }) catch |err| switch (err) { // The exit status is this check's verdict; its stdout is a // per-problem report — with `--no-dangling` an intact repository // writes none of it. So an overrun here is not an unreadable // answer, it is a repository with more broken links than the // report can hold. The named-overrun diagnostic is already // retained, so the operator sees both which check overran and the // verdict it produced. error.GitPreflightFailed, error.PreflightOutputTooLarge => return error.MissingObjects, else => return err, }; defer self.gpa.free(connectivity); const symbolic = try self.optionalQuery("symbolic-ref HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "symbolic-ref", "--quiet", "HEAD" }); if (symbolic) |branch_output| { defer self.gpa.free(branch_output); if (self.default_branch == null) { const branch_ref = std.mem.trim(u8, branch_output, " \t\r\n"); const prefix = "refs/heads/"; if (!std.mem.startsWith(u8, branch_ref, prefix) or branch_ref.len == prefix.len) return error.InvalidHead; self.default_branch = try self.gpa.dupe(u8, branch_ref[prefix.len..]); self.owns_default_branch = true; } } else if (self.default_branch == null) { return error.DetachedHead; } const partial_clone = try self.optionalQuery("config --get extensions.partialClone", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get", "extensions.partialClone" }); if (partial_clone) |configured_remote| { defer self.gpa.free(configured_remote); if (std.mem.trim(u8, configured_remote, " \t\r\n").len != 0) return error.PartialRepository; } const promisors = try self.optionalQuery("config --get-regexp remote.*.promisor", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get-regexp", "^remote\\..*\\.promisor$" }); if (promisors) |configured_promisors| { defer self.gpa.free(configured_promisors); if (try containsGitTrue(configured_promisors)) return error.PartialRepository; } self.analyzed_submanifest_k = try self.analyzeSubmanifestPin(source); } /// `git ls-tree` output the §6.3 analysis is willing to read. A monorepo /// at the ceiling lists several million paths; past it the pin falls back /// to the default rather than failing an import over an advisory /// measurement. const ls_tree_limit_bytes = 64 << 20; /// Measure §6.3's optimum for the source before a single object is read. /// The tip tree, not the root commit: an imported history's first commit /// is routinely a lone README, and pinning k from it would misconfigure /// every later commit in the repository. /// /// Advisory throughout — every failure path returns null and leaves the /// caller's default standing. `-z --name-only` is required, not stylistic: /// without it git quotes unusual bytes and the names would no longer be /// the ones §6.1's partition predicate hashes. The listing includes paths /// a check-in's F-set would exclude (§4.6.3 reserved names) and any path /// `update` would reject; both are rare enough not to move an optimum that /// tracks half the base-2 logarithm of the count. fn analyzeSubmanifestPin(self: *ProductionGit, source: []const u8) !?u4 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = &.{ "git", "-C", source, "--no-lazy-fetch", "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD" }, .environ_map = &env, .stdout_limit = .limited(ls_tree_limit_bytes), .stderr_limit = .limited(8 * 1024), }) catch return null; defer self.gpa.free(result.stderr); defer self.gpa.free(result.stdout); switch (result.term) { .exited => |code| if (code != 0) return null, else => return null, } var names: std.ArrayList([]const u8) = .empty; defer names.deinit(self.gpa); var listing = std.mem.splitScalar(u8, result.stdout, 0); while (listing.next()) |name| { if (name.len == 0) continue; try names.append(self.gpa, name); } return git_import.analyzeSubmanifestK(self.gpa, names.items, git_import.import_hash_algo); } fn containsGitTrue(configured: []const u8) !bool { var lines = std.mem.splitScalar(u8, configured, '\n'); while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, " \t\r"); if (trimmed.len == 0) continue; const value = trimmed[(std.mem.lastIndexOfAny(u8, trimmed, " \t") orelse return error.InvalidGitBoolean) + 1 ..]; if (try isGitTrue(value)) return true; } return false; } fn isGitTrue(value: []const u8) !bool { if (std.mem.eql(u8, value, "1") or std.ascii.eqlIgnoreCase(value, "true") or std.ascii.eqlIgnoreCase(value, "yes") or std.ascii.eqlIgnoreCase(value, "on")) return true; if (std.mem.eql(u8, value, "0") or std.ascii.eqlIgnoreCase(value, "false") or std.ascii.eqlIgnoreCase(value, "no") or std.ascii.eqlIgnoreCase(value, "off")) return false; return error.InvalidGitBoolean; } /// A limit overrun names the check that overran. `StreamTooLong` alone /// tells an operator nothing actionable: which of the preflight children /// produced too much output is the entire diagnostic. fn preflightOverran(self: *ProductionGit, check: []const u8) error{PreflightOutputTooLarge} { var buffer: [256]u8 = undefined; const message = std.fmt.bufPrint( &buffer, "import-git: preflight check 'git {s}' produced more than {d} bytes of output\n", .{ check, preflight_output_limit_bytes }, ) catch fallback: { // Every `check` is a short call-site literal, well inside the // buffer; a truncating format would be a bug here, not a wire // condition. Still total: report without the name rather than // dropping the diagnostic. break :fallback "import-git: a preflight check exceeded its output limit\n"; }; self.retainDiagnostics(message); return error.PreflightOutputTooLarge; } fn optionalQuery(self: *ProductionGit, check: []const u8, argv: []const []const u8) !?[]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| switch (code) { 0 => return result.stdout, 1 => { self.gpa.free(result.stdout); return null; }, else => {}, }, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn query(self: *ProductionGit, check: []const u8, argv: []const []const u8) ![]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| if (code == 0) return result.stdout, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn terminate(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); const child = &(self.child orelse return); switch (builtin.os.tag) { .windows => { const handle = child.id orelse return; switch (std.os.windows.ntdll.NtTerminateProcess(handle, @enumFromInt(1))) { .SUCCESS => {}, .PROCESS_IS_TERMINATING => return error.ChildAlreadyExited, else => return error.TerminateFailed, } }, .wasi => return error.UnsupportedTermination, else => { const pid = child.id orelse return; std.posix.kill(pid, .KILL) catch |err| switch (err) { error.ProcessNotFound => return error.ChildAlreadyExited, else => return err, }; }, } self.exit_code = 1; } fn drainStderr(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.joinStderr(); } fn joinStderr(self: *ProductionGit) void { if (self.stderr_thread) |thread| { thread.join(); self.stderr_thread = null; } } fn retainDiagnostics(self: *ProductionGit, bytes: []const u8) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); for (bytes) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } } fn writeRetainedDiagnostics(self: *ProductionGit, err: *std.Io.Writer) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); const retained_len = @min( self.stderr_ring_len, self.stderr_ring.len - cli_diagnostic_reserve_bytes, ); var index = (self.stderr_ring_start + self.stderr_ring_len - retained_len) % self.stderr_ring.len; for (0..retained_len) |_| { err.writeByte(self.stderr_ring[index]) catch return; index = (index + 1) % self.stderr_ring.len; } } fn writeStartupDiagnostics(context: *anyopaque, err: *std.Io.Writer) void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.writeRetainedDiagnostics(err); } fn drainStderrThread(self: *ProductionGit) void { const file = self.stderr_file orelse return; defer { file.close(self.io); self.stderr_file = null; } var buffer: [4096]u8 = undefined; while (true) { const n = file.readStreaming(self.io, &.{buffer[0..]}) catch break; if (n == 0) break; self.stderr_lock.lockUncancelable(self.io); for (buffer[0..n]) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } self.stderr_bytes_drained += n; self.stderr_lock.unlock(self.io); } } fn writeDiagnostics(pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void { const self = pipes.owner orelse return; if (!enabled) return; self.writeRetainedDiagnostics(err); } fn reap(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.reaped) return; self.reap_calls += 1; // No platform may reap or clear `child` until the independently owned // stderr descriptor is fully drained and its thread has exited. self.joinStderr(); const child = &(self.child orelse return); if (child.stdout) |stdout| { stdout.close(self.io); child.stdout = null; } const term = child.wait(self.io) catch |err| { child.kill(self.io); _ = child.wait(self.io) catch {}; self.exit_code = 1; self.reaped = true; self.child = null; return err; }; self.exit_code = switch (term) { .exited => |code| code, else => 1, }; self.reaped = true; self.child = null; } const stdout_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStdout }; const stderr_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStderr }; fn streamStdout(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stdout_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, true); } fn streamStderr(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stderr_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, false); } fn streamPipe( self: *ProductionGit, _: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit, comptime stdout: bool, ) std.Io.Reader.StreamError!usize { const child = &(self.child orelse return error.ReadFailed); const file = if (stdout) child.stdout orelse return error.ReadFailed else child.stderr orelse return error.ReadFailed; var temporary: [8192]u8 = undefined; const requested = if (limit.toInt()) |value| @min(value, temporary.len) else temporary.len; if (requested == 0) return 0; const n = file.readStreaming(self.io, &.{temporary[0..requested]}) catch |err| switch (err) { error.EndOfStream => { if (stdout) self.stdout_reached_eof = true; return error.EndOfStream; }, else => return error.ReadFailed, }; if (n == 0) { if (stdout) self.stdout_reached_eof = true; return 0; } writer.writeAll(temporary[0..n]) catch return error.WriteFailed; return n; } }; /// Fault injection is deliberately at publication boundaries, rather than in /// storage. This keeps the production storage path identical to the tested one. pub const PublicationHooks = struct { pub const FailurePoint = enum { create_store, before_flush_store, after_flush_store, before_sync_store, after_sync_store, before_close_store, after_close_store, before_flush_map, after_flush_map, before_sync_map, after_sync_map, before_close_map, after_close_map, // Generation 2. `create_store` above opens generation 1; these open, // write, and freeze the packed generation that is actually published. create_packed_store, before_pack, after_pack, before_sync_packed_store, after_sync_packed_store, before_close_packed_store, after_close_packed_store, // The pack filter is captured from the live importer, before the freeze // that invalidates it -- the earliest step that is packing's and not // generation 1's, and so its own boundary. capture_drop_set, // Generation 1 leaves staging between verification and the rename; // failing on either side of that must still leave DEST absent. before_discard_raw_generation, after_discard_raw_generation, before_sync_temporary_directory, after_sync_temporary_directory, before_rename_no_replace, after_rename_no_replace, verify_raw_store, verify_store, verify_map, sync_parent_after_publish, // Compatibility selectors retain the former one-sided fault API. flush_store, sync_store, close_store, flush_map, sync_map, close_map, sync_temporary_directory, rename_no_replace, }; pub const Observer = struct { context: *anyopaque, notify: *const fn (context: *anyopaque, point: FailurePoint) void, }; /// Test-only seam for replacing the staging sibling after it was opened but /// before its identity is checked at the publication boundary. pub const StagingObserver = struct { context: *anyopaque, replace: *const fn (context: *anyopaque, parent: std.Io.Dir, name: []const u8) void, }; fail_at: ?FailurePoint = null, observer: ?Observer = null, staging_observer: ?StagingObserver = null, pub const real_filesystem: PublicationHooks = .{}; fn check(self: PublicationHooks, point: FailurePoint) !void { const legacy: ?FailurePoint = switch (point) { .before_flush_store => .flush_store, .before_sync_store => .sync_store, .before_close_store => .close_store, .before_flush_map => .flush_map, .before_sync_map => .sync_map, .before_close_map => .close_map, .before_sync_temporary_directory => .sync_temporary_directory, .before_rename_no_replace => .rename_no_replace, else => null, }; if (self.fail_at == point or (legacy != null and self.fail_at == legacy)) return error.InjectedPublicationFailure; if (self.observer) |observer| observer.notify(observer.context, point); } fn observeStaging(self: PublicationHooks, parent: std.Io.Dir, name: []const u8) void { if (self.staging_observer) |observer| observer.replace(observer.context, parent, name); } }; pub const ImportLimits = struct { /// Cumulative `data <n>` payload one import may read, raised with /// `--max-export-payload-bytes`. /// /// It is not a memory bound, and reading it as one misjudges both ends. /// Payload is streamed and released per event, so resident size tracks the /// working set, not the total: importing sqlite — a 12.2 GiB export stream /// — held 21 MiB. What this actually bounds is the total history one /// import may carry. /// /// It defaulted to four GiB, which refused sqlite while over-provisioning /// memory by three orders of magnitude. That number could not simply be /// raised, because the per-commit file-command ceiling divided the same /// budget: a bigger allowance for a whole history was also a bigger /// allowance for one commit's retained command list, which unlike payload /// is held whole. The arena now has `max_command_arena_bytes` of its own /// and keeps the old four GiB, leaving this free to be what a total should /// be — large enough that an ordinary repository never meets it, still /// finite against a stream that never ends. max_export_payload_bytes: u64 = @as(u64, 1) << 40, /// Bytes one commit's retained file commands may occupy, which /// `fast_export.fileCommandCeiling` restates as a command count. No CLI /// flag raises it: at the default it admits ~104 million commands in a /// single commit, so anything that meets it is pathological rather than /// merely large, and the honest answer is that the importer cannot hold /// that commit whole. It is a field rather than a constant so the limit /// and its diagnostic stay testable from one place. max_command_arena_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, /// Bounds bytes cloned into importer state before any clone is attempted. /// This is the ONLY bound on retained state, and it is the principled one: /// `accountRetainedEvent` measures the bytes each event actually retains. /// /// A second bound used to sit beside it -- a 3,000,000 cap on the ENTRY /// COUNT -- and it was removed. It was a proxy for the same thing with an /// arbitrary number attached, and it bound first: pkgsrc at 185,280 commits /// over 123,843 files met it after 2h41m of translate, with 3.27 GB written /// and well inside this budget. A ceiling that stops a legitimate import /// before the real memory bound is reached is measuring the wrong thing. max_retained_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, }; /// A limit overrun names its numbers. `FileCommandLimitExceeded` alone tells an /// operator nothing actionable: what was observed, what the ceiling was, and /// what raises it is the entire diagnostic. Retained here and surfaced next to /// the error name, exactly as `ProductionGit.preflightOverran` does for /// preflight output. pub const LimitDiagnostic = struct { buffer: [512]u8 = undefined, len: usize = 0, fn retain(self: *LimitDiagnostic, comptime fmt: []const u8, args: anytype) void { const message = std.fmt.bufPrint(&self.buffer, fmt, args) catch { // Every argument is a bounded integer, well inside the buffer; a // truncating format would be a bug here, not a wire condition. // Still total: report without the numbers rather than drop it. const fallback = "import-git: a per-commit parser ceiling was exceeded; raise --max-export-payload-bytes\n"; @memcpy(self.buffer[0..fallback.len], fallback); self.len = fallback.len; return; }; self.len = message.len; } pub fn write(self: *const LimitDiagnostic, err: *std.Io.Writer) void { if (self.len == 0) return; err.writeAll(self.buffer[0..self.len]) catch {}; } }; pub const Dependencies = struct { git: ChildFactory, publication: PublicationHooks = .real_filesystem, limits: ImportLimits = .{}, }; pub const Request = struct { source: []const u8, destination_parent: std.Io.Dir, destination_name: []const u8, /// `.tree = null` selects the analyzed §6.3 optimum the child factory /// measured during preflight; `.tree = K` is the operator's explicit pin; /// `.flat` is `--no-submanifests`, §8.1's default posture. submanifest: git_import.SubmanifestChoice = .{ .tree = null }, attribution: git_import.Attribution = .committer, default_branch: ?[]const u8 = null, quiet: bool = false, /// Publish the import's instrumentation alongside the repository: the /// per-object provenance map (`.zit-import-map.jsonl`) and the source ref /// manifest (`refs`). Off by default -- a published checkout is the store /// and its checkout record, nothing else -- and opt-in for the workflows /// that measure imports rather than merely use them: every classifier /// census, drop audit and git-to-zit commit lookup this project has run /// reads the map from a PUBLISHED store, so the instrument must remain /// obtainable, just not ambient. provenance: bool = false, }; /// Generation 1: the append-only log the translator writes. It is never /// published, so it carries a name of its own and is deleted from staging /// before the directory is synced and renamed. Naming it `.zit` and renaming /// generation 2 over it inside staging would work too, and would be worse: the /// window in which `.zit` means "raw" is exactly the window a crash leaves /// behind, and a recovered staging directory would be indistinguishable from a /// published one. pub const raw_generation_name = ".zit-import-raw"; /// Generation 2: the packed log, and the only store any import publishes. pub const published_generation_name = ".zit"; pub const PublicationBundle = struct { /// The one-way transaction. Every step is a durability boundary the /// previous step has already crossed, so the order is the guarantee: a /// pack may not read a store still being written, verification may not read /// a store still being packed, and the rename may not commit a store /// nothing verified. `advance` admits exactly one successor, so a future /// edit that reorders the body fails at run time rather than publishing /// something unverified. /// Width is deliberate. `advance` computes `@intFromEnum(stage) + 1` in /// the tag type, so a tag exactly wide enough for today's six members /// would turn the seventh into a `ReleaseSafe` overflow PANIC instead of /// the `PublicationStageOutOfOrder` this returns -- a checked error /// silently becoming a crash on a future edit. pub const Stage = enum(u8) { writing_raw = 0, raw_frozen = 1, packing = 2, packed_frozen = 3, verified = 4, published = 5, }; gpa: std.mem.Allocator, io: std.Io, parent: std.Io.Dir, temporary: std.Io.Dir, temporary_inode: std.Io.File.INode, temporary_name: []u8, owned: bool = true, stage: Stage = .writing_raw, /// Whether generation 1 is still a file inside staging. Tracked so the /// discard is idempotent and so nothing has to re-derive "is it there?" /// from the filesystem at the one point where the answer decides what the /// rename ships. raw_present: bool = false, /// Set once, before staging fills: which manifest `verifyStagingManifest` /// holds the rename to, and whether `discardInstrumentation` runs. publish_instrumentation: bool = false, pub fn create(gpa: std.mem.Allocator, io: std.Io, parent: std.Io.Dir, destination: []const u8) !PublicationBundle { var writable_parent = try parent.openDir(io, ".", .{ .access_sub_paths = true, .follow_symlinks = false, }); errdefer writable_parent.close(io); const private_permissions: std.Io.Dir.Permissions = if (comptime builtin.os.tag == .windows or builtin.os.tag == .wasi) .default_dir else @enumFromInt(0o700); var attempt: usize = 0; while (attempt != 1024) : (attempt += 1) { var nonce: [16]u8 = undefined; try io.randomSecure(&nonce); const encoded_nonce = std.fmt.bytesToHex(nonce[0..], .lower); const temporary_name = try std.fmt.allocPrint(gpa, ".zit-tmp-import-{s}", .{encoded_nonce[0..]}); errdefer gpa.free(temporary_name); writable_parent.createDir(io, temporary_name, private_permissions) catch |err| switch (err) { error.PathAlreadyExists => { gpa.free(temporary_name); continue; }, else => return err, }; var temporary: ?std.Io.Dir = null; var temporary_inode: ?std.Io.File.INode = null; errdefer { if (temporary) |directory| directory.close(io); cleanupCreatedStaging(io, writable_parent, temporary_name, temporary_inode); } temporary = try writable_parent.openDir(io, temporary_name, .{ .access_sub_paths = true, .follow_symlinks = false, }); const temporary_stat = try temporary.?.stat(io); temporary_inode = temporary_stat.inode; if (temporary_stat.kind != .directory) return error.StagingEntryReplaced; // `destination` is intentionally part of this constructor's shape: // ownership is established only after the caller performed the // destination preflight, and publication has one atomic commit point. _ = destination; const result: PublicationBundle = .{ .gpa = gpa, .io = io, .parent = writable_parent, .temporary = temporary.?, .temporary_inode = temporary_stat.inode, .temporary_name = temporary_name, }; temporary = null; return result; } return error.NoTemporaryName; } /// The transaction's only forward edge. A skipped, repeated, or reversed /// stage is a CHECKED error rather than an assert: it gates an atomic /// rename that runs in `ReleaseSafe` and `ReleaseFast` alike, where an /// assert is worth nothing. pub fn advance(self: *PublicationBundle, to: Stage) !void { if (@intFromEnum(to) != @intFromEnum(self.stage) + 1) return error.PublicationStageOutOfOrder; self.stage = to; } fn createRawGeneration(self: *PublicationBundle) !std.Io.File { if (self.stage != .writing_raw) return error.PublicationStageOutOfOrder; const file = try self.temporary.createFile(self.io, raw_generation_name, .{ .read = true, .exclusive = true, }); self.raw_present = true; return file; } fn createPackedGeneration(self: *PublicationBundle) !std.Io.File { if (self.stage != .packing) return error.PublicationStageOutOfOrder; return self.temporary.createFile(self.io, published_generation_name, .{ .read = true, .exclusive = true, }); } /// Remove generation 1 from staging. The rename publishes the staging /// DIRECTORY whole, so this is what keeps the raw log out of `DEST` — not a /// cleanup nicety but the last step of the transaction's contract. It runs /// only after verification has read both generations, and it re-checks /// staging identity first, so a swapped sibling is never written through. /// Idempotent: a second call on an already-discarded generation is a no-op. fn discardRawGeneration(self: *PublicationBundle) !void { if (self.stage != .verified) return error.PublicationStageOutOfOrder; if (!self.raw_present) return; var current = try self.openVerifiedStaging(); defer current.close(self.io); current.deleteFile(self.io, raw_generation_name) catch |err| switch (err) { error.FileNotFound => {}, else => return err, }; self.raw_present = false; } /// Remove the instrumentation pair from staging once the verifier is done /// with them, at the same stage the raw generation leaves and for the same /// reason: the rename ships the directory whole, and without `--provenance` /// these are working files of the run, not payload of the repository. fn discardInstrumentation(self: *PublicationBundle) !void { if (self.stage != .verified) return error.PublicationStageOutOfOrder; if (self.publish_instrumentation) return; var current = try self.openVerifiedStaging(); defer current.close(self.io); for (instrumentation_manifest) |name| { current.deleteFile(self.io, name) catch |err| switch (err) { error.FileNotFound => {}, // `refs` is a directory tree. error.IsDir => current.deleteTree(self.io, name) catch return error.Storage, else => return err, }; } } /// Exactly what a published import contains. Named here rather than left /// implicit in the write order because the rename ships the staging /// directory WHOLE: this list is the publication's payload, and anything /// beside it is published by accident. /// /// The default is the repository and its checkout record, full stop -- /// fossil's own shape. The instrumentation pair is appended only under /// `--provenance`; both are always WRITTEN and verified in staging either /// way, because the in-run verifier reads them before publication. pub const publish_manifest = [_][]const u8{ published_generation_name, checkout.filename, }; pub const instrumentation_manifest = [_][]const u8{ ".zit-import-map.jsonl", "refs", }; /// The last checked statement before the commit point: staging holds the /// publish manifest and nothing else. /// /// The stage machine proves the ORDER of the transaction, and /// `discardRawGeneration` proves generation 1 was asked to leave. Neither /// proves what is actually on disk at the instant the directory is synced /// and renamed, and that is the only fact the rename acts on. One /// `readdir` per import buys it directly -- and it cannot be outlived by a /// later edit that adds another intermediate file, which a raw-generation /// check alone could be. fn verifyStagingManifest(self: *PublicationBundle) !void { var current = try self.openVerifiedStaging(); defer current.close(self.io); const expected: []const []const u8 = if (self.publish_instrumentation) &(publish_manifest ++ instrumentation_manifest) else &publish_manifest; var seen: [publish_manifest.len + instrumentation_manifest.len]bool = @splat(false); var entries = current.iterate(); while (try entries.next(self.io)) |entry| { const index = for (expected, 0..) |name, position| { if (std.mem.eql(u8, entry.name, name)) break position; } else return error.UnpublishableStagingEntry; // A repeat is impossible from a directory read; treating it as one // more unexpected entry keeps this total rather than assuming so. if (seen[index]) return error.UnpublishableStagingEntry; seen[index] = true; } for (seen[0..expected.len]) |present| { if (!present) return error.IncompletePublication; } } pub fn abort(self: *PublicationBundle) void { if (self.stage != .published and self.owned) { var current = self.openVerifiedStaging() catch null; if (current) |*directory| { const emptied = deleteOwnedContents(directory, self.io); directory.close(self.io); if (emptied) { var rechecked = self.openVerifiedStaging() catch null; if (rechecked) |*verified| { verified.close(self.io); self.parent.deleteDir(self.io, self.temporary_name) catch {}; } } } } self.temporary.close(self.io); self.parent.close(self.io); self.gpa.free(self.temporary_name); self.* = undefined; } pub fn publish(self: *PublicationBundle, destination: []const u8, hooks: PublicationHooks) !void { // The commit point admits exactly one predecessor. Reaching it from any // earlier stage would publish a store that was never packed, never // frozen, or never verified -- and after the rename there is nothing // left to unwind. if (self.stage != .verified) return error.PublicationNotVerified; // `.verified` is BOTH the pre-discard and post-discard stage, so the // stage alone does not say whether generation 1 already left staging. // The rename publishes the staging DIRECTORY whole, so a raw log still // sitting in it would ship into DEST; without this the only thing // stopping that is statement order in `importInto`, which is exactly // the reliance the stage machine exists to remove. if (self.raw_present) return error.PublicationRawGenerationPresent; if (!self.owned) return error.StagingEntryReplaced; try hooks.check(.before_rename_no_replace); hooks.observeStaging(self.parent, self.temporary_name); if (comptime builtin.os.tag == .macos) { const from = try self.gpa.dupeZ(u8, self.temporary_name); defer self.gpa.free(from); const to = try self.gpa.dupeZ(u8, destination); defer self.gpa.free(to); var current = try self.openVerifiedStaging(); defer current.close(self.io); // Darwin's standard `renamePreserve` emulates no-replace with a // hard link, which cannot publish a directory. `RENAME_EXCL` // supplies the required atomic directory commit point instead. if (renameatx_np(self.parent.handle, from.ptr, self.parent.handle, to.ptr, 0x0000_0004) != 0) { return error.RenameNoReplaceFailed; } } else { var current = try self.openVerifiedStaging(); defer current.close(self.io); try self.parent.renamePreserve(self.temporary_name, self.parent, destination, self.io); } self.owned = false; try hooks.check(.after_rename_no_replace); var published = self.parent.openDir(self.io, destination, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch return error.PublishedDestinationReplaced; defer published.close(self.io); const published_stat = published.stat(self.io) catch return error.PublishedDestinationReplaced; if (published_stat.kind != .directory or published_stat.inode != self.temporary_inode) { return error.PublishedDestinationReplaced; } // Through `advance` like every other edge. A direct assignment here // would leave the terminal transition -- the one that disarms `abort` // -- as the single in-module bypass of the machine. try self.advance(.published); } fn openVerifiedStaging(self: *PublicationBundle) !std.Io.Dir { if (!self.owned) return error.StagingEntryReplaced; const held = self.temporary.stat(self.io) catch { self.owned = false; return error.StagingEntryReplaced; }; if (held.kind != .directory or held.inode != self.temporary_inode) { self.owned = false; return error.StagingEntryReplaced; } var current = self.parent.openDir(self.io, self.temporary_name, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch { self.owned = false; return error.StagingEntryReplaced; }; errdefer current.close(self.io); const entry = current.stat(self.io) catch { self.owned = false; return error.StagingEntryReplaced; }; if (entry.kind != .directory or entry.inode != self.temporary_inode) { self.owned = false; return error.StagingEntryReplaced; } return current; } fn cleanupCreatedStaging( io: std.Io, parent: std.Io.Dir, name: []const u8, expected_inode: ?std.Io.File.INode, ) void { const inode = expected_inode orelse return; var current = parent.openDir(io, name, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch return; defer current.close(io); const entry = current.stat(io) catch return; if (entry.kind != .directory or entry.inode != inode) return; if (!deleteOwnedContents(¤t, io)) return; parent.deleteDir(io, name) catch {}; } fn deleteOwnedContents(directory: *std.Io.Dir, io: std.Io) bool { var entries = directory.iterate(); while (entries.next(io) catch return false) |entry| { directory.deleteTree(io, entry.name) catch return false; } return true; } }; const LogTreeStore = struct { gpa: std.mem.Allocator, log: *logstore.LogStore, fn treeStore(self: *LogTreeStore) editor.TreeStore { return .{ .ctx = self, .load_fn = load, .put_fn = put, .put_batch_fn = putBatch }; } fn load(context: *anyopaque, allocator: std.mem.Allocator, name: []const u8) editor.StoreError!editor.LoadedBytes { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var digest: [32]u8 = undefined; const raw = rawHashName(name, &digest) orelse return error.MissingNode; const rid = self.log.lookup(raw) orelse return error.MissingNode; const bytes = self.log.content(allocator, rid) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; return .{ .bytes = bytes orelse return error.MissingNode, .allocator = allocator }; } fn put(context: *anyopaque, name: []const u8, bytes: []const u8) editor.StoreError!void { const self: *LogTreeStore = @ptrCast(@alignCast(context)); // Algorithm-agnostic: `verify` picks the digest from the name width. if (!artifact.verify(name, bytes)) return error.StorageFailure; var digest: [32]u8 = undefined; const raw = rawHashName(name, &digest) orelse return error.StorageFailure; _ = self.log.putArtifact(raw, bytes, null) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; } fn putBatch(context: *anyopaque, nodes: []const editor.StoreBatchNode) editor.StoreError!void { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var digests: std.ArrayList([32]u8) = .empty; defer digests.deinit(self.gpa); var items: std.ArrayList(logstore.LogStore.ArtifactBatchItem) = .empty; defer items.deinit(self.gpa); digests.ensureTotalCapacity(self.gpa, nodes.len) catch return error.OutOfMemory; items.ensureTotalCapacity(self.gpa, nodes.len) catch return error.OutOfMemory; for (nodes) |node| { // Algorithm-agnostic: `verify` picks the digest from the name width. if (!artifact.verify(node.name, node.bytes)) return error.StorageFailure; var digest: [32]u8 = undefined; const raw = rawHashName(node.name, &digest) orelse return error.StorageFailure; digests.appendAssumeCapacity(digest); items.appendAssumeCapacity(.{ .hash = digests.items[digests.items.len - 1][0..raw.len], .bytes = node.bytes }); } self.log.putArtifactBatch(items.items) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; } }; /// Runs one all-or-nothing import. Until `renamePreserve` succeeds, every /// failure owns and removes only this invocation's temporary sibling. pub fn run( gpa: std.mem.Allocator, io: std.Io, request: Request, streams: cli.Streams, dependencies: Dependencies, ) cli.Exit { if (destinationExists(io, request.destination_parent, request.destination_name)) { streams.err.print("import-git: destination '{s}' already exists\n", .{request.destination_name}) catch {}; return .failure; } var pipes = dependencies.git.vtable.start_export(dependencies.git.context, request.source) catch |err| { if (dependencies.git.vtable.startup_diagnostics) |write_diagnostics| { write_diagnostics(dependencies.git.context, streams.err); } streams.err.print("import-git: cannot start git fast-export: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; var effective_request = request; if (effective_request.default_branch == null) effective_request.default_branch = pipes.detected_default_branch; var session_finished = false; defer if (!session_finished) { _ = finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch false; }; var publication = PublicationBundle.create(gpa, io, request.destination_parent, request.destination_name) catch |err| { streams.err.print("import-git: cannot create temporary destination: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; defer publication.abort(); publication.publish_instrumentation = request.provenance; var limit_diagnostic: LimitDiagnostic = .{}; importInto( gpa, io, effective_request, &publication, &pipes, streams, dependencies.git, &session_finished, dependencies.publication, dependencies.limits, &limit_diagnostic, ) catch |err| { if (err == error.DurabilityUncertain) { streams.err.print("import-git: destination '{s}' was published but parent-directory durability is uncertain\n", .{request.destination_name}) catch {}; } else { streams.err.print("import-git: import failed: {s}\n", .{@errorName(err)}) catch {}; limit_diagnostic.write(streams.err); } return .failure; }; // Pre-pay the derived-state cache while the import's work is still warm. // `Repository.open` derives `.zit-cache` on a miss, so without this the // FIRST read verb on an imported store eats a full-store derive -- on a // real 1M-artifact corpus that was ~342s of a 10-minute `open`. Clone has // warmed at completion since `warmCacheFromLog` existed; import simply // never did. After the atomic rename, against the published store, and // best-effort like every cache write: a failure costs the first reader // the derive it would have paid anyway. if (request.destination_parent.openDir(io, request.destination_name, .{})) |published_const| { var published = published_const; defer published.close(io); zit.Repository.warmCache(gpa, io, published); } else |_| {} streams.out.print("{s}\n", .{request.destination_name}) catch return .failure; if (!request.quiet) streams.err.print("import-git: imported '{s}'\n", .{request.source}) catch {}; return .ok; } fn importInto( gpa: std.mem.Allocator, io: std.Io, request: Request, publication: *PublicationBundle, pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, session_finished: *bool, hooks: PublicationHooks, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { const temporary = publication.temporary; try hooks.check(.create_store); // Generation 1, under a name of its own: translation never writes the file // that gets published. var store_file = try publication.createRawGeneration(); var store_file_open = true; defer if (store_file_open) store_file.close(io); var file_storage = storage.FileStorage.init(io, store_file, 0); // §7.1/§7.2 — this byte is a `HashPolicy` ordinal, NOT a `HashAlgo`. The two // enums overlap numerically and mean different things: `HashAlgo` is // {sha1=0, sha3_256=1}, `HashPolicy` is {sha1=0, auto=1, sha3=2, ...}. This // wrote `@intFromEnum(HashAlgo.sha1)`, so every imported repository // advertised the policy `sha1` by coincidence of ordinal — and the obvious // "switch it to sha3_256" repair would have yielded 1 = `auto`, silently // arming §7.4's promotion ratchet instead. Derived, never a literal. // // The import mints SHA-1 names throughout, so §7.2's rule gives `auto`: // a store that holds SHA-1 names and no SHA3 ones. When the naming // algorithm learns to follow its source, this call needs no edit — the // predicate changes, the derivation does not. const import_hash_policy: u8 = @intFromEnum(importDefaultPolicy()); var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), import_hash_policy); var log_live = true; defer if (log_live) log.deinit(gpa); var tree_store = LogTreeStore{ .gpa = gpa, .log = &log }; // One shape for the whole import, resolved before the first tree: flat when // the operator declined sub-manifests, else the operator's override if // given, else preflight's analyzed §6.3 optimum, else `resolveOptions`' own // default when neither is available. // // The analysis itself runs either way. Skipping it under `.flat` would make // tree emission the SECOND thing the two modes differ in -- the first being // a `git ls-tree` of the whole tip tree during preflight -- and the reason // this flag exists is to give a benchmark two imports that differ in // nothing else. One discarded measurement is the cheaper wart, and it is // not a new one: an explicit `--submanifest-k` already discards it. const resolved_choice: git_import.SubmanifestChoice = switch (request.submanifest) { .flat => .flat, .tree => |pin| .{ .tree = pin orelse pipes.analyzed_submanifest_k }, }; const shape = (try git_import.resolveOptions(.{ .submanifest = resolved_choice })).shape; const pinned_k: u4 = switch (shape) { // Not a k. `Settings.pinned()` reads zero as "no pin", which is the // truthful state for a repository that emitted no sub-manifest. .flat => 0, .tree => |tree| tree.k, }; switch (request.submanifest) { // Silence is the symmetric behavior for "do the default thing": a flat // import has no k to report, so it reports nothing. .flat => {}, .tree => |pin| if (!request.quiet and pin == null and pipes.analyzed_submanifest_k != null) { streams.err.print("import-git: analyzed sub-manifest k={d}\n", .{pinned_k}) catch {}; }, } var importer = try git_import.Importer.init(gpa, &log, tree_store.treeStore(), .{ .submanifest = switch (shape) { .flat => .flat, .tree => |tree| .{ .tree = tree.k }, }, .attribution = request.attribution, }); var importer_live = true; defer if (importer_live) importer.deinit(); var reader = fast_export.Reader.init(gpa, pipes.stdout, .{ .max_line = 1024 * 1024, .max_data = 1024 * 1024 * 1024, .max_path = 4096, // No event ceiling. `git fast-export` emits one event per blob, // commit, tag and reset, so the count scales with the source repo's // history rather than with anything this import spends: pkgsrc's // 185,280 commits are already a fifth of the 1,000,000 records this // used to admit, before a single blob event is counted. Memory is // bounded by bytes on both sides -- `max_total_data` for payload and // `max_retained_bytes` for retained state -- and those bound the // thing that actually costs. A stream of endless empty records can // still spin, but it retains nothing and stays interruptible. .max_events = std.math.maxInt(u64), .max_total_data = limits.max_export_payload_bytes, .max_command_arena_bytes = limits.max_command_arena_bytes, }); defer reader.deinit(); consumeExport(&reader, &importer, limits, diagnostic) catch |err| { const scripted_child_failed = pipes.exit_code_ref == null and terminalExitCode(pipes) != 0; const child_finished_before_cancellation = finishSession(pipes, streams, child, request.quiet, true) catch |lifecycle_err| { session_finished.* = true; return lifecycle_err; }; session_finished.* = true; if (scripted_child_failed or (child_finished_before_cancellation and terminalExitCode(pipes) != 0)) return error.GitExportFailed; return err; }; _ = finishSession(pipes, streams, child, request.quiet, false) catch |err| { session_finished.* = true; return err; }; session_finished.* = true; if (terminalExitCode(pipes) != 0) return error.GitExportFailed; var finalized = importer.finalizeRefs(.{ .default_branch = request.default_branch }) catch |err| { if (err == error.CanonicalRefCollision) if (importer.ref_collision) |collision| { diagnostic.retain( "import-git: '{s}' and '{s}' both name 'refs/{s}'; rename one in the source repository, or exclude it, and re-run\n", .{ collision.first_source, collision.second_source, collision.canonical }, ); }; return err; }; defer finalized.deinit(gpa); for (finalized.controls) |control| try putSha1Artifact(&log, control); // The strongest statement this code can make about a flat import, and it // costs nothing: `stored_submanifest` is `emitted_nodes > 0`, a whole-import // fact derived from work actually done, computed without consulting the // shape -- not "the flag was set", which is what an assertion on `shape` // would say. // // `emitted_nodes` counts nodes this import CREATED, so on its own it would // leave a root built entirely from reused nodes uncovered. It is not // uncovered here, because the raw generation is created by this run a few // dozen lines above (`createRawGeneration`, an exclusive create): // reuse can only hit a node this same import already emitted, so zero // emitted implies zero reused implies no S reference anywhere. // // A returned error rather than an assert because this aborts a // user-visible operation and must unwind the transaction so DEST is never // published. if (std.meta.activeTag(shape) == .flat and finalized.stored_submanifest) { return error.SubmanifestEmittedUnderFlatImport; } try log.putConfig(.{ // Same §7.2 derivation as the header above, and the same reason it is // not a literal: this field is a `HashPolicy`, and the `HashAlgo` // ordinal that used to sit here only looked right. .hash_policy = @intFromEnum(importDefaultPolicy()), .default_branch = finalized.primary_branch, .project_id = "", // These two are a pair, not two independent settings. Under `.flat`, // `submanifest_k` is deliberately zero -- outside `artifact.min_k .. // max_k` -- and that is only safe while `enable_submanifests` is false, // because `Settings.emission()` short-circuits on disabled before it // can range-check the pin. Enabled with a zero pin is the // `.out_of_range` misconfiguration, so the two must move together. .submanifest_k = pinned_k, .enable_submanifests = std.meta.activeTag(shape) == .tree, // The latch is one-way and means "an S reference is stored here", not // "trees are on" — see its doc in logstore.zig. Leaving it false while // the store holds S-bearing artifacts is a lie that stays hidden only // while `enable_submanifests` is true, because the §7.2 gate is the // disjunction of the two: a later `zit submanifest disable` would flip // the repository to advertising avoid-submanifests while holding trees. .seen_submanifest = finalized.stored_submanifest, }); try writeRefs(gpa, io, temporary, finalized.refs); try checkout.write(io, temporary, .{ // The imported store is complete, but no worktree files have been // materialized. Publishing the default tip here would make `update` // incorrectly treat that unmapped worktree as current. .version = "", .branch = finalized.primary_branch, .merge_parent = "", .special_op = "", .manifest_flags = "", }); // The pack filter, captured while the importer is still alive. // // `finalized.ledger.candidates[].name` BORROWS from the importer, which is // deinitialized at the freeze below -- long before packing could read it. // `mirrorDropNames` hands back owned copies of exactly the same set, so // this is the only capture that outlives the generation it describes. try hooks.check(.capture_drop_set); var drops = try MirrorDropSet.capture(gpa, &importer); defer drops.deinit(gpa); // What the packed generation is verified AGAINST, captured for the same // reason and at the same moment as the drop set. `finalized.ledger` cannot // serve: every string in it borrows the importer, which the freeze below // destroys before packing has even started. var witness = try importer.captureWitness(gpa, finalized); defer witness.deinit(); var map = try temporary.createFile(io, ".zit-import-map.jsonl", .{ .read = true, .exclusive = true }); var map_open = true; defer if (map_open) map.close(io); try hooks.check(.before_flush_map); try map.writeStreamingAll(io, finalized.provenance_jsonl); try hooks.check(.after_flush_map); try hooks.check(.before_flush_store); try log.commit(); try hooks.check(.after_flush_store); try hooks.check(.before_sync_store); try store_file.sync(io); try hooks.check(.after_sync_store); try hooks.check(.before_sync_map); try map.sync(io); try hooks.check(.after_sync_map); // Freeze: neither store nor map remains writable after this point. try hooks.check(.before_close_store); // Read before the importer is gone; reported after verification, where a // completion statement is finally safe to make. const checkin_shaped_blobs = importer.workCounters().checkin_shaped_blobs; importer.deinit(); importer_live = false; log.deinit(gpa); log_live = false; store_file.close(io); store_file_open = false; try hooks.check(.after_close_store); try hooks.check(.before_close_map); map.close(io); map_open = false; try hooks.check(.after_close_map); try publication.advance(.raw_frozen); // Generation 2. Packing is MANDATORY: an import publishes a packed store or // it publishes nothing. The drop set is the classifier's, and only the // classifier's -- every name in it has a counted zero retained-reference, // so nothing reachable from an emitted tree can be in it. try publication.advance(.packing); const stats = try packGeneration(gpa, io, publication, drops.filter(), hooks); try publication.advance(.packed_frozen); // The classifier authorized a set; the pack reports what it acted on. Equal // by construction -- every authorized name is an artifact generation 1 // holds, and the filter screens nothing else -- which is exactly why the // two being unequal means the classifier and the pack disagree about the // store's contents. Checking it makes the agreement a statement rather than // an inference from a matching pair of numbers nobody compared. if (stats.dropped != drops.set.count()) return error.MirrorDropSetNotApplied; try verifyFrozen(gpa, io, temporary, &witness, &drops, streams, hooks, request.quiet); try publication.advance(.verified); // Printed only once nothing further can invalidate it. Ahead of // verification this reads as a completion statement on a run that may still // fail, which is the one reading an operator must not get from it. if (!request.quiet) streams.err.print( "import-git: packed {d} artifacts into {d} bytes from {d} (dropped {d} mirror artifacts)\n", .{ stats.artifacts - stats.dropped, stats.bytes_after, stats.bytes_before, stats.dropped }, ) catch {}; // Stored verbatim, so each of these is a check-in of this repository now. // Said at import because the alternative is the operator meeting the class // for the first time as a refusal from `open`, with nothing connecting it // back to the file that carried it. Not a warning about a defect -- the // bytes are the source's and keeping them is deliberate. if (!request.quiet and checkin_shaped_blobs != 0) streams.err.print( "import-git: {d} tracked file(s) contain a complete check-in manifest and were stored " ++ "verbatim; each is now a check-in of this repository ('zit leaves' shows them)\n", .{checkin_shaped_blobs}, ) catch {}; // The pack's plaintext cache, reported for the same reason the sub-manifest // memo is: its budgets are fixed numbers, and a fixed number that is wrong // for a corpus is invisible from outside the process. A hit count without // its denominator cannot show that, which is why both are printed. if (!request.quiet) { const looks = stats.cache_hits + stats.cache_misses; if (looks != 0) streams.err.print( "import-git: pack plaintext cache {d}/{d} hits ({d}%), {d} evicted ({d} byte-cap, {d} entry-cap), {d} KiB in {d} entries resident\n", .{ stats.cache_hits, looks, stats.cache_hits * 100 / looks, stats.cache_evicted, stats.cache_evicted_for_bytes, stats.cache_evicted_for_entries, stats.cache_resident_bytes / 1024, stats.cache_resident_entries, }, ) catch {}; } // The rename ships the staging DIRECTORY, so generation 1 has to leave it // before the commit point or it publishes alongside generation 2. try hooks.check(.before_discard_raw_generation); try publication.discardRawGeneration(); try publication.discardInstrumentation(); try hooks.check(.after_discard_raw_generation); try publication.verifyStagingManifest(); try hooks.check(.before_sync_temporary_directory); try syncDirectory(temporary); try hooks.check(.after_sync_temporary_directory); try publication.publish(request.destination_name, hooks); // If this final durability boundary fails, the destination is intentionally // retained: its content was fully verified and atomically published. hooks.check(.sync_parent_after_publish) catch return error.DurabilityUncertain; syncDirectory(publication.parent) catch return error.DurabilityUncertain; } fn consumeExport( reader: *fast_export.Reader, importer: *git_import.Importer, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { var retained_bytes: u64 = 0; while (reader.next() catch |err| switch (err) { // Reader admits events and cumulative payload bytes before allocating // an event arena or payload, so its exact bound failures map directly. // This import sets no event ceiling, so the reader cannot raise this // one; the mapping stays because the reader can still express it and a // silent mistranslation would be worse than an arm that never runs. error.EventLimitExceeded => return error.ExportEventLimitExceeded, error.DataLimitExceeded => { const overrun = reader.payload_overrun.?; switch (overrun.bound) { .cumulative_total => diagnostic.retain( "import-git: the export stream's cumulative payload exceeded its budget: observed at least {d} bytes against a budget of {d} bytes; parsing stopped there, so the stream's full payload is larger than the observed total; re-run with --max-export-payload-bytes above the stream's full payload\n", .{ overrun.observed_at_least, overrun.ceiling }, ), // No flag reaches this ceiling, so naming one would be a lie. .single_payload => diagnostic.retain( "import-git: a single export payload of {d} bytes exceeded the {d}-byte per-payload ceiling; no option raises this ceiling\n", .{ overrun.observed_at_least, overrun.ceiling }, ), } return error.ExportPayloadLimitExceeded; }, error.FileCommandLimitExceeded => { const overrun = reader.file_command_overrun.?; diagnostic.retain( "import-git: a commit exceeded the per-commit file-command ceiling: observed at least {d}, ceiling {d}, derived from the {d}-byte per-commit arena budget at {d} minimum retained bytes per command. This is a structural bound on one commit, not a total: no option raises it, because a commit at this width is beyond what the importer can hold whole\n", .{ overrun.observed_at_least, overrun.ceiling, overrun.arena_budget_bytes, overrun.min_bytes_per_command, }, ); return error.ExportFileCommandLimitExceeded; }, else => return err, }) |lease| { var event = lease; defer event.deinit(); try accountRetainedEvent(&retained_bytes, event.value, limits.max_retained_bytes); try importer.ingest(event.value); } } const sha1_artifact_name_bytes: u64 = std.crypto.hash.Sha1.digest_length * 2; fn accountRetainedEvent(total: *u64, event: fast_export.Event, limit: u64) !void { switch (event) { .blob => |blob| { try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(blob.original_oid != null), limit); if (blob.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); }, .commit => |commit| { try accountRetainedFixed(total, @as(usize, 3) + @intFromBool(commit.original_oid != null) + @intFromBool(commit.from != null), limit); if (commit.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); try accountRetainedCopies(total, commit.ref, 3, limit); try accountRetainedIdentity(total, commit.author, limit); try accountRetainedIdentity(total, commit.committer, limit); try accountRetainedSlice(total, commit.message, limit); try accountRetainedPaths(total, commit.commands, limit); }, .reset => |reset| { try accountRetainedCopies(total, reset.ref, 2, limit); if (reset.target != null) try accountRetainedFixed(total, 1, limit); }, .tag => |tag| { const source_len = std.math.add(u64, @as(u64, "refs/tags/".len), std.math.cast(u64, tag.name.len) orelse return error.ImporterRetainedByteLimitExceeded) catch return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, source_len, 4, limit); try accountRetainedSlice(total, tag.name, limit); if (tag.original_oid) |oid| try accountRetainedSlice(total, oid, limit); try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(tag.mark != null), limit); try accountRetainedIdentity(total, tag.tagger, limit); try accountRetainedSlice(total, tag.message, limit); }, .done => {}, } } /// Returns the exact retained-state allocation charge for one parsed event. /// The import loop and limit-boundary tests share this calculation so a new /// retained field cannot silently create a guessed budget. pub fn retainedByteCharge(event: fast_export.Event) !u64 { var total: u64 = 0; try accountRetainedEvent(&total, event, std.math.maxInt(u64)); return total; } fn accountRetainedIdentity(total: *u64, identity: fast_export.Identity, limit: u64) !void { try accountRetainedSlice(total, identity.name, limit); try accountRetainedSlice(total, identity.email, limit); } fn accountRetainedPaths(total: *u64, commands: []const fast_export.FileCommand, limit: u64) !void { for (commands) |command| switch (command) { .modify => |modify| try accountRetainedSlice(total, modify.path, limit), .delete => |path| try accountRetainedSlice(total, path, limit), .rename => |rename| { try accountRetainedSlice(total, rename.old, limit); try accountRetainedSlice(total, rename.new, limit); }, .copy => |copy| { try accountRetainedSlice(total, copy.old, limit); try accountRetainedSlice(total, copy.new, limit); }, .delete_all => {}, }; } fn accountRetainedFixed(total: *u64, copies: usize, limit: u64) !void { try accountRetainedLengthCopies(total, sha1_artifact_name_bytes, copies, limit); } fn accountRetainedCopies(total: *u64, bytes: []const u8, copies: usize, limit: u64) !void { const len = std.math.cast(u64, bytes.len) orelse return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, len, copies, limit); } fn accountRetainedSlice(total: *u64, bytes: []const u8, limit: u64) !void { try accountRetainedCopies(total, bytes, 1, limit); } fn accountRetainedLengthCopies(total: *u64, len: u64, copies: usize, limit: u64) !void { var remaining = copies; while (remaining != 0) : (remaining -= 1) { if (total.* > limit or len > limit - total.*) return error.ImporterRetainedByteLimitExceeded; total.* += len; } } /// The exact candidate-only drop filter, owned for the whole pack. /// /// Two impedance mismatches meet here and both are load-bearing. The classifier /// reports HEX artifact names; `pack.Filter` screens RAW digest bytes and /// silently keeps anything that is not a complete 20- or 32-byte name, so a hex /// string handed straight through would drop nothing while reporting success. /// And `pack.NameSet` BORROWS its keys, so the decoded digests need storage /// that outlives the importer they came from. This owns that storage and /// nothing else. const MirrorDropSet = struct { /// One stable cell per name. `set`'s keys slice into these, so this must be /// allocated once at its final length and never resized. digests: [][32]u8 = &.{}, set: pack.NameSet = .{}, /// The same set in the hex spelling the verifier's name-set difference /// compares against. Owned here rather than re-derived: `digests` is a /// fixed-width cell array whose entries do not carry their own length, so /// hex could not be recovered from it without also carrying the widths. names: [][]u8 = &.{}, fn capture(gpa: std.mem.Allocator, importer: *const git_import.Importer) !MirrorDropSet { const names = try importer.mirrorDropNames(gpa); var owned = true; defer if (owned) { for (names) |name| gpa.free(name); gpa.free(names); }; var self: MirrorDropSet = .{}; errdefer self.deinit(gpa); self.digests = try gpa.alloc([32]u8, names.len); // `names` is repository-derived, and bounded: it is a subset of the // importer's mirror-candidate keys, which are retained state and so sit // under `limits.max_retained_bytes`, the budget that bounds everything // the importer holds. (An entry-count ceiling used to be named here too; // it was removed for measuring the wrong thing -- see that field's // note.) Keys of a hash map, so also distinct: `set.count()` // below equals `names.len`, which is what lets the caller compare the // set's size to the pack's own drop count. for (names, 0..) |name, index| { // A name the classifier authorized dropping but that will not // decode is a contradiction in this importer's own output, not an // operator error -- and the safe reading of "I cannot tell which // artifact this is" is to abort the import, never to pack on with a // filter that is quietly narrower than the one that was proven. const raw = rawHashName(name, &self.digests[index]) orelse return error.MirrorDropNameMalformed; self.set.add(gpa, raw) catch |err| switch (err) { error.BadKey => return error.MirrorDropNameMalformed, error.OutOfMemory => return error.OutOfMemory, }; } self.names = names; owned = false; return self; } fn deinit(self: *MirrorDropSet, gpa: std.mem.Allocator) void { self.set.deinit(gpa); gpa.free(self.digests); for (self.names) |name| gpa.free(name); gpa.free(self.names); self.* = undefined; } fn filter(self: *const MirrorDropSet) pack.Filter { return self.set.filter(); } }; /// Generation 1 -> generation 2. Opens the frozen raw log read-only, creates the /// packed generation as a separate file, and hands both to `packLog` with the /// classifier's drop set. /// /// `packLog` is the whole integrity story here and it is worth naming: it /// refuses a non-empty destination, §7.7-verifies every source frame it keeps, /// reconstructs and re-hashes every artifact it wrote, re-asks the filter over /// the destination's own entries so a pack that failed to drop what it was told /// to drop cannot return, and fsyncs the result. Any of those failing returns /// an error with the packed generation still a file inside staging, which the /// caller's `abort` removes along with everything else. fn packGeneration( gpa: std.mem.Allocator, io: std.Io, publication: *PublicationBundle, filter: pack.Filter, hooks: PublicationHooks, ) !pack.PackStats { try hooks.check(.create_packed_store); var packed_file = try publication.createPackedGeneration(); var packed_file_open = true; defer if (packed_file_open) packed_file.close(io); var raw_file = try publication.temporary.openFile(io, raw_generation_name, .{ .mode = .read_only }); defer raw_file.close(io); var raw_storage = storage.FileStorage.init(io, raw_file, (try raw_file.stat(io)).size); var raw_log = try logstore.LogStore.openReadOnly(gpa, raw_storage.storageFile()); defer raw_log.deinit(gpa); var packed_storage = storage.FileStorage.init(io, packed_file, 0); // The source's own recorded hash kind, not a constant: the packed // generation must reproduce generation 1's names, and a mismatched header // would make every artifact in it unaddressable. var packed_log = try logstore.LogStore.open(gpa, packed_storage.storageFile(), raw_log.hash_kind); var packed_log_live = true; defer if (packed_log_live) packed_log.deinit(gpa); try hooks.check(.before_pack); // No extra shuns. Every other caller that reclaims passes some, because it // is removing artifacts a published store already advertised and a peer may // already hold. Nothing here was ever externally visible: generation 1 has // existed only inside a staging directory that no reader can name, so there // is no history for a tombstone to reconcile with. The empty slice is the // statement, not an omission. const stats = try pack.packLog(gpa, &raw_log, &packed_log, filter, &.{}); try hooks.check(.after_pack); // `packLog` already committed (and so fsynced) the log. This is the file // handle's own durability boundary, kept symmetric with generation 1's so // both freezes are one shape and both are injectable. try hooks.check(.before_sync_packed_store); try packed_file.sync(io); try hooks.check(.after_sync_packed_store); try hooks.check(.before_close_packed_store); packed_log.deinit(gpa); packed_log_live = false; packed_file.close(io); packed_file_open = false; try hooks.check(.after_close_packed_store); return stats; } /// The complete pre-publication verification, over both frozen generations and /// everything staging holds beside them. /// /// Both stores open READ-ONLY. `openReadOnly` is not merely the polite choice /// here: `LogStore.open` is the append/repair path, and on a zero-length or /// torn-header file it writes a header or truncates -- which is the one thing a /// verifier must never do to the evidence it is judging. It also takes the /// store's own recorded hash kind rather than assuming one, for the same reason /// `packGeneration` propagates it: a mismatched header makes every artifact in /// the generation unaddressable. /// /// The clause pipeline itself lives in `git_import_verify.zig`. This function /// is the transaction's half of it: open the files, read the three staging /// artifacts, and translate a refusal into an aborted publication. fn verifyFrozen( gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, witness: *const git_import.ImportWitness, drops: *const MirrorDropSet, streams: cli.Streams, hooks: PublicationHooks, quiet: bool, ) !void { try hooks.check(.verify_raw_store); var raw_file = temporary.openFile(io, raw_generation_name, .{ .mode = .read_only }) catch |e| return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e); defer raw_file.close(io); const raw_size = (raw_file.stat(io) catch |e| return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e)).size; var raw_storage = storage.FileStorage.init(io, raw_file, raw_size); var raw_log = logstore.LogStore.openReadOnly(gpa, raw_storage.storageFile()) catch |e| return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e); defer raw_log.deinit(gpa); try hooks.check(.verify_store); var store_file = temporary.openFile(io, published_generation_name, .{ .mode = .read_only }) catch |e| return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e); defer store_file.close(io); const store_size = (store_file.stat(io) catch |e| return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e)).size; var file_storage = storage.FileStorage.init(io, store_file, store_size); var log = logstore.LogStore.openReadOnly(gpa, file_storage.storageFile()) catch |e| return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e); defer log.deinit(gpa); var checkout_buf: [checkout.max_ck_payload + 1]u8 = undefined; const state = try checkout.read(io, temporary, &checkout_buf) orelse return error.MissingCheckout; // Imports deliberately publish an unborn checkout: the repository is // discoverable, while the first update alone materializes its default tip. // Whether the branch it names is present and correct is the pipeline's // `checkout_invalid` clause, so it is not also pre-checked here -- one // statement of the rule, and the one that reaches the operator by name. try hooks.check(.verify_map); const map = temporary.readFileAlloc(io, ".zit-import-map.jsonl", gpa, .limited(max_verified_text_bytes)) catch |e| return reportUnreadable(streams, .map_unreadable, ".zit-import-map.jsonl", e); defer gpa.free(map); const refs = temporary.readFileAlloc(io, "refs", gpa, .limited(max_verified_text_bytes)) catch |e| return reportUnreadable(streams, .ref_target_mismatch, "refs", e); defer gpa.free(refs); try verifier.verify(gpa, .{ .raw = &raw_log, .packed_generation = &log, .witness = witness, .drop_names = drops.names, .map = map, .refs = refs, .refs_header = packed_refs_header, .checkout_branch = state.branch, .quiet = quiet, }, streams.err); } /// Refuse the way the pipeline refuses. Opening the evidence is as much a /// verification step as reading it, and an operator who is handed a bare /// `FileNotFound` from inside the transaction cannot tell a vanished generation /// from a torn header from a map that outgrew its ceiling. Same token shape, /// same error values, so the whole surface reads as one contract. fn reportUnreadable( streams: cli.Streams, clause: verifier.Clause, what: []const u8, cause: anyerror, ) verifier.Error { streams.err.print("import-git: verifier: {s}: {s}: {s}\n", .{ clause.token(), what, @errorName(cause) }) catch {}; return clause.err(); } /// The ceiling on the two staging text files the verifier reads whole. Both are /// bounded by the import's own object count -- one line per commit, blob, ref /// and tag -- so this is a runaway guard, not a policy: a repository that /// genuinely produces more than this has produced a map no operator can read /// either. const max_verified_text_bytes: usize = 1 << 30; fn syncDirectory(directory: std.Io.Dir) !void { if (std.c.fsync(directory.handle) != 0) return error.DirectorySyncFailed; } fn putSha1Artifact(log: *logstore.LogStore, bytes: []const u8) !void { var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined; std.crypto.hash.Sha1.hash(bytes, &digest, .{}); _ = try log.putArtifact(&digest, bytes, null); } /// The header every packed ref file opens with. Versioned because this is the /// only description of the format that ships with a repository. pub const packed_refs_header = "# zit packed-refs v1\n"; /// Writes every finalized ref into one sorted file at `refs`. /// /// One file per ref was the obvious encoding and the wrong one: a ref's name /// became a path, so the filesystem got a vote on which repositories are /// representable. Two refs differing only in case are legal and distinct in /// git — sqlite has a pair at different commits — and one silently landed on /// the other, surfacing as a bare `PathAlreadyExists` at publication after the /// whole import had run. Case is only the reachable instance; path length, /// reserved device names, and each platform's own Unicode fold table are the /// same bug wearing different clothes. /// /// A single file has no such vote. Lines are `<canonical> <target>`, sorted by /// canonical name, which makes the file byte-identical for a given ref set and /// so comparable across two imports of one repository. Ref names cannot /// contain a space or a control character, so the separator needs no escape. /// Nothing in zit reads this back — `refs` is a reserved root the working-tree /// scan skips, and branches resolve through repostate's `sym-` tags — so it is /// interop output, and its whole job is to be legible. fn writeRefs(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, refs: []const git_import.FinalizedRef) !void { const ordered = try gpa.alloc(git_import.FinalizedRef, refs.len); defer gpa.free(ordered); @memcpy(ordered, refs); std.mem.sort(git_import.FinalizedRef, ordered, {}, struct { fn lessThan(_: void, a: git_import.FinalizedRef, b: git_import.FinalizedRef) bool { return std.mem.lessThan(u8, a.canonical, b.canonical); } }.lessThan); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, packed_refs_header); for (ordered) |mapping| { try body.appendSlice(gpa, mapping.canonical); try body.append(gpa, ' '); try body.appendSlice(gpa, mapping.target); try body.append(gpa, '\n'); } var file = try temporary.createFile(io, "refs", .{ .read = true, .exclusive = true }); defer file.close(io); try file.writeStreamingAll(io, body.items); try file.sync(io); } fn rawHashName(name: []const u8, buffer: *[32]u8) ?[]const u8 { if (name.len != 40 and name.len != 64) return null; const raw = buffer[0 .. name.len / 2]; _ = std.fmt.hexToBytes(raw, name) catch return null; return raw; } fn finishSession( pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, quiet: bool, abort: bool, ) !bool { var lifecycle_error: ?anyerror = null; // EOF is observed by the importing thread before it starts cancellation. // A child that is a zombie can still accept `kill`, so this observation is // the race-free completion evidence; `reap` below supplies its actual code. var child_finished_before_cancellation = abort and if (pipes.owner) |owner| owner.stdout_reached_eof else false; if (abort) { child.vtable.terminate(child.context) catch |err| { if (err == error.ChildAlreadyExited) { child_finished_before_cancellation = true; } else { lifecycle_error = err; } }; } child.vtable.drain_stderr(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; child.vtable.reap(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; const enabled = abort or terminalExitCode(pipes) != 0 or !quiet; if (pipes.diagnostics) |write_diagnostics| { write_diagnostics(pipes, streams.err, enabled); } else { copyDiagnostics(&pipes.stderr, streams.err, enabled); } if (lifecycle_error) |err| return err; return child_finished_before_cancellation; } fn terminalExitCode(pipes: *const ExportPipes) u8 { return if (pipes.exit_code_ref) |code| code.* else pipes.exit_code; } fn fileExists(io: std.Io, dir: std.Io.Dir, name: []const u8) bool { var file = dir.openFile(io, name, .{}) catch return false; file.close(io); return true; } fn destinationExists(io: std.Io, parent: std.Io.Dir, name: []const u8) bool { var directory = parent.openDir(io, name, .{}) catch { var file = parent.openFile(io, name, .{}) catch return false; file.close(io); return true; }; directory.close(io); return true; } fn copyDiagnostics(reader: *std.Io.Reader, err: *std.Io.Writer, enabled: bool) void { if (enabled) _ = reader.streamRemaining(err) catch {} else _ = reader.discard(.unlimited) catch {}; } pub const CommandScript = struct { pub const Preflight = enum { branch_head, detached_head, git_missing, not_repository, missing_objects, shallow, partial_promisor }; /// The stream `cancel_export_mid_stream` substitutes for the configured export. /// /// Named rather than inlined because a test asserts what this stream DOES: /// one blob parses, then a record the reader rejects -- a truncation, not a /// rejection at record zero, so the raw generation is already open when the /// cancellation lands. A test that hand-copied these bytes would keep /// passing against a stale copy the moment this literal changed, and its /// claim about the window would quietly stop being true. pub const cancelled_export = "feature done\nblob\nmark :1\ndata 1\nx\ninvalid\n"; pub const StagingReplacement = enum { directory, symlink }; /// One deliberate divergence between the frozen generations and what the /// import decided, injected at `.verify_raw_store` -- after both /// generations are closed files and before the verifier reads either. /// /// This is the CORRUPTION seam, distinct from `PublicationHooks.fail_at`, /// which is the FAILURE seam. A failure proves the transaction unwinds; a /// corruption proves the verifier looks. Before this existed the two /// mutating seams (`replaceStaging`, `plantStrayStagingFile`) both fired /// after `advance(.verified)`, so nothing in the suite could reach the /// bytes the verifier is supposed to be reading. /// /// Each member is named for the invariant it must trip, NOT for the /// mechanism it uses. A fixture that fails at some other clause has not /// proven its clause exists. pub const VerifierFault = enum { /// An artifact the classifier authorized dropping survives in the /// packed generation. `packed = raw - drop_set` is violated upward. retained_omitted_artifact, /// An artifact NOT in the drop set is absent from the packed /// generation. The same equation, violated downward. dropped_required_artifact, /// A well-formed artifact that is in neither raw nor the drop set. added_artifact, /// A name that IS in raw, republished over bytes that do not hash to /// it. The name set is untouched; only the content moved. changed_artifact_bytes, /// A delta frame whose recorded base is not the base its delta was /// encoded against. Every name is right and every frame checksums; /// only reconstruction through the chain reveals it. wrong_delta_base, /// The check-in's F cards permuted onto different paths. Every blob /// still present, every artifact still hashes -- the flattened tree is /// the only thing that moved. tree_change_preserving_artifacts, /// A merge check-in's P card with its parents in the other order. parent_reorder, /// A ref row in `refs` pointed at a different check-in. ref_retarget, /// A tag row in `refs` retargeted, and its kind flipped between the /// lightweight and annotated spellings. tag_change, /// The finalized JSONL's author and committer identities exchanged, so /// the record disagrees with the U card the store actually carries. attribution_change, /// One hex digit of a `provenance_digest` value in the finalized JSONL /// flipped. Nothing else in the import moves. provenance_tamper, }; pub const Config = struct { preflight: Preflight = .branch_head, fast_export: []const u8 = "", stderr_bytes: usize = 0, diagnostic_ring_limit: usize = 4096, /// Substitute `cancelled_export` for `fast_export`: a stream that parses /// one blob and then hits a record the reader rejects. /// /// A BOOLEAN, not a count. It was declared as `?usize` and only ever /// tested for null, so the number was dead and the name promised a /// cancellation after N events that no code implements -- the mechanism /// is a stream substitution and nothing counts anything. cancel_export_mid_stream: bool = false, competing_destination_at_publish: bool = false, export_exit: u8 = 0, replace_staging_at_publish: ?StagingReplacement = null, replace_destination_after_rename: bool = false, /// Drop an unexpected file into staging after the raw generation has /// been discarded -- i.e. between the last step that removes anything /// and the rename that ships whatever is left. Stands in for a future /// edit that writes an intermediate file and forgets to clean it up. stray_staging_file_after_discard: bool = false, /// Corrupt staging between the freeze and the verifier's first read. verifier_fault: ?VerifierFault = null, limits: ImportLimits = .{}, }; pub const Publication = struct { store_reopened_for_verify: bool = false, map_reopened_for_verify: bool = false, verify_before_publish: bool = false, /// Two-generation ordering, each recorded as "was the PREVIOUS stage /// already complete when this one began?" rather than as a bare visit /// count -- a visit count would still be set by a body that ran the /// stages in the wrong order. raw_generation_frozen: bool = false, raw_frozen_before_packing: bool = false, packed_generation_created: bool = false, packed_generation_frozen: bool = false, packed_before_verify: bool = false, both_generations_present_before_discard: bool = false, raw_generation_discarded: bool = false, discard_before_rename: bool = false, }; pub const Environment = struct { pub fn get(_: Environment, name: []const u8) ?[]const u8 { if (std.mem.eql(u8, name, "GIT_TERMINAL_PROMPT")) return "0"; if (std.mem.eql(u8, name, "GIT_NO_LAZY_FETCH")) return "1"; return null; } }; preflight: Preflight, fast_export: []const u8, stderr_bytes: usize, diagnostic_ring_limit: usize, cancel_export_mid_stream: bool, competing_destination_at_publish: bool, export_exit: u8, replace_staging_at_publish: ?StagingReplacement, replace_destination_after_rename: bool, stray_staging_file_after_discard: bool, verifier_fault: ?VerifierFault, /// Whether the injection actually happened. A corruption fixture that /// asserts a verifier error without asserting this would pass on a run /// where the injection silently did nothing -- which is the one failure /// mode a corruption suite must not have. verifier_fault_injected: bool = false, /// The observer the caller handed to `dependencies`, chained behind the /// script's own rather than overwritten by it. caller_observer: ?PublicationHooks.Observer = null, limits: ImportLimits, default_branch: ?[]const u8 = null, detached_head_recorded: bool = false, export_starts: usize = 0, preflight_starts: usize = 0, lifecycle: ChildLifecycle = .{}, stderr_drained_bytes: usize = 0, environment: Environment = .{}, publication: Publication = .{}, competitor_destination_intact: bool = false, temporary_sibling_left: bool = false, staging_replacement_observed: bool = false, staging_replacement_name: [64]u8 = undefined, staging_replacement_name_len: usize = 0, destination_parent: ?std.Io.Dir = null, destination_name: []const u8 = "", io: ?std.Io = null, pub fn init(config: Config) CommandScript { return .{ .preflight = config.preflight, .fast_export = config.fast_export, .stderr_bytes = config.stderr_bytes, .diagnostic_ring_limit = config.diagnostic_ring_limit, .cancel_export_mid_stream = config.cancel_export_mid_stream, .competing_destination_at_publish = config.competing_destination_at_publish, .export_exit = config.export_exit, .replace_staging_at_publish = config.replace_staging_at_publish, .replace_destination_after_rename = config.replace_destination_after_rename, .stray_staging_file_after_discard = config.stray_staging_file_after_discard, .verifier_fault = config.verifier_fault, .limits = config.limits, }; } fn dependencies(self: *CommandScript, fault: PublicationHooks) Dependencies { var publication = fault; // The script needs its own notification to drive `publicationEvent`, // but a caller's observer is a QUESTION the caller asked, and dropping // it answers that question with silence: every assertion made through // a discarded observer passes vacuously and nothing says why. Chained // rather than replaced, so both are served. self.caller_observer = fault.observer; publication.observer = .{ .context = self, .notify = publicationEvent }; publication.staging_observer = .{ .context = self, .replace = replaceStaging }; return .{ .git = .{ .context = self, .vtable = &.{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, } }, .publication = publication, .limits = self.limits, }; } fn startExport(context: *anyopaque, _: []const u8) !ExportPipes { const self: *CommandScript = @ptrCast(@alignCast(context)); self.export_starts += 1; self.lifecycle.launched += 1; const export_bytes = if (self.cancel_export_mid_stream) cancelled_export else self.fast_export; return .{ .stdout = .fixed(export_bytes), .stderr = .fixed("scripted diagnostic\n"), .exit_code = self.export_exit }; } fn terminate(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.terminated += 1; } fn drainStderr(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.stderr_drained += 1; self.stderr_drained_bytes = self.stderr_bytes; } fn reap(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.reaped += 1; } fn publicationEvent(context: *anyopaque, point: PublicationHooks.FailurePoint) void { const self: *CommandScript = @ptrCast(@alignCast(context)); switch (point) { .after_close_store => self.publication.raw_generation_frozen = true, .create_packed_store => self.publication.raw_frozen_before_packing = self.publication.raw_generation_frozen, // Asked of the filesystem, not asserted. `create_packed_store` // fires BEFORE the create, so this is the first point at which the // question has an answer -- and an unconditional `true` here would // be exactly the vacuous observation this suite already got wrong // once with `temporary_sibling_left`. .before_pack => self.publication.packed_generation_created = self.stagingHolds(published_generation_name), .after_close_packed_store => self.publication.packed_generation_frozen = true, // The one instant at which both generations are closed files and // the verifier has read neither. .verify_raw_store => if (self.verifier_fault) |fault| self.injectVerifierFault(fault), .verify_store => { self.publication.store_reopened_for_verify = true; self.publication.packed_before_verify = self.publication.packed_generation_frozen; }, .verify_map => self.publication.map_reopened_for_verify = true, .before_discard_raw_generation => self.publication.both_generations_present_before_discard = self.stagingHoldsBothGenerations(), .after_discard_raw_generation => { self.publication.raw_generation_discarded = !self.stagingHolds(raw_generation_name); if (self.stray_staging_file_after_discard) self.plantStrayStagingFile(); }, .before_rename_no_replace => { self.publication.discard_before_rename = self.publication.raw_generation_discarded; self.publication.verify_before_publish = self.publication.store_reopened_for_verify and self.publication.map_reopened_for_verify; if (self.competing_destination_at_publish) { const parent = self.destination_parent orelse return; parent.createDir(self.io orelse return, self.destination_name, .default_dir) catch return; self.competitor_destination_intact = true; } }, .after_rename_no_replace => if (self.replace_destination_after_rename) { const parent = self.destination_parent orelse return; const io = self.io orelse return; parent.deleteTree(io, self.destination_name) catch return; parent.createDir(io, self.destination_name, .default_dir) catch return; var replacement = parent.openDir(io, self.destination_name, .{ .follow_symlinks = false }) catch return; defer replacement.close(io); var marker = replacement.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; self.competitor_destination_intact = true; }, else => {}, } // After the script's own bookkeeping: an observer that sees a boundary // sees it in the state the script left it in. if (self.caller_observer) |observer| observer.notify(observer.context, point); } /// Whether the staging sibling holds BOTH generation files right now. This /// is the direct evidence that the two generations are distinct files: it /// is asked at the one instant both are supposed to exist, from outside the /// transaction, by name. fn stagingHoldsBothGenerations(self: *CommandScript) bool { return self.stagingHolds(raw_generation_name) and self.stagingHolds(published_generation_name); } /// Write a file into staging that the publish manifest does not name. fn plantStrayStagingFile(self: *CommandScript) void { const parent = self.destination_parent orelse return; const io = self.io orelse return; var entries = parent.iterate(); while (entries.next(io) catch return) |entry| { if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue; var staging = parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return; defer staging.close(io); var stray = staging.createFile(io, "unpublishable-scratch", .{ .read = true, .exclusive = true }) catch return; defer stray.close(io); stray.writeStreamingAll(io, "scratch\n") catch return; return; } } // ---- Corruption seam ------------------------------------------------- // // Everything below runs at `.verify_raw_store`, from OUTSIDE the // transaction, against staging as a plain directory. It never touches the // production path; a run with `verifier_fault == null` executes none of it. /// Open the staging sibling. Caller closes. fn openStagingDir(self: *CommandScript) ?std.Io.Dir { const parent = self.destination_parent orelse return null; const io = self.io orelse return null; var entries = parent.iterate(); while (entries.next(io) catch return null) |entry| { if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue; return parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return null; } return null; } fn injectVerifierFault(self: *CommandScript, fault: VerifierFault) void { const io = self.io orelse return; var staging = self.openStagingDir() orelse return; defer staging.close(io); switch (fault) { .ref_retarget, .tag_change => self.rewriteStagingRefs(staging, fault), .attribution_change, .provenance_tamper => self.rewriteStagingProvenance(staging, fault), else => self.rewritePackedGeneration(staging, fault), } } fn readStagingText(self: *CommandScript, staging: std.Io.Dir, name: []const u8) ?[]u8 { const io = self.io orelse return null; return staging.readFileAlloc(io, name, std.heap.page_allocator, .limited(1 << 20)) catch null; } fn writeStagingText(self: *CommandScript, staging: std.Io.Dir, name: []const u8, bytes: []const u8) void { const io = self.io orelse return; staging.deleteFile(io, name) catch return; var file = staging.createFile(io, name, .{ .read = true, .exclusive = true }) catch return; defer file.close(io); file.writeStreamingAll(io, bytes) catch return; file.sync(io) catch return; self.verifier_fault_injected = true; } /// Flip one hex digit in place. The result is still a syntactically valid /// name, so nothing upstream of the clause under test can reject it on /// shape alone. fn flipHexDigit(byte: *u8) bool { byte.* = switch (byte.*) { '0' => '1', '1' => '0', 'a'...'e' => byte.* + 1, 'f' => 'a', '2'...'9' => byte.* - 1, else => return false, }; return true; } fn rewriteStagingRefs(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void { const body = self.readStagingText(staging, "refs") orelse return; defer std.heap.page_allocator.free(body); const want: []const u8 = if (fault == .tag_change) "sym-tags/" else "sym-heads/"; var cursor: usize = 0; while (std.mem.indexOfPos(u8, body, cursor, want)) |start| { const end = std.mem.indexOfScalarPos(u8, body, start, '\n') orelse body.len; const space = std.mem.lastIndexOfScalar(u8, body[start..end], ' ') orelse { cursor = end; continue; }; // Retarget: the row now names something that is not the check-in // the import resolved for it. if (!flipHexDigit(&body[start + space + 1])) { cursor = end; continue; } // Type change, for the tag case only: the same row reappears // spelled as a branch, which is a different kind of ref entirely. // The two prefixes differ in length, so this rebuilds the file // rather than poking it -- every other byte is carried through. if (fault == .tag_change) { const gpa = std.heap.page_allocator; var rebuilt: std.ArrayList(u8) = .empty; defer rebuilt.deinit(gpa); rebuilt.appendSlice(gpa, body[0..start]) catch return; rebuilt.appendSlice(gpa, "sym-heads/") catch return; rebuilt.appendSlice(gpa, body[start + want.len ..]) catch return; self.writeStagingText(staging, "refs", rebuilt.items); return; } self.writeStagingText(staging, "refs", body); return; } } fn rewriteStagingProvenance(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void { const gpa = std.heap.page_allocator; const body = self.readStagingText(staging, ".zit-import-map.jsonl") orelse return; defer gpa.free(body); switch (fault) { .provenance_tamper => { const marker = "\"provenance_digest\":\""; const at = std.mem.indexOf(u8, body, marker) orelse return; if (!flipHexDigit(&body[at + marker.len])) return; self.writeStagingText(staging, ".zit-import-map.jsonl", body); }, .attribution_change => { // Exchange the two identity objects on the first commit line. // Attribution mode selects ONE of them; swapping makes the // record name the identity the store's U card does not carry, // while every digit of both identities stays well-formed. const author_marker = "\"author\":"; const committer_marker = ",\"committer\":"; const tree_marker = ",\"tree_digest\":"; const a = std.mem.indexOf(u8, body, author_marker) orelse return; const c = std.mem.indexOfPos(u8, body, a, committer_marker) orelse return; const t = std.mem.indexOfPos(u8, body, c, tree_marker) orelse return; const author = body[a + author_marker.len .. c]; const committer = body[c + committer_marker.len .. t]; if (std.mem.eql(u8, author, committer)) return; var swapped: std.ArrayList(u8) = .empty; defer swapped.deinit(gpa); swapped.appendSlice(gpa, body[0 .. a + author_marker.len]) catch return; swapped.appendSlice(gpa, committer) catch return; swapped.appendSlice(gpa, committer_marker) catch return; swapped.appendSlice(gpa, author) catch return; swapped.appendSlice(gpa, body[t..]) catch return; self.writeStagingText(staging, ".zit-import-map.jsonl", swapped.items); }, else => {}, } } const StagedArtifact = struct { name: []u8, bytes: []u8 }; fn readGeneration( self: *CommandScript, staging: std.Io.Dir, name: []const u8, config: ?*?logstore.OwnedConfig, hash_kind: ?*u8, ) ?[]StagedArtifact { const io = self.io orelse return null; const gpa = std.heap.page_allocator; var file = staging.openFile(io, name, .{ .mode = .read_only }) catch return null; defer file.close(io); const size = (file.stat(io) catch return null).size; var file_storage = storage.FileStorage.init(io, file, size); var log = logstore.LogStore.openReadOnly(gpa, file_storage.storageFile()) catch return null; defer log.deinit(gpa); // Carried through the rebuild when the caller asks for it, so a // replacement generation differs from the original in the ONE property // its fault is named for and not also in its settings. The hash kind // most of all: hardcoding sha1 would make every store fault silently // inexpressible under any other algorithm -- the rebuilt header would // disagree with its own names and the whole suite would go red with no // diagnostic. if (config) |out| out.* = log.readConfig(gpa) catch null; if (hash_kind) |out| out.* = log.hash_kind; const rids = log.ridsInPersistedOrder(gpa) catch return null; defer gpa.free(rids); var out: std.ArrayList(StagedArtifact) = .empty; for (rids) |rid| { const bytes = (log.contentUnmasked(gpa, rid, null) catch continue) orelse continue; const hash = gpa.dupe(u8, log.hashOf(rid)) catch continue; out.append(gpa, .{ .name = hash, .bytes = bytes }) catch continue; } return out.toOwnedSlice(gpa) catch null; } fn freeGeneration(items: []StagedArtifact) void { const gpa = std.heap.page_allocator; for (items) |item| { gpa.free(item.name); gpa.free(item.bytes); } gpa.free(items); } /// Rewrite the packed generation from its own contents, applying `fault`. /// /// Rebuilding rather than byte-poking is deliberate: every frame the /// replacement contains checksums correctly and every name is well-formed, /// so the only thing a verifier can object to is the property the fault is /// named for. fn rewritePackedGeneration(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void { const io = self.io orelse return; const gpa = std.heap.page_allocator; var config: ?logstore.OwnedConfig = null; defer if (config) |*owned| owned.deinit(gpa); var hash_kind: u8 = 0; const items = self.readGeneration(staging, published_generation_name, &config, &hash_kind) orelse return; defer freeGeneration(items); if (items.len == 0) return; // `retained_omitted_artifact` needs a name the pack REMOVED, which // exists only as the difference between the two generations. var readd: ?StagedArtifact = null; const raw_items: ?[]StagedArtifact = if (fault == .retained_omitted_artifact) self.readGeneration(staging, raw_generation_name, null, null) else null; defer if (raw_items) |raw| freeGeneration(raw); if (raw_items) |raw| { outer: for (raw) |candidate| { for (items) |kept| { if (std.mem.eql(u8, kept.name, candidate.name)) continue :outer; } readd = candidate; break; } // No drop set means nothing was omitted, so this fault has no // referent. Leaving `verifier_fault_injected` false is the whole // point: the fixture asserts it and fails loudly rather than // passing on a corruption that never happened. if (readd == null) return; } staging.deleteFile(io, published_generation_name) catch return; var file = staging.createFile(io, published_generation_name, .{ .read = true, .exclusive = true }) catch return; var file_open = true; defer if (file_open) file.close(io); var file_storage = storage.FileStorage.init(io, file, 0); var log = logstore.LogStore.open(gpa, file_storage.storageFile(), hash_kind) catch return; var log_live = true; defer if (log_live) log.deinit(gpa); if (config) |owned| log.putConfig(owned.config) catch return; // When a check-in is re-emitted its name changes; every file that // names it must follow. const Renamed = struct { old: [2 * logstore.max_hash_len]u8, new: [2 * logstore.max_hash_len]u8, len: usize }; var renamed: ?Renamed = null; // A BLOB, chosen by kind rather than by position. Both faults below // want an ordinary artifact: aim either at a check-in and the fixture // starts proving `checkin-unresolvable` instead of the clause it is // named for -- loudly rather than falsely, but for the wrong reason. const target = lastBlob(items) orelse return; const skip: usize = if (fault == .dropped_required_artifact) target else std.math.maxInt(usize); const mutate: usize = switch (fault) { .changed_artifact_bytes, .wrong_delta_base => target, else => std.math.maxInt(usize), }; for (items, 0..) |item, index| { if (index == skip) { self.verifier_fault_injected = true; continue; } if (index == mutate and fault == .changed_artifact_bytes) { // Same name, different bytes. Nothing about the frame is // malformed; only `hash(content) == name` is false. _ = log.putArtifact(item.name, "verifier fixture: these are not the bytes\n", null) catch return; self.verifier_fault_injected = true; continue; } if (index == mutate and fault == .wrong_delta_base and index >= 2) { // A delta encoded against its NEIGHBOUR -- which is similar // enough to it to produce real copy segments -- but recorded // against rid 1, a much shorter artifact. Both frames exist, // both checksum, and the chain is within bounds; only walking // it lands anywhere wrong. // // The neighbour matters. A delta between two dissimilar // artifacts carries no copy segment at all, and such a delta // reconstructs the SAME bytes from any base whatsoever -- so // recording the wrong one would be an injection that changed // nothing, and this fixture would then be asserting a clause // about a store that is not actually wrong. if (self.plantWrongDeltaBase(&log, items[index - 1].bytes, item)) continue; } if (fault == .tree_change_preserving_artifacts or fault == .parent_reorder) { if (renamed == null) if (self.mutatedCheckin(item, fault)) |replacement| { defer gpa.free(replacement.name); defer gpa.free(replacement.bytes); _ = log.putArtifact(replacement.name, replacement.bytes, null) catch return; // Re-emitting a check-in necessarily renames it. Carrying // the rename into `refs` and the map is what keeps these // two fixtures SURGICAL: without it the ref rows and the // provenance lines would also dangle, and the fixture // would trip whichever clause happened to run first // instead of the one it is named for. var rename: Renamed = .{ .old = undefined, .new = undefined, .len = 2 * item.name.len }; _ = std.fmt.bufPrint(rename.old[0..rename.len], "{x}", .{item.name}) catch return; _ = std.fmt.bufPrint(rename.new[0..rename.len], "{x}", .{replacement.name}) catch return; renamed = rename; self.verifier_fault_injected = true; continue; }; } _ = log.putArtifact(item.name, item.bytes, null) catch return; } if (readd) |dropped| { _ = log.putArtifact(dropped.name, dropped.bytes, null) catch return; self.verifier_fault_injected = true; } if (fault == .added_artifact) { const novel = "verifier fixture: an artifact no import produced\n"; var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined; std.crypto.hash.Sha1.hash(novel, &digest, .{}); _ = log.putArtifact(&digest, novel, null) catch return; self.verifier_fault_injected = true; } log.commit() catch return; log.deinit(gpa); log_live = false; file.sync(io) catch {}; file.close(io); file_open = false; if (renamed) |rename| { self.substituteStagingName(staging, "refs", rename.old[0..rename.len], rename.new[0..rename.len]); self.substituteStagingName(staging, ".zit-import-map.jsonl", rename.old[0..rename.len], rename.new[0..rename.len]); } } /// The last artifact in persisted order whose bytes are NOT a check-in /// manifest, or null when a generation is all check-ins. fn lastBlob(items: []const StagedArtifact) ?usize { const gpa = std.heap.page_allocator; var index = items.len; while (index > 0) { index -= 1; var manifest = artifact.parse(gpa, items[index].bytes) catch return index; defer manifest.deinit(); if (manifest.kind != .manifest) return index; } return null; } /// Store `item` as a delta of `base` under a recorded base of rid 1. /// /// Returns whether the divergence is real, PROVEN rather than assumed: /// applying the encoded delta to the artifact the frame actually names must /// fail. Nothing is written and nothing is claimed when it does not, so the /// fixture reports "the injection did not happen" instead of asserting a /// verifier clause about a store that reconstructs perfectly. fn plantWrongDeltaBase( self: *CommandScript, log: *logstore.LogStore, base: []const u8, item: StagedArtifact, ) bool { const gpa = std.heap.page_allocator; const encoded = delta.create(gpa, base, item.bytes) catch return false; defer gpa.free(encoded); const recorded_base = (log.contentUnmasked(gpa, 1, null) catch return false) orelse return false; defer gpa.free(recorded_base); if (delta.apply(gpa, recorded_base, encoded)) |applied| { gpa.free(applied); return false; } else |_| {} _ = log.putDelta(item.name, 1, encoded) catch return false; self.verifier_fault_injected = true; return true; } /// Replace every occurrence of one artifact name with another in a staging /// text file. Equal lengths, so this is a straight in-place substitution. fn substituteStagingName( self: *CommandScript, staging: std.Io.Dir, file_name: []const u8, old_name: []const u8, new_name: []const u8, ) void { const injected = self.verifier_fault_injected; const body = self.readStagingText(staging, file_name) orelse return; defer std.heap.page_allocator.free(body); var cursor: usize = 0; var replaced = false; while (std.mem.indexOfPos(u8, body, cursor, old_name)) |at| { @memcpy(body[at .. at + new_name.len], new_name); cursor = at + new_name.len; replaced = true; } if (!replaced) return; self.writeStagingText(staging, file_name, body); // `writeStagingText` reports its own success; this helper only carries // a rename that a real injection already made, so it must not be what // turns the flag on. self.verifier_fault_injected = injected; } /// Reparse a check-in and re-emit it with `fault` applied. Returns null for /// anything that is not a check-in, so the caller can keep walking. /// /// The replacement is stored under its OWN correct name -- the point of /// both faults is that the store stays internally honest and only its /// meaning changed. fn mutatedCheckin(self: *CommandScript, item: StagedArtifact, fault: VerifierFault) ?StagedArtifact { _ = self; const gpa = std.heap.page_allocator; var manifest = artifact.parse(gpa, item.bytes) catch return null; defer manifest.deinit(); if (manifest.kind != .manifest) return null; switch (fault) { .tree_change_preserving_artifacts => { if (manifest.f.len < 2) return null; // Every blob still present and still named; two of them now // hang at each other's paths. const cards = @constCast(manifest.f); std.mem.swap(?[]const u8, &cards[0].hash, &cards[cards.len - 1].hash); }, .parent_reorder => { if (manifest.p.len == 0) return null; const card = manifest.p[0]; if (card.count < 2) return null; var parts = std.mem.splitScalar(u8, card.parents orelse return null, ' '); const first = parts.next() orelse return null; const second = parts.next() orelse return null; const swapped = std.fmt.allocPrint(gpa, "{s} {s}", .{ second, first }) catch return null; defer gpa.free(swapped); const cards = @constCast(manifest.p); cards[0].parents = swapped; const bytes = artifact.create(&manifest, gpa) catch return null; var name_buf: [64]u8 = undefined; const name = artifact.artifactName(git_import.import_hash_algo, bytes, &name_buf); var raw: [32]u8 = undefined; const digest = rawHashName(name, &raw) orelse { gpa.free(bytes); return null; }; return .{ .name = gpa.dupe(u8, digest) catch return null, .bytes = bytes }; }, else => return null, } const bytes = artifact.create(&manifest, gpa) catch return null; var name_buf: [64]u8 = undefined; const name = artifact.artifactName(git_import.import_hash_algo, bytes, &name_buf); var raw: [32]u8 = undefined; const digest = rawHashName(name, &raw) orelse { gpa.free(bytes); return null; }; return .{ .name = gpa.dupe(u8, digest) catch return null, .bytes = bytes }; } /// Whether the staging sibling holds `name` right now. fn stagingHolds(self: *CommandScript, name: []const u8) bool { const parent = self.destination_parent orelse return false; const io = self.io orelse return false; var entries = parent.iterate(); while (entries.next(io) catch return false) |entry| { if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue; var staging = parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return false; defer staging.close(io); return fileExists(io, staging, name); } return false; } /// Whether a staging sibling OTHER than a deliberately planted replacement /// survived the run. Read after `run` returns, so a true answer means this /// invocation leaked one. /// /// Fails CLOSED: an unreadable parent reports a leak. This backs a /// negative assertion -- `!temporary_sibling_left` -- so answering "no /// leak" on a question that could not be asked is how a leak assertion /// passes vacuously, which is the defect this field was added to fix. fn stagingSiblingSurvives(self: *CommandScript) bool { const parent = self.destination_parent orelse return true; const io = self.io orelse return true; const planted = self.stagingReplacementName(); var entries = parent.iterate(); while (entries.next(io) catch return true) |entry| { if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue; // A script that replaces the staging entry at the publication // boundary leaves its plant behind BY DESIGN -- the transaction // correctly refuses to delete an entry it no longer owns. Excluding // that one name keeps the assertion live for every other sibling, // rather than disabling it wholesale in exactly the shape where a // leak is most plausible. if (planted.len != 0 and std.mem.eql(u8, entry.name, planted)) continue; return true; } return false; } fn replaceStaging(context: *anyopaque, parent: std.Io.Dir, name: []const u8) void { const self: *CommandScript = @ptrCast(@alignCast(context)); const replacement = self.replace_staging_at_publish orelse return; const io = self.io orelse return; if (name.len > self.staging_replacement_name.len) return; parent.deleteTree(io, name) catch return; switch (replacement) { .directory => { parent.createDir(io, name, .default_dir) catch return; var directory = parent.openDir(io, name, .{ .follow_symlinks = false }) catch return; defer directory.close(io); var marker = directory.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; }, .symlink => parent.symLink(io, "outside", name, .{}) catch return, } @memcpy(self.staging_replacement_name[0..name.len], name); self.staging_replacement_name_len = name.len; self.staging_replacement_observed = true; } pub fn stagingReplacementName(self: *const CommandScript) []const u8 { return self.staging_replacement_name[0..self.staging_replacement_name_len]; } /// Derived from `exportArgv` so this fake cannot drift from the command /// the importer actually runs. Shows the default signature policy; the /// opt-out's effect is covered directly against `exportArgv`. pub fn exportCommandLine(_: *const CommandScript) []const u8 { return comptime line: { const argv = exportArgv("fixture.git", false); var joined: []const u8 = argv[0]; for (1..argv.len) |index| joined = joined ++ " " ++ argv[index]; break :line joined; }; } }; pub const ScriptResult = struct { code: cli.Exit, out: []const u8, err: []const u8, lifecycle: ChildLifecycle, retained_diagnostic: []const u8, published_durability_uncertain: bool = false, }; pub fn runScripted( gpa: std.mem.Allocator, io: std.Io, request: Request, script: *CommandScript, hooks: PublicationHooks, ) ScriptResult { var out_storage: [4096]u8 = undefined; var err_storage: [4096]u8 = undefined; var out = std.Io.Writer.fixed(&out_storage); var err = std.Io.Writer.fixed(&err_storage); const streams: cli.Streams = .{ .out = &out, .err = &err }; script.preflight_starts += 1; if (script.preflight == .git_missing or script.preflight == .not_repository or script.preflight == .missing_objects or script.preflight == .shallow or script.preflight == .partial_promisor or (script.preflight == .detached_head and script.default_branch == null)) { script.lifecycle.reaped += 1; return .{ .code = .failure, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "preflight rejected" }; } if (script.preflight == .detached_head) script.detached_head_recorded = true; script.destination_parent = request.destination_parent; script.destination_name = request.destination_name; script.io = io; var effective_request = request; effective_request.default_branch = script.default_branch; const code = run(gpa, io, effective_request, streams, script.dependencies(hooks)); // Asked after the run, from outside. `stagingSiblingSurvives` excludes a // deliberately planted replacement by name, so this stays a live leak // assertion even on the staging-replacement scripts. script.temporary_sibling_left = script.stagingSiblingSurvives(); if (code == .failure and hooks.fail_at == .sync_parent_after_publish) { return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "durability uncertain", .published_durability_uncertain = true }; } return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "scripted diagnostic" }; } pub fn dispatchForTest(args: []const []const u8) ScriptResult { if (args.len == 1 and std.mem.eql(u8, args[0], "--help")) { // Rendered from `cli.import_git_usage`, not transcribed: the hand copy // this replaced had silently fallen two options behind the real help. // The `usage: ` prefix is `main.zig`'s, carried here so this stub's // output is a true PREFIX of what the real help prints rather than // merely sharing its option surface. return .{ .code = .ok, .out = "usage: " ++ cli.import_git_usage ++ "\n", .err = "", .lifecycle = .{}, .retained_diagnostic = "" }; } _ = cli.parseImportGitArgs(args) catch { return .{ .code = .usage, .out = "", .err = "invalid import-git arguments\n", .lifecycle = .{}, .retained_diagnostic = "" }; }; return .{ .code = .failure, .out = "", .err = "test dispatch requires scripted dependencies\n", .lifecycle = .{}, .retained_diagnostic = "" }; }