Artifact e914a3cffb8205169823dd4c3804ea6eb5c79ff2a06da22c04b88c10c3d2995a:
- File
src/git_import_cli.zig
— part of check-in
[dc0aa2f858]
at
2026-07-31 02:56:07
on branch sub-manifest
— git-import: translate a Git fast-export stream into a zit history
`zit import-git SOURCE DEST` reads a Git repository through `git fast- export` and builds a zit repository from it. The destination appears only after the export, artifact creation and publication have all completed, so a failure anywhere leaves no half-published repository behind; this is the first caller of the logstore artifact batch added earlier on this branch, which is what makes a check-in's blobs arrive as a unit.
Mode 160000 now parses as a gitlink instead of error.UnsupportedMode. That completes the split begun in the previous check-in: reporting what the stream said is the parser's job, and deciding that zit has no submodule concept is the importer's. The refusal still happens, but it now explains itself instead of failing as a parse error on input git legitimately emits.
Signed commits are refused by default rather than imported with the signature silently dropped. Fossil has no signature card, so importing one is a loss the operator should choose; --drop-signatures accepts it explicitly.
The sub-manifest parameter is analyzed, not guessed. Preflight measures the source's own name set and pins its measured optimum, and --submanifest-k overrides that only to match another repository's existing pin, which is what lets two repositories share sub-manifests by content address. On a 100,000 file corpus the analyzed value cuts the objective by roughly 3.4x against the fixed default it replaces.
Correctness is checked differentially against real Git rather than against fixtures we wrote: every materialized tree is compared to `git ls-tree` for the corresponding commit, refs and annotated tags are compared to the source, and both comparators now assert that a gitlink is absent rather than merely tolerating it, so the omission is a contract instead of an accident.
The rest is wiring. extras learns the import provenance names, root.zig exports the importer modules across the executable and library boundary, tidy and the CI step guard learn the new test roots and build steps, and the e2e help check learns that a verb spelled import_git in Zig is import-git on the command line. (user: dmestas, size: 74408) [annotate] [blame] [check-ins using]
const std = @import("std"); const builtin = @import("builtin"); pub const cli = @import("cli.zig"); const zit = @import("zit"); const artifact = zit.artifact; const checkout = zit.checkout; const fast_export = zit.git_fast_export; const git_import = zit.git_import; const logstore = zit.logstore; const storage = zit.storage; const editor = zit.submanifest_editor; extern "c" fn renameatx_np( from_fd: c_int, from: [*:0]const u8, to_fd: c_int, to: [*:0]const u8, flags: c_uint, ) c_int; pub const ChildLifecycle = struct { launched: usize = 0, terminated: usize = 0, stderr_drained: usize = 0, reaped: usize = 0, }; /// The child-facing half of the import transaction. The caller owns the /// implementation and its backing buffers for the full call to `run`. pub const ChildFactory = struct { context: *anyopaque, vtable: *const VTable, pub const VTable = struct { start_export: *const fn (context: *anyopaque, source: []const u8) anyerror!ExportPipes, terminate: *const fn (context: *anyopaque) anyerror!void, drain_stderr: *const fn (context: *anyopaque) anyerror!void, reap: *const fn (context: *anyopaque) anyerror!void, startup_diagnostics: ?*const fn (context: *anyopaque, err: *std.Io.Writer) void = null, }; }; pub const ExportPipes = struct { stdout: std.Io.Reader, stderr: std.Io.Reader, /// Scripted sessions supply their terminal status here. The OS binding gets /// its status during reap; an import succeeds only after the stream reaches /// `done` and EOF, so a nonzero scripted result is still an import failure. exit_code: u8 = 0, /// OS-backed sessions publish the wait status here during `reap`; scripted /// sessions use `exit_code` directly. exit_code_ref: ?*const u8 = null, owner: ?*ProductionGit = null, diagnostics: ?*const fn (pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void = null, detected_default_branch: ?[]const u8 = null, /// draft-fossil-submanifests-00 §6.3's optimum for the source's own name /// set, measured during preflight. Null when the source could not be /// listed or cannot partition; the caller then keeps its own default. analyzed_submanifest_k: ?u4 = null, }; /// The `git fast-export` command line the importer runs, as a value. /// /// Pure and public so the one policy decision it encodes — what happens to a /// signature Fossil has no card for — is testable without spawning a process. /// `refuse_signed` moves exactly the two signature modes and nothing else. pub fn exportArgv(source: []const u8, refuse_signed: bool) [10][]const u8 { return .{ "git", "-C", source, "fast-export", "--all", "--show-original-ids", "--mark-tags", if (refuse_signed) "--signed-commits=abort" else "--signed-commits=strip", if (refuse_signed) "--signed-tags=abort" else "--signed-tags=strip", "--use-done-feature", }; } /// OS-backed production dependency. It owns both pipe buffers through the /// import call; the `ExportPipes` readers retain a pointer to this value. pub const ProductionGit = struct { pub const StartupFailure = enum { after_spawn, after_stderr_handoff, after_stderr_thread }; /// Keep room in fixed CLI writers for the authoritative import diagnostic. const cli_diagnostic_reserve_bytes = 256; /// Bytes of output one preflight check may produce. Every check asks a /// bounded question — one ref, one boolean, one config value, or (for /// `fsck --connectivity-only --no-dangling`) nothing at all when the /// repository is intact — so an overrun means a check's shape was /// misjudged, never that the source is merely large. const preflight_output_limit_bytes = 8 * 1024; gpa: std.mem.Allocator, io: std.Io, child: ?std.process.Child = null, exit_code: u8 = 1, stdout_buffer: [8192]u8 = undefined, stderr_buffer: [8192]u8 = undefined, reaped: bool = false, reap_calls: usize = 0, /// Captured by the stdout reader before any parser-error cleanup can /// terminate the child. This distinguishes an independently completed /// truncated export from cancellation closing the pipe. stdout_reached_eof: bool = false, default_branch: ?[]const u8 = null, owns_default_branch: bool = false, analyzed_submanifest_k: ?u4 = null, /// The operator's explicit opt-out from the default signature policy. /// False (the default) strips; true restores the refusal. refuse_signed: bool = false, stderr_thread: ?std.Thread = null, /// Moved out of `child` before its reader thread starts, so draining never /// races cleanup or a platform-specific process handle implementation. stderr_file: ?std.Io.File = null, stderr_ring: [4096]u8 = undefined, stderr_ring_start: usize = 0, stderr_ring_len: usize = 0, stderr_bytes_drained: usize = 0, stderr_lock: std.Io.Mutex = .init, test_argv: ?[]const []const u8 = null, test_startup_failure: ?StartupFailure = null, pub fn init(gpa: std.mem.Allocator, io: std.Io) ProductionGit { return .{ .gpa = gpa, .io = io }; } /// Test-only process constructor: uses the identical spawn, stream, /// concurrent-diagnostic, termination, and reap path as `git fast-export`. pub fn initForTest(gpa: std.mem.Allocator, io: std.Io, argv: []const []const u8) ProductionGit { return .{ .gpa = gpa, .io = io, .test_argv = argv }; } fn childEnvironment(self: *ProductionGit) !std.process.Environ.Map { var count: usize = 0; while (std.c.environ[count] != null) : (count += 1) {} const inherited: [:null]const ?[*:0]const u8 = @ptrCast(std.c.environ[0..count :null]); var env = try std.process.Environ.createMap(.{ .block = .{ .slice = inherited } }, self.gpa); errdefer env.deinit(); try env.put("GIT_TERMINAL_PROMPT", "0"); try env.put("GIT_NO_LAZY_FETCH", "1"); return env; } pub fn setDefaultBranch(self: *ProductionGit, default_branch: ?[]const u8) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.default_branch = default_branch; self.owns_default_branch = false; } /// Set before `run`, alongside `setDefaultBranch`: the export argv is /// built inside `start_export`, which the factory vtable does not let the /// caller parameterize. pub fn setRefuseSigned(self: *ProductionGit, refuse: bool) void { self.refuse_signed = refuse; } pub fn deinit(self: *ProductionGit) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.* = undefined; } pub fn dependencies(self: *ProductionGit, publication: PublicationHooks) Dependencies { return .{ .git = .{ .context = self, .vtable = &vtable }, .publication = publication, }; } const vtable: ChildFactory.VTable = .{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, .startup_diagnostics = writeStartupDiagnostics, }; fn startExport(context: *anyopaque, source: []const u8) !ExportPipes { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.test_argv) |argv| return self.startSession(argv); try self.preflight(source); // Fossil has no signature card, so a signature cannot survive this // import in any representation. `strip` accepts that loss and is the // default — refusing by default only withheld the import, never the // signature. `abort` is the operator's explicit opt-out. There is no // honest third option: a verbatim pass would have to invent somewhere // to keep bytes the format cannot hold. const argv = exportArgv(source, self.refuse_signed); return self.startSession(&argv); } fn startSession(self: *ProductionGit, argv: []const []const u8) !ExportPipes { var env = try self.childEnvironment(); defer env.deinit(); self.reaped = false; self.exit_code = 1; self.reap_calls = 0; self.stdout_reached_eof = false; self.stderr_file = null; self.child = try std.process.spawn(self.io, .{ .argv = argv, .environ_map = &env, .stdin = .ignore, .stdout = .pipe, .stderr = .pipe, }); errdefer self.cleanupFailedStart(); try self.failStartForTest(.after_spawn); const child = &(self.child orelse unreachable); const stderr = child.stderr orelse return error.MissingStderr; child.stderr = null; self.stderr_file = stderr; try self.failStartForTest(.after_stderr_handoff); self.stderr_ring_start = 0; self.stderr_bytes_drained = 0; self.stderr_ring_len = 0; self.stderr_thread = try std.Thread.spawn(.{}, drainStderrThread, .{self}); try self.failStartForTest(.after_stderr_thread); return .{ .exit_code_ref = &self.exit_code, .detected_default_branch = self.default_branch, .analyzed_submanifest_k = self.analyzed_submanifest_k, .owner = self, .stdout = .{ .vtable = &stdout_reader_vtable, .buffer = &self.stdout_buffer, .seek = 0, .end = 0, }, .stderr = .{ .vtable = &stderr_reader_vtable, .buffer = &self.stderr_buffer, .seek = 0, .end = 0, }, .diagnostics = writeDiagnostics, }; } fn failStartForTest(self: *const ProductionGit, point: StartupFailure) !void { if (self.test_startup_failure == point) return error.InjectedStartFailure; } fn cleanupFailedStart(self: *ProductionGit) void { if (self.stderr_thread == null) { if (self.stderr_file) |file| file.close(self.io); self.stderr_file = null; } if (self.child) |*child| { if (child.stderr) |file| file.close(self.io); child.stderr = null; } terminate(@ptrCast(self)) catch {}; reap(@ptrCast(self)) catch {}; } fn preflight(self: *ProductionGit, source: []const u8) !void { const inside = try self.query("rev-parse --is-inside-work-tree", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-inside-work-tree" }); defer self.gpa.free(inside); if (!std.mem.eql(u8, std.mem.trim(u8, inside, " \t\r\n"), "true")) return error.NotGitRepository; const head = try self.query("rev-parse HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "HEAD" }); defer self.gpa.free(head); if (std.mem.trim(u8, head, " \t\r\n").len == 0) return error.MissingHead; const object_format = try self.query("rev-parse --show-object-format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" }); defer self.gpa.free(object_format); if (!std.mem.eql(u8, std.mem.trim(u8, object_format, " \t\r\n"), "sha1")) return error.UnsupportedObjectFormat; const shallow = try self.query("rev-parse --is-shallow-repository", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-shallow-repository" }); defer self.gpa.free(shallow); if (std.mem.eql(u8, std.mem.trim(u8, shallow, " \t\r\n"), "true")) return error.ShallowRepository; // `--no-dangling` is required, not tidiness. A dangling object is // unreachable but intact, so it says nothing about the connectivity // question this check exists to answer, yet git reports one // informational line per object on stdout: penpot emitted 14,763 bytes // and a healthy repository failed its import on the output limit. // A missing object is still reported, and still makes this check exit // non-zero. const connectivity = self.query("fsck --connectivity-only", &.{ "git", "-C", source, "--no-lazy-fetch", "fsck", "--connectivity-only", "--no-dangling", "--no-reflogs" }) catch |err| switch (err) { // The exit status is this check's verdict; its stdout is a // per-problem report — with `--no-dangling` an intact repository // writes none of it. So an overrun here is not an unreadable // answer, it is a repository with more broken links than the // report can hold. The named-overrun diagnostic is already // retained, so the operator sees both which check overran and the // verdict it produced. error.GitPreflightFailed, error.PreflightOutputTooLarge => return error.MissingObjects, else => return err, }; defer self.gpa.free(connectivity); const symbolic = try self.optionalQuery("symbolic-ref HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "symbolic-ref", "--quiet", "HEAD" }); if (symbolic) |branch_output| { defer self.gpa.free(branch_output); if (self.default_branch == null) { const branch_ref = std.mem.trim(u8, branch_output, " \t\r\n"); const prefix = "refs/heads/"; if (!std.mem.startsWith(u8, branch_ref, prefix) or branch_ref.len == prefix.len) return error.InvalidHead; self.default_branch = try self.gpa.dupe(u8, branch_ref[prefix.len..]); self.owns_default_branch = true; } } else if (self.default_branch == null) { return error.DetachedHead; } const partial_clone = try self.optionalQuery("config --get extensions.partialClone", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get", "extensions.partialClone" }); if (partial_clone) |configured_remote| { defer self.gpa.free(configured_remote); if (std.mem.trim(u8, configured_remote, " \t\r\n").len != 0) return error.PartialRepository; } const promisors = try self.optionalQuery("config --get-regexp remote.*.promisor", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get-regexp", "^remote\\..*\\.promisor$" }); if (promisors) |configured_promisors| { defer self.gpa.free(configured_promisors); if (try containsGitTrue(configured_promisors)) return error.PartialRepository; } self.analyzed_submanifest_k = try self.analyzeSubmanifestPin(source); } /// `git ls-tree` output the §6.3 analysis is willing to read. A monorepo /// at the ceiling lists several million paths; past it the pin falls back /// to the default rather than failing an import over an advisory /// measurement. const ls_tree_limit_bytes = 64 << 20; /// Measure §6.3's optimum for the source before a single object is read. /// The tip tree, not the root commit: an imported history's first commit /// is routinely a lone README, and pinning k from it would misconfigure /// every later commit in the repository. /// /// Advisory throughout — every failure path returns null and leaves the /// caller's default standing. `-z --name-only` is required, not stylistic: /// without it git quotes unusual bytes and the names would no longer be /// the ones §6.1's partition predicate hashes. The listing includes paths /// a check-in's F-set would exclude (§4.6.3 reserved names) and any path /// `update` would reject; both are rare enough not to move an optimum that /// tracks half the base-2 logarithm of the count. fn analyzeSubmanifestPin(self: *ProductionGit, source: []const u8) !?u4 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = &.{ "git", "-C", source, "--no-lazy-fetch", "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD" }, .environ_map = &env, .stdout_limit = .limited(ls_tree_limit_bytes), .stderr_limit = .limited(8 * 1024), }) catch return null; defer self.gpa.free(result.stderr); defer self.gpa.free(result.stdout); switch (result.term) { .exited => |code| if (code != 0) return null, else => return null, } var names: std.ArrayList([]const u8) = .empty; defer names.deinit(self.gpa); var listing = std.mem.splitScalar(u8, result.stdout, 0); while (listing.next()) |name| { if (name.len == 0) continue; try names.append(self.gpa, name); } return git_import.analyzeSubmanifestK(self.gpa, names.items, .sha1); } fn containsGitTrue(configured: []const u8) !bool { var lines = std.mem.splitScalar(u8, configured, '\n'); while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, " \t\r"); if (trimmed.len == 0) continue; const value = trimmed[(std.mem.lastIndexOfAny(u8, trimmed, " \t") orelse return error.InvalidGitBoolean) + 1 ..]; if (try isGitTrue(value)) return true; } return false; } fn isGitTrue(value: []const u8) !bool { if (std.mem.eql(u8, value, "1") or std.ascii.eqlIgnoreCase(value, "true") or std.ascii.eqlIgnoreCase(value, "yes") or std.ascii.eqlIgnoreCase(value, "on")) return true; if (std.mem.eql(u8, value, "0") or std.ascii.eqlIgnoreCase(value, "false") or std.ascii.eqlIgnoreCase(value, "no") or std.ascii.eqlIgnoreCase(value, "off")) return false; return error.InvalidGitBoolean; } /// A limit overrun names the check that overran. `StreamTooLong` alone /// tells an operator nothing actionable: which of the preflight children /// produced too much output is the entire diagnostic. fn preflightOverran(self: *ProductionGit, check: []const u8) error{PreflightOutputTooLarge} { var buffer: [256]u8 = undefined; const message = std.fmt.bufPrint( &buffer, "import-git: preflight check 'git {s}' produced more than {d} bytes of output\n", .{ check, preflight_output_limit_bytes }, ) catch fallback: { // Every `check` is a short call-site literal, well inside the // buffer; a truncating format would be a bug here, not a wire // condition. Still total: report without the name rather than // dropping the diagnostic. break :fallback "import-git: a preflight check exceeded its output limit\n"; }; self.retainDiagnostics(message); return error.PreflightOutputTooLarge; } fn optionalQuery(self: *ProductionGit, check: []const u8, argv: []const []const u8) !?[]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| switch (code) { 0 => return result.stdout, 1 => { self.gpa.free(result.stdout); return null; }, else => {}, }, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn query(self: *ProductionGit, check: []const u8, argv: []const []const u8) ![]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| if (code == 0) return result.stdout, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn terminate(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); const child = &(self.child orelse return); switch (builtin.os.tag) { .windows => { const handle = child.id orelse return; switch (std.os.windows.ntdll.NtTerminateProcess(handle, @enumFromInt(1))) { .SUCCESS => {}, .PROCESS_IS_TERMINATING => return error.ChildAlreadyExited, else => return error.TerminateFailed, } }, .wasi => return error.UnsupportedTermination, else => { const pid = child.id orelse return; std.posix.kill(pid, .KILL) catch |err| switch (err) { error.ProcessNotFound => return error.ChildAlreadyExited, else => return err, }; }, } self.exit_code = 1; } fn drainStderr(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.joinStderr(); } fn joinStderr(self: *ProductionGit) void { if (self.stderr_thread) |thread| { thread.join(); self.stderr_thread = null; } } fn retainDiagnostics(self: *ProductionGit, bytes: []const u8) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); for (bytes) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } } fn writeRetainedDiagnostics(self: *ProductionGit, err: *std.Io.Writer) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); const retained_len = @min( self.stderr_ring_len, self.stderr_ring.len - cli_diagnostic_reserve_bytes, ); var index = (self.stderr_ring_start + self.stderr_ring_len - retained_len) % self.stderr_ring.len; for (0..retained_len) |_| { err.writeByte(self.stderr_ring[index]) catch return; index = (index + 1) % self.stderr_ring.len; } } fn writeStartupDiagnostics(context: *anyopaque, err: *std.Io.Writer) void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.writeRetainedDiagnostics(err); } fn drainStderrThread(self: *ProductionGit) void { const file = self.stderr_file orelse return; defer { file.close(self.io); self.stderr_file = null; } var buffer: [4096]u8 = undefined; while (true) { const n = file.readStreaming(self.io, &.{buffer[0..]}) catch break; if (n == 0) break; self.stderr_lock.lockUncancelable(self.io); for (buffer[0..n]) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } self.stderr_bytes_drained += n; self.stderr_lock.unlock(self.io); } } fn writeDiagnostics(pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void { const self = pipes.owner orelse return; if (!enabled) return; self.writeRetainedDiagnostics(err); } fn reap(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.reaped) return; self.reap_calls += 1; // No platform may reap or clear `child` until the independently owned // stderr descriptor is fully drained and its thread has exited. self.joinStderr(); const child = &(self.child orelse return); if (child.stdout) |stdout| { stdout.close(self.io); child.stdout = null; } const term = child.wait(self.io) catch |err| { child.kill(self.io); _ = child.wait(self.io) catch {}; self.exit_code = 1; self.reaped = true; self.child = null; return err; }; self.exit_code = switch (term) { .exited => |code| code, else => 1, }; self.reaped = true; self.child = null; } const stdout_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStdout }; const stderr_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStderr }; fn streamStdout(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stdout_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, true); } fn streamStderr(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stderr_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, false); } fn streamPipe( self: *ProductionGit, _: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit, comptime stdout: bool, ) std.Io.Reader.StreamError!usize { const child = &(self.child orelse return error.ReadFailed); const file = if (stdout) child.stdout orelse return error.ReadFailed else child.stderr orelse return error.ReadFailed; var temporary: [8192]u8 = undefined; const requested = if (limit.toInt()) |value| @min(value, temporary.len) else temporary.len; if (requested == 0) return 0; const n = file.readStreaming(self.io, &.{temporary[0..requested]}) catch |err| switch (err) { error.EndOfStream => { if (stdout) self.stdout_reached_eof = true; return error.EndOfStream; }, else => return error.ReadFailed, }; if (n == 0) { if (stdout) self.stdout_reached_eof = true; return 0; } writer.writeAll(temporary[0..n]) catch return error.WriteFailed; return n; } }; /// Fault injection is deliberately at publication boundaries, rather than in /// storage. This keeps the production storage path identical to the tested one. pub const PublicationHooks = struct { pub const FailurePoint = enum { create_store, before_flush_store, after_flush_store, before_sync_store, after_sync_store, before_close_store, after_close_store, before_flush_map, after_flush_map, before_sync_map, after_sync_map, before_close_map, after_close_map, before_sync_temporary_directory, after_sync_temporary_directory, before_rename_no_replace, after_rename_no_replace, verify_store, verify_map, sync_parent_after_publish, // Compatibility selectors retain the former one-sided fault API. flush_store, sync_store, close_store, flush_map, sync_map, close_map, sync_temporary_directory, rename_no_replace, }; pub const Observer = struct { context: *anyopaque, notify: *const fn (context: *anyopaque, point: FailurePoint) void, }; /// Test-only seam for replacing the staging sibling after it was opened but /// before its identity is checked at the publication boundary. pub const StagingObserver = struct { context: *anyopaque, replace: *const fn (context: *anyopaque, parent: std.Io.Dir, name: []const u8) void, }; fail_at: ?FailurePoint = null, observer: ?Observer = null, staging_observer: ?StagingObserver = null, pub const real_filesystem: PublicationHooks = .{}; fn check(self: PublicationHooks, point: FailurePoint) !void { const legacy: ?FailurePoint = switch (point) { .before_flush_store => .flush_store, .before_sync_store => .sync_store, .before_close_store => .close_store, .before_flush_map => .flush_map, .before_sync_map => .sync_map, .before_close_map => .close_map, .before_sync_temporary_directory => .sync_temporary_directory, .before_rename_no_replace => .rename_no_replace, else => null, }; if (self.fail_at == point or (legacy != null and self.fail_at == legacy)) return error.InjectedPublicationFailure; if (self.observer) |observer| observer.notify(observer.context, point); } fn observeStaging(self: PublicationHooks, parent: std.Io.Dir, name: []const u8) void { if (self.staging_observer) |observer| observer.replace(observer.context, parent, name); } }; pub const ImportLimits = struct { /// A record ceiling, not a memory one: it keeps a runaway or adversarial /// stream finite even when every record it emits is tiny. max_export_events: u64 = 1_000_000, /// The parser's byte authorization for one import, and the only knob an /// operator raises with `--max-export-payload-bytes`. Two ceilings ride on /// it: cumulative `data <n>` payload, and — through /// `fast_export.fileCommandCeiling` — the per-commit file-command count, /// which is this budget divided by the minimum bytes one retained command /// can occupy. Four GiB is the default because that is the working set an /// import may claim on a commodity machine without the operator asking. max_export_payload_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, /// A marked blob retains at most three entries and a marked commit at most /// four, so this caps importer mark state at three million slots. max_retained_entries: usize = 3_000_000, /// Bounds bytes cloned into importer state before any clone is attempted. max_retained_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, }; /// A limit overrun names its numbers. `FileCommandLimitExceeded` alone tells an /// operator nothing actionable: what was observed, what the ceiling was, and /// what raises it is the entire diagnostic. Retained here and surfaced next to /// the error name, exactly as `ProductionGit.preflightOverran` does for /// preflight output. pub const LimitDiagnostic = struct { buffer: [512]u8 = undefined, len: usize = 0, fn retain(self: *LimitDiagnostic, comptime fmt: []const u8, args: anytype) void { const message = std.fmt.bufPrint(&self.buffer, fmt, args) catch { // Every argument is a bounded integer, well inside the buffer; a // truncating format would be a bug here, not a wire condition. // Still total: report without the numbers rather than drop it. const fallback = "import-git: a per-commit parser ceiling was exceeded; raise --max-export-payload-bytes\n"; @memcpy(self.buffer[0..fallback.len], fallback); self.len = fallback.len; return; }; self.len = message.len; } pub fn write(self: *const LimitDiagnostic, err: *std.Io.Writer) void { if (self.len == 0) return; err.writeAll(self.buffer[0..self.len]) catch {}; } }; pub const Dependencies = struct { git: ChildFactory, publication: PublicationHooks = .real_filesystem, limits: ImportLimits = .{}, }; pub const Request = struct { source: []const u8, destination_parent: std.Io.Dir, destination_name: []const u8, /// Null selects the analyzed §6.3 optimum the child factory measured /// during preflight; a value is the operator's explicit pin. submanifest_k: ?u4 = null, attribution: git_import.Attribution = .committer, default_branch: ?[]const u8 = null, quiet: bool = false, }; const PublicationBundle = struct { gpa: std.mem.Allocator, io: std.Io, parent: std.Io.Dir, temporary: std.Io.Dir, temporary_inode: std.Io.File.INode, temporary_name: []u8, owned: bool = true, published: bool = false, fn create(gpa: std.mem.Allocator, io: std.Io, parent: std.Io.Dir, destination: []const u8) !PublicationBundle { var writable_parent = try parent.openDir(io, ".", .{ .access_sub_paths = true, .follow_symlinks = false, }); errdefer writable_parent.close(io); const private_permissions: std.Io.Dir.Permissions = if (comptime builtin.os.tag == .windows or builtin.os.tag == .wasi) .default_dir else @enumFromInt(0o700); var attempt: usize = 0; while (attempt != 1024) : (attempt += 1) { var nonce: [16]u8 = undefined; try io.randomSecure(&nonce); const encoded_nonce = std.fmt.bytesToHex(nonce[0..], .lower); const temporary_name = try std.fmt.allocPrint(gpa, ".zit-import-tmp-{s}", .{encoded_nonce[0..]}); errdefer gpa.free(temporary_name); writable_parent.createDir(io, temporary_name, private_permissions) catch |err| switch (err) { error.PathAlreadyExists => { gpa.free(temporary_name); continue; }, else => return err, }; var temporary: ?std.Io.Dir = null; var temporary_inode: ?std.Io.File.INode = null; errdefer { if (temporary) |directory| directory.close(io); cleanupCreatedStaging(io, writable_parent, temporary_name, temporary_inode); } temporary = try writable_parent.openDir(io, temporary_name, .{ .access_sub_paths = true, .follow_symlinks = false, }); const temporary_stat = try temporary.?.stat(io); temporary_inode = temporary_stat.inode; if (temporary_stat.kind != .directory) return error.StagingEntryReplaced; // `destination` is intentionally part of this constructor's shape: // ownership is established only after the caller performed the // destination preflight, and publication has one atomic commit point. _ = destination; const result: PublicationBundle = .{ .gpa = gpa, .io = io, .parent = writable_parent, .temporary = temporary.?, .temporary_inode = temporary_stat.inode, .temporary_name = temporary_name, }; temporary = null; return result; } return error.NoTemporaryName; } fn abort(self: *PublicationBundle) void { if (!self.published and self.owned) { var current = self.openVerifiedStaging() catch null; if (current) |*directory| { const emptied = deleteOwnedContents(directory, self.io); directory.close(self.io); if (emptied) { var rechecked = self.openVerifiedStaging() catch null; if (rechecked) |*verified| { verified.close(self.io); self.parent.deleteDir(self.io, self.temporary_name) catch {}; } } } } self.temporary.close(self.io); self.parent.close(self.io); self.gpa.free(self.temporary_name); self.* = undefined; } fn publish(self: *PublicationBundle, destination: []const u8, hooks: PublicationHooks) !void { if (!self.owned) return error.StagingEntryReplaced; try hooks.check(.before_rename_no_replace); hooks.observeStaging(self.parent, self.temporary_name); if (comptime builtin.os.tag == .macos) { const from = try self.gpa.dupeZ(u8, self.temporary_name); defer self.gpa.free(from); const to = try self.gpa.dupeZ(u8, destination); defer self.gpa.free(to); var current = try self.openVerifiedStaging(); defer current.close(self.io); // Darwin's standard `renamePreserve` emulates no-replace with a // hard link, which cannot publish a directory. `RENAME_EXCL` // supplies the required atomic directory commit point instead. if (renameatx_np(self.parent.handle, from.ptr, self.parent.handle, to.ptr, 0x0000_0004) != 0) { return error.RenameNoReplaceFailed; } } else { var current = try self.openVerifiedStaging(); defer current.close(self.io); try self.parent.renamePreserve(self.temporary_name, self.parent, destination, self.io); } self.owned = false; try hooks.check(.after_rename_no_replace); var published = self.parent.openDir(self.io, destination, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch return error.PublishedDestinationReplaced; defer published.close(self.io); const published_stat = published.stat(self.io) catch return error.PublishedDestinationReplaced; if (published_stat.kind != .directory or published_stat.inode != self.temporary_inode) { return error.PublishedDestinationReplaced; } self.published = true; } fn openVerifiedStaging(self: *PublicationBundle) !std.Io.Dir { if (!self.owned) return error.StagingEntryReplaced; const held = self.temporary.stat(self.io) catch { self.owned = false; return error.StagingEntryReplaced; }; if (held.kind != .directory or held.inode != self.temporary_inode) { self.owned = false; return error.StagingEntryReplaced; } var current = self.parent.openDir(self.io, self.temporary_name, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch { self.owned = false; return error.StagingEntryReplaced; }; errdefer current.close(self.io); const entry = current.stat(self.io) catch { self.owned = false; return error.StagingEntryReplaced; }; if (entry.kind != .directory or entry.inode != self.temporary_inode) { self.owned = false; return error.StagingEntryReplaced; } return current; } fn cleanupCreatedStaging( io: std.Io, parent: std.Io.Dir, name: []const u8, expected_inode: ?std.Io.File.INode, ) void { const inode = expected_inode orelse return; var current = parent.openDir(io, name, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch return; defer current.close(io); const entry = current.stat(io) catch return; if (entry.kind != .directory or entry.inode != inode) return; if (!deleteOwnedContents(¤t, io)) return; parent.deleteDir(io, name) catch {}; } fn deleteOwnedContents(directory: *std.Io.Dir, io: std.Io) bool { var entries = directory.iterate(); while (entries.next(io) catch return false) |entry| { directory.deleteTree(io, entry.name) catch return false; } return true; } }; const LogTreeStore = struct { gpa: std.mem.Allocator, log: *logstore.LogStore, fn treeStore(self: *LogTreeStore) editor.TreeStore { return .{ .ctx = self, .load_fn = load, .put_fn = put, .put_batch_fn = putBatch }; } fn load(context: *anyopaque, allocator: std.mem.Allocator, name: []const u8) editor.StoreError!editor.LoadedBytes { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var digest: [32]u8 = undefined; const raw = rawHashName(name, &digest) orelse return error.MissingNode; const rid = self.log.lookup(raw) orelse return error.MissingNode; const bytes = self.log.content(allocator, rid) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; return .{ .bytes = bytes orelse return error.MissingNode, .allocator = allocator }; } fn put(context: *anyopaque, name: []const u8, bytes: []const u8) editor.StoreError!void { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var expected_buf: [64]u8 = undefined; if (!std.mem.eql(u8, name, artifact.artifactName(.sha1, bytes, &expected_buf))) return error.StorageFailure; var digest: [32]u8 = undefined; const raw = rawHashName(name, &digest) orelse return error.StorageFailure; _ = self.log.putArtifact(raw, bytes, null) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; } fn putBatch(context: *anyopaque, nodes: []const editor.StoreBatchNode) editor.StoreError!void { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var digests: std.ArrayList([32]u8) = .empty; defer digests.deinit(self.gpa); var items: std.ArrayList(logstore.LogStore.ArtifactBatchItem) = .empty; defer items.deinit(self.gpa); digests.ensureTotalCapacity(self.gpa, nodes.len) catch return error.OutOfMemory; items.ensureTotalCapacity(self.gpa, nodes.len) catch return error.OutOfMemory; for (nodes) |node| { var expected_buf: [64]u8 = undefined; if (!std.mem.eql(u8, node.name, artifact.artifactName(.sha1, node.bytes, &expected_buf))) { return error.StorageFailure; } var digest: [32]u8 = undefined; const raw = rawHashName(node.name, &digest) orelse return error.StorageFailure; digests.appendAssumeCapacity(digest); items.appendAssumeCapacity(.{ .hash = digests.items[digests.items.len - 1][0..raw.len], .bytes = node.bytes }); } self.log.putArtifactBatch(items.items) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; } }; /// Runs one all-or-nothing import. Until `renamePreserve` succeeds, every /// failure owns and removes only this invocation's temporary sibling. pub fn run( gpa: std.mem.Allocator, io: std.Io, request: Request, streams: cli.Streams, dependencies: Dependencies, ) cli.Exit { if (destinationExists(io, request.destination_parent, request.destination_name)) { streams.err.print("import-git: destination '{s}' already exists\n", .{request.destination_name}) catch {}; return .failure; } var pipes = dependencies.git.vtable.start_export(dependencies.git.context, request.source) catch |err| { if (dependencies.git.vtable.startup_diagnostics) |write_diagnostics| { write_diagnostics(dependencies.git.context, streams.err); } streams.err.print("import-git: cannot start git fast-export: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; var effective_request = request; if (effective_request.default_branch == null) effective_request.default_branch = pipes.detected_default_branch; var session_finished = false; defer if (!session_finished) { _ = finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch false; }; var publication = PublicationBundle.create(gpa, io, request.destination_parent, request.destination_name) catch |err| { streams.err.print("import-git: cannot create temporary destination: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; defer publication.abort(); var limit_diagnostic: LimitDiagnostic = .{}; importInto( gpa, io, effective_request, &publication, &pipes, streams, dependencies.git, &session_finished, dependencies.publication, dependencies.limits, &limit_diagnostic, ) catch |err| { if (err == error.DurabilityUncertain) { streams.err.print("import-git: destination '{s}' was published but parent-directory durability is uncertain\n", .{request.destination_name}) catch {}; } else { streams.err.print("import-git: import failed: {s}\n", .{@errorName(err)}) catch {}; limit_diagnostic.write(streams.err); } return .failure; }; streams.out.print("{s}\n", .{request.destination_name}) catch return .failure; if (!request.quiet) streams.err.print("import-git: imported '{s}'\n", .{request.source}) catch {}; return .ok; } fn importInto( gpa: std.mem.Allocator, io: std.Io, request: Request, publication: *PublicationBundle, pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, session_finished: *bool, hooks: PublicationHooks, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { const temporary = publication.temporary; try hooks.check(.create_store); var store_file = try temporary.createFile(io, ".zit", .{ .read = true, .exclusive = true }); var store_file_open = true; defer if (store_file_open) store_file.close(io); var file_storage = storage.FileStorage.init(io, store_file, 0); var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), @intFromEnum(artifact.HashAlgo.sha1)); var log_live = true; defer if (log_live) log.deinit(gpa); var tree_store = LogTreeStore{ .gpa = gpa, .log = &log }; // One pin for the whole import, resolved before the first tree: the // operator's override if given, else preflight's analyzed §6.3 optimum, // else `resolveOptions`' own default when neither is available. const pinned_k = (try git_import.resolveOptions(.{ .submanifest_k = request.submanifest_k orelse pipes.analyzed_submanifest_k, })).submanifest_k; if (!request.quiet and request.submanifest_k == null and pipes.analyzed_submanifest_k != null) { streams.err.print("import-git: analyzed sub-manifest k={d}\n", .{pinned_k}) catch {}; } var importer = try git_import.Importer.init(gpa, &log, tree_store.treeStore(), .{ .submanifest_k = pinned_k, .attribution = request.attribution, }); var importer_live = true; defer if (importer_live) importer.deinit(); var reader = fast_export.Reader.init(gpa, pipes.stdout, .{ .max_line = 1024 * 1024, .max_data = 1024 * 1024 * 1024, .max_path = 4096, .max_events = limits.max_export_events, .max_total_data = limits.max_export_payload_bytes, }); defer reader.deinit(); consumeExport(&reader, &importer, limits, diagnostic) catch |err| { const scripted_child_failed = pipes.exit_code_ref == null and terminalExitCode(pipes) != 0; const child_finished_before_cancellation = finishSession(pipes, streams, child, request.quiet, true) catch |lifecycle_err| { session_finished.* = true; return lifecycle_err; }; session_finished.* = true; if (scripted_child_failed or (child_finished_before_cancellation and terminalExitCode(pipes) != 0)) return error.GitExportFailed; return err; }; _ = finishSession(pipes, streams, child, request.quiet, false) catch |err| { session_finished.* = true; return err; }; session_finished.* = true; if (terminalExitCode(pipes) != 0) return error.GitExportFailed; var finalized = try importer.finalizeRefs(.{ .default_branch = request.default_branch }); defer finalized.deinit(gpa); for (finalized.controls) |control| try putSha1Artifact(&log, control); try log.putConfig(.{ .hash_policy = @intFromEnum(artifact.HashAlgo.sha1), .default_branch = finalized.primary_branch, .project_id = "", .submanifest_k = pinned_k, .enable_submanifests = true, }); try writeRefs(gpa, io, temporary, finalized.refs); try checkout.write(io, temporary, .{ // The imported store is complete, but no worktree files have been // materialized. Publishing the default tip here would make `update` // incorrectly treat that unmapped worktree as current. .version = "", .branch = finalized.primary_branch, .merge_parent = "", .special_op = "", .manifest_flags = "", }); var map = try temporary.createFile(io, ".zit-import-map.jsonl", .{ .read = true, .exclusive = true }); var map_open = true; defer if (map_open) map.close(io); try hooks.check(.before_flush_map); try map.writeStreamingAll(io, finalized.provenance_jsonl); try hooks.check(.after_flush_map); try hooks.check(.before_flush_store); try log.commit(); try hooks.check(.after_flush_store); try hooks.check(.before_sync_store); try store_file.sync(io); try hooks.check(.after_sync_store); try hooks.check(.before_sync_map); try map.sync(io); try hooks.check(.after_sync_map); // Freeze: neither store nor map remains writable after this point. try hooks.check(.before_close_store); importer.deinit(); importer_live = false; log.deinit(gpa); log_live = false; store_file.close(io); store_file_open = false; try hooks.check(.after_close_store); try hooks.check(.before_close_map); map.close(io); map_open = false; try hooks.check(.after_close_map); try verifyFrozen(gpa, io, temporary, hooks); try hooks.check(.before_sync_temporary_directory); try syncDirectory(temporary); try hooks.check(.after_sync_temporary_directory); try publication.publish(request.destination_name, hooks); // If this final durability boundary fails, the destination is intentionally // retained: its content was fully verified and atomically published. hooks.check(.sync_parent_after_publish) catch return error.DurabilityUncertain; syncDirectory(publication.parent) catch return error.DurabilityUncertain; } fn consumeExport( reader: *fast_export.Reader, importer: *git_import.Importer, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { var retained_bytes: u64 = 0; while (reader.next() catch |err| switch (err) { // Reader admits events and cumulative payload bytes before allocating // an event arena or payload, so its exact bound failures map directly. error.EventLimitExceeded => return error.ExportEventLimitExceeded, error.DataLimitExceeded => return error.ExportPayloadLimitExceeded, error.FileCommandLimitExceeded => { const overrun = reader.file_command_overrun.?; diagnostic.retain( "import-git: a commit exceeded the per-commit file-command ceiling: observed at least {d}, ceiling {d}, derived from the {d}-byte export payload budget at {d} minimum retained bytes per command; re-run with --max-export-payload-bytes above {d}\n", .{ overrun.observed_at_least, overrun.ceiling, overrun.payload_budget_bytes, overrun.min_bytes_per_command, overrun.payload_budget_bytes, }, ); return error.ExportFileCommandLimitExceeded; }, else => return err, }) |lease| { var event = lease; defer event.deinit(); try accountRetainedEvent(&retained_bytes, event.value, limits.max_retained_bytes); try ensureRetainedEntryCapacity(importer, event.value, limits.max_retained_entries); try importer.ingest(event.value); } } const sha1_artifact_name_bytes: u64 = std.crypto.hash.Sha1.digest_length * 2; fn accountRetainedEvent(total: *u64, event: fast_export.Event, limit: u64) !void { switch (event) { .blob => |blob| { try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(blob.original_oid != null), limit); if (blob.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); }, .commit => |commit| { try accountRetainedFixed(total, @as(usize, 3) + @intFromBool(commit.original_oid != null) + @intFromBool(commit.from != null), limit); if (commit.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); try accountRetainedCopies(total, commit.ref, 3, limit); try accountRetainedIdentity(total, commit.author, limit); try accountRetainedIdentity(total, commit.committer, limit); try accountRetainedSlice(total, commit.message, limit); try accountRetainedPaths(total, commit.commands, limit); }, .reset => |reset| { try accountRetainedCopies(total, reset.ref, 2, limit); if (reset.target != null) try accountRetainedFixed(total, 1, limit); }, .tag => |tag| { const source_len = std.math.add(u64, @as(u64, "refs/tags/".len), std.math.cast(u64, tag.name.len) orelse return error.ImporterRetainedByteLimitExceeded) catch return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, source_len, 4, limit); try accountRetainedSlice(total, tag.name, limit); if (tag.original_oid) |oid| try accountRetainedSlice(total, oid, limit); try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(tag.mark != null), limit); try accountRetainedIdentity(total, tag.tagger, limit); try accountRetainedSlice(total, tag.message, limit); }, .done => {}, } } /// Returns the exact retained-state allocation charge for one parsed event. /// The import loop and limit-boundary tests share this calculation so a new /// retained field cannot silently create a guessed budget. pub fn retainedByteCharge(event: fast_export.Event) !u64 { var total: u64 = 0; try accountRetainedEvent(&total, event, std.math.maxInt(u64)); return total; } fn accountRetainedIdentity(total: *u64, identity: fast_export.Identity, limit: u64) !void { try accountRetainedSlice(total, identity.name, limit); try accountRetainedSlice(total, identity.email, limit); } fn accountRetainedPaths(total: *u64, commands: []const fast_export.FileCommand, limit: u64) !void { for (commands) |command| switch (command) { .modify => |modify| try accountRetainedSlice(total, modify.path, limit), .delete => |path| try accountRetainedSlice(total, path, limit), .rename => |rename| { try accountRetainedSlice(total, rename.old, limit); try accountRetainedSlice(total, rename.new, limit); }, .copy => |copy| { try accountRetainedSlice(total, copy.old, limit); try accountRetainedSlice(total, copy.new, limit); }, .delete_all => {}, }; } fn accountRetainedFixed(total: *u64, copies: usize, limit: u64) !void { try accountRetainedLengthCopies(total, sha1_artifact_name_bytes, copies, limit); } fn accountRetainedCopies(total: *u64, bytes: []const u8, copies: usize, limit: u64) !void { const len = std.math.cast(u64, bytes.len) orelse return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, len, copies, limit); } fn accountRetainedSlice(total: *u64, bytes: []const u8, limit: u64) !void { try accountRetainedCopies(total, bytes, 1, limit); } fn accountRetainedLengthCopies(total: *u64, len: u64, copies: usize, limit: u64) !void { var remaining = copies; while (remaining != 0) : (remaining -= 1) { if (total.* > limit or len > limit - total.*) return error.ImporterRetainedByteLimitExceeded; total.* += len; } } fn ensureRetainedEntryCapacity(importer: *const git_import.Importer, event: fast_export.Event, limit: usize) !void { const retained = importer.retainedStateEntries(); if (retained > limit) return error.ImporterRetainedEntryLimitExceeded; const growth: usize = switch (event) { .blob => 3, .commit => 4, .reset => 1, .tag => 3, .done => 0, }; if (growth > limit - retained) return error.ImporterRetainedEntryLimitExceeded; } fn verifyFrozen(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, hooks: PublicationHooks) !void { try hooks.check(.verify_store); var store_file = try temporary.openFile(io, ".zit", .{ .mode = .read_only }); defer store_file.close(io); var file_storage = storage.FileStorage.init(io, store_file, (try store_file.stat(io)).size); var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), @intFromEnum(artifact.HashAlgo.sha1)); defer log.deinit(gpa); var checkout_buf: [checkout.max_ck_payload + 1]u8 = undefined; const state = try checkout.read(io, temporary, &checkout_buf) orelse return error.MissingCheckout; // Imports deliberately publish an unborn checkout: the repository is // discoverable, while the first update alone materializes its default tip. if (state.branch.len == 0) return error.InvalidCheckout; try hooks.check(.verify_map); var map = try temporary.openFile(io, ".zit-import-map.jsonl", .{ .mode = .read_only }); map.close(io); } fn syncDirectory(directory: std.Io.Dir) !void { if (std.c.fsync(directory.handle) != 0) return error.DirectorySyncFailed; } fn putSha1Artifact(log: *logstore.LogStore, bytes: []const u8) !void { var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined; std.crypto.hash.Sha1.hash(bytes, &digest, .{}); _ = try log.putArtifact(&digest, bytes, null); } fn writeRefs(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, refs: []const git_import.FinalizedRef) !void { try temporary.createDir(io, "refs", .default_dir); // Finalized refs are constrained to these canonical namespaces. Create // them before any leaf so a single-component branch works without a // parent-directory race; nested branch components remain handled below. try temporary.createDir(io, "refs/sym-heads", .default_dir); try temporary.createDir(io, "refs/sym-tags", .default_dir); for (refs) |mapping| { const path = try std.fmt.allocPrint(gpa, "refs/{s}", .{mapping.canonical}); defer gpa.free(path); if (std.mem.lastIndexOfScalar(u8, path, '/')) |slash| try temporary.createDirPath(io, path[0..slash]); var file = try temporary.createFile(io, path, .{ .read = true, .exclusive = true }); defer file.close(io); try file.writeStreamingAll(io, mapping.target); try file.writeStreamingAll(io, "\n"); try file.sync(io); try syncRefAncestors(io, temporary, path); } var heads = try temporary.openDir(io, "refs/sym-heads", .{}); defer heads.close(io); try syncDirectory(heads); var tags = try temporary.openDir(io, "refs/sym-tags", .{}); defer tags.close(io); try syncDirectory(tags); var refs_dir = try temporary.openDir(io, "refs", .{}); defer refs_dir.close(io); try syncDirectory(refs_dir); } fn syncRefAncestors(io: std.Io, temporary: std.Io.Dir, path: []const u8) !void { var end = std.mem.lastIndexOfScalar(u8, path, '/') orelse return; while (true) { { var directory = try temporary.openDir(io, path[0..end], .{}); defer directory.close(io); try syncDirectory(directory); } end = std.mem.lastIndexOfScalar(u8, path[0..end], '/') orelse return; } } fn rawHashName(name: []const u8, buffer: *[32]u8) ?[]const u8 { if (name.len != 40 and name.len != 64) return null; const raw = buffer[0 .. name.len / 2]; _ = std.fmt.hexToBytes(raw, name) catch return null; return raw; } fn finishSession( pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, quiet: bool, abort: bool, ) !bool { var lifecycle_error: ?anyerror = null; // EOF is observed by the importing thread before it starts cancellation. // A child that is a zombie can still accept `kill`, so this observation is // the race-free completion evidence; `reap` below supplies its actual code. var child_finished_before_cancellation = abort and if (pipes.owner) |owner| owner.stdout_reached_eof else false; if (abort) { child.vtable.terminate(child.context) catch |err| { if (err == error.ChildAlreadyExited) { child_finished_before_cancellation = true; } else { lifecycle_error = err; } }; } child.vtable.drain_stderr(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; child.vtable.reap(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; const enabled = abort or terminalExitCode(pipes) != 0 or !quiet; if (pipes.diagnostics) |write_diagnostics| { write_diagnostics(pipes, streams.err, enabled); } else { copyDiagnostics(&pipes.stderr, streams.err, enabled); } if (lifecycle_error) |err| return err; return child_finished_before_cancellation; } fn terminalExitCode(pipes: *const ExportPipes) u8 { return if (pipes.exit_code_ref) |code| code.* else pipes.exit_code; } fn destinationExists(io: std.Io, parent: std.Io.Dir, name: []const u8) bool { var directory = parent.openDir(io, name, .{}) catch { var file = parent.openFile(io, name, .{}) catch return false; file.close(io); return true; }; directory.close(io); return true; } fn copyDiagnostics(reader: *std.Io.Reader, err: *std.Io.Writer, enabled: bool) void { if (enabled) _ = reader.streamRemaining(err) catch {} else _ = reader.discard(.unlimited) catch {}; } pub const CommandScript = struct { pub const Preflight = enum { branch_head, detached_head, git_missing, not_repository, missing_objects, shallow, partial_promisor }; pub const StagingReplacement = enum { directory, symlink }; pub const Config = struct { preflight: Preflight = .branch_head, fast_export: []const u8 = "", stderr_bytes: usize = 0, diagnostic_ring_limit: usize = 4096, cancel_after_events: ?usize = null, competing_destination_at_publish: bool = false, export_exit: u8 = 0, replace_staging_at_publish: ?StagingReplacement = null, replace_destination_after_rename: bool = false, limits: ImportLimits = .{}, }; pub const Publication = struct { store_reopened_for_verify: bool = false, map_reopened_for_verify: bool = false, verify_before_publish: bool = false, }; pub const Environment = struct { pub fn get(_: Environment, name: []const u8) ?[]const u8 { if (std.mem.eql(u8, name, "GIT_TERMINAL_PROMPT")) return "0"; if (std.mem.eql(u8, name, "GIT_NO_LAZY_FETCH")) return "1"; return null; } }; preflight: Preflight, fast_export: []const u8, stderr_bytes: usize, diagnostic_ring_limit: usize, cancel_after_events: ?usize, competing_destination_at_publish: bool, export_exit: u8, replace_staging_at_publish: ?StagingReplacement, replace_destination_after_rename: bool, limits: ImportLimits, default_branch: ?[]const u8 = null, detached_head_recorded: bool = false, export_starts: usize = 0, preflight_starts: usize = 0, lifecycle: ChildLifecycle = .{}, stderr_drained_bytes: usize = 0, environment: Environment = .{}, publication: Publication = .{}, competitor_destination_intact: bool = false, temporary_sibling_left: bool = false, staging_replacement_observed: bool = false, staging_replacement_name: [64]u8 = undefined, staging_replacement_name_len: usize = 0, destination_parent: ?std.Io.Dir = null, destination_name: []const u8 = "", io: ?std.Io = null, pub fn init(config: Config) CommandScript { return .{ .preflight = config.preflight, .fast_export = config.fast_export, .stderr_bytes = config.stderr_bytes, .diagnostic_ring_limit = config.diagnostic_ring_limit, .cancel_after_events = config.cancel_after_events, .competing_destination_at_publish = config.competing_destination_at_publish, .export_exit = config.export_exit, .replace_staging_at_publish = config.replace_staging_at_publish, .replace_destination_after_rename = config.replace_destination_after_rename, .limits = config.limits, }; } fn dependencies(self: *CommandScript, fault: PublicationHooks) Dependencies { var publication = fault; publication.observer = .{ .context = self, .notify = publicationEvent }; publication.staging_observer = .{ .context = self, .replace = replaceStaging }; return .{ .git = .{ .context = self, .vtable = &.{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, } }, .publication = publication, .limits = self.limits, }; } fn startExport(context: *anyopaque, _: []const u8) !ExportPipes { const self: *CommandScript = @ptrCast(@alignCast(context)); self.export_starts += 1; self.lifecycle.launched += 1; const export_bytes = if (self.cancel_after_events != null) "feature done\nblob\nmark :1\ndata 1\nx\ninvalid\n" else self.fast_export; return .{ .stdout = .fixed(export_bytes), .stderr = .fixed("scripted diagnostic\n"), .exit_code = self.export_exit }; } fn terminate(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.terminated += 1; } fn drainStderr(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.stderr_drained += 1; self.stderr_drained_bytes = self.stderr_bytes; } fn reap(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.reaped += 1; } fn publicationEvent(context: *anyopaque, point: PublicationHooks.FailurePoint) void { const self: *CommandScript = @ptrCast(@alignCast(context)); switch (point) { .verify_store => self.publication.store_reopened_for_verify = true, .verify_map => self.publication.map_reopened_for_verify = true, .before_rename_no_replace => { self.publication.verify_before_publish = self.publication.store_reopened_for_verify and self.publication.map_reopened_for_verify; if (self.competing_destination_at_publish) { const parent = self.destination_parent orelse return; parent.createDir(self.io orelse return, self.destination_name, .default_dir) catch return; self.competitor_destination_intact = true; } }, .after_rename_no_replace => if (self.replace_destination_after_rename) { const parent = self.destination_parent orelse return; const io = self.io orelse return; parent.deleteTree(io, self.destination_name) catch return; parent.createDir(io, self.destination_name, .default_dir) catch return; var replacement = parent.openDir(io, self.destination_name, .{ .follow_symlinks = false }) catch return; defer replacement.close(io); var marker = replacement.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; self.competitor_destination_intact = true; }, else => {}, } } fn replaceStaging(context: *anyopaque, parent: std.Io.Dir, name: []const u8) void { const self: *CommandScript = @ptrCast(@alignCast(context)); const replacement = self.replace_staging_at_publish orelse return; const io = self.io orelse return; if (name.len > self.staging_replacement_name.len) return; parent.deleteTree(io, name) catch return; switch (replacement) { .directory => { parent.createDir(io, name, .default_dir) catch return; var directory = parent.openDir(io, name, .{ .follow_symlinks = false }) catch return; defer directory.close(io); var marker = directory.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; }, .symlink => parent.symLink(io, "outside", name, .{}) catch return, } @memcpy(self.staging_replacement_name[0..name.len], name); self.staging_replacement_name_len = name.len; self.staging_replacement_observed = true; } pub fn stagingReplacementName(self: *const CommandScript) []const u8 { return self.staging_replacement_name[0..self.staging_replacement_name_len]; } /// Derived from `exportArgv` so this fake cannot drift from the command /// the importer actually runs. Shows the default signature policy; the /// opt-out's effect is covered directly against `exportArgv`. pub fn exportCommandLine(_: *const CommandScript) []const u8 { return comptime line: { const argv = exportArgv("fixture.git", false); var joined: []const u8 = argv[0]; for (1..argv.len) |index| joined = joined ++ " " ++ argv[index]; break :line joined; }; } }; pub const ScriptResult = struct { code: cli.Exit, out: []const u8, err: []const u8, lifecycle: ChildLifecycle, retained_diagnostic: []const u8, published_durability_uncertain: bool = false, }; pub fn runScripted( gpa: std.mem.Allocator, io: std.Io, request: Request, script: *CommandScript, hooks: PublicationHooks, ) ScriptResult { var out_storage: [4096]u8 = undefined; var err_storage: [4096]u8 = undefined; var out = std.Io.Writer.fixed(&out_storage); var err = std.Io.Writer.fixed(&err_storage); const streams: cli.Streams = .{ .out = &out, .err = &err }; script.preflight_starts += 1; if (script.preflight == .git_missing or script.preflight == .not_repository or script.preflight == .missing_objects or script.preflight == .shallow or script.preflight == .partial_promisor or (script.preflight == .detached_head and script.default_branch == null)) { script.lifecycle.reaped += 1; return .{ .code = .failure, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "preflight rejected" }; } if (script.preflight == .detached_head) script.detached_head_recorded = true; script.destination_parent = request.destination_parent; script.destination_name = request.destination_name; script.io = io; var effective_request = request; effective_request.default_branch = script.default_branch; const code = run(gpa, io, effective_request, streams, script.dependencies(hooks)); if (code == .failure and hooks.fail_at == .sync_parent_after_publish) { return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "durability uncertain", .published_durability_uncertain = true }; } return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "scripted diagnostic" }; } pub fn dispatchForTest(args: []const []const u8) ScriptResult { if (args.len == 1 and std.mem.eql(u8, args[0], "--help")) { return .{ .code = .ok, .out = "zit import-git SOURCE DEST [--submanifest-k K] [--attribution committer|author] [--default-branch NAME] [--quiet]\n", .err = "", .lifecycle = .{}, .retained_diagnostic = "" }; } _ = cli.parseImportGitArgs(args) catch { return .{ .code = .usage, .out = "", .err = "invalid import-git arguments\n", .lifecycle = .{}, .retained_diagnostic = "" }; }; return .{ .code = .failure, .out = "", .err = "test dispatch requires scripted dependencies\n", .lifecycle = .{}, .retained_diagnostic = "" }; }