Artifact e93dc056ab751189bf4cf87e52337dacb863c95a106752da80736f46ddddd893:
- File
src/git_import_cli.zig
— part of check-in
[8ae7041c2c]
at
2026-07-31 20:55:36
on branch sub-manifest
— import-git: pack the refs, and split the budget they shared with payload
zit now imports sqlite at defaults: 36,416 commits, 2,109 refs, 44m35s, no flags. Two independent walls stood in front of that.
Refs (issue 21). One file per ref made a ref's name a path, so the filesystem got a vote on which repositories are representable. sqlite has refs/remotes/origin/EQP-improvements and .../eqp-improvements at different commits -- legal, distinct git refs -- and on APFS the second landed on the first as a bare PathAlreadyExists, raised during publication after the whole import had run. Case was only the reachable instance; path length, reserved device names and each platform's own Unicode fold table are the same bug in other clothes.
They are one sorted file now, "refs", whose lines pair a canonical name with its target. Nothing reads it back -- "refs" is a reserved root the working-tree scan skips, and branches resolve through repostate's sym- tags -- so it is interop output and its job is to be legible. Sorting makes it byte-identical for a given ref set, so two imports of one repository can be compared.
This supersedes the ASCII-fold refusal from the previous commit. That was the right interim while a ref was a filename; with one file the collision is representable, so refusing it would be gratuitous. What survives is the genuine collision -- distinct sources normalizing to a byte-identical canonical name, refs/heads/./release against refs/heads/release -- which no layout can hold, and which now names both sources instead of only failing.
Budget (issue 20). The 4 GiB payload default refused sqlite's 12.2 GiB stream, and could not simply be raised: fileCommandCeiling divided the same number, so a larger allowance for a whole history was also a larger allowance for one commit's retained command list. Those bound different things and only one is memory -- payload streams and is released per event (sqlite's import held 21 MiB), while a commit's command list is held whole. The arena has max_command_arena_bytes of its own now and keeps the old 4 GiB; the cumulative total goes to 1 TiB.
The file-command ceiling is therefore structural: about 104 million commands in one commit, and no flag raises it. Its diagnostic says so rather than pointing at a knob that no longer moves it. The help text claimed the payload flag fixed both; it only ever fixed one. (user: dmestas, size: 77581) [annotate] [blame] [check-ins using]
const std = @import("std"); const builtin = @import("builtin"); pub const cli = @import("cli.zig"); const zit = @import("zit"); const artifact = zit.artifact; const checkout = zit.checkout; const fast_export = zit.git_fast_export; const git_import = zit.git_import; const logstore = zit.logstore; const storage = zit.storage; const editor = zit.submanifest_editor; extern "c" fn renameatx_np( from_fd: c_int, from: [*:0]const u8, to_fd: c_int, to: [*:0]const u8, flags: c_uint, ) c_int; pub const ChildLifecycle = struct { launched: usize = 0, terminated: usize = 0, stderr_drained: usize = 0, reaped: usize = 0, }; /// The child-facing half of the import transaction. The caller owns the /// implementation and its backing buffers for the full call to `run`. pub const ChildFactory = struct { context: *anyopaque, vtable: *const VTable, pub const VTable = struct { start_export: *const fn (context: *anyopaque, source: []const u8) anyerror!ExportPipes, terminate: *const fn (context: *anyopaque) anyerror!void, drain_stderr: *const fn (context: *anyopaque) anyerror!void, reap: *const fn (context: *anyopaque) anyerror!void, startup_diagnostics: ?*const fn (context: *anyopaque, err: *std.Io.Writer) void = null, }; }; pub const ExportPipes = struct { stdout: std.Io.Reader, stderr: std.Io.Reader, /// Scripted sessions supply their terminal status here. The OS binding gets /// its status during reap; an import succeeds only after the stream reaches /// `done` and EOF, so a nonzero scripted result is still an import failure. exit_code: u8 = 0, /// OS-backed sessions publish the wait status here during `reap`; scripted /// sessions use `exit_code` directly. exit_code_ref: ?*const u8 = null, owner: ?*ProductionGit = null, diagnostics: ?*const fn (pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void = null, detected_default_branch: ?[]const u8 = null, /// draft-fossil-submanifests-00 §6.3's optimum for the source's own name /// set, measured during preflight. Null when the source could not be /// listed or cannot partition; the caller then keeps its own default. analyzed_submanifest_k: ?u4 = null, }; /// The `git fast-export` command line the importer runs, as a value. /// /// Pure and public so the one policy decision it encodes — what happens to a /// signature Fossil has no card for — is testable without spawning a process. /// `refuse_signed` moves exactly the two signature modes and nothing else. pub fn exportArgv(source: []const u8, refuse_signed: bool) [10][]const u8 { return .{ "git", "-C", source, "fast-export", "--all", "--show-original-ids", "--mark-tags", if (refuse_signed) "--signed-commits=abort" else "--signed-commits=strip", if (refuse_signed) "--signed-tags=abort" else "--signed-tags=strip", "--use-done-feature", }; } /// OS-backed production dependency. It owns both pipe buffers through the /// import call; the `ExportPipes` readers retain a pointer to this value. pub const ProductionGit = struct { pub const StartupFailure = enum { after_spawn, after_stderr_handoff, after_stderr_thread }; /// Keep room in fixed CLI writers for the authoritative import diagnostic. const cli_diagnostic_reserve_bytes = 256; /// Bytes of output one preflight check may produce. Every check asks a /// bounded question — one ref, one boolean, one config value, or (for /// `fsck --connectivity-only --no-dangling`) nothing at all when the /// repository is intact — so an overrun means a check's shape was /// misjudged, never that the source is merely large. const preflight_output_limit_bytes = 8 * 1024; gpa: std.mem.Allocator, io: std.Io, child: ?std.process.Child = null, exit_code: u8 = 1, stdout_buffer: [8192]u8 = undefined, stderr_buffer: [8192]u8 = undefined, reaped: bool = false, reap_calls: usize = 0, /// Captured by the stdout reader before any parser-error cleanup can /// terminate the child. This distinguishes an independently completed /// truncated export from cancellation closing the pipe. stdout_reached_eof: bool = false, default_branch: ?[]const u8 = null, owns_default_branch: bool = false, analyzed_submanifest_k: ?u4 = null, /// The operator's explicit opt-out from the default signature policy. /// False (the default) strips; true restores the refusal. refuse_signed: bool = false, stderr_thread: ?std.Thread = null, /// Moved out of `child` before its reader thread starts, so draining never /// races cleanup or a platform-specific process handle implementation. stderr_file: ?std.Io.File = null, stderr_ring: [4096]u8 = undefined, stderr_ring_start: usize = 0, stderr_ring_len: usize = 0, stderr_bytes_drained: usize = 0, stderr_lock: std.Io.Mutex = .init, test_argv: ?[]const []const u8 = null, test_startup_failure: ?StartupFailure = null, pub fn init(gpa: std.mem.Allocator, io: std.Io) ProductionGit { return .{ .gpa = gpa, .io = io }; } /// Test-only process constructor: uses the identical spawn, stream, /// concurrent-diagnostic, termination, and reap path as `git fast-export`. pub fn initForTest(gpa: std.mem.Allocator, io: std.Io, argv: []const []const u8) ProductionGit { return .{ .gpa = gpa, .io = io, .test_argv = argv }; } fn childEnvironment(self: *ProductionGit) !std.process.Environ.Map { var count: usize = 0; while (std.c.environ[count] != null) : (count += 1) {} const inherited: [:null]const ?[*:0]const u8 = @ptrCast(std.c.environ[0..count :null]); var env = try std.process.Environ.createMap(.{ .block = .{ .slice = inherited } }, self.gpa); errdefer env.deinit(); try env.put("GIT_TERMINAL_PROMPT", "0"); try env.put("GIT_NO_LAZY_FETCH", "1"); return env; } pub fn setDefaultBranch(self: *ProductionGit, default_branch: ?[]const u8) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.default_branch = default_branch; self.owns_default_branch = false; } /// Set before `run`, alongside `setDefaultBranch`: the export argv is /// built inside `start_export`, which the factory vtable does not let the /// caller parameterize. pub fn setRefuseSigned(self: *ProductionGit, refuse: bool) void { self.refuse_signed = refuse; } pub fn deinit(self: *ProductionGit) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.* = undefined; } pub fn dependencies(self: *ProductionGit, publication: PublicationHooks) Dependencies { return .{ .git = .{ .context = self, .vtable = &vtable }, .publication = publication, }; } const vtable: ChildFactory.VTable = .{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, .startup_diagnostics = writeStartupDiagnostics, }; fn startExport(context: *anyopaque, source: []const u8) !ExportPipes { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.test_argv) |argv| return self.startSession(argv); try self.preflight(source); // Fossil has no signature card, so a signature cannot survive this // import in any representation. `strip` accepts that loss and is the // default — refusing by default only withheld the import, never the // signature. `abort` is the operator's explicit opt-out. There is no // honest third option: a verbatim pass would have to invent somewhere // to keep bytes the format cannot hold. const argv = exportArgv(source, self.refuse_signed); return self.startSession(&argv); } fn startSession(self: *ProductionGit, argv: []const []const u8) !ExportPipes { var env = try self.childEnvironment(); defer env.deinit(); self.reaped = false; self.exit_code = 1; self.reap_calls = 0; self.stdout_reached_eof = false; self.stderr_file = null; self.child = try std.process.spawn(self.io, .{ .argv = argv, .environ_map = &env, .stdin = .ignore, .stdout = .pipe, .stderr = .pipe, }); errdefer self.cleanupFailedStart(); try self.failStartForTest(.after_spawn); const child = &(self.child orelse unreachable); const stderr = child.stderr orelse return error.MissingStderr; child.stderr = null; self.stderr_file = stderr; try self.failStartForTest(.after_stderr_handoff); self.stderr_ring_start = 0; self.stderr_bytes_drained = 0; self.stderr_ring_len = 0; self.stderr_thread = try std.Thread.spawn(.{}, drainStderrThread, .{self}); try self.failStartForTest(.after_stderr_thread); return .{ .exit_code_ref = &self.exit_code, .detected_default_branch = self.default_branch, .analyzed_submanifest_k = self.analyzed_submanifest_k, .owner = self, .stdout = .{ .vtable = &stdout_reader_vtable, .buffer = &self.stdout_buffer, .seek = 0, .end = 0, }, .stderr = .{ .vtable = &stderr_reader_vtable, .buffer = &self.stderr_buffer, .seek = 0, .end = 0, }, .diagnostics = writeDiagnostics, }; } fn failStartForTest(self: *const ProductionGit, point: StartupFailure) !void { if (self.test_startup_failure == point) return error.InjectedStartFailure; } fn cleanupFailedStart(self: *ProductionGit) void { if (self.stderr_thread == null) { if (self.stderr_file) |file| file.close(self.io); self.stderr_file = null; } if (self.child) |*child| { if (child.stderr) |file| file.close(self.io); child.stderr = null; } terminate(@ptrCast(self)) catch {}; reap(@ptrCast(self)) catch {}; } fn preflight(self: *ProductionGit, source: []const u8) !void { const inside = try self.query("rev-parse --is-inside-work-tree", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-inside-work-tree" }); defer self.gpa.free(inside); if (!std.mem.eql(u8, std.mem.trim(u8, inside, " \t\r\n"), "true")) return error.NotGitRepository; const head = try self.query("rev-parse HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "HEAD" }); defer self.gpa.free(head); if (std.mem.trim(u8, head, " \t\r\n").len == 0) return error.MissingHead; const object_format = try self.query("rev-parse --show-object-format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" }); defer self.gpa.free(object_format); if (!std.mem.eql(u8, std.mem.trim(u8, object_format, " \t\r\n"), "sha1")) return error.UnsupportedObjectFormat; const shallow = try self.query("rev-parse --is-shallow-repository", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-shallow-repository" }); defer self.gpa.free(shallow); if (std.mem.eql(u8, std.mem.trim(u8, shallow, " \t\r\n"), "true")) return error.ShallowRepository; // `--no-dangling` is required, not tidiness. A dangling object is // unreachable but intact, so it says nothing about the connectivity // question this check exists to answer, yet git reports one // informational line per object on stdout: penpot emitted 14,763 bytes // and a healthy repository failed its import on the output limit. // A missing object is still reported, and still makes this check exit // non-zero. const connectivity = self.query("fsck --connectivity-only", &.{ "git", "-C", source, "--no-lazy-fetch", "fsck", "--connectivity-only", "--no-dangling", "--no-reflogs" }) catch |err| switch (err) { // The exit status is this check's verdict; its stdout is a // per-problem report — with `--no-dangling` an intact repository // writes none of it. So an overrun here is not an unreadable // answer, it is a repository with more broken links than the // report can hold. The named-overrun diagnostic is already // retained, so the operator sees both which check overran and the // verdict it produced. error.GitPreflightFailed, error.PreflightOutputTooLarge => return error.MissingObjects, else => return err, }; defer self.gpa.free(connectivity); const symbolic = try self.optionalQuery("symbolic-ref HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "symbolic-ref", "--quiet", "HEAD" }); if (symbolic) |branch_output| { defer self.gpa.free(branch_output); if (self.default_branch == null) { const branch_ref = std.mem.trim(u8, branch_output, " \t\r\n"); const prefix = "refs/heads/"; if (!std.mem.startsWith(u8, branch_ref, prefix) or branch_ref.len == prefix.len) return error.InvalidHead; self.default_branch = try self.gpa.dupe(u8, branch_ref[prefix.len..]); self.owns_default_branch = true; } } else if (self.default_branch == null) { return error.DetachedHead; } const partial_clone = try self.optionalQuery("config --get extensions.partialClone", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get", "extensions.partialClone" }); if (partial_clone) |configured_remote| { defer self.gpa.free(configured_remote); if (std.mem.trim(u8, configured_remote, " \t\r\n").len != 0) return error.PartialRepository; } const promisors = try self.optionalQuery("config --get-regexp remote.*.promisor", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get-regexp", "^remote\\..*\\.promisor$" }); if (promisors) |configured_promisors| { defer self.gpa.free(configured_promisors); if (try containsGitTrue(configured_promisors)) return error.PartialRepository; } self.analyzed_submanifest_k = try self.analyzeSubmanifestPin(source); } /// `git ls-tree` output the §6.3 analysis is willing to read. A monorepo /// at the ceiling lists several million paths; past it the pin falls back /// to the default rather than failing an import over an advisory /// measurement. const ls_tree_limit_bytes = 64 << 20; /// Measure §6.3's optimum for the source before a single object is read. /// The tip tree, not the root commit: an imported history's first commit /// is routinely a lone README, and pinning k from it would misconfigure /// every later commit in the repository. /// /// Advisory throughout — every failure path returns null and leaves the /// caller's default standing. `-z --name-only` is required, not stylistic: /// without it git quotes unusual bytes and the names would no longer be /// the ones §6.1's partition predicate hashes. The listing includes paths /// a check-in's F-set would exclude (§4.6.3 reserved names) and any path /// `update` would reject; both are rare enough not to move an optimum that /// tracks half the base-2 logarithm of the count. fn analyzeSubmanifestPin(self: *ProductionGit, source: []const u8) !?u4 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = &.{ "git", "-C", source, "--no-lazy-fetch", "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD" }, .environ_map = &env, .stdout_limit = .limited(ls_tree_limit_bytes), .stderr_limit = .limited(8 * 1024), }) catch return null; defer self.gpa.free(result.stderr); defer self.gpa.free(result.stdout); switch (result.term) { .exited => |code| if (code != 0) return null, else => return null, } var names: std.ArrayList([]const u8) = .empty; defer names.deinit(self.gpa); var listing = std.mem.splitScalar(u8, result.stdout, 0); while (listing.next()) |name| { if (name.len == 0) continue; try names.append(self.gpa, name); } return git_import.analyzeSubmanifestK(self.gpa, names.items, .sha1); } fn containsGitTrue(configured: []const u8) !bool { var lines = std.mem.splitScalar(u8, configured, '\n'); while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, " \t\r"); if (trimmed.len == 0) continue; const value = trimmed[(std.mem.lastIndexOfAny(u8, trimmed, " \t") orelse return error.InvalidGitBoolean) + 1 ..]; if (try isGitTrue(value)) return true; } return false; } fn isGitTrue(value: []const u8) !bool { if (std.mem.eql(u8, value, "1") or std.ascii.eqlIgnoreCase(value, "true") or std.ascii.eqlIgnoreCase(value, "yes") or std.ascii.eqlIgnoreCase(value, "on")) return true; if (std.mem.eql(u8, value, "0") or std.ascii.eqlIgnoreCase(value, "false") or std.ascii.eqlIgnoreCase(value, "no") or std.ascii.eqlIgnoreCase(value, "off")) return false; return error.InvalidGitBoolean; } /// A limit overrun names the check that overran. `StreamTooLong` alone /// tells an operator nothing actionable: which of the preflight children /// produced too much output is the entire diagnostic. fn preflightOverran(self: *ProductionGit, check: []const u8) error{PreflightOutputTooLarge} { var buffer: [256]u8 = undefined; const message = std.fmt.bufPrint( &buffer, "import-git: preflight check 'git {s}' produced more than {d} bytes of output\n", .{ check, preflight_output_limit_bytes }, ) catch fallback: { // Every `check` is a short call-site literal, well inside the // buffer; a truncating format would be a bug here, not a wire // condition. Still total: report without the name rather than // dropping the diagnostic. break :fallback "import-git: a preflight check exceeded its output limit\n"; }; self.retainDiagnostics(message); return error.PreflightOutputTooLarge; } fn optionalQuery(self: *ProductionGit, check: []const u8, argv: []const []const u8) !?[]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| switch (code) { 0 => return result.stdout, 1 => { self.gpa.free(result.stdout); return null; }, else => {}, }, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn query(self: *ProductionGit, check: []const u8, argv: []const []const u8) ![]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| if (code == 0) return result.stdout, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn terminate(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); const child = &(self.child orelse return); switch (builtin.os.tag) { .windows => { const handle = child.id orelse return; switch (std.os.windows.ntdll.NtTerminateProcess(handle, @enumFromInt(1))) { .SUCCESS => {}, .PROCESS_IS_TERMINATING => return error.ChildAlreadyExited, else => return error.TerminateFailed, } }, .wasi => return error.UnsupportedTermination, else => { const pid = child.id orelse return; std.posix.kill(pid, .KILL) catch |err| switch (err) { error.ProcessNotFound => return error.ChildAlreadyExited, else => return err, }; }, } self.exit_code = 1; } fn drainStderr(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.joinStderr(); } fn joinStderr(self: *ProductionGit) void { if (self.stderr_thread) |thread| { thread.join(); self.stderr_thread = null; } } fn retainDiagnostics(self: *ProductionGit, bytes: []const u8) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); for (bytes) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } } fn writeRetainedDiagnostics(self: *ProductionGit, err: *std.Io.Writer) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); const retained_len = @min( self.stderr_ring_len, self.stderr_ring.len - cli_diagnostic_reserve_bytes, ); var index = (self.stderr_ring_start + self.stderr_ring_len - retained_len) % self.stderr_ring.len; for (0..retained_len) |_| { err.writeByte(self.stderr_ring[index]) catch return; index = (index + 1) % self.stderr_ring.len; } } fn writeStartupDiagnostics(context: *anyopaque, err: *std.Io.Writer) void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.writeRetainedDiagnostics(err); } fn drainStderrThread(self: *ProductionGit) void { const file = self.stderr_file orelse return; defer { file.close(self.io); self.stderr_file = null; } var buffer: [4096]u8 = undefined; while (true) { const n = file.readStreaming(self.io, &.{buffer[0..]}) catch break; if (n == 0) break; self.stderr_lock.lockUncancelable(self.io); for (buffer[0..n]) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } self.stderr_bytes_drained += n; self.stderr_lock.unlock(self.io); } } fn writeDiagnostics(pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void { const self = pipes.owner orelse return; if (!enabled) return; self.writeRetainedDiagnostics(err); } fn reap(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.reaped) return; self.reap_calls += 1; // No platform may reap or clear `child` until the independently owned // stderr descriptor is fully drained and its thread has exited. self.joinStderr(); const child = &(self.child orelse return); if (child.stdout) |stdout| { stdout.close(self.io); child.stdout = null; } const term = child.wait(self.io) catch |err| { child.kill(self.io); _ = child.wait(self.io) catch {}; self.exit_code = 1; self.reaped = true; self.child = null; return err; }; self.exit_code = switch (term) { .exited => |code| code, else => 1, }; self.reaped = true; self.child = null; } const stdout_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStdout }; const stderr_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStderr }; fn streamStdout(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stdout_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, true); } fn streamStderr(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stderr_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, false); } fn streamPipe( self: *ProductionGit, _: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit, comptime stdout: bool, ) std.Io.Reader.StreamError!usize { const child = &(self.child orelse return error.ReadFailed); const file = if (stdout) child.stdout orelse return error.ReadFailed else child.stderr orelse return error.ReadFailed; var temporary: [8192]u8 = undefined; const requested = if (limit.toInt()) |value| @min(value, temporary.len) else temporary.len; if (requested == 0) return 0; const n = file.readStreaming(self.io, &.{temporary[0..requested]}) catch |err| switch (err) { error.EndOfStream => { if (stdout) self.stdout_reached_eof = true; return error.EndOfStream; }, else => return error.ReadFailed, }; if (n == 0) { if (stdout) self.stdout_reached_eof = true; return 0; } writer.writeAll(temporary[0..n]) catch return error.WriteFailed; return n; } }; /// Fault injection is deliberately at publication boundaries, rather than in /// storage. This keeps the production storage path identical to the tested one. pub const PublicationHooks = struct { pub const FailurePoint = enum { create_store, before_flush_store, after_flush_store, before_sync_store, after_sync_store, before_close_store, after_close_store, before_flush_map, after_flush_map, before_sync_map, after_sync_map, before_close_map, after_close_map, before_sync_temporary_directory, after_sync_temporary_directory, before_rename_no_replace, after_rename_no_replace, verify_store, verify_map, sync_parent_after_publish, // Compatibility selectors retain the former one-sided fault API. flush_store, sync_store, close_store, flush_map, sync_map, close_map, sync_temporary_directory, rename_no_replace, }; pub const Observer = struct { context: *anyopaque, notify: *const fn (context: *anyopaque, point: FailurePoint) void, }; /// Test-only seam for replacing the staging sibling after it was opened but /// before its identity is checked at the publication boundary. pub const StagingObserver = struct { context: *anyopaque, replace: *const fn (context: *anyopaque, parent: std.Io.Dir, name: []const u8) void, }; fail_at: ?FailurePoint = null, observer: ?Observer = null, staging_observer: ?StagingObserver = null, pub const real_filesystem: PublicationHooks = .{}; fn check(self: PublicationHooks, point: FailurePoint) !void { const legacy: ?FailurePoint = switch (point) { .before_flush_store => .flush_store, .before_sync_store => .sync_store, .before_close_store => .close_store, .before_flush_map => .flush_map, .before_sync_map => .sync_map, .before_close_map => .close_map, .before_sync_temporary_directory => .sync_temporary_directory, .before_rename_no_replace => .rename_no_replace, else => null, }; if (self.fail_at == point or (legacy != null and self.fail_at == legacy)) return error.InjectedPublicationFailure; if (self.observer) |observer| observer.notify(observer.context, point); } fn observeStaging(self: PublicationHooks, parent: std.Io.Dir, name: []const u8) void { if (self.staging_observer) |observer| observer.replace(observer.context, parent, name); } }; pub const ImportLimits = struct { /// A record ceiling, not a memory one: it keeps a runaway or adversarial /// stream finite even when every record it emits is tiny. max_export_events: u64 = 1_000_000, /// Cumulative `data <n>` payload one import may read, raised with /// `--max-export-payload-bytes`. /// /// It is not a memory bound, and reading it as one misjudges both ends. /// Payload is streamed and released per event, so resident size tracks the /// working set, not the total: importing sqlite — a 12.2 GiB export stream /// — held 21 MiB. What this actually bounds is the total history one /// import may carry. /// /// It defaulted to four GiB, which refused sqlite while over-provisioning /// memory by three orders of magnitude. That number could not simply be /// raised, because the per-commit file-command ceiling divided the same /// budget: a bigger allowance for a whole history was also a bigger /// allowance for one commit's retained command list, which unlike payload /// is held whole. The arena now has `max_command_arena_bytes` of its own /// and keeps the old four GiB, leaving this free to be what a total should /// be — large enough that an ordinary repository never meets it, still /// finite against a stream that never ends. max_export_payload_bytes: u64 = @as(u64, 1) << 40, /// Bytes one commit's retained file commands may occupy, which /// `fast_export.fileCommandCeiling` restates as a command count. No CLI /// flag raises it: at the default it admits ~104 million commands in a /// single commit, so anything that meets it is pathological rather than /// merely large, and the honest answer is that the importer cannot hold /// that commit whole. It is a field rather than a constant so the limit /// and its diagnostic stay testable from one place. max_command_arena_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, /// A marked blob retains at most three entries and a marked commit at most /// four, so this caps importer mark state at three million slots. max_retained_entries: usize = 3_000_000, /// Bounds bytes cloned into importer state before any clone is attempted. max_retained_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, }; /// A limit overrun names its numbers. `FileCommandLimitExceeded` alone tells an /// operator nothing actionable: what was observed, what the ceiling was, and /// what raises it is the entire diagnostic. Retained here and surfaced next to /// the error name, exactly as `ProductionGit.preflightOverran` does for /// preflight output. pub const LimitDiagnostic = struct { buffer: [512]u8 = undefined, len: usize = 0, fn retain(self: *LimitDiagnostic, comptime fmt: []const u8, args: anytype) void { const message = std.fmt.bufPrint(&self.buffer, fmt, args) catch { // Every argument is a bounded integer, well inside the buffer; a // truncating format would be a bug here, not a wire condition. // Still total: report without the numbers rather than drop it. const fallback = "import-git: a per-commit parser ceiling was exceeded; raise --max-export-payload-bytes\n"; @memcpy(self.buffer[0..fallback.len], fallback); self.len = fallback.len; return; }; self.len = message.len; } pub fn write(self: *const LimitDiagnostic, err: *std.Io.Writer) void { if (self.len == 0) return; err.writeAll(self.buffer[0..self.len]) catch {}; } }; pub const Dependencies = struct { git: ChildFactory, publication: PublicationHooks = .real_filesystem, limits: ImportLimits = .{}, }; pub const Request = struct { source: []const u8, destination_parent: std.Io.Dir, destination_name: []const u8, /// Null selects the analyzed §6.3 optimum the child factory measured /// during preflight; a value is the operator's explicit pin. submanifest_k: ?u4 = null, attribution: git_import.Attribution = .committer, default_branch: ?[]const u8 = null, quiet: bool = false, }; const PublicationBundle = struct { gpa: std.mem.Allocator, io: std.Io, parent: std.Io.Dir, temporary: std.Io.Dir, temporary_inode: std.Io.File.INode, temporary_name: []u8, owned: bool = true, published: bool = false, fn create(gpa: std.mem.Allocator, io: std.Io, parent: std.Io.Dir, destination: []const u8) !PublicationBundle { var writable_parent = try parent.openDir(io, ".", .{ .access_sub_paths = true, .follow_symlinks = false, }); errdefer writable_parent.close(io); const private_permissions: std.Io.Dir.Permissions = if (comptime builtin.os.tag == .windows or builtin.os.tag == .wasi) .default_dir else @enumFromInt(0o700); var attempt: usize = 0; while (attempt != 1024) : (attempt += 1) { var nonce: [16]u8 = undefined; try io.randomSecure(&nonce); const encoded_nonce = std.fmt.bytesToHex(nonce[0..], .lower); const temporary_name = try std.fmt.allocPrint(gpa, ".zit-import-tmp-{s}", .{encoded_nonce[0..]}); errdefer gpa.free(temporary_name); writable_parent.createDir(io, temporary_name, private_permissions) catch |err| switch (err) { error.PathAlreadyExists => { gpa.free(temporary_name); continue; }, else => return err, }; var temporary: ?std.Io.Dir = null; var temporary_inode: ?std.Io.File.INode = null; errdefer { if (temporary) |directory| directory.close(io); cleanupCreatedStaging(io, writable_parent, temporary_name, temporary_inode); } temporary = try writable_parent.openDir(io, temporary_name, .{ .access_sub_paths = true, .follow_symlinks = false, }); const temporary_stat = try temporary.?.stat(io); temporary_inode = temporary_stat.inode; if (temporary_stat.kind != .directory) return error.StagingEntryReplaced; // `destination` is intentionally part of this constructor's shape: // ownership is established only after the caller performed the // destination preflight, and publication has one atomic commit point. _ = destination; const result: PublicationBundle = .{ .gpa = gpa, .io = io, .parent = writable_parent, .temporary = temporary.?, .temporary_inode = temporary_stat.inode, .temporary_name = temporary_name, }; temporary = null; return result; } return error.NoTemporaryName; } fn abort(self: *PublicationBundle) void { if (!self.published and self.owned) { var current = self.openVerifiedStaging() catch null; if (current) |*directory| { const emptied = deleteOwnedContents(directory, self.io); directory.close(self.io); if (emptied) { var rechecked = self.openVerifiedStaging() catch null; if (rechecked) |*verified| { verified.close(self.io); self.parent.deleteDir(self.io, self.temporary_name) catch {}; } } } } self.temporary.close(self.io); self.parent.close(self.io); self.gpa.free(self.temporary_name); self.* = undefined; } fn publish(self: *PublicationBundle, destination: []const u8, hooks: PublicationHooks) !void { if (!self.owned) return error.StagingEntryReplaced; try hooks.check(.before_rename_no_replace); hooks.observeStaging(self.parent, self.temporary_name); if (comptime builtin.os.tag == .macos) { const from = try self.gpa.dupeZ(u8, self.temporary_name); defer self.gpa.free(from); const to = try self.gpa.dupeZ(u8, destination); defer self.gpa.free(to); var current = try self.openVerifiedStaging(); defer current.close(self.io); // Darwin's standard `renamePreserve` emulates no-replace with a // hard link, which cannot publish a directory. `RENAME_EXCL` // supplies the required atomic directory commit point instead. if (renameatx_np(self.parent.handle, from.ptr, self.parent.handle, to.ptr, 0x0000_0004) != 0) { return error.RenameNoReplaceFailed; } } else { var current = try self.openVerifiedStaging(); defer current.close(self.io); try self.parent.renamePreserve(self.temporary_name, self.parent, destination, self.io); } self.owned = false; try hooks.check(.after_rename_no_replace); var published = self.parent.openDir(self.io, destination, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch return error.PublishedDestinationReplaced; defer published.close(self.io); const published_stat = published.stat(self.io) catch return error.PublishedDestinationReplaced; if (published_stat.kind != .directory or published_stat.inode != self.temporary_inode) { return error.PublishedDestinationReplaced; } self.published = true; } fn openVerifiedStaging(self: *PublicationBundle) !std.Io.Dir { if (!self.owned) return error.StagingEntryReplaced; const held = self.temporary.stat(self.io) catch { self.owned = false; return error.StagingEntryReplaced; }; if (held.kind != .directory or held.inode != self.temporary_inode) { self.owned = false; return error.StagingEntryReplaced; } var current = self.parent.openDir(self.io, self.temporary_name, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch { self.owned = false; return error.StagingEntryReplaced; }; errdefer current.close(self.io); const entry = current.stat(self.io) catch { self.owned = false; return error.StagingEntryReplaced; }; if (entry.kind != .directory or entry.inode != self.temporary_inode) { self.owned = false; return error.StagingEntryReplaced; } return current; } fn cleanupCreatedStaging( io: std.Io, parent: std.Io.Dir, name: []const u8, expected_inode: ?std.Io.File.INode, ) void { const inode = expected_inode orelse return; var current = parent.openDir(io, name, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch return; defer current.close(io); const entry = current.stat(io) catch return; if (entry.kind != .directory or entry.inode != inode) return; if (!deleteOwnedContents(¤t, io)) return; parent.deleteDir(io, name) catch {}; } fn deleteOwnedContents(directory: *std.Io.Dir, io: std.Io) bool { var entries = directory.iterate(); while (entries.next(io) catch return false) |entry| { directory.deleteTree(io, entry.name) catch return false; } return true; } }; const LogTreeStore = struct { gpa: std.mem.Allocator, log: *logstore.LogStore, fn treeStore(self: *LogTreeStore) editor.TreeStore { return .{ .ctx = self, .load_fn = load, .put_fn = put, .put_batch_fn = putBatch }; } fn load(context: *anyopaque, allocator: std.mem.Allocator, name: []const u8) editor.StoreError!editor.LoadedBytes { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var digest: [32]u8 = undefined; const raw = rawHashName(name, &digest) orelse return error.MissingNode; const rid = self.log.lookup(raw) orelse return error.MissingNode; const bytes = self.log.content(allocator, rid) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; return .{ .bytes = bytes orelse return error.MissingNode, .allocator = allocator }; } fn put(context: *anyopaque, name: []const u8, bytes: []const u8) editor.StoreError!void { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var expected_buf: [64]u8 = undefined; if (!std.mem.eql(u8, name, artifact.artifactName(.sha1, bytes, &expected_buf))) return error.StorageFailure; var digest: [32]u8 = undefined; const raw = rawHashName(name, &digest) orelse return error.StorageFailure; _ = self.log.putArtifact(raw, bytes, null) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; } fn putBatch(context: *anyopaque, nodes: []const editor.StoreBatchNode) editor.StoreError!void { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var digests: std.ArrayList([32]u8) = .empty; defer digests.deinit(self.gpa); var items: std.ArrayList(logstore.LogStore.ArtifactBatchItem) = .empty; defer items.deinit(self.gpa); digests.ensureTotalCapacity(self.gpa, nodes.len) catch return error.OutOfMemory; items.ensureTotalCapacity(self.gpa, nodes.len) catch return error.OutOfMemory; for (nodes) |node| { var expected_buf: [64]u8 = undefined; if (!std.mem.eql(u8, node.name, artifact.artifactName(.sha1, node.bytes, &expected_buf))) { return error.StorageFailure; } var digest: [32]u8 = undefined; const raw = rawHashName(node.name, &digest) orelse return error.StorageFailure; digests.appendAssumeCapacity(digest); items.appendAssumeCapacity(.{ .hash = digests.items[digests.items.len - 1][0..raw.len], .bytes = node.bytes }); } self.log.putArtifactBatch(items.items) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; } }; /// Runs one all-or-nothing import. Until `renamePreserve` succeeds, every /// failure owns and removes only this invocation's temporary sibling. pub fn run( gpa: std.mem.Allocator, io: std.Io, request: Request, streams: cli.Streams, dependencies: Dependencies, ) cli.Exit { if (destinationExists(io, request.destination_parent, request.destination_name)) { streams.err.print("import-git: destination '{s}' already exists\n", .{request.destination_name}) catch {}; return .failure; } var pipes = dependencies.git.vtable.start_export(dependencies.git.context, request.source) catch |err| { if (dependencies.git.vtable.startup_diagnostics) |write_diagnostics| { write_diagnostics(dependencies.git.context, streams.err); } streams.err.print("import-git: cannot start git fast-export: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; var effective_request = request; if (effective_request.default_branch == null) effective_request.default_branch = pipes.detected_default_branch; var session_finished = false; defer if (!session_finished) { _ = finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch false; }; var publication = PublicationBundle.create(gpa, io, request.destination_parent, request.destination_name) catch |err| { streams.err.print("import-git: cannot create temporary destination: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; defer publication.abort(); var limit_diagnostic: LimitDiagnostic = .{}; importInto( gpa, io, effective_request, &publication, &pipes, streams, dependencies.git, &session_finished, dependencies.publication, dependencies.limits, &limit_diagnostic, ) catch |err| { if (err == error.DurabilityUncertain) { streams.err.print("import-git: destination '{s}' was published but parent-directory durability is uncertain\n", .{request.destination_name}) catch {}; } else { streams.err.print("import-git: import failed: {s}\n", .{@errorName(err)}) catch {}; limit_diagnostic.write(streams.err); } return .failure; }; streams.out.print("{s}\n", .{request.destination_name}) catch return .failure; if (!request.quiet) streams.err.print("import-git: imported '{s}'\n", .{request.source}) catch {}; return .ok; } fn importInto( gpa: std.mem.Allocator, io: std.Io, request: Request, publication: *PublicationBundle, pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, session_finished: *bool, hooks: PublicationHooks, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { const temporary = publication.temporary; try hooks.check(.create_store); var store_file = try temporary.createFile(io, ".zit", .{ .read = true, .exclusive = true }); var store_file_open = true; defer if (store_file_open) store_file.close(io); var file_storage = storage.FileStorage.init(io, store_file, 0); var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), @intFromEnum(artifact.HashAlgo.sha1)); var log_live = true; defer if (log_live) log.deinit(gpa); var tree_store = LogTreeStore{ .gpa = gpa, .log = &log }; // One pin for the whole import, resolved before the first tree: the // operator's override if given, else preflight's analyzed §6.3 optimum, // else `resolveOptions`' own default when neither is available. const pinned_k = (try git_import.resolveOptions(.{ .submanifest_k = request.submanifest_k orelse pipes.analyzed_submanifest_k, })).submanifest_k; if (!request.quiet and request.submanifest_k == null and pipes.analyzed_submanifest_k != null) { streams.err.print("import-git: analyzed sub-manifest k={d}\n", .{pinned_k}) catch {}; } var importer = try git_import.Importer.init(gpa, &log, tree_store.treeStore(), .{ .submanifest_k = pinned_k, .attribution = request.attribution, }); var importer_live = true; defer if (importer_live) importer.deinit(); var reader = fast_export.Reader.init(gpa, pipes.stdout, .{ .max_line = 1024 * 1024, .max_data = 1024 * 1024 * 1024, .max_path = 4096, .max_events = limits.max_export_events, .max_total_data = limits.max_export_payload_bytes, .max_command_arena_bytes = limits.max_command_arena_bytes, }); defer reader.deinit(); consumeExport(&reader, &importer, limits, diagnostic) catch |err| { const scripted_child_failed = pipes.exit_code_ref == null and terminalExitCode(pipes) != 0; const child_finished_before_cancellation = finishSession(pipes, streams, child, request.quiet, true) catch |lifecycle_err| { session_finished.* = true; return lifecycle_err; }; session_finished.* = true; if (scripted_child_failed or (child_finished_before_cancellation and terminalExitCode(pipes) != 0)) return error.GitExportFailed; return err; }; _ = finishSession(pipes, streams, child, request.quiet, false) catch |err| { session_finished.* = true; return err; }; session_finished.* = true; if (terminalExitCode(pipes) != 0) return error.GitExportFailed; var finalized = importer.finalizeRefs(.{ .default_branch = request.default_branch }) catch |err| { if (err == error.CanonicalRefCollision) if (importer.ref_collision) |collision| { diagnostic.retain( "import-git: '{s}' and '{s}' both name 'refs/{s}'; rename one in the source repository, or exclude it, and re-run\n", .{ collision.first_source, collision.second_source, collision.canonical }, ); }; return err; }; defer finalized.deinit(gpa); for (finalized.controls) |control| try putSha1Artifact(&log, control); try log.putConfig(.{ .hash_policy = @intFromEnum(artifact.HashAlgo.sha1), .default_branch = finalized.primary_branch, .project_id = "", .submanifest_k = pinned_k, .enable_submanifests = true, }); try writeRefs(gpa, io, temporary, finalized.refs); try checkout.write(io, temporary, .{ // The imported store is complete, but no worktree files have been // materialized. Publishing the default tip here would make `update` // incorrectly treat that unmapped worktree as current. .version = "", .branch = finalized.primary_branch, .merge_parent = "", .special_op = "", .manifest_flags = "", }); var map = try temporary.createFile(io, ".zit-import-map.jsonl", .{ .read = true, .exclusive = true }); var map_open = true; defer if (map_open) map.close(io); try hooks.check(.before_flush_map); try map.writeStreamingAll(io, finalized.provenance_jsonl); try hooks.check(.after_flush_map); try hooks.check(.before_flush_store); try log.commit(); try hooks.check(.after_flush_store); try hooks.check(.before_sync_store); try store_file.sync(io); try hooks.check(.after_sync_store); try hooks.check(.before_sync_map); try map.sync(io); try hooks.check(.after_sync_map); // Freeze: neither store nor map remains writable after this point. try hooks.check(.before_close_store); importer.deinit(); importer_live = false; log.deinit(gpa); log_live = false; store_file.close(io); store_file_open = false; try hooks.check(.after_close_store); try hooks.check(.before_close_map); map.close(io); map_open = false; try hooks.check(.after_close_map); try verifyFrozen(gpa, io, temporary, hooks); try hooks.check(.before_sync_temporary_directory); try syncDirectory(temporary); try hooks.check(.after_sync_temporary_directory); try publication.publish(request.destination_name, hooks); // If this final durability boundary fails, the destination is intentionally // retained: its content was fully verified and atomically published. hooks.check(.sync_parent_after_publish) catch return error.DurabilityUncertain; syncDirectory(publication.parent) catch return error.DurabilityUncertain; } fn consumeExport( reader: *fast_export.Reader, importer: *git_import.Importer, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { var retained_bytes: u64 = 0; while (reader.next() catch |err| switch (err) { // Reader admits events and cumulative payload bytes before allocating // an event arena or payload, so its exact bound failures map directly. error.EventLimitExceeded => return error.ExportEventLimitExceeded, error.DataLimitExceeded => { const overrun = reader.payload_overrun.?; switch (overrun.bound) { .cumulative_total => diagnostic.retain( "import-git: the export stream's cumulative payload exceeded its budget: observed at least {d} bytes against a budget of {d} bytes; parsing stopped there, so the stream's full payload is larger than the observed total; re-run with --max-export-payload-bytes above the stream's full payload\n", .{ overrun.observed_at_least, overrun.ceiling }, ), // No flag reaches this ceiling, so naming one would be a lie. .single_payload => diagnostic.retain( "import-git: a single export payload of {d} bytes exceeded the {d}-byte per-payload ceiling; no option raises this ceiling\n", .{ overrun.observed_at_least, overrun.ceiling }, ), } return error.ExportPayloadLimitExceeded; }, error.FileCommandLimitExceeded => { const overrun = reader.file_command_overrun.?; diagnostic.retain( "import-git: a commit exceeded the per-commit file-command ceiling: observed at least {d}, ceiling {d}, derived from the {d}-byte per-commit arena budget at {d} minimum retained bytes per command. This is a structural bound on one commit, not a total: no option raises it, because a commit at this width is beyond what the importer can hold whole\n", .{ overrun.observed_at_least, overrun.ceiling, overrun.arena_budget_bytes, overrun.min_bytes_per_command, }, ); return error.ExportFileCommandLimitExceeded; }, else => return err, }) |lease| { var event = lease; defer event.deinit(); try accountRetainedEvent(&retained_bytes, event.value, limits.max_retained_bytes); try ensureRetainedEntryCapacity(importer, event.value, limits.max_retained_entries); try importer.ingest(event.value); } } const sha1_artifact_name_bytes: u64 = std.crypto.hash.Sha1.digest_length * 2; fn accountRetainedEvent(total: *u64, event: fast_export.Event, limit: u64) !void { switch (event) { .blob => |blob| { try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(blob.original_oid != null), limit); if (blob.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); }, .commit => |commit| { try accountRetainedFixed(total, @as(usize, 3) + @intFromBool(commit.original_oid != null) + @intFromBool(commit.from != null), limit); if (commit.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); try accountRetainedCopies(total, commit.ref, 3, limit); try accountRetainedIdentity(total, commit.author, limit); try accountRetainedIdentity(total, commit.committer, limit); try accountRetainedSlice(total, commit.message, limit); try accountRetainedPaths(total, commit.commands, limit); }, .reset => |reset| { try accountRetainedCopies(total, reset.ref, 2, limit); if (reset.target != null) try accountRetainedFixed(total, 1, limit); }, .tag => |tag| { const source_len = std.math.add(u64, @as(u64, "refs/tags/".len), std.math.cast(u64, tag.name.len) orelse return error.ImporterRetainedByteLimitExceeded) catch return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, source_len, 4, limit); try accountRetainedSlice(total, tag.name, limit); if (tag.original_oid) |oid| try accountRetainedSlice(total, oid, limit); try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(tag.mark != null), limit); try accountRetainedIdentity(total, tag.tagger, limit); try accountRetainedSlice(total, tag.message, limit); }, .done => {}, } } /// Returns the exact retained-state allocation charge for one parsed event. /// The import loop and limit-boundary tests share this calculation so a new /// retained field cannot silently create a guessed budget. pub fn retainedByteCharge(event: fast_export.Event) !u64 { var total: u64 = 0; try accountRetainedEvent(&total, event, std.math.maxInt(u64)); return total; } fn accountRetainedIdentity(total: *u64, identity: fast_export.Identity, limit: u64) !void { try accountRetainedSlice(total, identity.name, limit); try accountRetainedSlice(total, identity.email, limit); } fn accountRetainedPaths(total: *u64, commands: []const fast_export.FileCommand, limit: u64) !void { for (commands) |command| switch (command) { .modify => |modify| try accountRetainedSlice(total, modify.path, limit), .delete => |path| try accountRetainedSlice(total, path, limit), .rename => |rename| { try accountRetainedSlice(total, rename.old, limit); try accountRetainedSlice(total, rename.new, limit); }, .copy => |copy| { try accountRetainedSlice(total, copy.old, limit); try accountRetainedSlice(total, copy.new, limit); }, .delete_all => {}, }; } fn accountRetainedFixed(total: *u64, copies: usize, limit: u64) !void { try accountRetainedLengthCopies(total, sha1_artifact_name_bytes, copies, limit); } fn accountRetainedCopies(total: *u64, bytes: []const u8, copies: usize, limit: u64) !void { const len = std.math.cast(u64, bytes.len) orelse return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, len, copies, limit); } fn accountRetainedSlice(total: *u64, bytes: []const u8, limit: u64) !void { try accountRetainedCopies(total, bytes, 1, limit); } fn accountRetainedLengthCopies(total: *u64, len: u64, copies: usize, limit: u64) !void { var remaining = copies; while (remaining != 0) : (remaining -= 1) { if (total.* > limit or len > limit - total.*) return error.ImporterRetainedByteLimitExceeded; total.* += len; } } fn ensureRetainedEntryCapacity(importer: *const git_import.Importer, event: fast_export.Event, limit: usize) !void { const retained = importer.retainedStateEntries(); if (retained > limit) return error.ImporterRetainedEntryLimitExceeded; const growth: usize = switch (event) { .blob => 3, .commit => 4, .reset => 1, .tag => 3, .done => 0, }; if (growth > limit - retained) return error.ImporterRetainedEntryLimitExceeded; } fn verifyFrozen(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, hooks: PublicationHooks) !void { try hooks.check(.verify_store); var store_file = try temporary.openFile(io, ".zit", .{ .mode = .read_only }); defer store_file.close(io); var file_storage = storage.FileStorage.init(io, store_file, (try store_file.stat(io)).size); var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), @intFromEnum(artifact.HashAlgo.sha1)); defer log.deinit(gpa); var checkout_buf: [checkout.max_ck_payload + 1]u8 = undefined; const state = try checkout.read(io, temporary, &checkout_buf) orelse return error.MissingCheckout; // Imports deliberately publish an unborn checkout: the repository is // discoverable, while the first update alone materializes its default tip. if (state.branch.len == 0) return error.InvalidCheckout; try hooks.check(.verify_map); var map = try temporary.openFile(io, ".zit-import-map.jsonl", .{ .mode = .read_only }); map.close(io); } fn syncDirectory(directory: std.Io.Dir) !void { if (std.c.fsync(directory.handle) != 0) return error.DirectorySyncFailed; } fn putSha1Artifact(log: *logstore.LogStore, bytes: []const u8) !void { var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined; std.crypto.hash.Sha1.hash(bytes, &digest, .{}); _ = try log.putArtifact(&digest, bytes, null); } /// The header every packed ref file opens with. Versioned because this is the /// only description of the format that ships with a repository. pub const packed_refs_header = "# zit packed-refs v1\n"; /// Writes every finalized ref into one sorted file at `refs`. /// /// One file per ref was the obvious encoding and the wrong one: a ref's name /// became a path, so the filesystem got a vote on which repositories are /// representable. Two refs differing only in case are legal and distinct in /// git — sqlite has a pair at different commits — and one silently landed on /// the other, surfacing as a bare `PathAlreadyExists` at publication after the /// whole import had run. Case is only the reachable instance; path length, /// reserved device names, and each platform's own Unicode fold table are the /// same bug wearing different clothes. /// /// A single file has no such vote. Lines are `<canonical> <target>`, sorted by /// canonical name, which makes the file byte-identical for a given ref set and /// so comparable across two imports of one repository. Ref names cannot /// contain a space or a control character, so the separator needs no escape. /// Nothing in zit reads this back — `refs` is a reserved root the working-tree /// scan skips, and branches resolve through repostate's `sym-` tags — so it is /// interop output, and its whole job is to be legible. fn writeRefs(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, refs: []const git_import.FinalizedRef) !void { const ordered = try gpa.alloc(git_import.FinalizedRef, refs.len); defer gpa.free(ordered); @memcpy(ordered, refs); std.mem.sort(git_import.FinalizedRef, ordered, {}, struct { fn lessThan(_: void, a: git_import.FinalizedRef, b: git_import.FinalizedRef) bool { return std.mem.lessThan(u8, a.canonical, b.canonical); } }.lessThan); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, packed_refs_header); for (ordered) |mapping| { try body.appendSlice(gpa, mapping.canonical); try body.append(gpa, ' '); try body.appendSlice(gpa, mapping.target); try body.append(gpa, '\n'); } var file = try temporary.createFile(io, "refs", .{ .read = true, .exclusive = true }); defer file.close(io); try file.writeStreamingAll(io, body.items); try file.sync(io); } fn rawHashName(name: []const u8, buffer: *[32]u8) ?[]const u8 { if (name.len != 40 and name.len != 64) return null; const raw = buffer[0 .. name.len / 2]; _ = std.fmt.hexToBytes(raw, name) catch return null; return raw; } fn finishSession( pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, quiet: bool, abort: bool, ) !bool { var lifecycle_error: ?anyerror = null; // EOF is observed by the importing thread before it starts cancellation. // A child that is a zombie can still accept `kill`, so this observation is // the race-free completion evidence; `reap` below supplies its actual code. var child_finished_before_cancellation = abort and if (pipes.owner) |owner| owner.stdout_reached_eof else false; if (abort) { child.vtable.terminate(child.context) catch |err| { if (err == error.ChildAlreadyExited) { child_finished_before_cancellation = true; } else { lifecycle_error = err; } }; } child.vtable.drain_stderr(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; child.vtable.reap(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; const enabled = abort or terminalExitCode(pipes) != 0 or !quiet; if (pipes.diagnostics) |write_diagnostics| { write_diagnostics(pipes, streams.err, enabled); } else { copyDiagnostics(&pipes.stderr, streams.err, enabled); } if (lifecycle_error) |err| return err; return child_finished_before_cancellation; } fn terminalExitCode(pipes: *const ExportPipes) u8 { return if (pipes.exit_code_ref) |code| code.* else pipes.exit_code; } fn destinationExists(io: std.Io, parent: std.Io.Dir, name: []const u8) bool { var directory = parent.openDir(io, name, .{}) catch { var file = parent.openFile(io, name, .{}) catch return false; file.close(io); return true; }; directory.close(io); return true; } fn copyDiagnostics(reader: *std.Io.Reader, err: *std.Io.Writer, enabled: bool) void { if (enabled) _ = reader.streamRemaining(err) catch {} else _ = reader.discard(.unlimited) catch {}; } pub const CommandScript = struct { pub const Preflight = enum { branch_head, detached_head, git_missing, not_repository, missing_objects, shallow, partial_promisor }; pub const StagingReplacement = enum { directory, symlink }; pub const Config = struct { preflight: Preflight = .branch_head, fast_export: []const u8 = "", stderr_bytes: usize = 0, diagnostic_ring_limit: usize = 4096, cancel_after_events: ?usize = null, competing_destination_at_publish: bool = false, export_exit: u8 = 0, replace_staging_at_publish: ?StagingReplacement = null, replace_destination_after_rename: bool = false, limits: ImportLimits = .{}, }; pub const Publication = struct { store_reopened_for_verify: bool = false, map_reopened_for_verify: bool = false, verify_before_publish: bool = false, }; pub const Environment = struct { pub fn get(_: Environment, name: []const u8) ?[]const u8 { if (std.mem.eql(u8, name, "GIT_TERMINAL_PROMPT")) return "0"; if (std.mem.eql(u8, name, "GIT_NO_LAZY_FETCH")) return "1"; return null; } }; preflight: Preflight, fast_export: []const u8, stderr_bytes: usize, diagnostic_ring_limit: usize, cancel_after_events: ?usize, competing_destination_at_publish: bool, export_exit: u8, replace_staging_at_publish: ?StagingReplacement, replace_destination_after_rename: bool, limits: ImportLimits, default_branch: ?[]const u8 = null, detached_head_recorded: bool = false, export_starts: usize = 0, preflight_starts: usize = 0, lifecycle: ChildLifecycle = .{}, stderr_drained_bytes: usize = 0, environment: Environment = .{}, publication: Publication = .{}, competitor_destination_intact: bool = false, temporary_sibling_left: bool = false, staging_replacement_observed: bool = false, staging_replacement_name: [64]u8 = undefined, staging_replacement_name_len: usize = 0, destination_parent: ?std.Io.Dir = null, destination_name: []const u8 = "", io: ?std.Io = null, pub fn init(config: Config) CommandScript { return .{ .preflight = config.preflight, .fast_export = config.fast_export, .stderr_bytes = config.stderr_bytes, .diagnostic_ring_limit = config.diagnostic_ring_limit, .cancel_after_events = config.cancel_after_events, .competing_destination_at_publish = config.competing_destination_at_publish, .export_exit = config.export_exit, .replace_staging_at_publish = config.replace_staging_at_publish, .replace_destination_after_rename = config.replace_destination_after_rename, .limits = config.limits, }; } fn dependencies(self: *CommandScript, fault: PublicationHooks) Dependencies { var publication = fault; publication.observer = .{ .context = self, .notify = publicationEvent }; publication.staging_observer = .{ .context = self, .replace = replaceStaging }; return .{ .git = .{ .context = self, .vtable = &.{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, } }, .publication = publication, .limits = self.limits, }; } fn startExport(context: *anyopaque, _: []const u8) !ExportPipes { const self: *CommandScript = @ptrCast(@alignCast(context)); self.export_starts += 1; self.lifecycle.launched += 1; const export_bytes = if (self.cancel_after_events != null) "feature done\nblob\nmark :1\ndata 1\nx\ninvalid\n" else self.fast_export; return .{ .stdout = .fixed(export_bytes), .stderr = .fixed("scripted diagnostic\n"), .exit_code = self.export_exit }; } fn terminate(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.terminated += 1; } fn drainStderr(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.stderr_drained += 1; self.stderr_drained_bytes = self.stderr_bytes; } fn reap(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.reaped += 1; } fn publicationEvent(context: *anyopaque, point: PublicationHooks.FailurePoint) void { const self: *CommandScript = @ptrCast(@alignCast(context)); switch (point) { .verify_store => self.publication.store_reopened_for_verify = true, .verify_map => self.publication.map_reopened_for_verify = true, .before_rename_no_replace => { self.publication.verify_before_publish = self.publication.store_reopened_for_verify and self.publication.map_reopened_for_verify; if (self.competing_destination_at_publish) { const parent = self.destination_parent orelse return; parent.createDir(self.io orelse return, self.destination_name, .default_dir) catch return; self.competitor_destination_intact = true; } }, .after_rename_no_replace => if (self.replace_destination_after_rename) { const parent = self.destination_parent orelse return; const io = self.io orelse return; parent.deleteTree(io, self.destination_name) catch return; parent.createDir(io, self.destination_name, .default_dir) catch return; var replacement = parent.openDir(io, self.destination_name, .{ .follow_symlinks = false }) catch return; defer replacement.close(io); var marker = replacement.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; self.competitor_destination_intact = true; }, else => {}, } } fn replaceStaging(context: *anyopaque, parent: std.Io.Dir, name: []const u8) void { const self: *CommandScript = @ptrCast(@alignCast(context)); const replacement = self.replace_staging_at_publish orelse return; const io = self.io orelse return; if (name.len > self.staging_replacement_name.len) return; parent.deleteTree(io, name) catch return; switch (replacement) { .directory => { parent.createDir(io, name, .default_dir) catch return; var directory = parent.openDir(io, name, .{ .follow_symlinks = false }) catch return; defer directory.close(io); var marker = directory.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; }, .symlink => parent.symLink(io, "outside", name, .{}) catch return, } @memcpy(self.staging_replacement_name[0..name.len], name); self.staging_replacement_name_len = name.len; self.staging_replacement_observed = true; } pub fn stagingReplacementName(self: *const CommandScript) []const u8 { return self.staging_replacement_name[0..self.staging_replacement_name_len]; } /// Derived from `exportArgv` so this fake cannot drift from the command /// the importer actually runs. Shows the default signature policy; the /// opt-out's effect is covered directly against `exportArgv`. pub fn exportCommandLine(_: *const CommandScript) []const u8 { return comptime line: { const argv = exportArgv("fixture.git", false); var joined: []const u8 = argv[0]; for (1..argv.len) |index| joined = joined ++ " " ++ argv[index]; break :line joined; }; } }; pub const ScriptResult = struct { code: cli.Exit, out: []const u8, err: []const u8, lifecycle: ChildLifecycle, retained_diagnostic: []const u8, published_durability_uncertain: bool = false, }; pub fn runScripted( gpa: std.mem.Allocator, io: std.Io, request: Request, script: *CommandScript, hooks: PublicationHooks, ) ScriptResult { var out_storage: [4096]u8 = undefined; var err_storage: [4096]u8 = undefined; var out = std.Io.Writer.fixed(&out_storage); var err = std.Io.Writer.fixed(&err_storage); const streams: cli.Streams = .{ .out = &out, .err = &err }; script.preflight_starts += 1; if (script.preflight == .git_missing or script.preflight == .not_repository or script.preflight == .missing_objects or script.preflight == .shallow or script.preflight == .partial_promisor or (script.preflight == .detached_head and script.default_branch == null)) { script.lifecycle.reaped += 1; return .{ .code = .failure, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "preflight rejected" }; } if (script.preflight == .detached_head) script.detached_head_recorded = true; script.destination_parent = request.destination_parent; script.destination_name = request.destination_name; script.io = io; var effective_request = request; effective_request.default_branch = script.default_branch; const code = run(gpa, io, effective_request, streams, script.dependencies(hooks)); if (code == .failure and hooks.fail_at == .sync_parent_after_publish) { return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "durability uncertain", .published_durability_uncertain = true }; } return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "scripted diagnostic" }; } pub fn dispatchForTest(args: []const []const u8) ScriptResult { if (args.len == 1 and std.mem.eql(u8, args[0], "--help")) { return .{ .code = .ok, .out = "zit import-git SOURCE DEST [--submanifest-k K] [--attribution committer|author] [--default-branch NAME] [--quiet]\n", .err = "", .lifecycle = .{}, .retained_diagnostic = "" }; } _ = cli.parseImportGitArgs(args) catch { return .{ .code = .usage, .out = "", .err = "invalid import-git arguments\n", .lifecycle = .{}, .retained_diagnostic = "" }; }; return .{ .code = .failure, .out = "", .err = "test dispatch requires scripted dependencies\n", .lifecycle = .{}, .retained_diagnostic = "" }; }