Artifact ea4ea80a0929534ee4d518960b34997d2d32d945be06e573d09145a5129a2d76:
- File
src/git_import_cli.zig
— part of check-in
[d189b0da1d]
at
2026-08-03 03:08:23
on branch sub-manifest
— import-git: classify Fossil mirror metadata as droppable
A Fossil->Git mirror carries root manifest/manifest.uuid/manifest.tags that are not project content: wapp has 252 distinct manifest blobs across 252 commits while its native history tracks no such file. The classifier proves per commit that those bytes are reconstructible from what the import already holds, so packing can drop them.
Fail-closed and cryptographic: stripped-manifest hash equals the uuid, F cards equal the tree, P cards equal validated parents, D card equals the committer instant. A real top-level file named manifest cannot satisfy that by chance, and any inconclusive signal retains every present candidate -- never a strict subset.
Three corrections to the design, all established against the corpus: presence is per-repository (fossil's manifest setting emits each of the three independently), gate 6 must exclude every present candidate since F cards list none of them, and gate 3 must strip the sterilizer line the export appends or the uuid never matches. update.zig's manifest_paths and sterilizer are now public rather than duplicated -- it already had all three names and the exact trailer.
Measured on the pinned wapp corpus: 252/252 commits validate, drop set 512 = 252 manifest + 252 manifest.uuid + 8 distinct manifest.tags.
test 3461/3461, gate, e2e 716/716, git-import-e2e 11/11, dst 6x1000. (user: dmestas, size: 81940) [annotate] [blame] [check-ins using]
const std = @import("std"); const builtin = @import("builtin"); pub const cli = @import("cli.zig"); const zit = @import("zit"); const artifact = zit.artifact; const checkout = zit.checkout; const fast_export = zit.git_fast_export; const git_import = zit.git_import; const logstore = zit.logstore; const storage = zit.storage; const editor = zit.submanifest_editor; extern "c" fn renameatx_np( from_fd: c_int, from: [*:0]const u8, to_fd: c_int, to: [*:0]const u8, flags: c_uint, ) c_int; pub const ChildLifecycle = struct { launched: usize = 0, terminated: usize = 0, stderr_drained: usize = 0, reaped: usize = 0, }; /// The child-facing half of the import transaction. The caller owns the /// implementation and its backing buffers for the full call to `run`. pub const ChildFactory = struct { context: *anyopaque, vtable: *const VTable, pub const VTable = struct { start_export: *const fn (context: *anyopaque, source: []const u8) anyerror!ExportPipes, terminate: *const fn (context: *anyopaque) anyerror!void, drain_stderr: *const fn (context: *anyopaque) anyerror!void, reap: *const fn (context: *anyopaque) anyerror!void, startup_diagnostics: ?*const fn (context: *anyopaque, err: *std.Io.Writer) void = null, }; }; pub const ExportPipes = struct { stdout: std.Io.Reader, stderr: std.Io.Reader, /// Scripted sessions supply their terminal status here. The OS binding gets /// its status during reap; an import succeeds only after the stream reaches /// `done` and EOF, so a nonzero scripted result is still an import failure. exit_code: u8 = 0, /// OS-backed sessions publish the wait status here during `reap`; scripted /// sessions use `exit_code` directly. exit_code_ref: ?*const u8 = null, owner: ?*ProductionGit = null, diagnostics: ?*const fn (pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void = null, detected_default_branch: ?[]const u8 = null, /// draft-fossil-submanifests-00 §6.3's optimum for the source's own name /// set, measured during preflight. Null when the source could not be /// listed or cannot partition; the caller then keeps its own default. analyzed_submanifest_k: ?u4 = null, }; /// The `git fast-export` command line the importer runs, as a value. /// /// Pure and public so the one policy decision it encodes — what happens to a /// signature Fossil has no card for — is testable without spawning a process. /// `refuse_signed` moves exactly the two signature modes and nothing else. pub fn exportArgv(source: []const u8, refuse_signed: bool) [10][]const u8 { return .{ "git", "-C", source, "fast-export", "--all", "--show-original-ids", "--mark-tags", if (refuse_signed) "--signed-commits=abort" else "--signed-commits=strip", if (refuse_signed) "--signed-tags=abort" else "--signed-tags=strip", "--use-done-feature", }; } /// OS-backed production dependency. It owns both pipe buffers through the /// import call; the `ExportPipes` readers retain a pointer to this value. pub const ProductionGit = struct { pub const StartupFailure = enum { after_spawn, after_stderr_handoff, after_stderr_thread }; /// Keep room in fixed CLI writers for the authoritative import diagnostic. const cli_diagnostic_reserve_bytes = 256; /// Bytes of output one preflight check may produce. Every check asks a /// bounded question — one ref, one boolean, one config value, or (for /// `fsck --connectivity-only --no-dangling`) nothing at all when the /// repository is intact — so an overrun means a check's shape was /// misjudged, never that the source is merely large. const preflight_output_limit_bytes = 8 * 1024; gpa: std.mem.Allocator, io: std.Io, child: ?std.process.Child = null, exit_code: u8 = 1, stdout_buffer: [8192]u8 = undefined, stderr_buffer: [8192]u8 = undefined, reaped: bool = false, reap_calls: usize = 0, /// Captured by the stdout reader before any parser-error cleanup can /// terminate the child. This distinguishes an independently completed /// truncated export from cancellation closing the pipe. stdout_reached_eof: bool = false, default_branch: ?[]const u8 = null, owns_default_branch: bool = false, analyzed_submanifest_k: ?u4 = null, /// The operator's explicit opt-out from the default signature policy. /// False (the default) strips; true restores the refusal. refuse_signed: bool = false, stderr_thread: ?std.Thread = null, /// Moved out of `child` before its reader thread starts, so draining never /// races cleanup or a platform-specific process handle implementation. stderr_file: ?std.Io.File = null, stderr_ring: [4096]u8 = undefined, stderr_ring_start: usize = 0, stderr_ring_len: usize = 0, stderr_bytes_drained: usize = 0, stderr_lock: std.Io.Mutex = .init, test_argv: ?[]const []const u8 = null, test_startup_failure: ?StartupFailure = null, pub fn init(gpa: std.mem.Allocator, io: std.Io) ProductionGit { return .{ .gpa = gpa, .io = io }; } /// Test-only process constructor: uses the identical spawn, stream, /// concurrent-diagnostic, termination, and reap path as `git fast-export`. pub fn initForTest(gpa: std.mem.Allocator, io: std.Io, argv: []const []const u8) ProductionGit { return .{ .gpa = gpa, .io = io, .test_argv = argv }; } fn childEnvironment(self: *ProductionGit) !std.process.Environ.Map { var count: usize = 0; while (std.c.environ[count] != null) : (count += 1) {} const inherited: [:null]const ?[*:0]const u8 = @ptrCast(std.c.environ[0..count :null]); var env = try std.process.Environ.createMap(.{ .block = .{ .slice = inherited } }, self.gpa); errdefer env.deinit(); try env.put("GIT_TERMINAL_PROMPT", "0"); try env.put("GIT_NO_LAZY_FETCH", "1"); return env; } pub fn setDefaultBranch(self: *ProductionGit, default_branch: ?[]const u8) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.default_branch = default_branch; self.owns_default_branch = false; } /// Set before `run`, alongside `setDefaultBranch`: the export argv is /// built inside `start_export`, which the factory vtable does not let the /// caller parameterize. pub fn setRefuseSigned(self: *ProductionGit, refuse: bool) void { self.refuse_signed = refuse; } pub fn deinit(self: *ProductionGit) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.* = undefined; } pub fn dependencies(self: *ProductionGit, publication: PublicationHooks) Dependencies { return .{ .git = .{ .context = self, .vtable = &vtable }, .publication = publication, }; } const vtable: ChildFactory.VTable = .{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, .startup_diagnostics = writeStartupDiagnostics, }; fn startExport(context: *anyopaque, source: []const u8) !ExportPipes { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.test_argv) |argv| return self.startSession(argv); try self.preflight(source); // Fossil has no signature card, so a signature cannot survive this // import in any representation. `strip` accepts that loss and is the // default — refusing by default only withheld the import, never the // signature. `abort` is the operator's explicit opt-out. There is no // honest third option: a verbatim pass would have to invent somewhere // to keep bytes the format cannot hold. const argv = exportArgv(source, self.refuse_signed); return self.startSession(&argv); } fn startSession(self: *ProductionGit, argv: []const []const u8) !ExportPipes { var env = try self.childEnvironment(); defer env.deinit(); self.reaped = false; self.exit_code = 1; self.reap_calls = 0; self.stdout_reached_eof = false; self.stderr_file = null; self.child = try std.process.spawn(self.io, .{ .argv = argv, .environ_map = &env, .stdin = .ignore, .stdout = .pipe, .stderr = .pipe, }); errdefer self.cleanupFailedStart(); try self.failStartForTest(.after_spawn); const child = &(self.child orelse unreachable); const stderr = child.stderr orelse return error.MissingStderr; child.stderr = null; self.stderr_file = stderr; try self.failStartForTest(.after_stderr_handoff); self.stderr_ring_start = 0; self.stderr_bytes_drained = 0; self.stderr_ring_len = 0; self.stderr_thread = try std.Thread.spawn(.{}, drainStderrThread, .{self}); try self.failStartForTest(.after_stderr_thread); return .{ .exit_code_ref = &self.exit_code, .detected_default_branch = self.default_branch, .analyzed_submanifest_k = self.analyzed_submanifest_k, .owner = self, .stdout = .{ .vtable = &stdout_reader_vtable, .buffer = &self.stdout_buffer, .seek = 0, .end = 0, }, .stderr = .{ .vtable = &stderr_reader_vtable, .buffer = &self.stderr_buffer, .seek = 0, .end = 0, }, .diagnostics = writeDiagnostics, }; } fn failStartForTest(self: *const ProductionGit, point: StartupFailure) !void { if (self.test_startup_failure == point) return error.InjectedStartFailure; } fn cleanupFailedStart(self: *ProductionGit) void { if (self.stderr_thread == null) { if (self.stderr_file) |file| file.close(self.io); self.stderr_file = null; } if (self.child) |*child| { if (child.stderr) |file| file.close(self.io); child.stderr = null; } terminate(@ptrCast(self)) catch {}; reap(@ptrCast(self)) catch {}; } fn preflight(self: *ProductionGit, source: []const u8) !void { const inside = try self.query("rev-parse --is-inside-work-tree", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-inside-work-tree" }); defer self.gpa.free(inside); if (!std.mem.eql(u8, std.mem.trim(u8, inside, " \t\r\n"), "true")) return error.NotGitRepository; const head = try self.query("rev-parse HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "HEAD" }); defer self.gpa.free(head); if (std.mem.trim(u8, head, " \t\r\n").len == 0) return error.MissingHead; const object_format = try self.query("rev-parse --show-object-format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" }); defer self.gpa.free(object_format); if (!std.mem.eql(u8, std.mem.trim(u8, object_format, " \t\r\n"), "sha1")) return error.UnsupportedObjectFormat; const shallow = try self.query("rev-parse --is-shallow-repository", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-shallow-repository" }); defer self.gpa.free(shallow); if (std.mem.eql(u8, std.mem.trim(u8, shallow, " \t\r\n"), "true")) return error.ShallowRepository; // `--no-dangling` is required, not tidiness. A dangling object is // unreachable but intact, so it says nothing about the connectivity // question this check exists to answer, yet git reports one // informational line per object on stdout: penpot emitted 14,763 bytes // and a healthy repository failed its import on the output limit. // A missing object is still reported, and still makes this check exit // non-zero. const connectivity = self.query("fsck --connectivity-only", &.{ "git", "-C", source, "--no-lazy-fetch", "fsck", "--connectivity-only", "--no-dangling", "--no-reflogs" }) catch |err| switch (err) { // The exit status is this check's verdict; its stdout is a // per-problem report — with `--no-dangling` an intact repository // writes none of it. So an overrun here is not an unreadable // answer, it is a repository with more broken links than the // report can hold. The named-overrun diagnostic is already // retained, so the operator sees both which check overran and the // verdict it produced. error.GitPreflightFailed, error.PreflightOutputTooLarge => return error.MissingObjects, else => return err, }; defer self.gpa.free(connectivity); const symbolic = try self.optionalQuery("symbolic-ref HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "symbolic-ref", "--quiet", "HEAD" }); if (symbolic) |branch_output| { defer self.gpa.free(branch_output); if (self.default_branch == null) { const branch_ref = std.mem.trim(u8, branch_output, " \t\r\n"); const prefix = "refs/heads/"; if (!std.mem.startsWith(u8, branch_ref, prefix) or branch_ref.len == prefix.len) return error.InvalidHead; self.default_branch = try self.gpa.dupe(u8, branch_ref[prefix.len..]); self.owns_default_branch = true; } } else if (self.default_branch == null) { return error.DetachedHead; } const partial_clone = try self.optionalQuery("config --get extensions.partialClone", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get", "extensions.partialClone" }); if (partial_clone) |configured_remote| { defer self.gpa.free(configured_remote); if (std.mem.trim(u8, configured_remote, " \t\r\n").len != 0) return error.PartialRepository; } const promisors = try self.optionalQuery("config --get-regexp remote.*.promisor", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get-regexp", "^remote\\..*\\.promisor$" }); if (promisors) |configured_promisors| { defer self.gpa.free(configured_promisors); if (try containsGitTrue(configured_promisors)) return error.PartialRepository; } self.analyzed_submanifest_k = try self.analyzeSubmanifestPin(source); } /// `git ls-tree` output the §6.3 analysis is willing to read. A monorepo /// at the ceiling lists several million paths; past it the pin falls back /// to the default rather than failing an import over an advisory /// measurement. const ls_tree_limit_bytes = 64 << 20; /// Measure §6.3's optimum for the source before a single object is read. /// The tip tree, not the root commit: an imported history's first commit /// is routinely a lone README, and pinning k from it would misconfigure /// every later commit in the repository. /// /// Advisory throughout — every failure path returns null and leaves the /// caller's default standing. `-z --name-only` is required, not stylistic: /// without it git quotes unusual bytes and the names would no longer be /// the ones §6.1's partition predicate hashes. The listing includes paths /// a check-in's F-set would exclude (§4.6.3 reserved names) and any path /// `update` would reject; both are rare enough not to move an optimum that /// tracks half the base-2 logarithm of the count. fn analyzeSubmanifestPin(self: *ProductionGit, source: []const u8) !?u4 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = &.{ "git", "-C", source, "--no-lazy-fetch", "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD" }, .environ_map = &env, .stdout_limit = .limited(ls_tree_limit_bytes), .stderr_limit = .limited(8 * 1024), }) catch return null; defer self.gpa.free(result.stderr); defer self.gpa.free(result.stdout); switch (result.term) { .exited => |code| if (code != 0) return null, else => return null, } var names: std.ArrayList([]const u8) = .empty; defer names.deinit(self.gpa); var listing = std.mem.splitScalar(u8, result.stdout, 0); while (listing.next()) |name| { if (name.len == 0) continue; try names.append(self.gpa, name); } return git_import.analyzeSubmanifestK(self.gpa, names.items, .sha1); } fn containsGitTrue(configured: []const u8) !bool { var lines = std.mem.splitScalar(u8, configured, '\n'); while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, " \t\r"); if (trimmed.len == 0) continue; const value = trimmed[(std.mem.lastIndexOfAny(u8, trimmed, " \t") orelse return error.InvalidGitBoolean) + 1 ..]; if (try isGitTrue(value)) return true; } return false; } fn isGitTrue(value: []const u8) !bool { if (std.mem.eql(u8, value, "1") or std.ascii.eqlIgnoreCase(value, "true") or std.ascii.eqlIgnoreCase(value, "yes") or std.ascii.eqlIgnoreCase(value, "on")) return true; if (std.mem.eql(u8, value, "0") or std.ascii.eqlIgnoreCase(value, "false") or std.ascii.eqlIgnoreCase(value, "no") or std.ascii.eqlIgnoreCase(value, "off")) return false; return error.InvalidGitBoolean; } /// A limit overrun names the check that overran. `StreamTooLong` alone /// tells an operator nothing actionable: which of the preflight children /// produced too much output is the entire diagnostic. fn preflightOverran(self: *ProductionGit, check: []const u8) error{PreflightOutputTooLarge} { var buffer: [256]u8 = undefined; const message = std.fmt.bufPrint( &buffer, "import-git: preflight check 'git {s}' produced more than {d} bytes of output\n", .{ check, preflight_output_limit_bytes }, ) catch fallback: { // Every `check` is a short call-site literal, well inside the // buffer; a truncating format would be a bug here, not a wire // condition. Still total: report without the name rather than // dropping the diagnostic. break :fallback "import-git: a preflight check exceeded its output limit\n"; }; self.retainDiagnostics(message); return error.PreflightOutputTooLarge; } fn optionalQuery(self: *ProductionGit, check: []const u8, argv: []const []const u8) !?[]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| switch (code) { 0 => return result.stdout, 1 => { self.gpa.free(result.stdout); return null; }, else => {}, }, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn query(self: *ProductionGit, check: []const u8, argv: []const []const u8) ![]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| if (code == 0) return result.stdout, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn terminate(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); const child = &(self.child orelse return); switch (builtin.os.tag) { .windows => { const handle = child.id orelse return; switch (std.os.windows.ntdll.NtTerminateProcess(handle, @enumFromInt(1))) { .SUCCESS => {}, .PROCESS_IS_TERMINATING => return error.ChildAlreadyExited, else => return error.TerminateFailed, } }, .wasi => return error.UnsupportedTermination, else => { const pid = child.id orelse return; std.posix.kill(pid, .KILL) catch |err| switch (err) { error.ProcessNotFound => return error.ChildAlreadyExited, else => return err, }; }, } self.exit_code = 1; } fn drainStderr(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.joinStderr(); } fn joinStderr(self: *ProductionGit) void { if (self.stderr_thread) |thread| { thread.join(); self.stderr_thread = null; } } fn retainDiagnostics(self: *ProductionGit, bytes: []const u8) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); for (bytes) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } } fn writeRetainedDiagnostics(self: *ProductionGit, err: *std.Io.Writer) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); const retained_len = @min( self.stderr_ring_len, self.stderr_ring.len - cli_diagnostic_reserve_bytes, ); var index = (self.stderr_ring_start + self.stderr_ring_len - retained_len) % self.stderr_ring.len; for (0..retained_len) |_| { err.writeByte(self.stderr_ring[index]) catch return; index = (index + 1) % self.stderr_ring.len; } } fn writeStartupDiagnostics(context: *anyopaque, err: *std.Io.Writer) void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.writeRetainedDiagnostics(err); } fn drainStderrThread(self: *ProductionGit) void { const file = self.stderr_file orelse return; defer { file.close(self.io); self.stderr_file = null; } var buffer: [4096]u8 = undefined; while (true) { const n = file.readStreaming(self.io, &.{buffer[0..]}) catch break; if (n == 0) break; self.stderr_lock.lockUncancelable(self.io); for (buffer[0..n]) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } self.stderr_bytes_drained += n; self.stderr_lock.unlock(self.io); } } fn writeDiagnostics(pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void { const self = pipes.owner orelse return; if (!enabled) return; self.writeRetainedDiagnostics(err); } fn reap(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.reaped) return; self.reap_calls += 1; // No platform may reap or clear `child` until the independently owned // stderr descriptor is fully drained and its thread has exited. self.joinStderr(); const child = &(self.child orelse return); if (child.stdout) |stdout| { stdout.close(self.io); child.stdout = null; } const term = child.wait(self.io) catch |err| { child.kill(self.io); _ = child.wait(self.io) catch {}; self.exit_code = 1; self.reaped = true; self.child = null; return err; }; self.exit_code = switch (term) { .exited => |code| code, else => 1, }; self.reaped = true; self.child = null; } const stdout_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStdout }; const stderr_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStderr }; fn streamStdout(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stdout_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, true); } fn streamStderr(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stderr_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, false); } fn streamPipe( self: *ProductionGit, _: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit, comptime stdout: bool, ) std.Io.Reader.StreamError!usize { const child = &(self.child orelse return error.ReadFailed); const file = if (stdout) child.stdout orelse return error.ReadFailed else child.stderr orelse return error.ReadFailed; var temporary: [8192]u8 = undefined; const requested = if (limit.toInt()) |value| @min(value, temporary.len) else temporary.len; if (requested == 0) return 0; const n = file.readStreaming(self.io, &.{temporary[0..requested]}) catch |err| switch (err) { error.EndOfStream => { if (stdout) self.stdout_reached_eof = true; return error.EndOfStream; }, else => return error.ReadFailed, }; if (n == 0) { if (stdout) self.stdout_reached_eof = true; return 0; } writer.writeAll(temporary[0..n]) catch return error.WriteFailed; return n; } }; /// Fault injection is deliberately at publication boundaries, rather than in /// storage. This keeps the production storage path identical to the tested one. pub const PublicationHooks = struct { pub const FailurePoint = enum { create_store, before_flush_store, after_flush_store, before_sync_store, after_sync_store, before_close_store, after_close_store, before_flush_map, after_flush_map, before_sync_map, after_sync_map, before_close_map, after_close_map, before_sync_temporary_directory, after_sync_temporary_directory, before_rename_no_replace, after_rename_no_replace, verify_store, verify_map, sync_parent_after_publish, // Compatibility selectors retain the former one-sided fault API. flush_store, sync_store, close_store, flush_map, sync_map, close_map, sync_temporary_directory, rename_no_replace, }; pub const Observer = struct { context: *anyopaque, notify: *const fn (context: *anyopaque, point: FailurePoint) void, }; /// Test-only seam for replacing the staging sibling after it was opened but /// before its identity is checked at the publication boundary. pub const StagingObserver = struct { context: *anyopaque, replace: *const fn (context: *anyopaque, parent: std.Io.Dir, name: []const u8) void, }; fail_at: ?FailurePoint = null, observer: ?Observer = null, staging_observer: ?StagingObserver = null, pub const real_filesystem: PublicationHooks = .{}; fn check(self: PublicationHooks, point: FailurePoint) !void { const legacy: ?FailurePoint = switch (point) { .before_flush_store => .flush_store, .before_sync_store => .sync_store, .before_close_store => .close_store, .before_flush_map => .flush_map, .before_sync_map => .sync_map, .before_close_map => .close_map, .before_sync_temporary_directory => .sync_temporary_directory, .before_rename_no_replace => .rename_no_replace, else => null, }; if (self.fail_at == point or (legacy != null and self.fail_at == legacy)) return error.InjectedPublicationFailure; if (self.observer) |observer| observer.notify(observer.context, point); } fn observeStaging(self: PublicationHooks, parent: std.Io.Dir, name: []const u8) void { if (self.staging_observer) |observer| observer.replace(observer.context, parent, name); } }; pub const ImportLimits = struct { /// A record ceiling, not a memory one: it keeps a runaway or adversarial /// stream finite even when every record it emits is tiny. max_export_events: u64 = 1_000_000, /// Cumulative `data <n>` payload one import may read, raised with /// `--max-export-payload-bytes`. /// /// It is not a memory bound, and reading it as one misjudges both ends. /// Payload is streamed and released per event, so resident size tracks the /// working set, not the total: importing sqlite — a 12.2 GiB export stream /// — held 21 MiB. What this actually bounds is the total history one /// import may carry. /// /// It defaulted to four GiB, which refused sqlite while over-provisioning /// memory by three orders of magnitude. That number could not simply be /// raised, because the per-commit file-command ceiling divided the same /// budget: a bigger allowance for a whole history was also a bigger /// allowance for one commit's retained command list, which unlike payload /// is held whole. The arena now has `max_command_arena_bytes` of its own /// and keeps the old four GiB, leaving this free to be what a total should /// be — large enough that an ordinary repository never meets it, still /// finite against a stream that never ends. max_export_payload_bytes: u64 = @as(u64, 1) << 40, /// Bytes one commit's retained file commands may occupy, which /// `fast_export.fileCommandCeiling` restates as a command count. No CLI /// flag raises it: at the default it admits ~104 million commands in a /// single commit, so anything that meets it is pathological rather than /// merely large, and the honest answer is that the importer cannot hold /// that commit whole. It is a field rather than a constant so the limit /// and its diagnostic stay testable from one place. max_command_arena_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, /// A marked blob retains at most three entries and a marked commit at most /// four, so this caps importer mark state at three million slots. max_retained_entries: usize = 3_000_000, /// Bounds bytes cloned into importer state before any clone is attempted. max_retained_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, }; /// A limit overrun names its numbers. `FileCommandLimitExceeded` alone tells an /// operator nothing actionable: what was observed, what the ceiling was, and /// what raises it is the entire diagnostic. Retained here and surfaced next to /// the error name, exactly as `ProductionGit.preflightOverran` does for /// preflight output. pub const LimitDiagnostic = struct { buffer: [512]u8 = undefined, len: usize = 0, fn retain(self: *LimitDiagnostic, comptime fmt: []const u8, args: anytype) void { const message = std.fmt.bufPrint(&self.buffer, fmt, args) catch { // Every argument is a bounded integer, well inside the buffer; a // truncating format would be a bug here, not a wire condition. // Still total: report without the numbers rather than drop it. const fallback = "import-git: a per-commit parser ceiling was exceeded; raise --max-export-payload-bytes\n"; @memcpy(self.buffer[0..fallback.len], fallback); self.len = fallback.len; return; }; self.len = message.len; } pub fn write(self: *const LimitDiagnostic, err: *std.Io.Writer) void { if (self.len == 0) return; err.writeAll(self.buffer[0..self.len]) catch {}; } }; pub const Dependencies = struct { git: ChildFactory, publication: PublicationHooks = .real_filesystem, limits: ImportLimits = .{}, }; pub const Request = struct { source: []const u8, destination_parent: std.Io.Dir, destination_name: []const u8, /// `.tree = null` selects the analyzed §6.3 optimum the child factory /// measured during preflight; `.tree = K` is the operator's explicit pin; /// `.flat` is `--no-submanifests`, §8.1's default posture. submanifest: git_import.SubmanifestChoice = .{ .tree = null }, attribution: git_import.Attribution = .committer, default_branch: ?[]const u8 = null, quiet: bool = false, }; const PublicationBundle = struct { gpa: std.mem.Allocator, io: std.Io, parent: std.Io.Dir, temporary: std.Io.Dir, temporary_inode: std.Io.File.INode, temporary_name: []u8, owned: bool = true, published: bool = false, fn create(gpa: std.mem.Allocator, io: std.Io, parent: std.Io.Dir, destination: []const u8) !PublicationBundle { var writable_parent = try parent.openDir(io, ".", .{ .access_sub_paths = true, .follow_symlinks = false, }); errdefer writable_parent.close(io); const private_permissions: std.Io.Dir.Permissions = if (comptime builtin.os.tag == .windows or builtin.os.tag == .wasi) .default_dir else @enumFromInt(0o700); var attempt: usize = 0; while (attempt != 1024) : (attempt += 1) { var nonce: [16]u8 = undefined; try io.randomSecure(&nonce); const encoded_nonce = std.fmt.bytesToHex(nonce[0..], .lower); const temporary_name = try std.fmt.allocPrint(gpa, ".zit-import-tmp-{s}", .{encoded_nonce[0..]}); errdefer gpa.free(temporary_name); writable_parent.createDir(io, temporary_name, private_permissions) catch |err| switch (err) { error.PathAlreadyExists => { gpa.free(temporary_name); continue; }, else => return err, }; var temporary: ?std.Io.Dir = null; var temporary_inode: ?std.Io.File.INode = null; errdefer { if (temporary) |directory| directory.close(io); cleanupCreatedStaging(io, writable_parent, temporary_name, temporary_inode); } temporary = try writable_parent.openDir(io, temporary_name, .{ .access_sub_paths = true, .follow_symlinks = false, }); const temporary_stat = try temporary.?.stat(io); temporary_inode = temporary_stat.inode; if (temporary_stat.kind != .directory) return error.StagingEntryReplaced; // `destination` is intentionally part of this constructor's shape: // ownership is established only after the caller performed the // destination preflight, and publication has one atomic commit point. _ = destination; const result: PublicationBundle = .{ .gpa = gpa, .io = io, .parent = writable_parent, .temporary = temporary.?, .temporary_inode = temporary_stat.inode, .temporary_name = temporary_name, }; temporary = null; return result; } return error.NoTemporaryName; } fn abort(self: *PublicationBundle) void { if (!self.published and self.owned) { var current = self.openVerifiedStaging() catch null; if (current) |*directory| { const emptied = deleteOwnedContents(directory, self.io); directory.close(self.io); if (emptied) { var rechecked = self.openVerifiedStaging() catch null; if (rechecked) |*verified| { verified.close(self.io); self.parent.deleteDir(self.io, self.temporary_name) catch {}; } } } } self.temporary.close(self.io); self.parent.close(self.io); self.gpa.free(self.temporary_name); self.* = undefined; } fn publish(self: *PublicationBundle, destination: []const u8, hooks: PublicationHooks) !void { if (!self.owned) return error.StagingEntryReplaced; try hooks.check(.before_rename_no_replace); hooks.observeStaging(self.parent, self.temporary_name); if (comptime builtin.os.tag == .macos) { const from = try self.gpa.dupeZ(u8, self.temporary_name); defer self.gpa.free(from); const to = try self.gpa.dupeZ(u8, destination); defer self.gpa.free(to); var current = try self.openVerifiedStaging(); defer current.close(self.io); // Darwin's standard `renamePreserve` emulates no-replace with a // hard link, which cannot publish a directory. `RENAME_EXCL` // supplies the required atomic directory commit point instead. if (renameatx_np(self.parent.handle, from.ptr, self.parent.handle, to.ptr, 0x0000_0004) != 0) { return error.RenameNoReplaceFailed; } } else { var current = try self.openVerifiedStaging(); defer current.close(self.io); try self.parent.renamePreserve(self.temporary_name, self.parent, destination, self.io); } self.owned = false; try hooks.check(.after_rename_no_replace); var published = self.parent.openDir(self.io, destination, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch return error.PublishedDestinationReplaced; defer published.close(self.io); const published_stat = published.stat(self.io) catch return error.PublishedDestinationReplaced; if (published_stat.kind != .directory or published_stat.inode != self.temporary_inode) { return error.PublishedDestinationReplaced; } self.published = true; } fn openVerifiedStaging(self: *PublicationBundle) !std.Io.Dir { if (!self.owned) return error.StagingEntryReplaced; const held = self.temporary.stat(self.io) catch { self.owned = false; return error.StagingEntryReplaced; }; if (held.kind != .directory or held.inode != self.temporary_inode) { self.owned = false; return error.StagingEntryReplaced; } var current = self.parent.openDir(self.io, self.temporary_name, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch { self.owned = false; return error.StagingEntryReplaced; }; errdefer current.close(self.io); const entry = current.stat(self.io) catch { self.owned = false; return error.StagingEntryReplaced; }; if (entry.kind != .directory or entry.inode != self.temporary_inode) { self.owned = false; return error.StagingEntryReplaced; } return current; } fn cleanupCreatedStaging( io: std.Io, parent: std.Io.Dir, name: []const u8, expected_inode: ?std.Io.File.INode, ) void { const inode = expected_inode orelse return; var current = parent.openDir(io, name, .{ .access_sub_paths = true, .follow_symlinks = false, }) catch return; defer current.close(io); const entry = current.stat(io) catch return; if (entry.kind != .directory or entry.inode != inode) return; if (!deleteOwnedContents(¤t, io)) return; parent.deleteDir(io, name) catch {}; } fn deleteOwnedContents(directory: *std.Io.Dir, io: std.Io) bool { var entries = directory.iterate(); while (entries.next(io) catch return false) |entry| { directory.deleteTree(io, entry.name) catch return false; } return true; } }; const LogTreeStore = struct { gpa: std.mem.Allocator, log: *logstore.LogStore, fn treeStore(self: *LogTreeStore) editor.TreeStore { return .{ .ctx = self, .load_fn = load, .put_fn = put, .put_batch_fn = putBatch }; } fn load(context: *anyopaque, allocator: std.mem.Allocator, name: []const u8) editor.StoreError!editor.LoadedBytes { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var digest: [32]u8 = undefined; const raw = rawHashName(name, &digest) orelse return error.MissingNode; const rid = self.log.lookup(raw) orelse return error.MissingNode; const bytes = self.log.content(allocator, rid) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; return .{ .bytes = bytes orelse return error.MissingNode, .allocator = allocator }; } fn put(context: *anyopaque, name: []const u8, bytes: []const u8) editor.StoreError!void { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var expected_buf: [64]u8 = undefined; if (!std.mem.eql(u8, name, artifact.artifactName(.sha1, bytes, &expected_buf))) return error.StorageFailure; var digest: [32]u8 = undefined; const raw = rawHashName(name, &digest) orelse return error.StorageFailure; _ = self.log.putArtifact(raw, bytes, null) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; } fn putBatch(context: *anyopaque, nodes: []const editor.StoreBatchNode) editor.StoreError!void { const self: *LogTreeStore = @ptrCast(@alignCast(context)); var digests: std.ArrayList([32]u8) = .empty; defer digests.deinit(self.gpa); var items: std.ArrayList(logstore.LogStore.ArtifactBatchItem) = .empty; defer items.deinit(self.gpa); digests.ensureTotalCapacity(self.gpa, nodes.len) catch return error.OutOfMemory; items.ensureTotalCapacity(self.gpa, nodes.len) catch return error.OutOfMemory; for (nodes) |node| { var expected_buf: [64]u8 = undefined; if (!std.mem.eql(u8, node.name, artifact.artifactName(.sha1, node.bytes, &expected_buf))) { return error.StorageFailure; } var digest: [32]u8 = undefined; const raw = rawHashName(node.name, &digest) orelse return error.StorageFailure; digests.appendAssumeCapacity(digest); items.appendAssumeCapacity(.{ .hash = digests.items[digests.items.len - 1][0..raw.len], .bytes = node.bytes }); } self.log.putArtifactBatch(items.items) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.StorageFailure, }; } }; /// Runs one all-or-nothing import. Until `renamePreserve` succeeds, every /// failure owns and removes only this invocation's temporary sibling. pub fn run( gpa: std.mem.Allocator, io: std.Io, request: Request, streams: cli.Streams, dependencies: Dependencies, ) cli.Exit { if (destinationExists(io, request.destination_parent, request.destination_name)) { streams.err.print("import-git: destination '{s}' already exists\n", .{request.destination_name}) catch {}; return .failure; } var pipes = dependencies.git.vtable.start_export(dependencies.git.context, request.source) catch |err| { if (dependencies.git.vtable.startup_diagnostics) |write_diagnostics| { write_diagnostics(dependencies.git.context, streams.err); } streams.err.print("import-git: cannot start git fast-export: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; var effective_request = request; if (effective_request.default_branch == null) effective_request.default_branch = pipes.detected_default_branch; var session_finished = false; defer if (!session_finished) { _ = finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch false; }; var publication = PublicationBundle.create(gpa, io, request.destination_parent, request.destination_name) catch |err| { streams.err.print("import-git: cannot create temporary destination: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; defer publication.abort(); var limit_diagnostic: LimitDiagnostic = .{}; importInto( gpa, io, effective_request, &publication, &pipes, streams, dependencies.git, &session_finished, dependencies.publication, dependencies.limits, &limit_diagnostic, ) catch |err| { if (err == error.DurabilityUncertain) { streams.err.print("import-git: destination '{s}' was published but parent-directory durability is uncertain\n", .{request.destination_name}) catch {}; } else { streams.err.print("import-git: import failed: {s}\n", .{@errorName(err)}) catch {}; limit_diagnostic.write(streams.err); } return .failure; }; streams.out.print("{s}\n", .{request.destination_name}) catch return .failure; if (!request.quiet) streams.err.print("import-git: imported '{s}'\n", .{request.source}) catch {}; return .ok; } fn importInto( gpa: std.mem.Allocator, io: std.Io, request: Request, publication: *PublicationBundle, pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, session_finished: *bool, hooks: PublicationHooks, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { const temporary = publication.temporary; try hooks.check(.create_store); var store_file = try temporary.createFile(io, ".zit", .{ .read = true, .exclusive = true }); var store_file_open = true; defer if (store_file_open) store_file.close(io); var file_storage = storage.FileStorage.init(io, store_file, 0); var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), @intFromEnum(artifact.HashAlgo.sha1)); var log_live = true; defer if (log_live) log.deinit(gpa); var tree_store = LogTreeStore{ .gpa = gpa, .log = &log }; // One shape for the whole import, resolved before the first tree: flat when // the operator declined sub-manifests, else the operator's override if // given, else preflight's analyzed §6.3 optimum, else `resolveOptions`' own // default when neither is available. // // The analysis itself runs either way. Skipping it under `.flat` would make // tree emission the SECOND thing the two modes differ in -- the first being // a `git ls-tree` of the whole tip tree during preflight -- and the reason // this flag exists is to give a benchmark two imports that differ in // nothing else. One discarded measurement is the cheaper wart, and it is // not a new one: an explicit `--submanifest-k` already discards it. const resolved_choice: git_import.SubmanifestChoice = switch (request.submanifest) { .flat => .flat, .tree => |pin| .{ .tree = pin orelse pipes.analyzed_submanifest_k }, }; const shape = (try git_import.resolveOptions(.{ .submanifest = resolved_choice })).shape; const pinned_k: u4 = switch (shape) { // Not a k. `Settings.pinned()` reads zero as "no pin", which is the // truthful state for a repository that emitted no sub-manifest. .flat => 0, .tree => |tree| tree.k, }; switch (request.submanifest) { // Silence is the symmetric behavior for "do the default thing": a flat // import has no k to report, so it reports nothing. .flat => {}, .tree => |pin| if (!request.quiet and pin == null and pipes.analyzed_submanifest_k != null) { streams.err.print("import-git: analyzed sub-manifest k={d}\n", .{pinned_k}) catch {}; }, } var importer = try git_import.Importer.init(gpa, &log, tree_store.treeStore(), .{ .submanifest = switch (shape) { .flat => .flat, .tree => |tree| .{ .tree = tree.k }, }, .attribution = request.attribution, }); var importer_live = true; defer if (importer_live) importer.deinit(); var reader = fast_export.Reader.init(gpa, pipes.stdout, .{ .max_line = 1024 * 1024, .max_data = 1024 * 1024 * 1024, .max_path = 4096, .max_events = limits.max_export_events, .max_total_data = limits.max_export_payload_bytes, .max_command_arena_bytes = limits.max_command_arena_bytes, }); defer reader.deinit(); consumeExport(&reader, &importer, limits, diagnostic) catch |err| { const scripted_child_failed = pipes.exit_code_ref == null and terminalExitCode(pipes) != 0; const child_finished_before_cancellation = finishSession(pipes, streams, child, request.quiet, true) catch |lifecycle_err| { session_finished.* = true; return lifecycle_err; }; session_finished.* = true; if (scripted_child_failed or (child_finished_before_cancellation and terminalExitCode(pipes) != 0)) return error.GitExportFailed; return err; }; _ = finishSession(pipes, streams, child, request.quiet, false) catch |err| { session_finished.* = true; return err; }; session_finished.* = true; if (terminalExitCode(pipes) != 0) return error.GitExportFailed; var finalized = importer.finalizeRefs(.{ .default_branch = request.default_branch }) catch |err| { if (err == error.CanonicalRefCollision) if (importer.ref_collision) |collision| { diagnostic.retain( "import-git: '{s}' and '{s}' both name 'refs/{s}'; rename one in the source repository, or exclude it, and re-run\n", .{ collision.first_source, collision.second_source, collision.canonical }, ); }; return err; }; defer finalized.deinit(gpa); for (finalized.controls) |control| try putSha1Artifact(&log, control); // The strongest statement this code can make about a flat import, and it // costs nothing: `stored_submanifest` is `emitted_nodes > 0`, a whole-import // fact derived from work actually done, computed without consulting the // shape -- not "the flag was set", which is what an assertion on `shape` // would say. // // `emitted_nodes` counts nodes this import CREATED, so on its own it would // leave a root built entirely from reused nodes uncovered. It is not // uncovered here, because DEST's store is created by this run a few dozen // lines above (`temporary.createFile(".zit", .{ .exclusive = true })`): // reuse can only hit a node this same import already emitted, so zero // emitted implies zero reused implies no S reference anywhere. // // A returned error rather than an assert because this aborts a // user-visible operation and must unwind the transaction so DEST is never // published. if (std.meta.activeTag(shape) == .flat and finalized.stored_submanifest) { return error.SubmanifestEmittedUnderFlatImport; } try log.putConfig(.{ .hash_policy = @intFromEnum(artifact.HashAlgo.sha1), .default_branch = finalized.primary_branch, .project_id = "", // These two are a pair, not two independent settings. Under `.flat`, // `submanifest_k` is deliberately zero -- outside `artifact.min_k .. // max_k` -- and that is only safe while `enable_submanifests` is false, // because `Settings.emission()` short-circuits on disabled before it // can range-check the pin. Enabled with a zero pin is the // `.out_of_range` misconfiguration, so the two must move together. .submanifest_k = pinned_k, .enable_submanifests = std.meta.activeTag(shape) == .tree, // The latch is one-way and means "an S reference is stored here", not // "trees are on" — see its doc in logstore.zig. Leaving it false while // the store holds S-bearing artifacts is a lie that stays hidden only // while `enable_submanifests` is true, because the §7.2 gate is the // disjunction of the two: a later `zit submanifest disable` would flip // the repository to advertising avoid-submanifests while holding trees. .seen_submanifest = finalized.stored_submanifest, }); try writeRefs(gpa, io, temporary, finalized.refs); try checkout.write(io, temporary, .{ // The imported store is complete, but no worktree files have been // materialized. Publishing the default tip here would make `update` // incorrectly treat that unmapped worktree as current. .version = "", .branch = finalized.primary_branch, .merge_parent = "", .special_op = "", .manifest_flags = "", }); var map = try temporary.createFile(io, ".zit-import-map.jsonl", .{ .read = true, .exclusive = true }); var map_open = true; defer if (map_open) map.close(io); try hooks.check(.before_flush_map); try map.writeStreamingAll(io, finalized.provenance_jsonl); try hooks.check(.after_flush_map); try hooks.check(.before_flush_store); try log.commit(); try hooks.check(.after_flush_store); try hooks.check(.before_sync_store); try store_file.sync(io); try hooks.check(.after_sync_store); try hooks.check(.before_sync_map); try map.sync(io); try hooks.check(.after_sync_map); // Freeze: neither store nor map remains writable after this point. try hooks.check(.before_close_store); importer.deinit(); importer_live = false; log.deinit(gpa); log_live = false; store_file.close(io); store_file_open = false; try hooks.check(.after_close_store); try hooks.check(.before_close_map); map.close(io); map_open = false; try hooks.check(.after_close_map); try verifyFrozen(gpa, io, temporary, hooks); try hooks.check(.before_sync_temporary_directory); try syncDirectory(temporary); try hooks.check(.after_sync_temporary_directory); try publication.publish(request.destination_name, hooks); // If this final durability boundary fails, the destination is intentionally // retained: its content was fully verified and atomically published. hooks.check(.sync_parent_after_publish) catch return error.DurabilityUncertain; syncDirectory(publication.parent) catch return error.DurabilityUncertain; } fn consumeExport( reader: *fast_export.Reader, importer: *git_import.Importer, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { var retained_bytes: u64 = 0; while (reader.next() catch |err| switch (err) { // Reader admits events and cumulative payload bytes before allocating // an event arena or payload, so its exact bound failures map directly. error.EventLimitExceeded => return error.ExportEventLimitExceeded, error.DataLimitExceeded => { const overrun = reader.payload_overrun.?; switch (overrun.bound) { .cumulative_total => diagnostic.retain( "import-git: the export stream's cumulative payload exceeded its budget: observed at least {d} bytes against a budget of {d} bytes; parsing stopped there, so the stream's full payload is larger than the observed total; re-run with --max-export-payload-bytes above the stream's full payload\n", .{ overrun.observed_at_least, overrun.ceiling }, ), // No flag reaches this ceiling, so naming one would be a lie. .single_payload => diagnostic.retain( "import-git: a single export payload of {d} bytes exceeded the {d}-byte per-payload ceiling; no option raises this ceiling\n", .{ overrun.observed_at_least, overrun.ceiling }, ), } return error.ExportPayloadLimitExceeded; }, error.FileCommandLimitExceeded => { const overrun = reader.file_command_overrun.?; diagnostic.retain( "import-git: a commit exceeded the per-commit file-command ceiling: observed at least {d}, ceiling {d}, derived from the {d}-byte per-commit arena budget at {d} minimum retained bytes per command. This is a structural bound on one commit, not a total: no option raises it, because a commit at this width is beyond what the importer can hold whole\n", .{ overrun.observed_at_least, overrun.ceiling, overrun.arena_budget_bytes, overrun.min_bytes_per_command, }, ); return error.ExportFileCommandLimitExceeded; }, else => return err, }) |lease| { var event = lease; defer event.deinit(); try accountRetainedEvent(&retained_bytes, event.value, limits.max_retained_bytes); try ensureRetainedEntryCapacity(importer, event.value, limits.max_retained_entries); try importer.ingest(event.value); } } const sha1_artifact_name_bytes: u64 = std.crypto.hash.Sha1.digest_length * 2; fn accountRetainedEvent(total: *u64, event: fast_export.Event, limit: u64) !void { switch (event) { .blob => |blob| { try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(blob.original_oid != null), limit); if (blob.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); }, .commit => |commit| { try accountRetainedFixed(total, @as(usize, 3) + @intFromBool(commit.original_oid != null) + @intFromBool(commit.from != null), limit); if (commit.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); try accountRetainedCopies(total, commit.ref, 3, limit); try accountRetainedIdentity(total, commit.author, limit); try accountRetainedIdentity(total, commit.committer, limit); try accountRetainedSlice(total, commit.message, limit); try accountRetainedPaths(total, commit.commands, limit); }, .reset => |reset| { try accountRetainedCopies(total, reset.ref, 2, limit); if (reset.target != null) try accountRetainedFixed(total, 1, limit); }, .tag => |tag| { const source_len = std.math.add(u64, @as(u64, "refs/tags/".len), std.math.cast(u64, tag.name.len) orelse return error.ImporterRetainedByteLimitExceeded) catch return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, source_len, 4, limit); try accountRetainedSlice(total, tag.name, limit); if (tag.original_oid) |oid| try accountRetainedSlice(total, oid, limit); try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(tag.mark != null), limit); try accountRetainedIdentity(total, tag.tagger, limit); try accountRetainedSlice(total, tag.message, limit); }, .done => {}, } } /// Returns the exact retained-state allocation charge for one parsed event. /// The import loop and limit-boundary tests share this calculation so a new /// retained field cannot silently create a guessed budget. pub fn retainedByteCharge(event: fast_export.Event) !u64 { var total: u64 = 0; try accountRetainedEvent(&total, event, std.math.maxInt(u64)); return total; } fn accountRetainedIdentity(total: *u64, identity: fast_export.Identity, limit: u64) !void { try accountRetainedSlice(total, identity.name, limit); try accountRetainedSlice(total, identity.email, limit); } fn accountRetainedPaths(total: *u64, commands: []const fast_export.FileCommand, limit: u64) !void { for (commands) |command| switch (command) { .modify => |modify| try accountRetainedSlice(total, modify.path, limit), .delete => |path| try accountRetainedSlice(total, path, limit), .rename => |rename| { try accountRetainedSlice(total, rename.old, limit); try accountRetainedSlice(total, rename.new, limit); }, .copy => |copy| { try accountRetainedSlice(total, copy.old, limit); try accountRetainedSlice(total, copy.new, limit); }, .delete_all => {}, }; } fn accountRetainedFixed(total: *u64, copies: usize, limit: u64) !void { try accountRetainedLengthCopies(total, sha1_artifact_name_bytes, copies, limit); } fn accountRetainedCopies(total: *u64, bytes: []const u8, copies: usize, limit: u64) !void { const len = std.math.cast(u64, bytes.len) orelse return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, len, copies, limit); } fn accountRetainedSlice(total: *u64, bytes: []const u8, limit: u64) !void { try accountRetainedCopies(total, bytes, 1, limit); } fn accountRetainedLengthCopies(total: *u64, len: u64, copies: usize, limit: u64) !void { var remaining = copies; while (remaining != 0) : (remaining -= 1) { if (total.* > limit or len > limit - total.*) return error.ImporterRetainedByteLimitExceeded; total.* += len; } } fn ensureRetainedEntryCapacity(importer: *const git_import.Importer, event: fast_export.Event, limit: usize) !void { const retained = importer.retainedStateEntries(); if (retained > limit) return error.ImporterRetainedEntryLimitExceeded; const growth: usize = switch (event) { // Mark, OID, blob record, and the blob's content-facts entry. .blob => 4, // Mark, OID, object record, ref state -- and the mirror tables, which // grow on the commit that *references* a blob rather than on the blob // itself. A commit adds at most one retained-reference entry per file // command, plus three candidate entries and one tags-grammar entry. // Bounding those by the blob budget would state a per-event guarantee // this check does not make: it predicts one event's growth, it does not // reserve across events. .commit => |value| 11 +| value.commands.len, .reset => 1, .tag => 3, .done => 0, }; if (growth > limit - retained) return error.ImporterRetainedEntryLimitExceeded; } fn verifyFrozen(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, hooks: PublicationHooks) !void { try hooks.check(.verify_store); var store_file = try temporary.openFile(io, ".zit", .{ .mode = .read_only }); defer store_file.close(io); var file_storage = storage.FileStorage.init(io, store_file, (try store_file.stat(io)).size); var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), @intFromEnum(artifact.HashAlgo.sha1)); defer log.deinit(gpa); var checkout_buf: [checkout.max_ck_payload + 1]u8 = undefined; const state = try checkout.read(io, temporary, &checkout_buf) orelse return error.MissingCheckout; // Imports deliberately publish an unborn checkout: the repository is // discoverable, while the first update alone materializes its default tip. if (state.branch.len == 0) return error.InvalidCheckout; try hooks.check(.verify_map); var map = try temporary.openFile(io, ".zit-import-map.jsonl", .{ .mode = .read_only }); map.close(io); } fn syncDirectory(directory: std.Io.Dir) !void { if (std.c.fsync(directory.handle) != 0) return error.DirectorySyncFailed; } fn putSha1Artifact(log: *logstore.LogStore, bytes: []const u8) !void { var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined; std.crypto.hash.Sha1.hash(bytes, &digest, .{}); _ = try log.putArtifact(&digest, bytes, null); } /// The header every packed ref file opens with. Versioned because this is the /// only description of the format that ships with a repository. pub const packed_refs_header = "# zit packed-refs v1\n"; /// Writes every finalized ref into one sorted file at `refs`. /// /// One file per ref was the obvious encoding and the wrong one: a ref's name /// became a path, so the filesystem got a vote on which repositories are /// representable. Two refs differing only in case are legal and distinct in /// git — sqlite has a pair at different commits — and one silently landed on /// the other, surfacing as a bare `PathAlreadyExists` at publication after the /// whole import had run. Case is only the reachable instance; path length, /// reserved device names, and each platform's own Unicode fold table are the /// same bug wearing different clothes. /// /// A single file has no such vote. Lines are `<canonical> <target>`, sorted by /// canonical name, which makes the file byte-identical for a given ref set and /// so comparable across two imports of one repository. Ref names cannot /// contain a space or a control character, so the separator needs no escape. /// Nothing in zit reads this back — `refs` is a reserved root the working-tree /// scan skips, and branches resolve through repostate's `sym-` tags — so it is /// interop output, and its whole job is to be legible. fn writeRefs(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, refs: []const git_import.FinalizedRef) !void { const ordered = try gpa.alloc(git_import.FinalizedRef, refs.len); defer gpa.free(ordered); @memcpy(ordered, refs); std.mem.sort(git_import.FinalizedRef, ordered, {}, struct { fn lessThan(_: void, a: git_import.FinalizedRef, b: git_import.FinalizedRef) bool { return std.mem.lessThan(u8, a.canonical, b.canonical); } }.lessThan); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, packed_refs_header); for (ordered) |mapping| { try body.appendSlice(gpa, mapping.canonical); try body.append(gpa, ' '); try body.appendSlice(gpa, mapping.target); try body.append(gpa, '\n'); } var file = try temporary.createFile(io, "refs", .{ .read = true, .exclusive = true }); defer file.close(io); try file.writeStreamingAll(io, body.items); try file.sync(io); } fn rawHashName(name: []const u8, buffer: *[32]u8) ?[]const u8 { if (name.len != 40 and name.len != 64) return null; const raw = buffer[0 .. name.len / 2]; _ = std.fmt.hexToBytes(raw, name) catch return null; return raw; } fn finishSession( pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, quiet: bool, abort: bool, ) !bool { var lifecycle_error: ?anyerror = null; // EOF is observed by the importing thread before it starts cancellation. // A child that is a zombie can still accept `kill`, so this observation is // the race-free completion evidence; `reap` below supplies its actual code. var child_finished_before_cancellation = abort and if (pipes.owner) |owner| owner.stdout_reached_eof else false; if (abort) { child.vtable.terminate(child.context) catch |err| { if (err == error.ChildAlreadyExited) { child_finished_before_cancellation = true; } else { lifecycle_error = err; } }; } child.vtable.drain_stderr(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; child.vtable.reap(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; const enabled = abort or terminalExitCode(pipes) != 0 or !quiet; if (pipes.diagnostics) |write_diagnostics| { write_diagnostics(pipes, streams.err, enabled); } else { copyDiagnostics(&pipes.stderr, streams.err, enabled); } if (lifecycle_error) |err| return err; return child_finished_before_cancellation; } fn terminalExitCode(pipes: *const ExportPipes) u8 { return if (pipes.exit_code_ref) |code| code.* else pipes.exit_code; } fn destinationExists(io: std.Io, parent: std.Io.Dir, name: []const u8) bool { var directory = parent.openDir(io, name, .{}) catch { var file = parent.openFile(io, name, .{}) catch return false; file.close(io); return true; }; directory.close(io); return true; } fn copyDiagnostics(reader: *std.Io.Reader, err: *std.Io.Writer, enabled: bool) void { if (enabled) _ = reader.streamRemaining(err) catch {} else _ = reader.discard(.unlimited) catch {}; } pub const CommandScript = struct { pub const Preflight = enum { branch_head, detached_head, git_missing, not_repository, missing_objects, shallow, partial_promisor }; pub const StagingReplacement = enum { directory, symlink }; pub const Config = struct { preflight: Preflight = .branch_head, fast_export: []const u8 = "", stderr_bytes: usize = 0, diagnostic_ring_limit: usize = 4096, cancel_after_events: ?usize = null, competing_destination_at_publish: bool = false, export_exit: u8 = 0, replace_staging_at_publish: ?StagingReplacement = null, replace_destination_after_rename: bool = false, limits: ImportLimits = .{}, }; pub const Publication = struct { store_reopened_for_verify: bool = false, map_reopened_for_verify: bool = false, verify_before_publish: bool = false, }; pub const Environment = struct { pub fn get(_: Environment, name: []const u8) ?[]const u8 { if (std.mem.eql(u8, name, "GIT_TERMINAL_PROMPT")) return "0"; if (std.mem.eql(u8, name, "GIT_NO_LAZY_FETCH")) return "1"; return null; } }; preflight: Preflight, fast_export: []const u8, stderr_bytes: usize, diagnostic_ring_limit: usize, cancel_after_events: ?usize, competing_destination_at_publish: bool, export_exit: u8, replace_staging_at_publish: ?StagingReplacement, replace_destination_after_rename: bool, limits: ImportLimits, default_branch: ?[]const u8 = null, detached_head_recorded: bool = false, export_starts: usize = 0, preflight_starts: usize = 0, lifecycle: ChildLifecycle = .{}, stderr_drained_bytes: usize = 0, environment: Environment = .{}, publication: Publication = .{}, competitor_destination_intact: bool = false, temporary_sibling_left: bool = false, staging_replacement_observed: bool = false, staging_replacement_name: [64]u8 = undefined, staging_replacement_name_len: usize = 0, destination_parent: ?std.Io.Dir = null, destination_name: []const u8 = "", io: ?std.Io = null, pub fn init(config: Config) CommandScript { return .{ .preflight = config.preflight, .fast_export = config.fast_export, .stderr_bytes = config.stderr_bytes, .diagnostic_ring_limit = config.diagnostic_ring_limit, .cancel_after_events = config.cancel_after_events, .competing_destination_at_publish = config.competing_destination_at_publish, .export_exit = config.export_exit, .replace_staging_at_publish = config.replace_staging_at_publish, .replace_destination_after_rename = config.replace_destination_after_rename, .limits = config.limits, }; } fn dependencies(self: *CommandScript, fault: PublicationHooks) Dependencies { var publication = fault; publication.observer = .{ .context = self, .notify = publicationEvent }; publication.staging_observer = .{ .context = self, .replace = replaceStaging }; return .{ .git = .{ .context = self, .vtable = &.{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, } }, .publication = publication, .limits = self.limits, }; } fn startExport(context: *anyopaque, _: []const u8) !ExportPipes { const self: *CommandScript = @ptrCast(@alignCast(context)); self.export_starts += 1; self.lifecycle.launched += 1; const export_bytes = if (self.cancel_after_events != null) "feature done\nblob\nmark :1\ndata 1\nx\ninvalid\n" else self.fast_export; return .{ .stdout = .fixed(export_bytes), .stderr = .fixed("scripted diagnostic\n"), .exit_code = self.export_exit }; } fn terminate(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.terminated += 1; } fn drainStderr(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.stderr_drained += 1; self.stderr_drained_bytes = self.stderr_bytes; } fn reap(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.reaped += 1; } fn publicationEvent(context: *anyopaque, point: PublicationHooks.FailurePoint) void { const self: *CommandScript = @ptrCast(@alignCast(context)); switch (point) { .verify_store => self.publication.store_reopened_for_verify = true, .verify_map => self.publication.map_reopened_for_verify = true, .before_rename_no_replace => { self.publication.verify_before_publish = self.publication.store_reopened_for_verify and self.publication.map_reopened_for_verify; if (self.competing_destination_at_publish) { const parent = self.destination_parent orelse return; parent.createDir(self.io orelse return, self.destination_name, .default_dir) catch return; self.competitor_destination_intact = true; } }, .after_rename_no_replace => if (self.replace_destination_after_rename) { const parent = self.destination_parent orelse return; const io = self.io orelse return; parent.deleteTree(io, self.destination_name) catch return; parent.createDir(io, self.destination_name, .default_dir) catch return; var replacement = parent.openDir(io, self.destination_name, .{ .follow_symlinks = false }) catch return; defer replacement.close(io); var marker = replacement.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; self.competitor_destination_intact = true; }, else => {}, } } fn replaceStaging(context: *anyopaque, parent: std.Io.Dir, name: []const u8) void { const self: *CommandScript = @ptrCast(@alignCast(context)); const replacement = self.replace_staging_at_publish orelse return; const io = self.io orelse return; if (name.len > self.staging_replacement_name.len) return; parent.deleteTree(io, name) catch return; switch (replacement) { .directory => { parent.createDir(io, name, .default_dir) catch return; var directory = parent.openDir(io, name, .{ .follow_symlinks = false }) catch return; defer directory.close(io); var marker = directory.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; }, .symlink => parent.symLink(io, "outside", name, .{}) catch return, } @memcpy(self.staging_replacement_name[0..name.len], name); self.staging_replacement_name_len = name.len; self.staging_replacement_observed = true; } pub fn stagingReplacementName(self: *const CommandScript) []const u8 { return self.staging_replacement_name[0..self.staging_replacement_name_len]; } /// Derived from `exportArgv` so this fake cannot drift from the command /// the importer actually runs. Shows the default signature policy; the /// opt-out's effect is covered directly against `exportArgv`. pub fn exportCommandLine(_: *const CommandScript) []const u8 { return comptime line: { const argv = exportArgv("fixture.git", false); var joined: []const u8 = argv[0]; for (1..argv.len) |index| joined = joined ++ " " ++ argv[index]; break :line joined; }; } }; pub const ScriptResult = struct { code: cli.Exit, out: []const u8, err: []const u8, lifecycle: ChildLifecycle, retained_diagnostic: []const u8, published_durability_uncertain: bool = false, }; pub fn runScripted( gpa: std.mem.Allocator, io: std.Io, request: Request, script: *CommandScript, hooks: PublicationHooks, ) ScriptResult { var out_storage: [4096]u8 = undefined; var err_storage: [4096]u8 = undefined; var out = std.Io.Writer.fixed(&out_storage); var err = std.Io.Writer.fixed(&err_storage); const streams: cli.Streams = .{ .out = &out, .err = &err }; script.preflight_starts += 1; if (script.preflight == .git_missing or script.preflight == .not_repository or script.preflight == .missing_objects or script.preflight == .shallow or script.preflight == .partial_promisor or (script.preflight == .detached_head and script.default_branch == null)) { script.lifecycle.reaped += 1; return .{ .code = .failure, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "preflight rejected" }; } if (script.preflight == .detached_head) script.detached_head_recorded = true; script.destination_parent = request.destination_parent; script.destination_name = request.destination_name; script.io = io; var effective_request = request; effective_request.default_branch = script.default_branch; const code = run(gpa, io, effective_request, streams, script.dependencies(hooks)); if (code == .failure and hooks.fail_at == .sync_parent_after_publish) { return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "durability uncertain", .published_durability_uncertain = true }; } return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "scripted diagnostic" }; } pub fn dispatchForTest(args: []const []const u8) ScriptResult { if (args.len == 1 and std.mem.eql(u8, args[0], "--help")) { // Rendered from `cli.import_git_usage`, not transcribed: the hand copy // this replaced had silently fallen two options behind the real help. // The `usage: ` prefix is `main.zig`'s, carried here so this stub's // output is a true PREFIX of what the real help prints rather than // merely sharing its option surface. return .{ .code = .ok, .out = "usage: " ++ cli.import_git_usage ++ "\n", .err = "", .lifecycle = .{}, .retained_diagnostic = "" }; } _ = cli.parseImportGitArgs(args) catch { return .{ .code = .usage, .out = "", .err = "invalid import-git arguments\n", .lifecycle = .{}, .retained_diagnostic = "" }; }; return .{ .code = .failure, .out = "", .err = "test dispatch requires scripted dependencies\n", .lifecycle = .{}, .retained_diagnostic = "" }; }