Artifact 1a0d36a188aae0575a9fa385eebe177721b7e351a302629f57bdc6748fbc375a:
- File
src/git/git_import_cli.zig
— part of check-in
[3873759cfa]
at
2026-09-28 12:53:35
on branch trunk
— pack, shun --force: fsync the directory after the rename; model it in DST
The rename that installs a new generation was never made durable, so a power loss after a reported success could bring the old store back, and with it content a shun had removed. The fault layer now reverts creates, renames and deletes no directory fsync made durable, and the shun sweep requires a rewrite that succeeded to survive a power cut. (user: dmestas, size: 129795) [annotate] [blame] [check-ins using]
const std = @import("std"); const builtin = @import("builtin"); pub const cli = @import("zit").cli; const zit = @import("zit"); const artifact = zit.artifact; const checkout = zit.checkout; const delta = zit.delta; const fast_export = zit.git_fast_export; const git_import = zit.git_import; const logstore = zit.logstore; const pack = zit.pack; const storage = zit.storage; const editor = zit.submanifest_editor; const verifier = @import("git_import_verify.zig"); pub const ChildLifecycle = struct { launched: usize = 0, terminated: usize = 0, stderr_drained: usize = 0, reaped: usize = 0, }; /// The child-facing half of the import transaction. The caller owns the /// implementation and its backing buffers for the full call to `run`. pub const ChildFactory = struct { context: *anyopaque, vtable: *const VTable, pub const VTable = struct { start_export: *const fn (context: *anyopaque, source: []const u8) anyerror!ExportPipes, terminate: *const fn (context: *anyopaque) anyerror!void, drain_stderr: *const fn (context: *anyopaque) anyerror!void, reap: *const fn (context: *anyopaque) anyerror!void, startup_diagnostics: ?*const fn (context: *anyopaque, err: *std.Io.Writer) void = null, }; }; pub const ExportPipes = struct { stdout: std.Io.Reader, stderr: std.Io.Reader, /// Scripted sessions supply their terminal status here. The OS binding gets /// its status during reap; an import succeeds only after the stream reaches /// `done` and EOF, so a nonzero scripted result is still an import failure. exit_code: u8 = 0, /// OS-backed sessions publish the wait status here during `reap`; scripted /// sessions use `exit_code` directly. exit_code_ref: ?*const u8 = null, owner: ?*ProductionGit = null, diagnostics: ?*const fn (pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void = null, detected_default_branch: ?[]const u8 = null, /// draft-fossil-submanifests-00 §6.3's optimum for the source's own name /// set, measured during preflight. Null when the source could not be /// listed or cannot partition; the caller then keeps its own default. analyzed_submanifest_k: ?u4 = null, /// The source's commit count, measured during preflight so the ingest /// loop can report progress against a real denominator. Null when the /// count could not be taken; progress then reports without a total. commit_total: ?u64 = null, }; /// The store's §7.2 default policy for a fresh import, derived from the /// algorithm the import actually names artifacts under rather than from a /// guess about what it produced. Both call sites read it, so the header and /// the persisted setting cannot drift apart. fn importDefaultPolicy() artifact.HashPolicy { const algo = git_import.import_hash_algo; return artifact.defaultPolicy(algo == .sha1, algo == .sha3_256); } /// The `git fast-export` command line the importer runs, as a value. /// /// Pure and public so the one policy decision it encodes — what happens to a /// signature Fossil has no card for — is testable without spawning a process. /// `refuse_signed` moves exactly the two signature modes and nothing else. pub fn exportArgv(source: []const u8, refuse_signed: bool) [10][]const u8 { return .{ "git", "-C", source, "fast-export", "--all", "--show-original-ids", "--mark-tags", if (refuse_signed) "--signed-commits=abort" else "--signed-commits=strip", if (refuse_signed) "--signed-tags=abort" else "--signed-tags=strip", "--use-done-feature", }; } /// OS-backed production dependency. It owns both pipe buffers through the /// import call; the `ExportPipes` readers retain a pointer to this value. pub const ProductionGit = struct { pub const StartupFailure = enum { after_spawn, after_stderr_handoff, after_stderr_thread }; /// Keep room in fixed CLI writers for the authoritative import diagnostic. const cli_diagnostic_reserve_bytes = 256; /// Bytes of output one preflight check may produce. Every check asks a /// bounded question — one ref, one boolean, one config value, or (for /// `fsck --connectivity-only --no-dangling`) nothing at all when the /// repository is intact — so an overrun means a check's shape was /// misjudged, never that the source is merely large. const preflight_output_limit_bytes = 8 * 1024; gpa: std.mem.Allocator, io: std.Io, /// The process environment the spawned git children inherit (plus the /// GIT_* pins `childEnvironment` adds). Threaded from main's `Init` in /// production and `std.testing.environ` in tests. environ: std.process.Environ, child: ?std.process.Child = null, exit_code: u8 = 1, stdout_buffer: [8192]u8 = undefined, stderr_buffer: [8192]u8 = undefined, reaped: bool = false, reap_calls: usize = 0, /// Captured by the stdout reader before any parser-error cleanup can /// terminate the child. This distinguishes an independently completed /// truncated export from cancellation closing the pipe. stdout_reached_eof: bool = false, default_branch: ?[]const u8 = null, owns_default_branch: bool = false, analyzed_submanifest_k: ?u4 = null, commit_total: ?u64 = null, /// The operator's explicit opt-out from the default signature policy. /// False (the default) strips; true restores the refusal. refuse_signed: bool = false, stderr_thread: ?std.Thread = null, /// Moved out of `child` before its reader thread starts, so draining never /// races cleanup or a platform-specific process handle implementation. stderr_file: ?std.Io.File = null, stderr_ring: [4096]u8 = undefined, stderr_ring_start: usize = 0, stderr_ring_len: usize = 0, stderr_bytes_drained: usize = 0, stderr_lock: std.Io.Mutex = .init, test_argv: ?[]const []const u8 = null, test_startup_failure: ?StartupFailure = null, pub fn init(gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ) ProductionGit { return .{ .gpa = gpa, .io = io, .environ = environ }; } /// Test-only process constructor: uses the identical spawn, stream, /// concurrent-diagnostic, termination, and reap path as `git fast-export`. pub fn initForTest(gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ, argv: []const []const u8) ProductionGit { return .{ .gpa = gpa, .io = io, .environ = environ, .test_argv = argv }; } fn childEnvironment(self: *ProductionGit) !std.process.Environ.Map { // The process environ, threaded from main's `Init` — never `std.c`: // libc is implicit on macOS but must be explicitly linked on Linux, // and the environment does not need libc to read. var env = try self.environ.createMap(self.gpa); errdefer env.deinit(); try env.put("GIT_TERMINAL_PROMPT", "0"); try env.put("GIT_NO_LAZY_FETCH", "1"); return env; } pub fn setDefaultBranch(self: *ProductionGit, default_branch: ?[]const u8) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.default_branch = default_branch; self.owns_default_branch = false; } /// Set before `run`, alongside `setDefaultBranch`: the export argv is /// built inside `start_export`, which the factory vtable does not let the /// caller parameterize. pub fn setRefuseSigned(self: *ProductionGit, refuse: bool) void { self.refuse_signed = refuse; } pub fn deinit(self: *ProductionGit) void { if (self.owns_default_branch) self.gpa.free(self.default_branch.?); self.* = undefined; } pub fn dependencies(self: *ProductionGit, publication: PublicationHooks) Dependencies { return .{ .git = .{ .context = self, .vtable = &vtable }, .publication = publication, }; } const vtable: ChildFactory.VTable = .{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, .startup_diagnostics = writeStartupDiagnostics, }; fn startExport(context: *anyopaque, source: []const u8) !ExportPipes { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.test_argv) |argv| return self.startSession(argv); try self.preflight(source); // Fossil has no signature card, so a signature cannot survive this // import in any representation. `strip` accepts that loss and is the // default — refusing by default only withheld the import, never the // signature. `abort` is the operator's explicit opt-out. There is no // honest third option: a verbatim pass would have to invent somewhere // to keep bytes the format cannot hold. const argv = exportArgv(source, self.refuse_signed); return self.startSession(&argv); } fn startSession(self: *ProductionGit, argv: []const []const u8) !ExportPipes { var env = try self.childEnvironment(); defer env.deinit(); self.reaped = false; self.exit_code = 1; self.reap_calls = 0; self.stdout_reached_eof = false; self.stderr_file = null; self.child = try std.process.spawn(self.io, .{ .argv = argv, .environ_map = &env, .stdin = .ignore, .stdout = .pipe, .stderr = .pipe, }); errdefer self.cleanupFailedStart(); try self.failStartForTest(.after_spawn); const child = &(self.child orelse unreachable); const stderr = child.stderr orelse return error.MissingStderr; child.stderr = null; self.stderr_file = stderr; try self.failStartForTest(.after_stderr_handoff); self.stderr_ring_start = 0; self.stderr_bytes_drained = 0; self.stderr_ring_len = 0; self.stderr_thread = try std.Thread.spawn(.{}, drainStderrThread, .{self}); try self.failStartForTest(.after_stderr_thread); return .{ .exit_code_ref = &self.exit_code, .detected_default_branch = self.default_branch, .analyzed_submanifest_k = self.analyzed_submanifest_k, .commit_total = self.commit_total, .owner = self, .stdout = .{ .vtable = &stdout_reader_vtable, .buffer = &self.stdout_buffer, .seek = 0, .end = 0, }, .stderr = .{ .vtable = &stderr_reader_vtable, .buffer = &self.stderr_buffer, .seek = 0, .end = 0, }, .diagnostics = writeDiagnostics, }; } fn failStartForTest(self: *const ProductionGit, point: StartupFailure) !void { if (self.test_startup_failure == point) return error.InjectedStartFailure; } fn cleanupFailedStart(self: *ProductionGit) void { if (self.stderr_thread == null) { if (self.stderr_file) |file| file.close(self.io); self.stderr_file = null; } if (self.child) |*child| { if (child.stderr) |file| file.close(self.io); child.stderr = null; } terminate(@ptrCast(self)) catch {}; reap(@ptrCast(self)) catch {}; } fn preflight(self: *ProductionGit, source: []const u8) !void { // FIRST, and without --no-lazy-fetch: every other preflight query // carries that flag, and a git older than 2.45 rejects the whole // command line with a usage dump — 13 opaque e2e failures on stock // Ubuntu 24.04 read that way before this check named the cause. // The requirement stays hard rather than version-conditional argv: // the flag's guarantee is already re-provided by the shallow/partial // rejections below, and a second code path whose only purpose is // re-proving that would be complexity with no carrier. const version_out = try self.query("--version", &.{ "git", "--version" }); defer self.gpa.free(version_out); const version_line = std.mem.trim(u8, version_out, " \t\r\n"); if (gitOlderThan(version_line, 2, 45)) { var buf: [160]u8 = undefined; const msg = std.fmt.bufPrint( &buf, "zit import-git needs git >= 2.45 (for --no-lazy-fetch); found: {s}\n", .{version_line}, ) catch "zit import-git needs git >= 2.45 (for --no-lazy-fetch)\n"; self.retainDiagnostics(msg); return error.GitTooOld; } const inside = try self.query("rev-parse --is-inside-work-tree", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-inside-work-tree" }); defer self.gpa.free(inside); if (!std.mem.eql(u8, std.mem.trim(u8, inside, " \t\r\n"), "true")) return error.NotGitRepository; const head = try self.query("rev-parse HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "HEAD" }); defer self.gpa.free(head); if (std.mem.trim(u8, head, " \t\r\n").len == 0) return error.MissingHead; const object_format = try self.query("rev-parse --show-object-format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" }); defer self.gpa.free(object_format); if (!std.mem.eql(u8, std.mem.trim(u8, object_format, " \t\r\n"), "sha1")) return error.UnsupportedObjectFormat; const shallow = try self.query("rev-parse --is-shallow-repository", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-shallow-repository" }); defer self.gpa.free(shallow); if (std.mem.eql(u8, std.mem.trim(u8, shallow, " \t\r\n"), "true")) return error.ShallowRepository; // `--no-dangling` is required, not tidiness. A dangling object is // unreachable but intact, so it says nothing about the connectivity // question this check exists to answer, yet git reports one // informational line per object on stdout: penpot emitted 14,763 bytes // and a healthy repository failed its import on the output limit. // A missing object is still reported, and still makes this check exit // non-zero. const connectivity = self.query("fsck --connectivity-only", &.{ "git", "-C", source, "--no-lazy-fetch", "fsck", "--connectivity-only", "--no-dangling", "--no-reflogs" }) catch |err| switch (err) { // The exit status is this check's verdict; its stdout is a // per-problem report — with `--no-dangling` an intact repository // writes none of it. So an overrun here is not an unreadable // answer, it is a repository with more broken links than the // report can hold. The named-overrun diagnostic is already // retained, so the operator sees both which check overran and the // verdict it produced. error.GitPreflightFailed, error.PreflightOutputTooLarge => return error.MissingObjects, else => return err, }; defer self.gpa.free(connectivity); const symbolic = try self.optionalQuery("symbolic-ref HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "symbolic-ref", "--quiet", "HEAD" }); if (symbolic) |branch_output| { defer self.gpa.free(branch_output); if (self.default_branch == null) { const branch_ref = std.mem.trim(u8, branch_output, " \t\r\n"); const prefix = "refs/heads/"; if (!std.mem.startsWith(u8, branch_ref, prefix) or branch_ref.len == prefix.len) return error.InvalidHead; self.default_branch = try self.gpa.dupe(u8, branch_ref[prefix.len..]); self.owns_default_branch = true; } } else if (self.default_branch == null) { return error.DetachedHead; } const partial_clone = try self.optionalQuery("config --get extensions.partialClone", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get", "extensions.partialClone" }); if (partial_clone) |configured_remote| { defer self.gpa.free(configured_remote); if (std.mem.trim(u8, configured_remote, " \t\r\n").len != 0) return error.PartialRepository; } const promisors = try self.optionalQuery("config --get-regexp remote.*.promisor", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get-regexp", "^remote\\..*\\.promisor$" }); if (promisors) |configured_promisors| { defer self.gpa.free(configured_promisors); if (try containsGitTrue(configured_promisors)) return error.PartialRepository; } const counted = try self.optionalQuery("rev-list --count --all", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-list", "--count", "--all" }); if (counted) |count_output| { defer self.gpa.free(count_output); self.commit_total = std.fmt.parseInt(u64, std.mem.trim(u8, count_output, " \t\r\n"), 10) catch null; } self.analyzed_submanifest_k = try self.analyzeSubmanifestPin(source); } /// `git ls-tree` output the §6.3 analysis is willing to read. A monorepo /// at the ceiling lists several million paths; past it the pin falls back /// to the default rather than failing an import over an advisory /// measurement. const ls_tree_limit_bytes = 64 << 20; /// Measure §6.3's optimum for the source before a single object is read. /// The tip tree, not the root commit: an imported history's first commit /// is routinely a lone README, and pinning k from it would misconfigure /// every later commit in the repository. /// /// Advisory throughout — every failure path returns null and leaves the /// caller's default standing. `-z --name-only` is required, not stylistic: /// without it git quotes unusual bytes and the names would no longer be /// the ones §6.1's partition predicate hashes. The listing includes paths /// a check-in's F-set would exclude (§4.6.3 reserved names) and any path /// `update` would reject; both are rare enough not to move an optimum that /// tracks half the base-2 logarithm of the count. fn analyzeSubmanifestPin(self: *ProductionGit, source: []const u8) !?u4 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = &.{ "git", "-C", source, "--no-lazy-fetch", "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD" }, .environ_map = &env, .stdout_limit = .limited(ls_tree_limit_bytes), .stderr_limit = .limited(8 * 1024), }) catch return null; defer self.gpa.free(result.stderr); defer self.gpa.free(result.stdout); switch (result.term) { .exited => |code| if (code != 0) return null, else => return null, } var names: std.ArrayList([]const u8) = .empty; defer names.deinit(self.gpa); var listing = std.mem.splitScalar(u8, result.stdout, 0); while (listing.next()) |name| { if (name.len == 0) continue; try names.append(self.gpa, name); } return git_import.analyzeSubmanifestK(self.gpa, names.items, git_import.import_hash_algo); } fn containsGitTrue(configured: []const u8) !bool { var lines = std.mem.splitScalar(u8, configured, '\n'); while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, " \t\r"); if (trimmed.len == 0) continue; const value = trimmed[(std.mem.lastIndexOfAny(u8, trimmed, " \t") orelse return error.InvalidGitBoolean) + 1 ..]; if (try isGitTrue(value)) return true; } return false; } fn isGitTrue(value: []const u8) !bool { if (std.mem.eql(u8, value, "1") or std.ascii.eqlIgnoreCase(value, "true") or std.ascii.eqlIgnoreCase(value, "yes") or std.ascii.eqlIgnoreCase(value, "on")) return true; if (std.mem.eql(u8, value, "0") or std.ascii.eqlIgnoreCase(value, "false") or std.ascii.eqlIgnoreCase(value, "no") or std.ascii.eqlIgnoreCase(value, "off")) return false; return error.InvalidGitBoolean; } /// A limit overrun names the check that overran. `StreamTooLong` alone /// tells an operator nothing actionable: which of the preflight children /// produced too much output is the entire diagnostic. fn preflightOverran(self: *ProductionGit, check: []const u8) error{PreflightOutputTooLarge} { var buffer: [256]u8 = undefined; const message = std.fmt.bufPrint( &buffer, "import-git: preflight check 'git {s}' produced more than {d} bytes of output\n", .{ check, preflight_output_limit_bytes }, ) catch fallback: { // Every `check` is a short call-site literal, well inside the // buffer; a truncating format would be a bug here, not a wire // condition. Still total: report without the name rather than // dropping the diagnostic. break :fallback "import-git: a preflight check exceeded its output limit\n"; }; self.retainDiagnostics(message); return error.PreflightOutputTooLarge; } fn optionalQuery(self: *ProductionGit, check: []const u8, argv: []const []const u8) !?[]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| switch (code) { 0 => return result.stdout, 1 => { self.gpa.free(result.stdout); return null; }, else => {}, }, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn query(self: *ProductionGit, check: []const u8, argv: []const []const u8) ![]u8 { var env = try self.childEnvironment(); defer env.deinit(); const result = std.process.run(self.gpa, self.io, .{ .argv = argv, .environ_map = &env, .stdout_limit = .limited(preflight_output_limit_bytes), .stderr_limit = .limited(preflight_output_limit_bytes), }) catch |err| switch (err) { error.StreamTooLong => return self.preflightOverran(check), else => return err, }; defer self.gpa.free(result.stderr); switch (result.term) { .exited => |code| if (code == 0) return result.stdout, else => {}, } self.retainDiagnostics(result.stderr); self.gpa.free(result.stdout); return error.GitPreflightFailed; } fn terminate(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); const child = &(self.child orelse return); switch (builtin.os.tag) { .windows => { const handle = child.id orelse return; switch (std.os.windows.ntdll.NtTerminateProcess(handle, @enumFromInt(1))) { .SUCCESS => {}, .PROCESS_IS_TERMINATING => return error.ChildAlreadyExited, else => return error.TerminateFailed, } }, .wasi => return error.UnsupportedTermination, else => { const pid = child.id orelse return; std.posix.kill(pid, .KILL) catch |err| switch (err) { error.ProcessNotFound => return error.ChildAlreadyExited, else => return err, }; }, } self.exit_code = 1; } fn drainStderr(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.joinStderr(); } fn joinStderr(self: *ProductionGit) void { if (self.stderr_thread) |thread| { thread.join(); self.stderr_thread = null; } } fn retainDiagnostics(self: *ProductionGit, bytes: []const u8) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); for (bytes) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } } fn writeRetainedDiagnostics(self: *ProductionGit, err: *std.Io.Writer) void { self.stderr_lock.lockUncancelable(self.io); defer self.stderr_lock.unlock(self.io); const retained_len = @min( self.stderr_ring_len, self.stderr_ring.len - cli_diagnostic_reserve_bytes, ); var index = (self.stderr_ring_start + self.stderr_ring_len - retained_len) % self.stderr_ring.len; for (0..retained_len) |_| { err.writeByte(self.stderr_ring[index]) catch return; index = (index + 1) % self.stderr_ring.len; } } fn writeStartupDiagnostics(context: *anyopaque, err: *std.Io.Writer) void { const self: *ProductionGit = @ptrCast(@alignCast(context)); self.writeRetainedDiagnostics(err); } fn drainStderrThread(self: *ProductionGit) void { const file = self.stderr_file orelse return; defer { file.close(self.io); self.stderr_file = null; } var buffer: [4096]u8 = undefined; while (true) { const n = file.readStreaming(self.io, &.{buffer[0..]}) catch break; if (n == 0) break; self.stderr_lock.lockUncancelable(self.io); for (buffer[0..n]) |byte| { if (self.stderr_ring_len < self.stderr_ring.len) { self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte; self.stderr_ring_len += 1; } else { self.stderr_ring[self.stderr_ring_start] = byte; self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len; } } self.stderr_bytes_drained += n; self.stderr_lock.unlock(self.io); } } fn writeDiagnostics(pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void { const self = pipes.owner orelse return; if (!enabled) return; self.writeRetainedDiagnostics(err); } fn reap(context: *anyopaque) !void { const self: *ProductionGit = @ptrCast(@alignCast(context)); if (self.reaped) return; self.reap_calls += 1; // No platform may reap or clear `child` until the independently owned // stderr descriptor is fully drained and its thread has exited. self.joinStderr(); const child = &(self.child orelse return); if (child.stdout) |stdout| { stdout.close(self.io); child.stdout = null; } const term = child.wait(self.io) catch |err| { child.kill(self.io); _ = child.wait(self.io) catch {}; self.exit_code = 1; self.reaped = true; self.child = null; return err; }; self.exit_code = switch (term) { .exited => |code| code, else => 1, }; self.reaped = true; self.child = null; } const stdout_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStdout }; const stderr_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStderr }; fn streamStdout(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stdout_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, true); } fn streamStderr(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *ProductionGit = @alignCast(@fieldParentPtr("stderr_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr)))); return streamPipe(self, reader, writer, limit, false); } fn streamPipe( self: *ProductionGit, _: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit, comptime stdout: bool, ) std.Io.Reader.StreamError!usize { const child = &(self.child orelse return error.ReadFailed); const file = if (stdout) child.stdout orelse return error.ReadFailed else child.stderr orelse return error.ReadFailed; var temporary: [8192]u8 = undefined; const requested = if (limit.toInt()) |value| @min(value, temporary.len) else temporary.len; if (requested == 0) return 0; const n = file.readStreaming(self.io, &.{temporary[0..requested]}) catch |err| switch (err) { error.EndOfStream => { if (stdout) self.stdout_reached_eof = true; return error.EndOfStream; }, else => return error.ReadFailed, }; if (n == 0) { if (stdout) self.stdout_reached_eof = true; return 0; } writer.writeAll(temporary[0..n]) catch return error.WriteFailed; return n; } }; /// Fault injection is deliberately at publication boundaries, rather than in /// storage. This keeps the production storage path identical to the tested one. pub const PublicationHooks = zit.publication.PublicationHooks; pub const ImportLimits = struct { /// Cumulative `data <n>` payload one import may read, raised with /// `--max-export-payload-bytes`. /// /// It is not a memory bound, and reading it as one misjudges both ends. /// Payload is streamed and released per event, so resident size tracks the /// working set, not the total: importing sqlite — a 12.2 GiB export stream /// — held 21 MiB. What this actually bounds is the total history one /// import may carry. /// /// It defaulted to four GiB, which refused sqlite while over-provisioning /// memory by three orders of magnitude. That number could not simply be /// raised, because the per-commit file-command ceiling divided the same /// budget: a bigger allowance for a whole history was also a bigger /// allowance for one commit's retained command list, which unlike payload /// is held whole. The arena now has `max_command_arena_bytes` of its own /// and keeps the old four GiB, leaving this free to be what a total should /// be — large enough that an ordinary repository never meets it, still /// finite against a stream that never ends. max_export_payload_bytes: u64 = @as(u64, 1) << 40, /// Bytes one commit's retained file commands may occupy, which /// `fast_export.fileCommandCeiling` restates as a command count. No CLI /// flag raises it: at the default it admits ~104 million commands in a /// single commit, so anything that meets it is pathological rather than /// merely large, and the honest answer is that the importer cannot hold /// that commit whole. It is a field rather than a constant so the limit /// and its diagnostic stay testable from one place. max_command_arena_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, /// Bounds bytes cloned into importer state before any clone is attempted. /// This is the ONLY bound on retained state, and it is the principled one: /// `accountRetainedEvent` measures the bytes each event actually retains. /// /// A second bound used to sit beside it -- a 3,000,000 cap on the ENTRY /// COUNT -- and it was removed. It was a proxy for the same thing with an /// arbitrary number attached, and it bound first: pkgsrc at 185,280 commits /// over 123,843 files met it after 2h41m of translate, with 3.27 GB written /// and well inside this budget. A ceiling that stops a legitimate import /// before the real memory bound is reached is measuring the wrong thing. max_retained_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024, }; /// A limit overrun names its numbers. `FileCommandLimitExceeded` alone tells an /// operator nothing actionable: what was observed, what the ceiling was, and /// what raises it is the entire diagnostic. Retained here and surfaced next to /// the error name, exactly as `ProductionGit.preflightOverran` does for /// preflight output. pub const LimitDiagnostic = struct { buffer: [512]u8 = undefined, len: usize = 0, fn retain(self: *LimitDiagnostic, comptime fmt: []const u8, args: anytype) void { const message = std.fmt.bufPrint(&self.buffer, fmt, args) catch { // Every argument is a bounded integer, well inside the buffer; a // truncating format would be a bug here, not a wire condition. // Still total: report without the numbers rather than drop it. const fallback = "import-git: a per-commit parser ceiling was exceeded; raise --max-export-payload-bytes\n"; @memcpy(self.buffer[0..fallback.len], fallback); self.len = fallback.len; return; }; self.len = message.len; } pub fn write(self: *const LimitDiagnostic, err: *std.Io.Writer) void { if (self.len == 0) return; err.writeAll(self.buffer[0..self.len]) catch {}; } }; pub const Dependencies = struct { git: ChildFactory, publication: PublicationHooks = .real_filesystem, limits: ImportLimits = .{}, }; pub const Request = struct { source: []const u8, destination_parent: std.Io.Dir, destination_name: []const u8, /// sync §9.1 — the new repository's project code, which every peer checks /// in the handshake and refuses a repository without. Required, not /// defaulted: the source is git, so there is no code to carry over, and a /// store written without one cannot be served, cloned or adopted. The /// caller derives it as `init` does, from the destination's absolute path. project_code: zit.sync.HashHex, /// `.tree = null` selects the analyzed §6.3 optimum the child factory /// measured during preflight; `.tree = K` is the operator's explicit pin; /// `.flat` is `--no-submanifests`, §8.1's default posture. submanifest: git_import.SubmanifestChoice = .{ .tree = null }, attribution: git_import.Attribution = .committer, default_branch: ?[]const u8 = null, quiet: bool = false, /// Publish the import's instrumentation alongside the repository: the /// per-object provenance map (`.zit-import-map.jsonl`) and the source ref /// manifest (`refs`). Off by default -- a published checkout is the store /// and its checkout record, nothing else -- and opt-in for the workflows /// that measure imports rather than merely use them: every classifier /// census, drop audit and git-to-zit commit lookup this project has run /// reads the map from a PUBLISHED store, so the instrument must remain /// obtainable, just not ambient. provenance: bool = false, }; /// Generation 1: the append-only log the translator writes. It is never /// published, so it carries a name of its own and is deleted from staging /// before the directory is synced and renamed. Naming it `.zit` and renaming /// generation 2 over it inside staging would work too, and would be worse: the /// window in which `.zit` means "raw" is exactly the window a crash leaves /// behind, and a recovered staging directory would be indistinguishable from a /// published one. pub const raw_generation_name = zit.publication.raw_generation_name; pub const published_generation_name = zit.publication.published_generation_name; pub const PublicationBundle = zit.publication.PublicationBundle; const LogTreeStore = zit.log_tree_store.LogTreeStore; /// Runs one all-or-nothing import. Until `renamePreserve` succeeds, every /// failure owns and removes only this invocation's temporary sibling. pub fn run( gpa: std.mem.Allocator, io: std.Io, request: Request, streams: cli.Streams, dependencies: Dependencies, ) cli.Exit { if (destinationExists(io, request.destination_parent, request.destination_name)) { streams.err.print("import-git: destination '{s}' already exists\n", .{request.destination_name}) catch {}; return .failure; } var pipes = dependencies.git.vtable.start_export(dependencies.git.context, request.source) catch |err| { if (dependencies.git.vtable.startup_diagnostics) |write_diagnostics| { write_diagnostics(dependencies.git.context, streams.err); } streams.err.print("import-git: cannot start git fast-export: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; var effective_request = request; if (effective_request.default_branch == null) effective_request.default_branch = pipes.detected_default_branch; var session_finished = false; defer if (!session_finished) { _ = finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch false; }; var publication = PublicationBundle.create(gpa, io, request.destination_parent, request.destination_name) catch |err| { streams.err.print("import-git: cannot create temporary destination: {s}\n", .{@errorName(err)}) catch {}; return .failure; }; defer publication.abort(); publication.publish_instrumentation = request.provenance; var limit_diagnostic: LimitDiagnostic = .{}; importInto( gpa, io, effective_request, &publication, &pipes, streams, dependencies.git, &session_finished, dependencies.publication, dependencies.limits, &limit_diagnostic, ) catch |err| { if (err == error.DurabilityUncertain) { streams.err.print("import-git: destination '{s}' was published but parent-directory durability is uncertain\n", .{request.destination_name}) catch {}; } else { streams.err.print("import-git: import failed: {s}\n", .{@errorName(err)}) catch {}; limit_diagnostic.write(streams.err); } return .failure; }; // Pre-pay the derived-state cache while the import's work is still warm. // `Repository.open` derives `.zit-cache` on a miss, so without this the // FIRST read verb on an imported store eats a full-store derive -- on a // real 1M-artifact corpus that was ~342s of a 10-minute `open`. Clone has // warmed at completion since `warmCacheFromLog` existed; import simply // never did. After the atomic rename, against the published store, and // best-effort like every cache write: a failure costs the first reader // the derive it would have paid anyway. if (request.destination_parent.openDir(io, request.destination_name, .{})) |published_const| { var published = published_const; defer published.close(io); zit.Repository.warmCache(gpa, io, published); } else |_| {} streams.out.print("{s}\n", .{request.destination_name}) catch return .failure; if (!request.quiet) streams.err.print("import-git: imported '{s}'\n", .{request.source}) catch {}; return .ok; } fn importInto( gpa: std.mem.Allocator, io: std.Io, request: Request, publication: *PublicationBundle, pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, session_finished: *bool, hooks: PublicationHooks, limits: ImportLimits, diagnostic: *LimitDiagnostic, ) !void { const temporary = publication.temporary; try hooks.check(.create_store); // Generation 1, under a name of its own: translation never writes the file // that gets published. var store_file = try publication.createRawGeneration(); var store_file_open = true; defer if (store_file_open) store_file.close(io); var file_storage = storage.FileStorage.init(io, store_file, 0); // §7.1/§7.2 — this byte is a `HashPolicy` ordinal, NOT a `HashAlgo`. The two // enums overlap numerically and mean different things: `HashAlgo` is // {sha1=0, sha3_256=1}, `HashPolicy` is {sha1=0, auto=1, sha3=2, ...}. This // wrote `@intFromEnum(HashAlgo.sha1)`, so every imported repository // advertised the policy `sha1` by coincidence of ordinal — and the obvious // "switch it to sha3_256" repair would have yielded 1 = `auto`, silently // arming §7.4's promotion ratchet instead. Derived, never a literal. // // The import mints SHA-1 names throughout, so §7.2's rule gives `auto`: // a store that holds SHA-1 names and no SHA3 ones. When the naming // algorithm learns to follow its source, this call needs no edit — the // predicate changes, the derivation does not. const import_hash_policy: u8 = @intFromEnum(importDefaultPolicy()); var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), import_hash_policy); var log_live = true; defer if (log_live) log.deinit(gpa); var tree_store = LogTreeStore{ .gpa = gpa, .log = &log }; defer tree_store.deinitCache(); // One shape for the whole import, resolved before the first tree: flat when // the operator declined sub-manifests, else the operator's override if // given, else preflight's analyzed §6.3 optimum, else `resolveOptions`' own // default when neither is available. // // The analysis itself runs either way. Skipping it under `.flat` would make // tree emission the SECOND thing the two modes differ in -- the first being // a `git ls-tree` of the whole tip tree during preflight -- and the reason // this flag exists is to give a benchmark two imports that differ in // nothing else. One discarded measurement is the cheaper wart, and it is // not a new one: an explicit `--submanifest-k` already discards it. const resolved_choice: git_import.SubmanifestChoice = switch (request.submanifest) { .flat => .flat, .tree => |pin| .{ .tree = pin orelse pipes.analyzed_submanifest_k }, }; const shape = (try git_import.resolveOptions(.{ .submanifest = resolved_choice })).shape; const pinned_k: u4 = switch (shape) { // Not a k. `Settings.pinned()` reads zero as "no pin", which is the // truthful state for a repository that emitted no sub-manifest. .flat => 0, .tree => |tree| tree.k, }; switch (request.submanifest) { // Silence is the symmetric behavior for "do the default thing": a flat // import has no k to report, so it reports nothing. .flat => {}, .tree => |pin| if (!request.quiet and pin == null and pipes.analyzed_submanifest_k != null) { streams.err.print("import-git: analyzed sub-manifest k={d}\n", .{pinned_k}) catch {}; }, } var importer = try git_import.Importer.init(gpa, &log, tree_store.treeStore(), .{ .submanifest = switch (shape) { .flat => .flat, .tree => |tree| .{ .tree = tree.k }, }, .attribution = request.attribution, }); var importer_live = true; defer if (importer_live) importer.deinit(); var reader = fast_export.Reader.init(gpa, pipes.stdout, .{ .max_line = 1024 * 1024, .max_data = 1024 * 1024 * 1024, .max_path = 4096, // No event ceiling. `git fast-export` emits one event per blob, // commit, tag and reset, so the count scales with the source repo's // history rather than with anything this import spends: pkgsrc's // 185,280 commits are already a fifth of the 1,000,000 records this // used to admit, before a single blob event is counted. Memory is // bounded by bytes on both sides -- `max_total_data` for payload and // `max_retained_bytes` for retained state -- and those bound the // thing that actually costs. A stream of endless empty records can // still spin, but it retains nothing and stays interruptible. .max_events = std.math.maxInt(u64), .max_total_data = limits.max_export_payload_bytes, .max_command_arena_bytes = limits.max_command_arena_bytes, }); defer reader.deinit(); consumeExport(&reader, &importer, limits, diagnostic, .{ .streams = streams, .io = io, .quiet = request.quiet, .commit_total = pipes.commit_total, }) catch |err| { const scripted_child_failed = pipes.exit_code_ref == null and terminalExitCode(pipes) != 0; const child_finished_before_cancellation = finishSession(pipes, streams, child, request.quiet, true) catch |lifecycle_err| { session_finished.* = true; return lifecycle_err; }; session_finished.* = true; if (scripted_child_failed or (child_finished_before_cancellation and terminalExitCode(pipes) != 0)) return error.GitExportFailed; return err; }; _ = finishSession(pipes, streams, child, request.quiet, false) catch |err| { session_finished.* = true; return err; }; session_finished.* = true; if (terminalExitCode(pipes) != 0) return error.GitExportFailed; var finalized = importer.finalizeRefs(.{ .default_branch = request.default_branch }) catch |err| { if (err == error.CanonicalRefCollision) if (importer.ref_collision) |collision| { diagnostic.retain( "import-git: '{s}' and '{s}' both name 'refs/{s}'; rename one in the source repository, or exclude it, and re-run\n", .{ collision.first_source, collision.second_source, collision.canonical }, ); }; return err; }; defer finalized.deinit(gpa); for (finalized.controls) |control| try putSha1Artifact(&log, control); // The strongest statement this code can make about a flat import, and it // costs nothing: `stored_submanifest` is `emitted_nodes > 0`, a whole-import // fact derived from work actually done, computed without consulting the // shape -- not "the flag was set", which is what an assertion on `shape` // would say. // // `emitted_nodes` counts nodes this import CREATED, so on its own it would // leave a root built entirely from reused nodes uncovered. It is not // uncovered here, because the raw generation is created by this run a few // dozen lines above (`createRawGeneration`, an exclusive create): // reuse can only hit a node this same import already emitted, so zero // emitted implies zero reused implies no S reference anywhere. // // A returned error rather than an assert because this aborts a // user-visible operation and must unwind the transaction so DEST is never // published. if (std.meta.activeTag(shape) == .flat and finalized.stored_submanifest) { return error.SubmanifestEmittedUnderFlatImport; } try log.putConfig(.{ // Same §7.2 derivation as the header above, and the same reason it is // not a literal: this field is a `HashPolicy`, and the `HashAlgo` // ordinal that used to sit here only looked right. .hash_policy = @intFromEnum(importDefaultPolicy()), .default_branch = finalized.primary_branch, .project_id = "", .project_code = &request.project_code, // These two are a pair, not two independent settings. Under `.flat`, // `submanifest_k` is deliberately zero -- outside `artifact.min_k .. // max_k` -- and that is only safe while `enable_submanifests` is false, // because `Settings.emission()` short-circuits on disabled before it // can range-check the pin. Enabled with a zero pin is the // `.out_of_range` misconfiguration, so the two must move together. .submanifest_k = pinned_k, .enable_submanifests = std.meta.activeTag(shape) == .tree, // The latch is one-way and means "an S reference is stored here", not // "trees are on" — see its doc in logstore.zig. Leaving it false while // the store holds S-bearing artifacts is a lie that stays hidden only // while `enable_submanifests` is true, because the §7.2 gate is the // disjunction of the two: a later `zit submanifest disable` would flip // the repository to advertising avoid-submanifests while holding trees. .seen_submanifest = finalized.stored_submanifest, }); try writeRefs(gpa, io, temporary, finalized.refs); try checkout.write(io, temporary, .{ // The imported store is complete, but no worktree files have been // materialized. Publishing the default tip here would make `update` // incorrectly treat that unmapped worktree as current. .version = "", .branch = finalized.primary_branch, .merge_parent = "", .special_op = "", .manifest_flags = "", }); // The pack filter, captured while the importer is still alive. // // `finalized.ledger.candidates[].name` BORROWS from the importer, which is // deinitialized at the freeze below -- long before packing could read it. // `mirrorDropNames` hands back owned copies of exactly the same set, so // this is the only capture that outlives the generation it describes. try hooks.check(.capture_drop_set); var drops = try MirrorDropSet.capture(gpa, &importer); defer drops.deinit(gpa); // What the packed generation is verified AGAINST, captured for the same // reason and at the same moment as the drop set. `finalized.ledger` cannot // serve: every string in it borrows the importer, which the freeze below // destroys before packing has even started. var witness = try importer.captureWitness(gpa, finalized); defer witness.deinit(); var map = try temporary.createFile(io, ".zit-import-map.jsonl", .{ .read = true, .exclusive = true }); var map_open = true; defer if (map_open) map.close(io); try hooks.check(.before_flush_map); try map.writeStreamingAll(io, finalized.provenance_jsonl); try hooks.check(.after_flush_map); try hooks.check(.before_flush_store); try log.commit(); try hooks.check(.after_flush_store); try hooks.check(.before_sync_store); try store_file.sync(io); try hooks.check(.after_sync_store); try hooks.check(.before_sync_map); try map.sync(io); try hooks.check(.after_sync_map); // Freeze: neither store nor map remains writable after this point. try hooks.check(.before_close_store); // Read before the importer is gone; reported after verification, where a // completion statement is finally safe to make. const checkin_shaped_blobs = importer.workCounters().checkin_shaped_blobs; importer.deinit(); importer_live = false; log.deinit(gpa); log_live = false; store_file.close(io); store_file_open = false; try hooks.check(.after_close_store); try hooks.check(.before_close_map); map.close(io); map_open = false; try hooks.check(.after_close_map); try publication.advance(.raw_frozen); // Generation 2. Packing is MANDATORY: an import publishes a packed store or // it publishes nothing. The drop set is the classifier's, and only the // classifier's -- every name in it has a counted zero retained-reference, // so nothing reachable from an emitted tree can be in it. try publication.advance(.packing); // The operator has watched commits count up for the whole ingest; these // two phase lines keep the narration alive through the store-wide passes // that follow, which are the stretches that otherwise read as a hang. if (!request.quiet) streams.err.print("import-git: ingest complete, packing\n", .{}) catch {}; const stats = try packGeneration(gpa, io, publication, drops.filter(), hooks); try publication.advance(.packed_frozen); // The classifier authorized a set; the pack reports what it acted on. Equal // by construction -- every authorized name is an artifact generation 1 // holds, and the filter screens nothing else -- which is exactly why the // two being unequal means the classifier and the pack disagree about the // store's contents. Checking it makes the agreement a statement rather than // an inference from a matching pair of numbers nobody compared. if (stats.dropped != drops.set.count()) return error.MirrorDropSetNotApplied; if (!request.quiet) streams.err.print("import-git: packed, verifying\n", .{}) catch {}; try verifyFrozen(gpa, io, temporary, &witness, &drops, streams, hooks, request.quiet); try publication.advance(.verified); // Printed only once nothing further can invalidate it. Ahead of // verification this reads as a completion statement on a run that may still // fail, which is the one reading an operator must not get from it. if (!request.quiet) streams.err.print( "import-git: packed {d} artifacts into {d} bytes from {d} (dropped {d} mirror artifacts)\n", .{ stats.artifacts - stats.dropped, stats.bytes_after, stats.bytes_before, stats.dropped }, ) catch {}; // Stored verbatim, so each of these is a check-in of this repository now. // Said at import because the alternative is the operator meeting the class // for the first time as a refusal from `open`, with nothing connecting it // back to the file that carried it. Not a warning about a defect -- the // bytes are the source's and keeping them is deliberate. if (!request.quiet and checkin_shaped_blobs != 0) streams.err.print( "import-git: {d} tracked file(s) contain a complete check-in manifest and were stored " ++ "verbatim; each is now a check-in of this repository ('zit leaves' shows them)\n", .{checkin_shaped_blobs}, ) catch {}; // The pack's plaintext cache, reported for the same reason the sub-manifest // memo is: its budgets are fixed numbers, and a fixed number that is wrong // for a corpus is invisible from outside the process. A hit count without // its denominator cannot show that, which is why both are printed. if (!request.quiet) { const looks = stats.cache_hits + stats.cache_misses; if (looks != 0) streams.err.print( "import-git: pack plaintext cache {d}/{d} hits ({d}%), {d} evicted ({d} byte-cap, {d} entry-cap), {d} KiB in {d} entries resident\n", .{ stats.cache_hits, looks, stats.cache_hits * 100 / looks, stats.cache_evicted, stats.cache_evicted_for_bytes, stats.cache_evicted_for_entries, stats.cache_resident_bytes / 1024, stats.cache_resident_entries, }, ) catch {}; } // The rename ships the staging DIRECTORY, so generation 1 has to leave it // before the commit point or it publishes alongside generation 2. try hooks.check(.before_discard_raw_generation); try publication.discardRawGeneration(); try publication.discardInstrumentation(); try hooks.check(.after_discard_raw_generation); try publication.verifyStagingManifest(); try hooks.check(.before_sync_temporary_directory); try storage.syncDirectory(io, temporary); try hooks.check(.after_sync_temporary_directory); try publication.publish(request.destination_name, hooks); // If this final durability boundary fails, the destination is intentionally // retained: its content was fully verified and atomically published. hooks.check(.sync_parent_after_publish) catch return error.DurabilityUncertain; storage.syncDirectory(io, publication.parent) catch return error.DurabilityUncertain; } fn consumeExport( reader: *fast_export.Reader, importer: *git_import.Importer, limits: ImportLimits, diagnostic: *LimitDiagnostic, progress: Progress, ) !void { var retained_bytes: u64 = 0; var reporter = ProgressReporter.init(progress); while (reader.next() catch |err| switch (err) { // Reader admits events and cumulative payload bytes before allocating // an event arena or payload, so its exact bound failures map directly. // This import sets no event ceiling, so the reader cannot raise this // one; the mapping stays because the reader can still express it and a // silent mistranslation would be worse than an arm that never runs. error.EventLimitExceeded => return error.ExportEventLimitExceeded, error.DataLimitExceeded => { const overrun = reader.payload_overrun.?; switch (overrun.bound) { .cumulative_total => diagnostic.retain( "import-git: the export stream's cumulative payload exceeded its budget: observed at least {d} bytes against a budget of {d} bytes; parsing stopped there, so the stream's full payload is larger than the observed total; re-run with --max-export-payload-bytes above the stream's full payload\n", .{ overrun.observed_at_least, overrun.ceiling }, ), // No flag reaches this ceiling, so naming one would be a lie. .single_payload => diagnostic.retain( "import-git: a single export payload of {d} bytes exceeded the {d}-byte per-payload ceiling; no option raises this ceiling\n", .{ overrun.observed_at_least, overrun.ceiling }, ), } return error.ExportPayloadLimitExceeded; }, error.FileCommandLimitExceeded => { const overrun = reader.file_command_overrun.?; diagnostic.retain( "import-git: a commit exceeded the per-commit file-command ceiling: observed at least {d}, ceiling {d}, derived from the {d}-byte per-commit arena budget at {d} minimum retained bytes per command. This is a structural bound on one commit, not a total: no option raises it, because a commit at this width is beyond what the importer can hold whole\n", .{ overrun.observed_at_least, overrun.ceiling, overrun.arena_budget_bytes, overrun.min_bytes_per_command, }, ); return error.ExportFileCommandLimitExceeded; }, else => return err, }) |lease| { var event = lease; defer event.deinit(); try accountRetainedEvent(&retained_bytes, event.value, limits.max_retained_bytes); try importer.ingest(event.value); if (event.value == .commit) reporter.commitIngested(); } } /// What the ingest loop needs to narrate itself: where to write, whether the /// operator asked for silence, and the preflight's commit count when it took /// one. const Progress = struct { streams: cli.Streams, io: std.Io, quiet: bool, commit_total: ?u64, }; /// Strided progress for the hours-long phase. One stderr line at most every /// two seconds, so a pkgsrc-sized import narrates itself a few hundred times /// rather than 185,000 -- and a short import usually says nothing at all. const ProgressReporter = struct { progress: Progress, commits: u64 = 0, reported_at: u64 = 0, reported_at_ns: i96 = 0, const report_interval_ns: u64 = 2 * std.time.ns_per_s; // Clock reads go through a syscall, so they are strided the same way the // lines are: one read per checked batch, not one per commit. const check_stride: u64 = 64; fn init(progress: Progress) ProgressReporter { return .{ .progress = progress, .reported_at_ns = if (progress.quiet) 0 else std.Io.Clock.now(.awake, progress.io).nanoseconds, }; } fn commitIngested(self: *ProgressReporter) void { self.commits += 1; if (self.progress.quiet) return; if (self.commits % check_stride != 0) return; const now_ns = std.Io.Clock.now(.awake, self.progress.io).nanoseconds; const elapsed_ns: u64 = @intCast(@max(now_ns - self.reported_at_ns, 1)); if (elapsed_ns < report_interval_ns) return; self.reported_at_ns = now_ns; const rate = (self.commits - self.reported_at) * std.time.ns_per_s / elapsed_ns; self.reported_at = self.commits; const err = self.progress.streams.err; if (self.progress.commit_total) |total| { err.print("import-git: {d}/{d} commits ({d}%), {d}/s\n", .{ self.commits, total, self.commits * 100 / @max(total, 1), rate, }) catch {}; } else { err.print("import-git: {d} commits, {d}/s\n", .{ self.commits, rate }) catch {}; } } }; const sha1_artifact_name_bytes: u64 = std.crypto.hash.Sha1.digest_length * 2; fn accountRetainedEvent(total: *u64, event: fast_export.Event, limit: u64) !void { switch (event) { .blob => |blob| { try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(blob.original_oid != null), limit); if (blob.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); }, .commit => |commit| { try accountRetainedFixed(total, @as(usize, 3) + @intFromBool(commit.original_oid != null) + @intFromBool(commit.from != null), limit); if (commit.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit); try accountRetainedCopies(total, commit.ref, 3, limit); try accountRetainedIdentity(total, commit.author, limit); try accountRetainedIdentity(total, commit.committer, limit); try accountRetainedSlice(total, commit.message, limit); try accountRetainedPaths(total, commit.commands, limit); }, .reset => |reset| { try accountRetainedCopies(total, reset.ref, 2, limit); if (reset.target != null) try accountRetainedFixed(total, 1, limit); }, .tag => |tag| { const source_len = std.math.add(u64, @as(u64, "refs/tags/".len), std.math.cast(u64, tag.name.len) orelse return error.ImporterRetainedByteLimitExceeded) catch return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, source_len, 4, limit); try accountRetainedSlice(total, tag.name, limit); if (tag.original_oid) |oid| try accountRetainedSlice(total, oid, limit); try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(tag.mark != null), limit); try accountRetainedIdentity(total, tag.tagger, limit); try accountRetainedSlice(total, tag.message, limit); }, .done => {}, } } /// Returns the exact retained-state allocation charge for one parsed event. /// The import loop and limit-boundary tests share this calculation so a new /// retained field cannot silently create a guessed budget. pub fn retainedByteCharge(event: fast_export.Event) !u64 { var total: u64 = 0; try accountRetainedEvent(&total, event, std.math.maxInt(u64)); return total; } fn accountRetainedIdentity(total: *u64, identity: fast_export.Identity, limit: u64) !void { try accountRetainedSlice(total, identity.name, limit); try accountRetainedSlice(total, identity.email, limit); } fn accountRetainedPaths(total: *u64, commands: []const fast_export.FileCommand, limit: u64) !void { for (commands) |command| switch (command) { .modify => |modify| try accountRetainedSlice(total, modify.path, limit), .delete => |path| try accountRetainedSlice(total, path, limit), .rename => |rename| { try accountRetainedSlice(total, rename.old, limit); try accountRetainedSlice(total, rename.new, limit); }, .copy => |copy| { try accountRetainedSlice(total, copy.old, limit); try accountRetainedSlice(total, copy.new, limit); }, .delete_all => {}, }; } fn accountRetainedFixed(total: *u64, copies: usize, limit: u64) !void { try accountRetainedLengthCopies(total, sha1_artifact_name_bytes, copies, limit); } fn accountRetainedCopies(total: *u64, bytes: []const u8, copies: usize, limit: u64) !void { const len = std.math.cast(u64, bytes.len) orelse return error.ImporterRetainedByteLimitExceeded; try accountRetainedLengthCopies(total, len, copies, limit); } fn accountRetainedSlice(total: *u64, bytes: []const u8, limit: u64) !void { try accountRetainedCopies(total, bytes, 1, limit); } fn accountRetainedLengthCopies(total: *u64, len: u64, copies: usize, limit: u64) !void { var remaining = copies; while (remaining != 0) : (remaining -= 1) { if (total.* > limit or len > limit - total.*) return error.ImporterRetainedByteLimitExceeded; total.* += len; } } /// The exact candidate-only drop filter, owned for the whole pack. /// /// Two impedance mismatches meet here and both are load-bearing. The classifier /// reports HEX artifact names; `pack.Filter` screens RAW digest bytes and /// silently keeps anything that is not a complete 20- or 32-byte name, so a hex /// string handed straight through would drop nothing while reporting success. /// And `pack.NameSet` BORROWS its keys, so the decoded digests need storage /// that outlives the importer they came from. This owns that storage and /// nothing else. const MirrorDropSet = struct { /// One stable cell per name. `set`'s keys slice into these, so this must be /// allocated once at its final length and never resized. digests: [][32]u8 = &.{}, set: pack.NameSet = .{}, /// The same set in the hex spelling the verifier's name-set difference /// compares against. Owned here rather than re-derived: `digests` is a /// fixed-width cell array whose entries do not carry their own length, so /// hex could not be recovered from it without also carrying the widths. names: [][]u8 = &.{}, fn capture(gpa: std.mem.Allocator, importer: *const git_import.Importer) !MirrorDropSet { const names = try importer.mirrorDropNames(gpa); var owned = true; defer if (owned) { for (names) |name| gpa.free(name); gpa.free(names); }; var self: MirrorDropSet = .{}; errdefer self.deinit(gpa); self.digests = try gpa.alloc([32]u8, names.len); // `names` is repository-derived, and bounded: it is a subset of the // importer's mirror-candidate keys, which are retained state and so sit // under `limits.max_retained_bytes`, the budget that bounds everything // the importer holds. (An entry-count ceiling used to be named here too; // it was removed for measuring the wrong thing -- see that field's // note.) Keys of a hash map, so also distinct: `set.count()` // below equals `names.len`, which is what lets the caller compare the // set's size to the pack's own drop count. for (names, 0..) |name, index| { // A name the classifier authorized dropping but that will not // decode is a contradiction in this importer's own output, not an // operator error -- and the safe reading of "I cannot tell which // artifact this is" is to abort the import, never to pack on with a // filter that is quietly narrower than the one that was proven. const raw = rawHashName(name, &self.digests[index]) orelse return error.MirrorDropNameMalformed; self.set.add(gpa, raw) catch |err| switch (err) { error.BadKey => return error.MirrorDropNameMalformed, error.OutOfMemory => return error.OutOfMemory, }; } self.names = names; owned = false; return self; } fn deinit(self: *MirrorDropSet, gpa: std.mem.Allocator) void { self.set.deinit(gpa); gpa.free(self.digests); for (self.names) |name| gpa.free(name); gpa.free(self.names); self.* = undefined; } fn filter(self: *const MirrorDropSet) pack.Filter { return self.set.filter(); } }; /// Generation 1 -> generation 2. Opens the frozen raw log read-only, creates the /// packed generation as a separate file, and hands both to `packLog` with the /// classifier's drop set. /// /// `packLog` is the whole integrity story here and it is worth naming: it /// refuses a non-empty destination, §7.7-verifies every source frame it keeps, /// reconstructs and re-hashes every artifact it wrote, re-asks the filter over /// the destination's own entries so a pack that failed to drop what it was told /// to drop cannot return, and fsyncs the result. Any of those failing returns /// an error with the packed generation still a file inside staging, which the /// caller's `abort` removes along with everything else. fn packGeneration( gpa: std.mem.Allocator, io: std.Io, publication: *PublicationBundle, filter: pack.Filter, hooks: PublicationHooks, ) !pack.PackStats { try hooks.check(.create_packed_store); var packed_file = try publication.createPackedGeneration(); var packed_file_open = true; defer if (packed_file_open) packed_file.close(io); var raw_file = try publication.temporary.openFile(io, raw_generation_name, .{ .mode = .read_only }); defer raw_file.close(io); var raw_storage = storage.FileStorage.init(io, raw_file, (try raw_file.stat(io)).size); var raw_log = try logstore.LogStore.openReadOnly(gpa, raw_storage.storageFile()); defer raw_log.deinit(gpa); var packed_storage = storage.FileStorage.init(io, packed_file, 0); // The source's own recorded hash kind, not a constant: the packed // generation must reproduce generation 1's names, and a mismatched header // would make every artifact in it unaddressable. var packed_log = try logstore.LogStore.open(gpa, packed_storage.storageFile(), raw_log.hash_kind); var packed_log_live = true; defer if (packed_log_live) packed_log.deinit(gpa); try hooks.check(.before_pack); // No extra shuns. Every other caller that reclaims passes some, because it // is removing artifacts a published store already advertised and a peer may // already hold. Nothing here was ever externally visible: generation 1 has // existed only inside a staging directory that no reader can name, so there // is no history for a tombstone to reconcile with. The empty slice is the // statement, not an omission. const stats = try pack.packLog(gpa, &raw_log, &packed_log, filter, &.{}); try hooks.check(.after_pack); // `packLog` already committed (and so fsynced) the log. This is the file // handle's own durability boundary, kept symmetric with generation 1's so // both freezes are one shape and both are injectable. try hooks.check(.before_sync_packed_store); try packed_file.sync(io); try hooks.check(.after_sync_packed_store); try hooks.check(.before_close_packed_store); packed_log.deinit(gpa); packed_log_live = false; packed_file.close(io); packed_file_open = false; try hooks.check(.after_close_packed_store); return stats; } /// The complete pre-publication verification, over both frozen generations and /// everything staging holds beside them. /// /// Both stores open READ-ONLY. `openReadOnly` is not merely the polite choice /// here: `LogStore.open` is the append/repair path, and on a zero-length or /// torn-header file it writes a header or truncates -- which is the one thing a /// verifier must never do to the evidence it is judging. It also takes the /// store's own recorded hash kind rather than assuming one, for the same reason /// `packGeneration` propagates it: a mismatched header makes every artifact in /// the generation unaddressable. /// /// The clause pipeline itself lives in `git_import_verify.zig`. This function /// is the transaction's half of it: open the files, read the three staging /// artifacts, and translate a refusal into an aborted publication. fn verifyFrozen( gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, witness: *const git_import.ImportWitness, drops: *const MirrorDropSet, streams: cli.Streams, hooks: PublicationHooks, quiet: bool, ) !void { try hooks.check(.verify_raw_store); var raw_file = temporary.openFile(io, raw_generation_name, .{ .mode = .read_only }) catch |e| return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e); defer raw_file.close(io); const raw_size = (raw_file.stat(io) catch |e| return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e)).size; var raw_storage = storage.FileStorage.init(io, raw_file, raw_size); var raw_log = logstore.LogStore.openReadOnly(gpa, raw_storage.storageFile()) catch |e| return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e); defer raw_log.deinit(gpa); try hooks.check(.verify_store); var store_file = temporary.openFile(io, published_generation_name, .{ .mode = .read_only }) catch |e| return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e); defer store_file.close(io); const store_size = (store_file.stat(io) catch |e| return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e)).size; var file_storage = storage.FileStorage.init(io, store_file, store_size); var log = logstore.LogStore.openReadOnly(gpa, file_storage.storageFile()) catch |e| return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e); defer log.deinit(gpa); var checkout_buf: [checkout.max_ck_payload + 1]u8 = undefined; const state = try checkout.read(io, temporary, &checkout_buf) orelse return error.MissingCheckout; // Imports deliberately publish an unborn checkout: the repository is // discoverable, while the first update alone materializes its default tip. // Whether the branch it names is present and correct is the pipeline's // `checkout_invalid` clause, so it is not also pre-checked here -- one // statement of the rule, and the one that reaches the operator by name. try hooks.check(.verify_map); const map = temporary.readFileAlloc(io, ".zit-import-map.jsonl", gpa, .limited(max_verified_text_bytes)) catch |e| return reportUnreadable(streams, .map_unreadable, ".zit-import-map.jsonl", e); defer gpa.free(map); const refs = temporary.readFileAlloc(io, "refs", gpa, .limited(max_verified_text_bytes)) catch |e| return reportUnreadable(streams, .ref_target_mismatch, "refs", e); defer gpa.free(refs); try verifier.verify(gpa, .{ .raw = &raw_log, .packed_generation = &log, .witness = witness, .drop_names = drops.names, .map = map, .refs = refs, .refs_header = packed_refs_header, .checkout_branch = state.branch, .quiet = quiet, }, streams.err); } /// Refuse the way the pipeline refuses. Opening the evidence is as much a /// verification step as reading it, and an operator who is handed a bare /// `FileNotFound` from inside the transaction cannot tell a vanished generation /// from a torn header from a map that outgrew its ceiling. Same token shape, /// same error values, so the whole surface reads as one contract. fn reportUnreadable( streams: cli.Streams, clause: verifier.Clause, what: []const u8, cause: anyerror, ) verifier.Error { streams.err.print("import-git: verifier: {s}: {s}: {s}\n", .{ clause.token(), what, @errorName(cause) }) catch {}; return clause.err(); } /// The ceiling on the two staging text files the verifier reads whole. Both are /// bounded by the import's own object count -- one line per commit, blob, ref /// and tag -- so this is a runaway guard, not a policy: a repository that /// genuinely produces more than this has produced a map no operator can read /// either. const max_verified_text_bytes: usize = 1 << 30; /// Whether a `git --version` line names a git older than `major.minor`. /// Tolerant by design: an unparseable line answers FALSE, so an exotic build /// string never blocks an import the real version check could not judge — /// the flag rejection it would hit next still fails honestly. Handles the /// common suffixed forms ("git version 2.39.3 (Apple Git-146)"). fn gitOlderThan(version_line: []const u8, major: u32, minor: u32) bool { const prefix = "git version "; if (!std.mem.startsWith(u8, version_line, prefix)) return false; var it = std.mem.splitScalar(u8, version_line[prefix.len..], '.'); const got_major = std.fmt.parseUnsigned(u32, it.first(), 10) catch return false; const minor_text = it.next() orelse return false; // The minor field may carry a non-numeric tail in exotic builds; digits // are read greedily and anything after them ignored. var digits: usize = 0; while (digits < minor_text.len and std.ascii.isDigit(minor_text[digits])) digits += 1; if (digits == 0) return false; const got_minor = std.fmt.parseUnsigned(u32, minor_text[0..digits], 10) catch return false; if (got_major != major) return got_major < major; return got_minor < minor; } test "gitOlderThan: distro and exotic version lines judge correctly" { try std.testing.expect(gitOlderThan("git version 2.43.0", 2, 45)); try std.testing.expect(gitOlderThan("git version 2.39.3 (Apple Git-146)", 2, 45)); try std.testing.expect(!gitOlderThan("git version 2.45.0", 2, 45)); try std.testing.expect(!gitOlderThan("git version 2.55.0", 2, 45)); try std.testing.expect(!gitOlderThan("git version 3.0", 2, 45)); // Unparseable lines never block: the answer is "not provably old". try std.testing.expect(!gitOlderThan("gobbledygook", 2, 45)); try std.testing.expect(!gitOlderThan("git version future", 2, 45)); } fn putSha1Artifact(log: *logstore.LogStore, bytes: []const u8) !void { var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined; std.crypto.hash.Sha1.hash(bytes, &digest, .{}); _ = try log.putArtifact(&digest, bytes, null); } /// The header every packed ref file opens with. Versioned because this is the /// only description of the format that ships with a repository. pub const packed_refs_header = "# zit packed-refs v1\n"; /// Writes every finalized ref into one sorted file at `refs`. /// /// One file per ref was the obvious encoding and the wrong one: a ref's name /// became a path, so the filesystem got a vote on which repositories are /// representable. Two refs differing only in case are legal and distinct in /// git — sqlite has a pair at different commits — and one silently landed on /// the other, surfacing as a bare `PathAlreadyExists` at publication after the /// whole import had run. Case is only the reachable instance; path length, /// reserved device names, and each platform's own Unicode fold table are the /// same bug wearing different clothes. /// /// A single file has no such vote. Lines are `<canonical> <target>`, sorted by /// canonical name, which makes the file byte-identical for a given ref set and /// so comparable across two imports of one repository. Ref names cannot /// contain a space or a control character, so the separator needs no escape. /// Nothing in zit reads this back — `refs` is a reserved root the working-tree /// scan skips, and branches resolve through repostate's `sym-` tags — so it is /// interop output, and its whole job is to be legible. fn writeRefs(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, refs: []const git_import.FinalizedRef) !void { const ordered = try gpa.alloc(git_import.FinalizedRef, refs.len); defer gpa.free(ordered); @memcpy(ordered, refs); std.mem.sort(git_import.FinalizedRef, ordered, {}, struct { fn lessThan(_: void, a: git_import.FinalizedRef, b: git_import.FinalizedRef) bool { return std.mem.lessThan(u8, a.canonical, b.canonical); } }.lessThan); var body: std.ArrayList(u8) = .empty; defer body.deinit(gpa); try body.appendSlice(gpa, packed_refs_header); for (ordered) |mapping| { try body.appendSlice(gpa, mapping.canonical); try body.append(gpa, ' '); try body.appendSlice(gpa, mapping.target); try body.append(gpa, '\n'); } var file = try temporary.createFile(io, "refs", .{ .read = true, .exclusive = true }); defer file.close(io); try file.writeStreamingAll(io, body.items); try file.sync(io); } fn rawHashName(name: []const u8, buffer: *[32]u8) ?[]const u8 { if (name.len != 40 and name.len != 64) return null; const raw = buffer[0 .. name.len / 2]; _ = std.fmt.hexToBytes(raw, name) catch return null; return raw; } fn finishSession( pipes: *ExportPipes, streams: cli.Streams, child: ChildFactory, quiet: bool, abort: bool, ) !bool { var lifecycle_error: ?anyerror = null; // EOF is observed by the importing thread before it starts cancellation. // A child that is a zombie can still accept `kill`, so this observation is // the race-free completion evidence; `reap` below supplies its actual code. var child_finished_before_cancellation = abort and if (pipes.owner) |owner| owner.stdout_reached_eof else false; if (abort) { child.vtable.terminate(child.context) catch |err| { if (err == error.ChildAlreadyExited) { child_finished_before_cancellation = true; } else { lifecycle_error = err; } }; } child.vtable.drain_stderr(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; child.vtable.reap(child.context) catch |err| { if (lifecycle_error == null) lifecycle_error = err; }; const enabled = abort or terminalExitCode(pipes) != 0 or !quiet; if (pipes.diagnostics) |write_diagnostics| { write_diagnostics(pipes, streams.err, enabled); } else { copyDiagnostics(&pipes.stderr, streams.err, enabled); } if (lifecycle_error) |err| return err; return child_finished_before_cancellation; } fn terminalExitCode(pipes: *const ExportPipes) u8 { return if (pipes.exit_code_ref) |code| code.* else pipes.exit_code; } fn fileExists(io: std.Io, dir: std.Io.Dir, name: []const u8) bool { var file = dir.openFile(io, name, .{}) catch return false; file.close(io); return true; } fn destinationExists(io: std.Io, parent: std.Io.Dir, name: []const u8) bool { var directory = parent.openDir(io, name, .{}) catch { var file = parent.openFile(io, name, .{}) catch return false; file.close(io); return true; }; directory.close(io); return true; } fn copyDiagnostics(reader: *std.Io.Reader, err: *std.Io.Writer, enabled: bool) void { if (enabled) _ = reader.streamRemaining(err) catch {} else _ = reader.discard(.unlimited) catch {}; } pub const CommandScript = struct { pub const Preflight = enum { branch_head, detached_head, git_missing, not_repository, missing_objects, shallow, partial_promisor }; /// The stream `cancel_export_mid_stream` substitutes for the configured export. /// /// Named rather than inlined because a test asserts what this stream DOES: /// one blob parses, then a record the reader rejects -- a truncation, not a /// rejection at record zero, so the raw generation is already open when the /// cancellation lands. A test that hand-copied these bytes would keep /// passing against a stale copy the moment this literal changed, and its /// claim about the window would quietly stop being true. pub const cancelled_export = "feature done\nblob\nmark :1\ndata 1\nx\ninvalid\n"; pub const StagingReplacement = enum { directory, symlink }; /// One deliberate divergence between the frozen generations and what the /// import decided, injected at `.verify_raw_store` -- after both /// generations are closed files and before the verifier reads either. /// /// This is the CORRUPTION seam, distinct from `PublicationHooks.fail_at`, /// which is the FAILURE seam. A failure proves the transaction unwinds; a /// corruption proves the verifier looks. Before this existed the two /// mutating seams (`replaceStaging`, `plantStrayStagingFile`) both fired /// after `advance(.verified)`, so nothing in the suite could reach the /// bytes the verifier is supposed to be reading. /// /// Each member is named for the invariant it must trip, NOT for the /// mechanism it uses. A fixture that fails at some other clause has not /// proven its clause exists. pub const VerifierFault = enum { /// An artifact the classifier authorized dropping survives in the /// packed generation. `packed = raw - drop_set` is violated upward. retained_omitted_artifact, /// An artifact NOT in the drop set is absent from the packed /// generation. The same equation, violated downward. dropped_required_artifact, /// A well-formed artifact that is in neither raw nor the drop set. added_artifact, /// A name that IS in raw, republished over bytes that do not hash to /// it. The name set is untouched; only the content moved. changed_artifact_bytes, /// A delta frame whose recorded base is not the base its delta was /// encoded against. Every name is right and every frame checksums; /// only reconstruction through the chain reveals it. wrong_delta_base, /// The check-in's F cards permuted onto different paths. Every blob /// still present, every artifact still hashes -- the flattened tree is /// the only thing that moved. tree_change_preserving_artifacts, /// A merge check-in's P card with its parents in the other order. parent_reorder, /// A ref row in `refs` pointed at a different check-in. ref_retarget, /// A tag row in `refs` retargeted, and its kind flipped between the /// lightweight and annotated spellings. tag_change, /// The finalized JSONL's author and committer identities exchanged, so /// the record disagrees with the U card the store actually carries. attribution_change, /// One hex digit of a `provenance_digest` value in the finalized JSONL /// flipped. Nothing else in the import moves. provenance_tamper, }; pub const Config = struct { preflight: Preflight = .branch_head, fast_export: []const u8 = "", stderr_bytes: usize = 0, diagnostic_ring_limit: usize = 4096, /// Substitute `cancelled_export` for `fast_export`: a stream that parses /// one blob and then hits a record the reader rejects. /// /// A BOOLEAN, not a count. It was declared as `?usize` and only ever /// tested for null, so the number was dead and the name promised a /// cancellation after N events that no code implements -- the mechanism /// is a stream substitution and nothing counts anything. cancel_export_mid_stream: bool = false, competing_destination_at_publish: bool = false, export_exit: u8 = 0, replace_staging_at_publish: ?StagingReplacement = null, replace_destination_after_rename: bool = false, /// Drop an unexpected file into staging after the raw generation has /// been discarded -- i.e. between the last step that removes anything /// and the rename that ships whatever is left. Stands in for a future /// edit that writes an intermediate file and forgets to clean it up. stray_staging_file_after_discard: bool = false, /// Corrupt staging between the freeze and the verifier's first read. verifier_fault: ?VerifierFault = null, limits: ImportLimits = .{}, }; pub const Publication = struct { store_reopened_for_verify: bool = false, map_reopened_for_verify: bool = false, verify_before_publish: bool = false, /// Two-generation ordering, each recorded as "was the PREVIOUS stage /// already complete when this one began?" rather than as a bare visit /// count -- a visit count would still be set by a body that ran the /// stages in the wrong order. raw_generation_frozen: bool = false, raw_frozen_before_packing: bool = false, packed_generation_created: bool = false, packed_generation_frozen: bool = false, packed_before_verify: bool = false, both_generations_present_before_discard: bool = false, raw_generation_discarded: bool = false, discard_before_rename: bool = false, }; pub const Environment = struct { pub fn get(_: Environment, name: []const u8) ?[]const u8 { if (std.mem.eql(u8, name, "GIT_TERMINAL_PROMPT")) return "0"; if (std.mem.eql(u8, name, "GIT_NO_LAZY_FETCH")) return "1"; return null; } }; preflight: Preflight, fast_export: []const u8, stderr_bytes: usize, diagnostic_ring_limit: usize, cancel_export_mid_stream: bool, competing_destination_at_publish: bool, export_exit: u8, replace_staging_at_publish: ?StagingReplacement, replace_destination_after_rename: bool, stray_staging_file_after_discard: bool, verifier_fault: ?VerifierFault, /// Whether the injection actually happened. A corruption fixture that /// asserts a verifier error without asserting this would pass on a run /// where the injection silently did nothing -- which is the one failure /// mode a corruption suite must not have. verifier_fault_injected: bool = false, /// The observer the caller handed to `dependencies`, chained behind the /// script's own rather than overwritten by it. caller_observer: ?PublicationHooks.Observer = null, limits: ImportLimits, default_branch: ?[]const u8 = null, detached_head_recorded: bool = false, export_starts: usize = 0, preflight_starts: usize = 0, lifecycle: ChildLifecycle = .{}, stderr_drained_bytes: usize = 0, environment: Environment = .{}, publication: Publication = .{}, competitor_destination_intact: bool = false, temporary_sibling_left: bool = false, staging_replacement_observed: bool = false, staging_replacement_name: [64]u8 = undefined, staging_replacement_name_len: usize = 0, destination_parent: ?std.Io.Dir = null, destination_name: []const u8 = "", io: ?std.Io = null, pub fn init(config: Config) CommandScript { return .{ .preflight = config.preflight, .fast_export = config.fast_export, .stderr_bytes = config.stderr_bytes, .diagnostic_ring_limit = config.diagnostic_ring_limit, .cancel_export_mid_stream = config.cancel_export_mid_stream, .competing_destination_at_publish = config.competing_destination_at_publish, .export_exit = config.export_exit, .replace_staging_at_publish = config.replace_staging_at_publish, .replace_destination_after_rename = config.replace_destination_after_rename, .stray_staging_file_after_discard = config.stray_staging_file_after_discard, .verifier_fault = config.verifier_fault, .limits = config.limits, }; } fn dependencies(self: *CommandScript, fault: PublicationHooks) Dependencies { var publication = fault; // The script needs its own notification to drive `publicationEvent`, // but a caller's observer is a QUESTION the caller asked, and dropping // it answers that question with silence: every assertion made through // a discarded observer passes vacuously and nothing says why. Chained // rather than replaced, so both are served. self.caller_observer = fault.observer; publication.observer = .{ .context = self, .notify = publicationEvent }; publication.staging_observer = .{ .context = self, .replace = replaceStaging }; return .{ .git = .{ .context = self, .vtable = &.{ .start_export = startExport, .terminate = terminate, .drain_stderr = drainStderr, .reap = reap, } }, .publication = publication, .limits = self.limits, }; } fn startExport(context: *anyopaque, _: []const u8) !ExportPipes { const self: *CommandScript = @ptrCast(@alignCast(context)); self.export_starts += 1; self.lifecycle.launched += 1; const export_bytes = if (self.cancel_export_mid_stream) cancelled_export else self.fast_export; return .{ .stdout = .fixed(export_bytes), .stderr = .fixed("scripted diagnostic\n"), .exit_code = self.export_exit }; } fn terminate(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.terminated += 1; } fn drainStderr(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.stderr_drained += 1; self.stderr_drained_bytes = self.stderr_bytes; } fn reap(context: *anyopaque) !void { const self: *CommandScript = @ptrCast(@alignCast(context)); self.lifecycle.reaped += 1; } fn publicationEvent(context: *anyopaque, point: PublicationHooks.FailurePoint) void { const self: *CommandScript = @ptrCast(@alignCast(context)); switch (point) { .after_close_store => self.publication.raw_generation_frozen = true, .create_packed_store => self.publication.raw_frozen_before_packing = self.publication.raw_generation_frozen, // Asked of the filesystem, not asserted. `create_packed_store` // fires BEFORE the create, so this is the first point at which the // question has an answer -- and an unconditional `true` here would // be exactly the vacuous observation this suite already got wrong // once with `temporary_sibling_left`. .before_pack => self.publication.packed_generation_created = self.stagingHolds(published_generation_name), .after_close_packed_store => self.publication.packed_generation_frozen = true, // The one instant at which both generations are closed files and // the verifier has read neither. .verify_raw_store => if (self.verifier_fault) |fault| self.injectVerifierFault(fault), .verify_store => { self.publication.store_reopened_for_verify = true; self.publication.packed_before_verify = self.publication.packed_generation_frozen; }, .verify_map => self.publication.map_reopened_for_verify = true, .before_discard_raw_generation => self.publication.both_generations_present_before_discard = self.stagingHoldsBothGenerations(), .after_discard_raw_generation => { self.publication.raw_generation_discarded = !self.stagingHolds(raw_generation_name); if (self.stray_staging_file_after_discard) self.plantStrayStagingFile(); }, .before_rename_no_replace => { self.publication.discard_before_rename = self.publication.raw_generation_discarded; self.publication.verify_before_publish = self.publication.store_reopened_for_verify and self.publication.map_reopened_for_verify; if (self.competing_destination_at_publish) { const parent = self.destination_parent orelse return; parent.createDir(self.io orelse return, self.destination_name, .default_dir) catch return; self.competitor_destination_intact = true; } }, .after_rename_no_replace => if (self.replace_destination_after_rename) { const parent = self.destination_parent orelse return; const io = self.io orelse return; parent.deleteTree(io, self.destination_name) catch return; parent.createDir(io, self.destination_name, .default_dir) catch return; var replacement = parent.openDir(io, self.destination_name, .{ .follow_symlinks = false }) catch return; defer replacement.close(io); var marker = replacement.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; self.competitor_destination_intact = true; }, else => {}, } // After the script's own bookkeeping: an observer that sees a boundary // sees it in the state the script left it in. if (self.caller_observer) |observer| observer.notify(observer.context, point); } /// Whether the staging sibling holds BOTH generation files right now. This /// is the direct evidence that the two generations are distinct files: it /// is asked at the one instant both are supposed to exist, from outside the /// transaction, by name. fn stagingHoldsBothGenerations(self: *CommandScript) bool { return self.stagingHolds(raw_generation_name) and self.stagingHolds(published_generation_name); } /// Write a file into staging that the publish manifest does not name. fn plantStrayStagingFile(self: *CommandScript) void { const parent = self.destination_parent orelse return; const io = self.io orelse return; var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return; defer it_dir.close(io); var entries = it_dir.iterate(); while (entries.next(io) catch return) |entry| { if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue; var staging = parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return; defer staging.close(io); var stray = staging.createFile(io, "unpublishable-scratch", .{ .read = true, .exclusive = true }) catch return; defer stray.close(io); stray.writeStreamingAll(io, "scratch\n") catch return; return; } } // ---- Corruption seam ------------------------------------------------- // // Everything below runs at `.verify_raw_store`, from OUTSIDE the // transaction, against staging as a plain directory. It never touches the // production path; a run with `verifier_fault == null` executes none of it. /// Open the staging sibling. Caller closes. fn openStagingDir(self: *CommandScript) ?std.Io.Dir { const parent = self.destination_parent orelse return null; const io = self.io orelse return null; var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return null; defer it_dir.close(io); var entries = it_dir.iterate(); while (entries.next(io) catch return null) |entry| { if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue; return parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return null; } return null; } fn injectVerifierFault(self: *CommandScript, fault: VerifierFault) void { const io = self.io orelse return; var staging = self.openStagingDir() orelse return; defer staging.close(io); switch (fault) { .ref_retarget, .tag_change => self.rewriteStagingRefs(staging, fault), .attribution_change, .provenance_tamper => self.rewriteStagingProvenance(staging, fault), else => self.rewritePackedGeneration(staging, fault), } } fn readStagingText(self: *CommandScript, staging: std.Io.Dir, name: []const u8) ?[]u8 { const io = self.io orelse return null; return staging.readFileAlloc(io, name, std.heap.page_allocator, .limited(1 << 20)) catch null; } fn writeStagingText(self: *CommandScript, staging: std.Io.Dir, name: []const u8, bytes: []const u8) void { const io = self.io orelse return; staging.deleteFile(io, name) catch return; var file = staging.createFile(io, name, .{ .read = true, .exclusive = true }) catch return; defer file.close(io); file.writeStreamingAll(io, bytes) catch return; file.sync(io) catch return; self.verifier_fault_injected = true; } /// Flip one hex digit in place. The result is still a syntactically valid /// name, so nothing upstream of the clause under test can reject it on /// shape alone. fn flipHexDigit(byte: *u8) bool { byte.* = switch (byte.*) { '0' => '1', '1' => '0', 'a'...'e' => byte.* + 1, 'f' => 'a', '2'...'9' => byte.* - 1, else => return false, }; return true; } fn rewriteStagingRefs(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void { const body = self.readStagingText(staging, "refs") orelse return; defer std.heap.page_allocator.free(body); const want: []const u8 = if (fault == .tag_change) "sym-tags/" else "sym-heads/"; var cursor: usize = 0; while (std.mem.indexOfPos(u8, body, cursor, want)) |start| { const end = std.mem.indexOfScalarPos(u8, body, start, '\n') orelse body.len; const space = std.mem.lastIndexOfScalar(u8, body[start..end], ' ') orelse { cursor = end; continue; }; // Retarget: the row now names something that is not the check-in // the import resolved for it. if (!flipHexDigit(&body[start + space + 1])) { cursor = end; continue; } // Type change, for the tag case only: the same row reappears // spelled as a branch, which is a different kind of ref entirely. // The two prefixes differ in length, so this rebuilds the file // rather than poking it -- every other byte is carried through. if (fault == .tag_change) { const gpa = std.heap.page_allocator; var rebuilt: std.ArrayList(u8) = .empty; defer rebuilt.deinit(gpa); rebuilt.appendSlice(gpa, body[0..start]) catch return; rebuilt.appendSlice(gpa, "sym-heads/") catch return; rebuilt.appendSlice(gpa, body[start + want.len ..]) catch return; self.writeStagingText(staging, "refs", rebuilt.items); return; } self.writeStagingText(staging, "refs", body); return; } } fn rewriteStagingProvenance(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void { const gpa = std.heap.page_allocator; const body = self.readStagingText(staging, ".zit-import-map.jsonl") orelse return; defer gpa.free(body); switch (fault) { .provenance_tamper => { const marker = "\"provenance_digest\":\""; const at = std.mem.indexOf(u8, body, marker) orelse return; if (!flipHexDigit(&body[at + marker.len])) return; self.writeStagingText(staging, ".zit-import-map.jsonl", body); }, .attribution_change => { // Exchange the two identity objects on the first commit line. // Attribution mode selects ONE of them; swapping makes the // record name the identity the store's U card does not carry, // while every digit of both identities stays well-formed. const author_marker = "\"author\":"; const committer_marker = ",\"committer\":"; const tree_marker = ",\"tree_digest\":"; const a = std.mem.indexOf(u8, body, author_marker) orelse return; const c = std.mem.indexOfPos(u8, body, a, committer_marker) orelse return; const t = std.mem.indexOfPos(u8, body, c, tree_marker) orelse return; const author = body[a + author_marker.len .. c]; const committer = body[c + committer_marker.len .. t]; if (std.mem.eql(u8, author, committer)) return; var swapped: std.ArrayList(u8) = .empty; defer swapped.deinit(gpa); swapped.appendSlice(gpa, body[0 .. a + author_marker.len]) catch return; swapped.appendSlice(gpa, committer) catch return; swapped.appendSlice(gpa, committer_marker) catch return; swapped.appendSlice(gpa, author) catch return; swapped.appendSlice(gpa, body[t..]) catch return; self.writeStagingText(staging, ".zit-import-map.jsonl", swapped.items); }, else => {}, } } const StagedArtifact = struct { name: []u8, bytes: []u8 }; fn readGeneration( self: *CommandScript, staging: std.Io.Dir, name: []const u8, config: ?*?logstore.OwnedConfig, hash_kind: ?*u8, ) ?[]StagedArtifact { const io = self.io orelse return null; const gpa = std.heap.page_allocator; var file = staging.openFile(io, name, .{ .mode = .read_only }) catch return null; defer file.close(io); const size = (file.stat(io) catch return null).size; var file_storage = storage.FileStorage.init(io, file, size); var log = logstore.LogStore.openReadOnly(gpa, file_storage.storageFile()) catch return null; defer log.deinit(gpa); // Carried through the rebuild when the caller asks for it, so a // replacement generation differs from the original in the ONE property // its fault is named for and not also in its settings. The hash kind // most of all: hardcoding sha1 would make every store fault silently // inexpressible under any other algorithm -- the rebuilt header would // disagree with its own names and the whole suite would go red with no // diagnostic. if (config) |out| out.* = log.readConfig(gpa) catch null; if (hash_kind) |out| out.* = log.hash_kind; const rids = log.ridsInPersistedOrder(gpa) catch return null; defer gpa.free(rids); var out: std.ArrayList(StagedArtifact) = .empty; for (rids) |rid| { const bytes = (log.contentUnmasked(gpa, rid, null) catch continue) orelse continue; const hash = gpa.dupe(u8, log.hashOf(rid)) catch continue; out.append(gpa, .{ .name = hash, .bytes = bytes }) catch continue; } return out.toOwnedSlice(gpa) catch null; } fn freeGeneration(items: []StagedArtifact) void { const gpa = std.heap.page_allocator; for (items) |item| { gpa.free(item.name); gpa.free(item.bytes); } gpa.free(items); } /// Rewrite the packed generation from its own contents, applying `fault`. /// /// Rebuilding rather than byte-poking is deliberate: every frame the /// replacement contains checksums correctly and every name is well-formed, /// so the only thing a verifier can object to is the property the fault is /// named for. fn rewritePackedGeneration(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void { const io = self.io orelse return; const gpa = std.heap.page_allocator; var config: ?logstore.OwnedConfig = null; defer if (config) |*owned| owned.deinit(gpa); var hash_kind: u8 = 0; const items = self.readGeneration(staging, published_generation_name, &config, &hash_kind) orelse return; defer freeGeneration(items); if (items.len == 0) return; // `retained_omitted_artifact` needs a name the pack REMOVED, which // exists only as the difference between the two generations. var readd: ?StagedArtifact = null; const raw_items: ?[]StagedArtifact = if (fault == .retained_omitted_artifact) self.readGeneration(staging, raw_generation_name, null, null) else null; defer if (raw_items) |raw| freeGeneration(raw); if (raw_items) |raw| { outer: for (raw) |candidate| { for (items) |kept| { if (std.mem.eql(u8, kept.name, candidate.name)) continue :outer; } readd = candidate; break; } // No drop set means nothing was omitted, so this fault has no // referent. Leaving `verifier_fault_injected` false is the whole // point: the fixture asserts it and fails loudly rather than // passing on a corruption that never happened. if (readd == null) return; } staging.deleteFile(io, published_generation_name) catch return; var file = staging.createFile(io, published_generation_name, .{ .read = true, .exclusive = true }) catch return; var file_open = true; defer if (file_open) file.close(io); var file_storage = storage.FileStorage.init(io, file, 0); var log = logstore.LogStore.open(gpa, file_storage.storageFile(), hash_kind) catch return; var log_live = true; defer if (log_live) log.deinit(gpa); if (config) |owned| log.putConfig(owned.config) catch return; // When a check-in is re-emitted its name changes; every file that // names it must follow. const Renamed = struct { old: [2 * logstore.max_hash_len]u8, new: [2 * logstore.max_hash_len]u8, len: usize }; var renamed: ?Renamed = null; // A BLOB, chosen by kind rather than by position. Both faults below // want an ordinary artifact: aim either at a check-in and the fixture // starts proving `checkin-unresolvable` instead of the clause it is // named for -- loudly rather than falsely, but for the wrong reason. const target = lastBlob(items) orelse return; const skip: usize = if (fault == .dropped_required_artifact) target else std.math.maxInt(usize); const mutate: usize = switch (fault) { .changed_artifact_bytes, .wrong_delta_base => target, else => std.math.maxInt(usize), }; for (items, 0..) |item, index| { if (index == skip) { self.verifier_fault_injected = true; continue; } if (index == mutate and fault == .changed_artifact_bytes) { // Same name, different bytes. Nothing about the frame is // malformed; only `hash(content) == name` is false. _ = log.putArtifact(item.name, "verifier fixture: these are not the bytes\n", null) catch return; self.verifier_fault_injected = true; continue; } if (index == mutate and fault == .wrong_delta_base and index >= 2) { // A delta encoded against its NEIGHBOUR -- which is similar // enough to it to produce real copy segments -- but recorded // against rid 1, a much shorter artifact. Both frames exist, // both checksum, and the chain is within bounds; only walking // it lands anywhere wrong. // // The neighbour matters. A delta between two dissimilar // artifacts carries no copy segment at all, and such a delta // reconstructs the SAME bytes from any base whatsoever -- so // recording the wrong one would be an injection that changed // nothing, and this fixture would then be asserting a clause // about a store that is not actually wrong. if (self.plantWrongDeltaBase(&log, items[index - 1].bytes, item)) continue; } if (fault == .tree_change_preserving_artifacts or fault == .parent_reorder) { if (renamed == null) if (self.mutatedCheckin(item, fault)) |replacement| { defer gpa.free(replacement.name); defer gpa.free(replacement.bytes); _ = log.putArtifact(replacement.name, replacement.bytes, null) catch return; // Re-emitting a check-in necessarily renames it. Carrying // the rename into `refs` and the map is what keeps these // two fixtures SURGICAL: without it the ref rows and the // provenance lines would also dangle, and the fixture // would trip whichever clause happened to run first // instead of the one it is named for. var rename: Renamed = .{ .old = undefined, .new = undefined, .len = 2 * item.name.len }; _ = std.fmt.bufPrint(rename.old[0..rename.len], "{x}", .{item.name}) catch return; _ = std.fmt.bufPrint(rename.new[0..rename.len], "{x}", .{replacement.name}) catch return; renamed = rename; self.verifier_fault_injected = true; continue; }; } _ = log.putArtifact(item.name, item.bytes, null) catch return; } if (readd) |dropped| { _ = log.putArtifact(dropped.name, dropped.bytes, null) catch return; self.verifier_fault_injected = true; } if (fault == .added_artifact) { const novel = "verifier fixture: an artifact no import produced\n"; var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined; std.crypto.hash.Sha1.hash(novel, &digest, .{}); _ = log.putArtifact(&digest, novel, null) catch return; self.verifier_fault_injected = true; } log.commit() catch return; log.deinit(gpa); log_live = false; file.sync(io) catch {}; file.close(io); file_open = false; if (renamed) |rename| { self.substituteStagingName(staging, "refs", rename.old[0..rename.len], rename.new[0..rename.len]); self.substituteStagingName(staging, ".zit-import-map.jsonl", rename.old[0..rename.len], rename.new[0..rename.len]); } } /// The last artifact in persisted order whose bytes are NOT a check-in /// manifest, or null when a generation is all check-ins. fn lastBlob(items: []const StagedArtifact) ?usize { const gpa = std.heap.page_allocator; var index = items.len; while (index > 0) { index -= 1; var manifest = artifact.parse(gpa, items[index].bytes) catch return index; defer manifest.deinit(); if (manifest.kind != .manifest) return index; } return null; } /// Store `item` as a delta of `base` under a recorded base of rid 1. /// /// Returns whether the divergence is real, PROVEN rather than assumed: /// applying the encoded delta to the artifact the frame actually names must /// fail. Nothing is written and nothing is claimed when it does not, so the /// fixture reports "the injection did not happen" instead of asserting a /// verifier clause about a store that reconstructs perfectly. fn plantWrongDeltaBase( self: *CommandScript, log: *logstore.LogStore, base: []const u8, item: StagedArtifact, ) bool { const gpa = std.heap.page_allocator; const encoded = delta.create(gpa, base, item.bytes) catch return false; defer gpa.free(encoded); const recorded_base = (log.contentUnmasked(gpa, 1, null) catch return false) orelse return false; defer gpa.free(recorded_base); if (delta.apply(gpa, recorded_base, encoded)) |applied| { gpa.free(applied); return false; } else |_| {} _ = log.putDelta(item.name, 1, encoded) catch return false; self.verifier_fault_injected = true; return true; } /// Replace every occurrence of one artifact name with another in a staging /// text file. Equal lengths, so this is a straight in-place substitution. fn substituteStagingName( self: *CommandScript, staging: std.Io.Dir, file_name: []const u8, old_name: []const u8, new_name: []const u8, ) void { const injected = self.verifier_fault_injected; const body = self.readStagingText(staging, file_name) orelse return; defer std.heap.page_allocator.free(body); var cursor: usize = 0; var replaced = false; while (std.mem.indexOfPos(u8, body, cursor, old_name)) |at| { @memcpy(body[at .. at + new_name.len], new_name); cursor = at + new_name.len; replaced = true; } if (!replaced) return; self.writeStagingText(staging, file_name, body); // `writeStagingText` reports its own success; this helper only carries // a rename that a real injection already made, so it must not be what // turns the flag on. self.verifier_fault_injected = injected; } /// Reparse a check-in and re-emit it with `fault` applied. Returns null for /// anything that is not a check-in, so the caller can keep walking. /// /// The replacement is stored under its OWN correct name -- the point of /// both faults is that the store stays internally honest and only its /// meaning changed. fn mutatedCheckin(self: *CommandScript, item: StagedArtifact, fault: VerifierFault) ?StagedArtifact { _ = self; const gpa = std.heap.page_allocator; var manifest = artifact.parse(gpa, item.bytes) catch return null; defer manifest.deinit(); if (manifest.kind != .manifest) return null; switch (fault) { .tree_change_preserving_artifacts => { if (manifest.f.len < 2) return null; // Every blob still present and still named; two of them now // hang at each other's paths. const cards = @constCast(manifest.f); std.mem.swap(?[]const u8, &cards[0].hash, &cards[cards.len - 1].hash); }, .parent_reorder => { if (manifest.p.len == 0) return null; const card = manifest.p[0]; if (card.count < 2) return null; var parts = std.mem.splitScalar(u8, card.parents orelse return null, ' '); const first = parts.next() orelse return null; const second = parts.next() orelse return null; const swapped = std.fmt.allocPrint(gpa, "{s} {s}", .{ second, first }) catch return null; defer gpa.free(swapped); const cards = @constCast(manifest.p); cards[0].parents = swapped; const bytes = artifact.create(&manifest, gpa) catch return null; var name_buf: [64]u8 = undefined; const name = artifact.artifactName(git_import.import_hash_algo, bytes, &name_buf); var raw: [32]u8 = undefined; const digest = rawHashName(name, &raw) orelse { gpa.free(bytes); return null; }; return .{ .name = gpa.dupe(u8, digest) catch return null, .bytes = bytes }; }, else => return null, } const bytes = artifact.create(&manifest, gpa) catch return null; var name_buf: [64]u8 = undefined; const name = artifact.artifactName(git_import.import_hash_algo, bytes, &name_buf); var raw: [32]u8 = undefined; const digest = rawHashName(name, &raw) orelse { gpa.free(bytes); return null; }; return .{ .name = gpa.dupe(u8, digest) catch return null, .bytes = bytes }; } /// Whether the staging sibling holds `name` right now. fn stagingHolds(self: *CommandScript, name: []const u8) bool { const parent = self.destination_parent orelse return false; const io = self.io orelse return false; // Reopened for iteration: Linux enforces iterate rights at readdir // time (a plain handle is O_PATH there); macOS does not. var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return false; defer it_dir.close(io); var entries = it_dir.iterate(); while (entries.next(io) catch return false) |entry| { if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue; var staging = parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return false; defer staging.close(io); return fileExists(io, staging, name); } return false; } /// Whether a staging sibling OTHER than a deliberately planted replacement /// survived the run. Read after `run` returns, so a true answer means this /// invocation leaked one. /// /// Fails CLOSED: an unreadable parent reports a leak. This backs a /// negative assertion -- `!temporary_sibling_left` -- so answering "no /// leak" on a question that could not be asked is how a leak assertion /// passes vacuously, which is the defect this field was added to fix. fn stagingSiblingSurvives(self: *CommandScript) bool { const parent = self.destination_parent orelse return true; const io = self.io orelse return true; const planted = self.stagingReplacementName(); var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return true; defer it_dir.close(io); var entries = it_dir.iterate(); while (entries.next(io) catch return true) |entry| { if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue; // A script that replaces the staging entry at the publication // boundary leaves its plant behind BY DESIGN -- the transaction // correctly refuses to delete an entry it no longer owns. Excluding // that one name keeps the assertion live for every other sibling, // rather than disabling it wholesale in exactly the shape where a // leak is most plausible. if (planted.len != 0 and std.mem.eql(u8, entry.name, planted)) continue; return true; } return false; } fn replaceStaging(context: *anyopaque, parent: std.Io.Dir, name: []const u8) void { const self: *CommandScript = @ptrCast(@alignCast(context)); const replacement = self.replace_staging_at_publish orelse return; const io = self.io orelse return; if (name.len > self.staging_replacement_name.len) return; parent.deleteTree(io, name) catch return; switch (replacement) { .directory => { parent.createDir(io, name, .default_dir) catch return; var directory = parent.openDir(io, name, .{ .follow_symlinks = false }) catch return; defer directory.close(io); var marker = directory.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return; defer marker.close(io); marker.writeStreamingAll(io, "survives\n") catch return; }, .symlink => parent.symLink(io, "outside", name, .{}) catch return, } @memcpy(self.staging_replacement_name[0..name.len], name); self.staging_replacement_name_len = name.len; self.staging_replacement_observed = true; } pub fn stagingReplacementName(self: *const CommandScript) []const u8 { return self.staging_replacement_name[0..self.staging_replacement_name_len]; } /// Derived from `exportArgv` so this fake cannot drift from the command /// the importer actually runs. Shows the default signature policy; the /// opt-out's effect is covered directly against `exportArgv`. pub fn exportCommandLine(_: *const CommandScript) []const u8 { return comptime line: { const argv = exportArgv("fixture.git", false); var joined: []const u8 = argv[0]; for (1..argv.len) |index| joined = joined ++ " " ++ argv[index]; break :line joined; }; } }; /// Caller-owned capture storage for `runScripted`'s stdout/stderr. The /// returned `ScriptResult` slices point INTO this struct, so it must outlive /// every read of the result — the storage used to live on `runScripted`'s own /// frame, which made the returned slices dangle (visible as garbled stderr on /// Linux, silent luck on macOS). pub const ScriptCapture = struct { out: [4096]u8 = undefined, err: [4096]u8 = undefined, }; pub const ScriptResult = struct { code: cli.Exit, out: []const u8, err: []const u8, lifecycle: ChildLifecycle, retained_diagnostic: []const u8, published_durability_uncertain: bool = false, }; pub fn runScripted( gpa: std.mem.Allocator, io: std.Io, request: Request, script: *CommandScript, hooks: PublicationHooks, capture: *ScriptCapture, ) ScriptResult { var out = std.Io.Writer.fixed(&capture.out); var err = std.Io.Writer.fixed(&capture.err); const streams: cli.Streams = .{ .out = &out, .err = &err }; script.preflight_starts += 1; if (script.preflight == .git_missing or script.preflight == .not_repository or script.preflight == .missing_objects or script.preflight == .shallow or script.preflight == .partial_promisor or (script.preflight == .detached_head and script.default_branch == null)) { script.lifecycle.reaped += 1; return .{ .code = .failure, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "preflight rejected" }; } if (script.preflight == .detached_head) script.detached_head_recorded = true; script.destination_parent = request.destination_parent; script.destination_name = request.destination_name; script.io = io; var effective_request = request; effective_request.default_branch = script.default_branch; const code = run(gpa, io, effective_request, streams, script.dependencies(hooks)); // Asked after the run, from outside. `stagingSiblingSurvives` excludes a // deliberately planted replacement by name, so this stays a live leak // assertion even on the staging-replacement scripts. script.temporary_sibling_left = script.stagingSiblingSurvives(); if (code == .failure and hooks.fail_at == .sync_parent_after_publish) { return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "durability uncertain", .published_durability_uncertain = true }; } return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "scripted diagnostic" }; } pub fn dispatchForTest(args: []const []const u8) ScriptResult { if (args.len == 1 and std.mem.eql(u8, args[0], "--help")) { // Rendered from `cli.import_git_usage`, not transcribed: the hand copy // this replaced had silently fallen two options behind the real help. // The `usage: ` prefix is `main.zig`'s, carried here so this stub's // output is a true PREFIX of what the real help prints rather than // merely sharing its option surface. return .{ .code = .ok, .out = "usage: " ++ cli.import_git_usage ++ "\n", .err = "", .lifecycle = .{}, .retained_diagnostic = "" }; } _ = cli.parseImportGitArgs(args) catch { return .{ .code = .usage, .out = "", .err = "invalid import-git arguments\n", .lifecycle = .{}, .retained_diagnostic = "" }; }; return .{ .code = .failure, .out = "", .err = "test dispatch requires scripted dependencies\n", .lifecycle = .{}, .retained_diagnostic = "" }; }