const std = @import("std");
const builtin = @import("builtin");
pub const cli = @import("zit").cli;
const zit = @import("zit");
const artifact = zit.artifact;
const checkout = zit.checkout;
const delta = zit.delta;
const fast_export = zit.git_fast_export;
const git_import = zit.git_import;
const logstore = zit.logstore;
const pack = zit.pack;
const storage = zit.storage;
const editor = zit.submanifest_editor;
const verifier = @import("git_import_verify.zig");
pub const ChildLifecycle = struct {
launched: usize = 0,
terminated: usize = 0,
stderr_drained: usize = 0,
reaped: usize = 0,
};
/// The child-facing half of the import transaction. The caller owns the
/// implementation and its backing buffers for the full call to `run`.
pub const ChildFactory = struct {
context: *anyopaque,
vtable: *const VTable,
pub const VTable = struct {
start_export: *const fn (context: *anyopaque, source: []const u8) anyerror!ExportPipes,
terminate: *const fn (context: *anyopaque) anyerror!void,
drain_stderr: *const fn (context: *anyopaque) anyerror!void,
reap: *const fn (context: *anyopaque) anyerror!void,
startup_diagnostics: ?*const fn (context: *anyopaque, err: *std.Io.Writer) void = null,
};
};
pub const ExportPipes = struct {
stdout: std.Io.Reader,
stderr: std.Io.Reader,
/// Scripted sessions supply their terminal status here. The OS binding gets
/// its status during reap; an import succeeds only after the stream reaches
/// `done` and EOF, so a nonzero scripted result is still an import failure.
exit_code: u8 = 0,
/// OS-backed sessions publish the wait status here during `reap`; scripted
/// sessions use `exit_code` directly.
exit_code_ref: ?*const u8 = null,
owner: ?*ProductionGit = null,
diagnostics: ?*const fn (pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void = null,
detected_default_branch: ?[]const u8 = null,
/// draft-fossil-submanifests-00 §6.3's optimum for the source's own name
/// set, measured during preflight. Null when the source could not be
/// listed or cannot partition; the caller then keeps its own default.
analyzed_submanifest_k: ?u4 = null,
/// The source's commit count, measured during preflight so the ingest
/// loop can report progress against a real denominator. Null when the
/// count could not be taken; progress then reports without a total.
commit_total: ?u64 = null,
};
/// The store's §7.2 default policy for a fresh import, derived from the
/// algorithm the import actually names artifacts under rather than from a
/// guess about what it produced. Both call sites read it, so the header and
/// the persisted setting cannot drift apart.
fn importDefaultPolicy() artifact.HashPolicy {
const algo = git_import.import_hash_algo;
return artifact.defaultPolicy(algo == .sha1, algo == .sha3_256);
}
/// The `git fast-export` command line the importer runs, as a value.
///
/// Pure and public so the one policy decision it encodes — what happens to a
/// signature Fossil has no card for — is testable without spawning a process.
/// `refuse_signed` moves exactly the two signature modes and nothing else.
pub fn exportArgv(source: []const u8, refuse_signed: bool) [10][]const u8 {
return .{
"git",
"-C",
source,
"fast-export",
"--all",
"--show-original-ids",
"--mark-tags",
if (refuse_signed) "--signed-commits=abort" else "--signed-commits=strip",
if (refuse_signed) "--signed-tags=abort" else "--signed-tags=strip",
"--use-done-feature",
};
}
/// OS-backed production dependency. It owns both pipe buffers through the
/// import call; the `ExportPipes` readers retain a pointer to this value.
pub const ProductionGit = struct {
pub const StartupFailure = enum { after_spawn, after_stderr_handoff, after_stderr_thread };
/// Keep room in fixed CLI writers for the authoritative import diagnostic.
const cli_diagnostic_reserve_bytes = 256;
/// Bytes of output one preflight check may produce. Every check asks a
/// bounded question — one ref, one boolean, one config value, or (for
/// `fsck --connectivity-only --no-dangling`) nothing at all when the
/// repository is intact — so an overrun means a check's shape was
/// misjudged, never that the source is merely large.
const preflight_output_limit_bytes = 8 * 1024;
gpa: std.mem.Allocator,
io: std.Io,
/// The process environment the spawned git children inherit (plus the
/// GIT_* pins `childEnvironment` adds). Threaded from main's `Init` in
/// production and `std.testing.environ` in tests.
environ: std.process.Environ,
child: ?std.process.Child = null,
exit_code: u8 = 1,
stdout_buffer: [8192]u8 = undefined,
stderr_buffer: [8192]u8 = undefined,
reaped: bool = false,
reap_calls: usize = 0,
/// Captured by the stdout reader before any parser-error cleanup can
/// terminate the child. This distinguishes an independently completed
/// truncated export from cancellation closing the pipe.
stdout_reached_eof: bool = false,
default_branch: ?[]const u8 = null,
owns_default_branch: bool = false,
analyzed_submanifest_k: ?u4 = null,
commit_total: ?u64 = null,
/// The operator's explicit opt-out from the default signature policy.
/// False (the default) strips; true restores the refusal.
refuse_signed: bool = false,
stderr_thread: ?std.Thread = null,
/// Moved out of `child` before its reader thread starts, so draining never
/// races cleanup or a platform-specific process handle implementation.
stderr_file: ?std.Io.File = null,
stderr_ring: [4096]u8 = undefined,
stderr_ring_start: usize = 0,
stderr_ring_len: usize = 0,
stderr_bytes_drained: usize = 0,
stderr_lock: std.Io.Mutex = .init,
test_argv: ?[]const []const u8 = null,
test_startup_failure: ?StartupFailure = null,
pub fn init(gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ) ProductionGit {
return .{ .gpa = gpa, .io = io, .environ = environ };
}
/// Test-only process constructor: uses the identical spawn, stream,
/// concurrent-diagnostic, termination, and reap path as `git fast-export`.
pub fn initForTest(gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ, argv: []const []const u8) ProductionGit {
return .{ .gpa = gpa, .io = io, .environ = environ, .test_argv = argv };
}
fn childEnvironment(self: *ProductionGit) !std.process.Environ.Map {
// The process environ, threaded from main's `Init` — never `std.c`:
// libc is implicit on macOS but must be explicitly linked on Linux,
// and the environment does not need libc to read.
var env = try self.environ.createMap(self.gpa);
errdefer env.deinit();
try env.put("GIT_TERMINAL_PROMPT", "0");
try env.put("GIT_NO_LAZY_FETCH", "1");
return env;
}
pub fn setDefaultBranch(self: *ProductionGit, default_branch: ?[]const u8) void {
if (self.owns_default_branch) self.gpa.free(self.default_branch.?);
self.default_branch = default_branch;
self.owns_default_branch = false;
}
/// Set before `run`, alongside `setDefaultBranch`: the export argv is
/// built inside `start_export`, which the factory vtable does not let the
/// caller parameterize.
pub fn setRefuseSigned(self: *ProductionGit, refuse: bool) void {
self.refuse_signed = refuse;
}
pub fn deinit(self: *ProductionGit) void {
if (self.owns_default_branch) self.gpa.free(self.default_branch.?);
self.* = undefined;
}
pub fn dependencies(self: *ProductionGit, publication: PublicationHooks) Dependencies {
return .{
.git = .{ .context = self, .vtable = &vtable },
.publication = publication,
};
}
const vtable: ChildFactory.VTable = .{
.start_export = startExport,
.terminate = terminate,
.drain_stderr = drainStderr,
.reap = reap,
.startup_diagnostics = writeStartupDiagnostics,
};
fn startExport(context: *anyopaque, source: []const u8) !ExportPipes {
const self: *ProductionGit = @ptrCast(@alignCast(context));
if (self.test_argv) |argv| return self.startSession(argv);
try self.preflight(source);
// Fossil has no signature card, so a signature cannot survive this
// import in any representation. `strip` accepts that loss and is the
// default — refusing by default only withheld the import, never the
// signature. `abort` is the operator's explicit opt-out. There is no
// honest third option: a verbatim pass would have to invent somewhere
// to keep bytes the format cannot hold.
const argv = exportArgv(source, self.refuse_signed);
return self.startSession(&argv);
}
fn startSession(self: *ProductionGit, argv: []const []const u8) !ExportPipes {
var env = try self.childEnvironment();
defer env.deinit();
self.reaped = false;
self.exit_code = 1;
self.reap_calls = 0;
self.stdout_reached_eof = false;
self.stderr_file = null;
self.child = try std.process.spawn(self.io, .{
.argv = argv,
.environ_map = &env,
.stdin = .ignore,
.stdout = .pipe,
.stderr = .pipe,
});
errdefer self.cleanupFailedStart();
try self.failStartForTest(.after_spawn);
const child = &(self.child orelse unreachable);
const stderr = child.stderr orelse return error.MissingStderr;
child.stderr = null;
self.stderr_file = stderr;
try self.failStartForTest(.after_stderr_handoff);
self.stderr_ring_start = 0;
self.stderr_bytes_drained = 0;
self.stderr_ring_len = 0;
self.stderr_thread = try std.Thread.spawn(.{}, drainStderrThread, .{self});
try self.failStartForTest(.after_stderr_thread);
return .{
.exit_code_ref = &self.exit_code,
.detected_default_branch = self.default_branch,
.analyzed_submanifest_k = self.analyzed_submanifest_k,
.commit_total = self.commit_total,
.owner = self,
.stdout = .{
.vtable = &stdout_reader_vtable,
.buffer = &self.stdout_buffer,
.seek = 0,
.end = 0,
},
.stderr = .{
.vtable = &stderr_reader_vtable,
.buffer = &self.stderr_buffer,
.seek = 0,
.end = 0,
},
.diagnostics = writeDiagnostics,
};
}
fn failStartForTest(self: *const ProductionGit, point: StartupFailure) !void {
if (self.test_startup_failure == point) return error.InjectedStartFailure;
}
fn cleanupFailedStart(self: *ProductionGit) void {
if (self.stderr_thread == null) {
if (self.stderr_file) |file| file.close(self.io);
self.stderr_file = null;
}
if (self.child) |*child| {
if (child.stderr) |file| file.close(self.io);
child.stderr = null;
}
terminate(@ptrCast(self)) catch {};
reap(@ptrCast(self)) catch {};
}
fn preflight(self: *ProductionGit, source: []const u8) !void {
// FIRST, and without --no-lazy-fetch: every other preflight query
// carries that flag, and a git older than 2.45 rejects the whole
// command line with a usage dump — 13 opaque e2e failures on stock
// Ubuntu 24.04 read that way before this check named the cause.
// The requirement stays hard rather than version-conditional argv:
// the flag's guarantee is already re-provided by the shallow/partial
// rejections below, and a second code path whose only purpose is
// re-proving that would be complexity with no carrier.
const version_out = try self.query("--version", &.{ "git", "--version" });
defer self.gpa.free(version_out);
const version_line = std.mem.trim(u8, version_out, " \t\r\n");
if (gitOlderThan(version_line, 2, 45)) {
var buf: [160]u8 = undefined;
const msg = std.fmt.bufPrint(
&buf,
"zit import-git needs git >= 2.45 (for --no-lazy-fetch); found: {s}\n",
.{version_line},
) catch "zit import-git needs git >= 2.45 (for --no-lazy-fetch)\n";
self.retainDiagnostics(msg);
return error.GitTooOld;
}
const inside = try self.query("rev-parse --is-inside-work-tree", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-inside-work-tree" });
defer self.gpa.free(inside);
if (!std.mem.eql(u8, std.mem.trim(u8, inside, " \t\r\n"), "true")) return error.NotGitRepository;
const head = try self.query("rev-parse HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "HEAD" });
defer self.gpa.free(head);
if (std.mem.trim(u8, head, " \t\r\n").len == 0) return error.MissingHead;
const object_format = try self.query("rev-parse --show-object-format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" });
defer self.gpa.free(object_format);
if (!std.mem.eql(u8, std.mem.trim(u8, object_format, " \t\r\n"), "sha1")) return error.UnsupportedObjectFormat;
const shallow = try self.query("rev-parse --is-shallow-repository", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-shallow-repository" });
defer self.gpa.free(shallow);
if (std.mem.eql(u8, std.mem.trim(u8, shallow, " \t\r\n"), "true")) return error.ShallowRepository;
// `--no-dangling` is required, not tidiness. A dangling object is
// unreachable but intact, so it says nothing about the connectivity
// question this check exists to answer, yet git reports one
// informational line per object on stdout: penpot emitted 14,763 bytes
// and a healthy repository failed its import on the output limit.
// A missing object is still reported, and still makes this check exit
// non-zero.
const connectivity = self.query("fsck --connectivity-only", &.{ "git", "-C", source, "--no-lazy-fetch", "fsck", "--connectivity-only", "--no-dangling", "--no-reflogs" }) catch |err| switch (err) {
// The exit status is this check's verdict; its stdout is a
// per-problem report — with `--no-dangling` an intact repository
// writes none of it. So an overrun here is not an unreadable
// answer, it is a repository with more broken links than the
// report can hold. The named-overrun diagnostic is already
// retained, so the operator sees both which check overran and the
// verdict it produced.
error.GitPreflightFailed, error.PreflightOutputTooLarge => return error.MissingObjects,
else => return err,
};
defer self.gpa.free(connectivity);
const symbolic = try self.optionalQuery("symbolic-ref HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "symbolic-ref", "--quiet", "HEAD" });
if (symbolic) |branch_output| {
defer self.gpa.free(branch_output);
if (self.default_branch == null) {
const branch_ref = std.mem.trim(u8, branch_output, " \t\r\n");
const prefix = "refs/heads/";
if (!std.mem.startsWith(u8, branch_ref, prefix) or branch_ref.len == prefix.len) return error.InvalidHead;
self.default_branch = try self.gpa.dupe(u8, branch_ref[prefix.len..]);
self.owns_default_branch = true;
}
} else if (self.default_branch == null) {
return error.DetachedHead;
}
const partial_clone = try self.optionalQuery("config --get extensions.partialClone", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get", "extensions.partialClone" });
if (partial_clone) |configured_remote| {
defer self.gpa.free(configured_remote);
if (std.mem.trim(u8, configured_remote, " \t\r\n").len != 0) return error.PartialRepository;
}
const promisors = try self.optionalQuery("config --get-regexp remote.*.promisor", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get-regexp", "^remote\\..*\\.promisor$" });
if (promisors) |configured_promisors| {
defer self.gpa.free(configured_promisors);
if (try containsGitTrue(configured_promisors)) return error.PartialRepository;
}
const counted = try self.optionalQuery("rev-list --count --all", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-list", "--count", "--all" });
if (counted) |count_output| {
defer self.gpa.free(count_output);
self.commit_total = std.fmt.parseInt(u64, std.mem.trim(u8, count_output, " \t\r\n"), 10) catch null;
}
self.analyzed_submanifest_k = try self.analyzeSubmanifestPin(source);
}
/// `git ls-tree` output the §6.3 analysis is willing to read. A monorepo
/// at the ceiling lists several million paths; past it the pin falls back
/// to the default rather than failing an import over an advisory
/// measurement.
const ls_tree_limit_bytes = 64 << 20;
/// Measure §6.3's optimum for the source before a single object is read.
/// The tip tree, not the root commit: an imported history's first commit
/// is routinely a lone README, and pinning k from it would misconfigure
/// every later commit in the repository.
///
/// Advisory throughout — every failure path returns null and leaves the
/// caller's default standing. `-z --name-only` is required, not stylistic:
/// without it git quotes unusual bytes and the names would no longer be
/// the ones §6.1's partition predicate hashes. The listing includes paths
/// a check-in's F-set would exclude (§4.6.3 reserved names) and any path
/// `update` would reject; both are rare enough not to move an optimum that
/// tracks half the base-2 logarithm of the count.
fn analyzeSubmanifestPin(self: *ProductionGit, source: []const u8) !?u4 {
var env = try self.childEnvironment();
defer env.deinit();
const result = std.process.run(self.gpa, self.io, .{
.argv = &.{ "git", "-C", source, "--no-lazy-fetch", "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD" },
.environ_map = &env,
.stdout_limit = .limited(ls_tree_limit_bytes),
.stderr_limit = .limited(8 * 1024),
}) catch return null;
defer self.gpa.free(result.stderr);
defer self.gpa.free(result.stdout);
switch (result.term) {
.exited => |code| if (code != 0) return null,
else => return null,
}
var names: std.ArrayList([]const u8) = .empty;
defer names.deinit(self.gpa);
var listing = std.mem.splitScalar(u8, result.stdout, 0);
while (listing.next()) |name| {
if (name.len == 0) continue;
try names.append(self.gpa, name);
}
return git_import.analyzeSubmanifestK(self.gpa, names.items, git_import.import_hash_algo);
}
fn containsGitTrue(configured: []const u8) !bool {
var lines = std.mem.splitScalar(u8, configured, '\n');
while (lines.next()) |line| {
const trimmed = std.mem.trim(u8, line, " \t\r");
if (trimmed.len == 0) continue;
const value = trimmed[(std.mem.lastIndexOfAny(u8, trimmed, " \t") orelse return error.InvalidGitBoolean) + 1 ..];
if (try isGitTrue(value)) return true;
}
return false;
}
fn isGitTrue(value: []const u8) !bool {
if (std.mem.eql(u8, value, "1") or std.ascii.eqlIgnoreCase(value, "true") or
std.ascii.eqlIgnoreCase(value, "yes") or std.ascii.eqlIgnoreCase(value, "on")) return true;
if (std.mem.eql(u8, value, "0") or std.ascii.eqlIgnoreCase(value, "false") or
std.ascii.eqlIgnoreCase(value, "no") or std.ascii.eqlIgnoreCase(value, "off")) return false;
return error.InvalidGitBoolean;
}
/// A limit overrun names the check that overran. `StreamTooLong` alone
/// tells an operator nothing actionable: which of the preflight children
/// produced too much output is the entire diagnostic.
fn preflightOverran(self: *ProductionGit, check: []const u8) error{PreflightOutputTooLarge} {
var buffer: [256]u8 = undefined;
const message = std.fmt.bufPrint(
&buffer,
"import-git: preflight check 'git {s}' produced more than {d} bytes of output\n",
.{ check, preflight_output_limit_bytes },
) catch fallback: {
// Every `check` is a short call-site literal, well inside the
// buffer; a truncating format would be a bug here, not a wire
// condition. Still total: report without the name rather than
// dropping the diagnostic.
break :fallback "import-git: a preflight check exceeded its output limit\n";
};
self.retainDiagnostics(message);
return error.PreflightOutputTooLarge;
}
fn optionalQuery(self: *ProductionGit, check: []const u8, argv: []const []const u8) !?[]u8 {
var env = try self.childEnvironment();
defer env.deinit();
const result = std.process.run(self.gpa, self.io, .{
.argv = argv,
.environ_map = &env,
.stdout_limit = .limited(preflight_output_limit_bytes),
.stderr_limit = .limited(preflight_output_limit_bytes),
}) catch |err| switch (err) {
error.StreamTooLong => return self.preflightOverran(check),
else => return err,
};
defer self.gpa.free(result.stderr);
switch (result.term) {
.exited => |code| switch (code) {
0 => return result.stdout,
1 => {
self.gpa.free(result.stdout);
return null;
},
else => {},
},
else => {},
}
self.retainDiagnostics(result.stderr);
self.gpa.free(result.stdout);
return error.GitPreflightFailed;
}
fn query(self: *ProductionGit, check: []const u8, argv: []const []const u8) ![]u8 {
var env = try self.childEnvironment();
defer env.deinit();
const result = std.process.run(self.gpa, self.io, .{
.argv = argv,
.environ_map = &env,
.stdout_limit = .limited(preflight_output_limit_bytes),
.stderr_limit = .limited(preflight_output_limit_bytes),
}) catch |err| switch (err) {
error.StreamTooLong => return self.preflightOverran(check),
else => return err,
};
defer self.gpa.free(result.stderr);
switch (result.term) {
.exited => |code| if (code == 0) return result.stdout,
else => {},
}
self.retainDiagnostics(result.stderr);
self.gpa.free(result.stdout);
return error.GitPreflightFailed;
}
fn terminate(context: *anyopaque) !void {
const self: *ProductionGit = @ptrCast(@alignCast(context));
const child = &(self.child orelse return);
switch (builtin.os.tag) {
.windows => {
const handle = child.id orelse return;
switch (std.os.windows.ntdll.NtTerminateProcess(handle, @fromBackingInt(@intCast(1)))) {
.SUCCESS => {},
.PROCESS_IS_TERMINATING => return error.ChildAlreadyExited,
else => return error.TerminateFailed,
}
},
.wasi => return error.UnsupportedTermination,
else => {
const pid = child.id orelse return;
std.posix.kill(pid, .KILL) catch |err| switch (err) {
error.ProcessNotFound => return error.ChildAlreadyExited,
else => return err,
};
},
}
self.exit_code = 1;
}
fn drainStderr(context: *anyopaque) !void {
const self: *ProductionGit = @ptrCast(@alignCast(context));
self.joinStderr();
}
fn joinStderr(self: *ProductionGit) void {
if (self.stderr_thread) |thread| {
thread.join();
self.stderr_thread = null;
}
}
fn retainDiagnostics(self: *ProductionGit, bytes: []const u8) void {
self.stderr_lock.lockUncancelable(self.io);
defer self.stderr_lock.unlock(self.io);
for (bytes) |byte| {
if (self.stderr_ring_len < self.stderr_ring.len) {
self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte;
self.stderr_ring_len += 1;
} else {
self.stderr_ring[self.stderr_ring_start] = byte;
self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len;
}
}
}
fn writeRetainedDiagnostics(self: *ProductionGit, err: *std.Io.Writer) void {
self.stderr_lock.lockUncancelable(self.io);
defer self.stderr_lock.unlock(self.io);
const retained_len = @min(
self.stderr_ring_len,
self.stderr_ring.len - cli_diagnostic_reserve_bytes,
);
var index = (self.stderr_ring_start + self.stderr_ring_len - retained_len) % self.stderr_ring.len;
for (0..retained_len) |_| {
err.writeByte(self.stderr_ring[index]) catch return;
index = (index + 1) % self.stderr_ring.len;
}
}
fn writeStartupDiagnostics(context: *anyopaque, err: *std.Io.Writer) void {
const self: *ProductionGit = @ptrCast(@alignCast(context));
self.writeRetainedDiagnostics(err);
}
fn drainStderrThread(self: *ProductionGit) void {
const file = self.stderr_file orelse return;
defer {
file.close(self.io);
self.stderr_file = null;
}
var buffer: [4096]u8 = undefined;
while (true) {
const n = file.readStreaming(self.io, &.{buffer[0..]}) catch break;
if (n == 0) break;
self.stderr_lock.lockUncancelable(self.io);
for (buffer[0..n]) |byte| {
if (self.stderr_ring_len < self.stderr_ring.len) {
self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte;
self.stderr_ring_len += 1;
} else {
self.stderr_ring[self.stderr_ring_start] = byte;
self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len;
}
}
self.stderr_bytes_drained += n;
self.stderr_lock.unlock(self.io);
}
}
fn writeDiagnostics(pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void {
const self = pipes.owner orelse return;
if (!enabled) return;
self.writeRetainedDiagnostics(err);
}
fn reap(context: *anyopaque) !void {
const self: *ProductionGit = @ptrCast(@alignCast(context));
if (self.reaped) return;
self.reap_calls += 1;
// No platform may reap or clear `child` until the independently owned
// stderr descriptor is fully drained and its thread has exited.
self.joinStderr();
const child = &(self.child orelse return);
if (child.stdout) |stdout| {
stdout.close(self.io);
child.stdout = null;
}
const term = child.wait(self.io) catch |err| {
child.kill(self.io);
_ = child.wait(self.io) catch {};
self.exit_code = 1;
self.reaped = true;
self.child = null;
return err;
};
self.exit_code = switch (term) {
.exited => |code| code,
else => 1,
};
self.reaped = true;
self.child = null;
}
const stdout_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStdout };
const stderr_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStderr };
fn streamStdout(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize {
const self: *ProductionGit = @alignCast(@fieldParentPtr("stdout_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr))));
return streamPipe(self, reader, writer, limit, true);
}
fn streamStderr(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize {
const self: *ProductionGit = @alignCast(@fieldParentPtr("stderr_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr))));
return streamPipe(self, reader, writer, limit, false);
}
fn streamPipe(
self: *ProductionGit,
_: *std.Io.Reader,
writer: *std.Io.Writer,
limit: std.Io.Limit,
comptime stdout: bool,
) std.Io.Reader.StreamError!usize {
const child = &(self.child orelse return error.ReadFailed);
const file = if (stdout) child.stdout orelse return error.ReadFailed else child.stderr orelse return error.ReadFailed;
var temporary: [8192]u8 = undefined;
const requested = if (limit.toInt()) |value| @min(value, temporary.len) else temporary.len;
if (requested == 0) return 0;
const n = file.readStreaming(self.io, &.{temporary[0..requested]}) catch |err| switch (err) {
error.EndOfStream => {
if (stdout) self.stdout_reached_eof = true;
return error.EndOfStream;
},
else => return error.ReadFailed,
};
if (n == 0) {
if (stdout) self.stdout_reached_eof = true;
return 0;
}
writer.writeAll(temporary[0..n]) catch return error.WriteFailed;
return n;
}
};
/// Fault injection is deliberately at publication boundaries, rather than in
/// storage. This keeps the production storage path identical to the tested one.
pub const PublicationHooks = zit.publication.PublicationHooks;
pub const ImportLimits = struct {
/// Cumulative `data <n>` payload one import may read, raised with
/// `--max-export-payload-bytes`.
///
/// It is not a memory bound, and reading it as one misjudges both ends.
/// Payload is streamed and released per event, so resident size tracks the
/// working set, not the total: importing sqlite — a 12.2 GiB export stream
/// — held 21 MiB. What this actually bounds is the total history one
/// import may carry.
///
/// It defaulted to four GiB, which refused sqlite while over-provisioning
/// memory by three orders of magnitude. That number could not simply be
/// raised, because the per-commit file-command ceiling divided the same
/// budget: a bigger allowance for a whole history was also a bigger
/// allowance for one commit's retained command list, which unlike payload
/// is held whole. The arena now has `max_command_arena_bytes` of its own
/// and keeps the old four GiB, leaving this free to be what a total should
/// be — large enough that an ordinary repository never meets it, still
/// finite against a stream that never ends.
max_export_payload_bytes: u64 = @as(u64, 1) << 40,
/// Bytes one commit's retained file commands may occupy, which
/// `fast_export.fileCommandCeiling` restates as a command count. No CLI
/// flag raises it: at the default it admits ~104 million commands in a
/// single commit, so anything that meets it is pathological rather than
/// merely large, and the honest answer is that the importer cannot hold
/// that commit whole. It is a field rather than a constant so the limit
/// and its diagnostic stay testable from one place.
max_command_arena_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024,
/// Bounds bytes cloned into importer state before any clone is attempted.
/// This is the ONLY bound on retained state, and it is the principled one:
/// `accountRetainedEvent` measures the bytes each event actually retains.
///
/// A second bound used to sit beside it -- a 3,000,000 cap on the ENTRY
/// COUNT -- and it was removed. It was a proxy for the same thing with an
/// arbitrary number attached, and it bound first: pkgsrc at 185,280 commits
/// over 123,843 files met it after 2h41m of translate, with 3.27 GB written
/// and well inside this budget. A ceiling that stops a legitimate import
/// before the real memory bound is reached is measuring the wrong thing.
max_retained_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024,
};
/// A limit overrun names its numbers. `FileCommandLimitExceeded` alone tells an
/// operator nothing actionable: what was observed, what the ceiling was, and
/// what raises it is the entire diagnostic. Retained here and surfaced next to
/// the error name, exactly as `ProductionGit.preflightOverran` does for
/// preflight output.
pub const LimitDiagnostic = struct {
buffer: [512]u8 = undefined,
len: usize = 0,
fn retain(self: *LimitDiagnostic, comptime fmt: []const u8, args: anytype) void {
const message = std.fmt.bufPrint(&self.buffer, fmt, args) catch {
// Every argument is a bounded integer, well inside the buffer; a
// truncating format would be a bug here, not a wire condition.
// Still total: report without the numbers rather than drop it.
const fallback = "import-git: a per-commit parser ceiling was exceeded; raise --max-export-payload-bytes\n";
@memcpy(self.buffer[0..fallback.len], fallback);
self.len = fallback.len;
return;
};
self.len = message.len;
}
pub fn write(self: *const LimitDiagnostic, err: *std.Io.Writer) void {
if (self.len == 0) return;
err.writeAll(self.buffer[0..self.len]) catch {};
}
};
pub const Dependencies = struct {
git: ChildFactory,
publication: PublicationHooks = .real_filesystem,
limits: ImportLimits = .{},
};
pub const Request = struct {
source: []const u8,
destination_parent: std.Io.Dir,
destination_name: []const u8,
/// sync §9.1 — the new repository's project code, which every peer checks
/// in the handshake and refuses a repository without. Required, not
/// defaulted: the source is git, so there is no code to carry over, and a
/// store written without one cannot be served, cloned or adopted. The
/// caller derives it as `init` does, from the destination's absolute path.
project_code: zit.sync.HashHex,
/// `.tree = null` selects the analyzed §6.3 optimum the child factory
/// measured during preflight; `.tree = K` is the operator's explicit pin;
/// `.flat` is `--no-submanifests`, §8.1's default posture.
submanifest: git_import.SubmanifestChoice = .{ .tree = null },
attribution: git_import.Attribution = .committer,
default_branch: ?[]const u8 = null,
quiet: bool = false,
/// Publish the import's instrumentation alongside the repository: the
/// per-object provenance map (`.zit-import-map.jsonl`) and the source ref
/// manifest (`refs`). Off by default -- a published checkout is the store
/// and its checkout record, nothing else -- and opt-in for the workflows
/// that measure imports rather than merely use them: every classifier
/// census, drop audit and git-to-zit commit lookup this project has run
/// reads the map from a PUBLISHED store, so the instrument must remain
/// obtainable, just not ambient.
provenance: bool = false,
};
/// Generation 1: the append-only log the translator writes. It is never
/// published, so it carries a name of its own and is deleted from staging
/// before the directory is synced and renamed. Naming it `.zit` and renaming
/// generation 2 over it inside staging would work too, and would be worse: the
/// window in which `.zit` means "raw" is exactly the window a crash leaves
/// behind, and a recovered staging directory would be indistinguishable from a
/// published one.
pub const raw_generation_name = zit.publication.raw_generation_name;
pub const published_generation_name = zit.publication.published_generation_name;
pub const PublicationBundle = zit.publication.PublicationBundle;
const LogTreeStore = zit.log_tree_store.LogTreeStore;
/// Runs one all-or-nothing import. Until `renamePreserve` succeeds, every
/// failure owns and removes only this invocation's temporary sibling.
pub fn run(
gpa: std.mem.Allocator,
io: std.Io,
request: Request,
streams: cli.Streams,
dependencies: Dependencies,
) cli.Exit {
if (destinationExists(io, request.destination_parent, request.destination_name)) {
streams.err.print("import-git: destination '{s}' already exists\n", .{request.destination_name}) catch {};
return .failure;
}
var pipes = dependencies.git.vtable.start_export(dependencies.git.context, request.source) catch |err| {
if (dependencies.git.vtable.startup_diagnostics) |write_diagnostics| {
write_diagnostics(dependencies.git.context, streams.err);
}
streams.err.print("import-git: cannot start git fast-export: {s}\n", .{@errorName(err)}) catch {};
return .failure;
};
var effective_request = request;
if (effective_request.default_branch == null) effective_request.default_branch = pipes.detected_default_branch;
var session_finished = false;
defer if (!session_finished) {
_ = finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch false;
};
var publication = PublicationBundle.create(gpa, io, request.destination_parent, request.destination_name) catch |err| {
streams.err.print("import-git: cannot create temporary destination: {s}\n", .{@errorName(err)}) catch {};
return .failure;
};
defer publication.abort();
publication.publish_instrumentation = request.provenance;
var limit_diagnostic: LimitDiagnostic = .{};
importInto(
gpa,
io,
effective_request,
&publication,
&pipes,
streams,
dependencies.git,
&session_finished,
dependencies.publication,
dependencies.limits,
&limit_diagnostic,
) catch |err| {
if (err == error.DurabilityUncertain) {
streams.err.print("import-git: destination '{s}' was published but parent-directory durability is uncertain\n", .{request.destination_name}) catch {};
} else {
streams.err.print("import-git: import failed: {s}\n", .{@errorName(err)}) catch {};
limit_diagnostic.write(streams.err);
}
return .failure;
};
// Pre-pay the derived-state cache while the import's work is still warm.
// `Repository.open` derives `.zit-cache` on a miss, so without this the
// FIRST read verb on an imported store eats a full-store derive -- on a
// real 1M-artifact corpus that was ~342s of a 10-minute `open`. Clone has
// warmed at completion since `warmCacheFromLog` existed; import simply
// never did. After the atomic rename, against the published store, and
// best-effort like every cache write: a failure costs the first reader
// the derive it would have paid anyway.
if (request.destination_parent.openDir(io, request.destination_name, .{})) |published_const| {
var published = published_const;
defer published.close(io);
zit.Repository.warmCache(gpa, io, published);
} else |_| {}
streams.out.print("{s}\n", .{request.destination_name}) catch return .failure;
if (!request.quiet) streams.err.print("import-git: imported '{s}'\n", .{request.source}) catch {};
return .ok;
}
fn importInto(
gpa: std.mem.Allocator,
io: std.Io,
request: Request,
publication: *PublicationBundle,
pipes: *ExportPipes,
streams: cli.Streams,
child: ChildFactory,
session_finished: *bool,
hooks: PublicationHooks,
limits: ImportLimits,
diagnostic: *LimitDiagnostic,
) !void {
const temporary = publication.temporary;
try hooks.check(.create_store);
// Generation 1, under a name of its own: translation never writes the file
// that gets published.
var store_file = try publication.createRawGeneration();
var store_file_open = true;
defer if (store_file_open) store_file.close(io);
var file_storage = storage.FileStorage.init(io, store_file, 0);
// §7.1/§7.2 — this byte is a `HashPolicy` ordinal, NOT a `HashAlgo`. The two
// enums overlap numerically and mean different things: `HashAlgo` is
// {sha1=0, sha3_256=1}, `HashPolicy` is {sha1=0, auto=1, sha3=2, ...}. This
// wrote `@intFromEnum(HashAlgo.sha1)`, so every imported repository
// advertised the policy `sha1` by coincidence of ordinal — and the obvious
// "switch it to sha3_256" repair would have yielded 1 = `auto`, silently
// arming §7.4's promotion ratchet instead. Derived, never a literal.
//
// The import mints SHA-1 names throughout, so §7.2's rule gives `auto`:
// a store that holds SHA-1 names and no SHA3 ones. When the naming
// algorithm learns to follow its source, this call needs no edit — the
// predicate changes, the derivation does not.
const import_hash_policy: u8 = @backingInt(importDefaultPolicy());
var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), import_hash_policy);
var log_live = true;
defer if (log_live) log.deinit(gpa);
var tree_store = LogTreeStore{ .gpa = gpa, .log = &log };
defer tree_store.deinitCache();
// One shape for the whole import, resolved before the first tree: flat when
// the operator declined sub-manifests, else the operator's override if
// given, else preflight's analyzed §6.3 optimum, else `resolveOptions`' own
// default when neither is available.
//
// The analysis itself runs either way. Skipping it under `.flat` would make
// tree emission the SECOND thing the two modes differ in -- the first being
// a `git ls-tree` of the whole tip tree during preflight -- and the reason
// this flag exists is to give a benchmark two imports that differ in
// nothing else. One discarded measurement is the cheaper wart, and it is
// not a new one: an explicit `--submanifest-k` already discards it.
const resolved_choice: git_import.SubmanifestChoice = switch (request.submanifest) {
.flat => .flat,
.tree => |pin| .{ .tree = pin orelse pipes.analyzed_submanifest_k },
};
const shape = (try git_import.resolveOptions(.{ .submanifest = resolved_choice })).shape;
const pinned_k: u4 = switch (shape) {
// Not a k. `Settings.pinned()` reads zero as "no pin", which is the
// truthful state for a repository that emitted no sub-manifest.
.flat => 0,
.tree => |tree| tree.k,
};
switch (request.submanifest) {
// Silence is the symmetric behavior for "do the default thing": a flat
// import has no k to report, so it reports nothing.
.flat => {},
.tree => |pin| if (!request.quiet and pin == null and pipes.analyzed_submanifest_k != null) {
streams.err.print("import-git: analyzed sub-manifest k={d}\n", .{pinned_k}) catch {};
},
}
var importer = try git_import.Importer.init(gpa, &log, tree_store.treeStore(), .{
.submanifest = switch (shape) {
.flat => .flat,
.tree => |tree| .{ .tree = tree.k },
},
.attribution = request.attribution,
});
var importer_live = true;
defer if (importer_live) importer.deinit();
var reader = fast_export.Reader.init(gpa, pipes.stdout, .{
.max_line = 1024 * 1024,
.max_data = 1024 * 1024 * 1024,
.max_path = 4096,
// No event ceiling. `git fast-export` emits one event per blob,
// commit, tag and reset, so the count scales with the source repo's
// history rather than with anything this import spends: pkgsrc's
// 185,280 commits are already a fifth of the 1,000,000 records this
// used to admit, before a single blob event is counted. Memory is
// bounded by bytes on both sides -- `max_total_data` for payload and
// `max_retained_bytes` for retained state -- and those bound the
// thing that actually costs. A stream of endless empty records can
// still spin, but it retains nothing and stays interruptible.
.max_events = std.math.maxInt(u64),
.max_total_data = limits.max_export_payload_bytes,
.max_command_arena_bytes = limits.max_command_arena_bytes,
});
defer reader.deinit();
consumeExport(&reader, &importer, limits, diagnostic, .{
.streams = streams,
.io = io,
.quiet = request.quiet,
.commit_total = pipes.commit_total,
}) catch |err| {
const scripted_child_failed = pipes.exit_code_ref == null and terminalExitCode(pipes) != 0;
const child_finished_before_cancellation = finishSession(pipes, streams, child, request.quiet, true) catch |lifecycle_err| {
session_finished.* = true;
return lifecycle_err;
};
session_finished.* = true;
if (scripted_child_failed or (child_finished_before_cancellation and terminalExitCode(pipes) != 0)) return error.GitExportFailed;
return err;
};
_ = finishSession(pipes, streams, child, request.quiet, false) catch |err| {
session_finished.* = true;
return err;
};
session_finished.* = true;
if (terminalExitCode(pipes) != 0) return error.GitExportFailed;
var finalized = importer.finalizeRefs(.{ .default_branch = request.default_branch }) catch |err| {
if (err == error.CanonicalRefCollision) if (importer.ref_collision) |collision| {
diagnostic.retain(
"import-git: '{s}' and '{s}' both name 'refs/{s}'; rename one in the source repository, or exclude it, and re-run\n",
.{ collision.first_source, collision.second_source, collision.canonical },
);
};
return err;
};
defer finalized.deinit(gpa);
for (finalized.controls) |control| try putSha1Artifact(&log, control);
// The strongest statement this code can make about a flat import, and it
// costs nothing: `stored_submanifest` is `emitted_nodes > 0`, a whole-import
// fact derived from work actually done, computed without consulting the
// shape -- not "the flag was set", which is what an assertion on `shape`
// would say.
//
// `emitted_nodes` counts nodes this import CREATED, so on its own it would
// leave a root built entirely from reused nodes uncovered. It is not
// uncovered here, because the raw generation is created by this run a few
// dozen lines above (`createRawGeneration`, an exclusive create):
// reuse can only hit a node this same import already emitted, so zero
// emitted implies zero reused implies no S reference anywhere.
//
// A returned error rather than an assert because this aborts a
// user-visible operation and must unwind the transaction so DEST is never
// published.
if (std.meta.activeTag(shape) == .flat and finalized.stored_submanifest) {
return error.SubmanifestEmittedUnderFlatImport;
}
try log.putConfig(.{
// Same §7.2 derivation as the header above, and the same reason it is
// not a literal: this field is a `HashPolicy`, and the `HashAlgo`
// ordinal that used to sit here only looked right.
.hash_policy = @backingInt(importDefaultPolicy()),
.default_branch = finalized.primary_branch,
.project_id = "",
.project_code = &request.project_code,
// These two are a pair, not two independent settings. Under `.flat`,
// `submanifest_k` is deliberately zero -- outside `artifact.min_k ..
// max_k` -- and that is only safe while `enable_submanifests` is false,
// because `Settings.emission()` short-circuits on disabled before it
// can range-check the pin. Enabled with a zero pin is the
// `.out_of_range` misconfiguration, so the two must move together.
.submanifest_k = pinned_k,
.enable_submanifests = std.meta.activeTag(shape) == .tree,
// The latch is one-way and means "an S reference is stored here", not
// "trees are on" — see its doc in logstore.zig. Leaving it false while
// the store holds S-bearing artifacts is a lie that stays hidden only
// while `enable_submanifests` is true, because the §7.2 gate is the
// disjunction of the two: a later `zit submanifest disable` would flip
// the repository to advertising avoid-submanifests while holding trees.
.seen_submanifest = finalized.stored_submanifest,
});
try writeRefs(gpa, io, temporary, finalized.refs);
try checkout.write(io, temporary, .{
// The imported store is complete, but no worktree files have been
// materialized. Publishing the default tip here would make `update`
// incorrectly treat that unmapped worktree as current.
.version = "",
.branch = finalized.primary_branch,
.merge_parent = "",
.special_op = "",
.manifest_flags = "",
});
// The pack filter, captured while the importer is still alive.
//
// `finalized.ledger.candidates[].name` BORROWS from the importer, which is
// deinitialized at the freeze below -- long before packing could read it.
// `mirrorDropNames` hands back owned copies of exactly the same set, so
// this is the only capture that outlives the generation it describes.
try hooks.check(.capture_drop_set);
var drops = try MirrorDropSet.capture(gpa, &importer);
defer drops.deinit(gpa);
// What the packed generation is verified AGAINST, captured for the same
// reason and at the same moment as the drop set. `finalized.ledger` cannot
// serve: every string in it borrows the importer, which the freeze below
// destroys before packing has even started.
var witness = try importer.captureWitness(gpa, finalized);
defer witness.deinit();
var map = try temporary.createFile(io, ".zit-import-map.jsonl", .{ .read = true, .exclusive = true });
var map_open = true;
defer if (map_open) map.close(io);
try hooks.check(.before_flush_map);
try map.writeStreamingAll(io, finalized.provenance_jsonl);
try hooks.check(.after_flush_map);
try hooks.check(.before_flush_store);
try log.commit();
try hooks.check(.after_flush_store);
try hooks.check(.before_sync_store);
try store_file.sync(io);
try hooks.check(.after_sync_store);
try hooks.check(.before_sync_map);
try map.sync(io);
try hooks.check(.after_sync_map);
// Freeze: neither store nor map remains writable after this point.
try hooks.check(.before_close_store);
// Read before the importer is gone; reported after verification, where a
// completion statement is finally safe to make.
const checkin_shaped_blobs = importer.workCounters().checkin_shaped_blobs;
importer.deinit();
importer_live = false;
log.deinit(gpa);
log_live = false;
store_file.close(io);
store_file_open = false;
try hooks.check(.after_close_store);
try hooks.check(.before_close_map);
map.close(io);
map_open = false;
try hooks.check(.after_close_map);
try publication.advance(.raw_frozen);
// Generation 2. Packing is MANDATORY: an import publishes a packed store or
// it publishes nothing. The drop set is the classifier's, and only the
// classifier's -- every name in it has a counted zero retained-reference,
// so nothing reachable from an emitted tree can be in it.
try publication.advance(.packing);
// The operator has watched commits count up for the whole ingest; these
// two phase lines keep the narration alive through the store-wide passes
// that follow, which are the stretches that otherwise read as a hang.
if (!request.quiet) streams.err.print("import-git: ingest complete, packing\n", .{}) catch {};
const stats = try packGeneration(gpa, io, publication, drops.filter(), hooks);
try publication.advance(.packed_frozen);
// The classifier authorized a set; the pack reports what it acted on. Equal
// by construction -- every authorized name is an artifact generation 1
// holds, and the filter screens nothing else -- which is exactly why the
// two being unequal means the classifier and the pack disagree about the
// store's contents. Checking it makes the agreement a statement rather than
// an inference from a matching pair of numbers nobody compared.
if (stats.dropped != drops.set.count()) return error.MirrorDropSetNotApplied;
if (!request.quiet) streams.err.print("import-git: packed, verifying\n", .{}) catch {};
try verifyFrozen(gpa, io, temporary, &witness, &drops, streams, hooks, request.quiet);
try publication.advance(.verified);
// Printed only once nothing further can invalidate it. Ahead of
// verification this reads as a completion statement on a run that may still
// fail, which is the one reading an operator must not get from it.
if (!request.quiet) streams.err.print(
"import-git: packed {d} artifacts into {d} bytes from {d} (dropped {d} mirror artifacts)\n",
.{ stats.artifacts - stats.dropped, stats.bytes_after, stats.bytes_before, stats.dropped },
) catch {};
// Stored verbatim, so each of these is a check-in of this repository now.
// Said at import because the alternative is the operator meeting the class
// for the first time as a refusal from `open`, with nothing connecting it
// back to the file that carried it. Not a warning about a defect -- the
// bytes are the source's and keeping them is deliberate.
if (!request.quiet and checkin_shaped_blobs != 0) streams.err.print(
"import-git: {d} tracked file(s) contain a complete check-in manifest and were stored " ++
"verbatim; each is now a check-in of this repository ('zit leaves' shows them)\n",
.{checkin_shaped_blobs},
) catch {};
// The pack's plaintext cache, reported for the same reason the sub-manifest
// memo is: its budgets are fixed numbers, and a fixed number that is wrong
// for a corpus is invisible from outside the process. A hit count without
// its denominator cannot show that, which is why both are printed.
if (!request.quiet) {
const looks = stats.cache_hits + stats.cache_misses;
if (looks != 0) streams.err.print(
"import-git: pack plaintext cache {d}/{d} hits ({d}%), {d} evicted ({d} byte-cap, {d} entry-cap), {d} KiB in {d} entries resident\n",
.{
stats.cache_hits,
looks,
stats.cache_hits * 100 / looks,
stats.cache_evicted,
stats.cache_evicted_for_bytes,
stats.cache_evicted_for_entries,
stats.cache_resident_bytes / 1024,
stats.cache_resident_entries,
},
) catch {};
}
// The rename ships the staging DIRECTORY, so generation 1 has to leave it
// before the commit point or it publishes alongside generation 2.
try hooks.check(.before_discard_raw_generation);
try publication.discardRawGeneration();
try publication.discardInstrumentation();
try hooks.check(.after_discard_raw_generation);
try publication.verifyStagingManifest();
try hooks.check(.before_sync_temporary_directory);
try storage.syncDirectory(io, temporary);
try hooks.check(.after_sync_temporary_directory);
try publication.publish(request.destination_name, hooks);
// If this final durability boundary fails, the destination is intentionally
// retained: its content was fully verified and atomically published.
hooks.check(.sync_parent_after_publish) catch return error.DurabilityUncertain;
storage.syncDirectory(io, publication.parent) catch return error.DurabilityUncertain;
}
fn consumeExport(
reader: *fast_export.Reader,
importer: *git_import.Importer,
limits: ImportLimits,
diagnostic: *LimitDiagnostic,
progress: Progress,
) !void {
var retained_bytes: u64 = 0;
var reporter = ProgressReporter.init(progress);
while (reader.next() catch |err| switch (err) {
// Reader admits events and cumulative payload bytes before allocating
// an event arena or payload, so its exact bound failures map directly.
// This import sets no event ceiling, so the reader cannot raise this
// one; the mapping stays because the reader can still express it and a
// silent mistranslation would be worse than an arm that never runs.
error.EventLimitExceeded => return error.ExportEventLimitExceeded,
error.DataLimitExceeded => {
const overrun = reader.payload_overrun.?;
switch (overrun.bound) {
.cumulative_total => diagnostic.retain(
"import-git: the export stream's cumulative payload exceeded its budget: observed at least {d} bytes against a budget of {d} bytes; parsing stopped there, so the stream's full payload is larger than the observed total; re-run with --max-export-payload-bytes above the stream's full payload\n",
.{ overrun.observed_at_least, overrun.ceiling },
),
// No flag reaches this ceiling, so naming one would be a lie.
.single_payload => diagnostic.retain(
"import-git: a single export payload of {d} bytes exceeded the {d}-byte per-payload ceiling; no option raises this ceiling\n",
.{ overrun.observed_at_least, overrun.ceiling },
),
}
return error.ExportPayloadLimitExceeded;
},
error.FileCommandLimitExceeded => {
const overrun = reader.file_command_overrun.?;
diagnostic.retain(
"import-git: a commit exceeded the per-commit file-command ceiling: observed at least {d}, ceiling {d}, derived from the {d}-byte per-commit arena budget at {d} minimum retained bytes per command. This is a structural bound on one commit, not a total: no option raises it, because a commit at this width is beyond what the importer can hold whole\n",
.{
overrun.observed_at_least,
overrun.ceiling,
overrun.arena_budget_bytes,
overrun.min_bytes_per_command,
},
);
return error.ExportFileCommandLimitExceeded;
},
else => return err,
}) |lease| {
var event = lease;
defer event.deinit();
try accountRetainedEvent(&retained_bytes, event.value, limits.max_retained_bytes);
try importer.ingest(event.value);
if (event.value == .commit) reporter.commitIngested();
}
}
/// What the ingest loop needs to narrate itself: where to write, whether the
/// operator asked for silence, and the preflight's commit count when it took
/// one.
const Progress = struct {
streams: cli.Streams,
io: std.Io,
quiet: bool,
commit_total: ?u64,
};
/// Strided progress for the hours-long phase. One stderr line at most every
/// two seconds, so a pkgsrc-sized import narrates itself a few hundred times
/// rather than 185,000 -- and a short import usually says nothing at all.
const ProgressReporter = struct {
progress: Progress,
commits: u64 = 0,
reported_at: u64 = 0,
reported_at_ns: i96 = 0,
const report_interval_ns: u64 = 2 * std.time.ns_per_s;
// Clock reads go through a syscall, so they are strided the same way the
// lines are: one read per checked batch, not one per commit.
const check_stride: u64 = 64;
fn init(progress: Progress) ProgressReporter {
return .{
.progress = progress,
.reported_at_ns = if (progress.quiet) 0 else std.Io.Clock.now(.awake, progress.io).nanoseconds,
};
}
fn commitIngested(self: *ProgressReporter) void {
self.commits += 1;
if (self.progress.quiet) return;
if (self.commits % check_stride != 0) return;
const now_ns = std.Io.Clock.now(.awake, self.progress.io).nanoseconds;
const elapsed_ns: u64 = @intCast(@max(now_ns - self.reported_at_ns, 1));
if (elapsed_ns < report_interval_ns) return;
self.reported_at_ns = now_ns;
const rate = (self.commits - self.reported_at) * std.time.ns_per_s / elapsed_ns;
self.reported_at = self.commits;
const err = self.progress.streams.err;
if (self.progress.commit_total) |total| {
err.print("import-git: {d}/{d} commits ({d}%), {d}/s\n", .{
self.commits,
total,
self.commits * 100 / @max(total, 1),
rate,
}) catch {};
} else {
err.print("import-git: {d} commits, {d}/s\n", .{ self.commits, rate }) catch {};
}
}
};
const sha1_artifact_name_bytes: u64 = std.crypto.hash.Sha1.digest_length * 2;
fn accountRetainedEvent(total: *u64, event: fast_export.Event, limit: u64) !void {
switch (event) {
.blob => |blob| {
try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(blob.original_oid != null), limit);
if (blob.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit);
},
.commit => |commit| {
try accountRetainedFixed(total, @as(usize, 3) + @intFromBool(commit.original_oid != null) + @intFromBool(commit.from != null), limit);
if (commit.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit);
try accountRetainedCopies(total, commit.ref, 3, limit);
try accountRetainedIdentity(total, commit.author, limit);
try accountRetainedIdentity(total, commit.committer, limit);
try accountRetainedSlice(total, commit.message, limit);
try accountRetainedPaths(total, commit.commands, limit);
},
.reset => |reset| {
try accountRetainedCopies(total, reset.ref, 2, limit);
if (reset.target != null) try accountRetainedFixed(total, 1, limit);
},
.tag => |tag| {
const source_len = std.math.add(u64, @as(u64, "refs/tags/".len), std.math.cast(u64, tag.name.len) orelse return error.ImporterRetainedByteLimitExceeded) catch return error.ImporterRetainedByteLimitExceeded;
try accountRetainedLengthCopies(total, source_len, 4, limit);
try accountRetainedSlice(total, tag.name, limit);
if (tag.original_oid) |oid| try accountRetainedSlice(total, oid, limit);
try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(tag.mark != null), limit);
try accountRetainedIdentity(total, tag.tagger, limit);
try accountRetainedSlice(total, tag.message, limit);
},
.done => {},
}
}
/// Returns the exact retained-state allocation charge for one parsed event.
/// The import loop and limit-boundary tests share this calculation so a new
/// retained field cannot silently create a guessed budget.
pub fn retainedByteCharge(event: fast_export.Event) !u64 {
var total: u64 = 0;
try accountRetainedEvent(&total, event, std.math.maxInt(u64));
return total;
}
fn accountRetainedIdentity(total: *u64, identity: fast_export.Identity, limit: u64) !void {
try accountRetainedSlice(total, identity.name, limit);
try accountRetainedSlice(total, identity.email, limit);
}
fn accountRetainedPaths(total: *u64, commands: []const fast_export.FileCommand, limit: u64) !void {
for (commands) |command| switch (command) {
.modify => |modify| try accountRetainedSlice(total, modify.path, limit),
.delete => |path| try accountRetainedSlice(total, path, limit),
.rename => |rename| {
try accountRetainedSlice(total, rename.old, limit);
try accountRetainedSlice(total, rename.new, limit);
},
.copy => |copy| {
try accountRetainedSlice(total, copy.old, limit);
try accountRetainedSlice(total, copy.new, limit);
},
.delete_all => {},
};
}
fn accountRetainedFixed(total: *u64, copies: usize, limit: u64) !void {
try accountRetainedLengthCopies(total, sha1_artifact_name_bytes, copies, limit);
}
fn accountRetainedCopies(total: *u64, bytes: []const u8, copies: usize, limit: u64) !void {
const len = std.math.cast(u64, bytes.len) orelse return error.ImporterRetainedByteLimitExceeded;
try accountRetainedLengthCopies(total, len, copies, limit);
}
fn accountRetainedSlice(total: *u64, bytes: []const u8, limit: u64) !void {
try accountRetainedCopies(total, bytes, 1, limit);
}
fn accountRetainedLengthCopies(total: *u64, len: u64, copies: usize, limit: u64) !void {
var remaining = copies;
while (remaining != 0) : (remaining -= 1) {
if (total.* > limit or len > limit - total.*) return error.ImporterRetainedByteLimitExceeded;
total.* += len;
}
}
/// The exact candidate-only drop filter, owned for the whole pack.
///
/// Two impedance mismatches meet here and both are load-bearing. The classifier
/// reports HEX artifact names; `pack.Filter` screens RAW digest bytes and
/// silently keeps anything that is not a complete 20- or 32-byte name, so a hex
/// string handed straight through would drop nothing while reporting success.
/// And `pack.NameSet` BORROWS its keys, so the decoded digests need storage
/// that outlives the importer they came from. This owns that storage and
/// nothing else.
const MirrorDropSet = struct {
/// One stable cell per name. `set`'s keys slice into these, so this must be
/// allocated once at its final length and never resized.
digests: [][32]u8 = &.{},
set: pack.NameSet = .{},
/// The same set in the hex spelling the verifier's name-set difference
/// compares against. Owned here rather than re-derived: `digests` is a
/// fixed-width cell array whose entries do not carry their own length, so
/// hex could not be recovered from it without also carrying the widths.
names: [][]u8 = &.{},
fn capture(gpa: std.mem.Allocator, importer: *const git_import.Importer) !MirrorDropSet {
const names = try importer.mirrorDropNames(gpa);
var owned = true;
defer if (owned) {
for (names) |name| gpa.free(name);
gpa.free(names);
};
var self: MirrorDropSet = .{};
errdefer self.deinit(gpa);
self.digests = try gpa.alloc([32]u8, names.len);
// `names` is repository-derived, and bounded: it is a subset of the
// importer's mirror-candidate keys, which are retained state and so sit
// under `limits.max_retained_bytes`, the budget that bounds everything
// the importer holds. (An entry-count ceiling used to be named here too;
// it was removed for measuring the wrong thing -- see that field's
// note.) Keys of a hash map, so also distinct: `set.count()`
// below equals `names.len`, which is what lets the caller compare the
// set's size to the pack's own drop count.
for (names, 0..) |name, index| {
// A name the classifier authorized dropping but that will not
// decode is a contradiction in this importer's own output, not an
// operator error -- and the safe reading of "I cannot tell which
// artifact this is" is to abort the import, never to pack on with a
// filter that is quietly narrower than the one that was proven.
const raw = rawHashName(name, &self.digests[index]) orelse return error.MirrorDropNameMalformed;
self.set.add(gpa, raw) catch |err| switch (err) {
error.BadKey => return error.MirrorDropNameMalformed,
error.OutOfMemory => return error.OutOfMemory,
};
}
self.names = names;
owned = false;
return self;
}
fn deinit(self: *MirrorDropSet, gpa: std.mem.Allocator) void {
self.set.deinit(gpa);
gpa.free(self.digests);
for (self.names) |name| gpa.free(name);
gpa.free(self.names);
self.* = undefined;
}
fn filter(self: *const MirrorDropSet) pack.Filter {
return self.set.filter();
}
};
/// Generation 1 -> generation 2. Opens the frozen raw log read-only, creates the
/// packed generation as a separate file, and hands both to `packLog` with the
/// classifier's drop set.
///
/// `packLog` is the whole integrity story here and it is worth naming: it
/// refuses a non-empty destination, §7.7-verifies every source frame it keeps,
/// reconstructs and re-hashes every artifact it wrote, re-asks the filter over
/// the destination's own entries so a pack that failed to drop what it was told
/// to drop cannot return, and fsyncs the result. Any of those failing returns
/// an error with the packed generation still a file inside staging, which the
/// caller's `abort` removes along with everything else.
fn packGeneration(
gpa: std.mem.Allocator,
io: std.Io,
publication: *PublicationBundle,
filter: pack.Filter,
hooks: PublicationHooks,
) !pack.PackStats {
try hooks.check(.create_packed_store);
var packed_file = try publication.createPackedGeneration();
var packed_file_open = true;
defer if (packed_file_open) packed_file.close(io);
var raw_file = try publication.temporary.openFile(io, raw_generation_name, .{ .mode = .read_only });
defer raw_file.close(io);
var raw_storage = storage.FileStorage.init(io, raw_file, (try raw_file.stat(io)).size);
var raw_log = try logstore.LogStore.openReadOnly(gpa, raw_storage.storageFile());
defer raw_log.deinit(gpa);
var packed_storage = storage.FileStorage.init(io, packed_file, 0);
// The source's own recorded hash kind, not a constant: the packed
// generation must reproduce generation 1's names, and a mismatched header
// would make every artifact in it unaddressable.
var packed_log = try logstore.LogStore.open(gpa, packed_storage.storageFile(), raw_log.hash_kind);
var packed_log_live = true;
defer if (packed_log_live) packed_log.deinit(gpa);
try hooks.check(.before_pack);
// No extra shuns. Every other caller that reclaims passes some, because it
// is removing artifacts a published store already advertised and a peer may
// already hold. Nothing here was ever externally visible: generation 1 has
// existed only inside a staging directory that no reader can name, so there
// is no history for a tombstone to reconcile with. The empty slice is the
// statement, not an omission.
const stats = try pack.packLog(gpa, &raw_log, &packed_log, filter, &.{});
try hooks.check(.after_pack);
// `packLog` already committed (and so fsynced) the log. This is the file
// handle's own durability boundary, kept symmetric with generation 1's so
// both freezes are one shape and both are injectable.
try hooks.check(.before_sync_packed_store);
try packed_file.sync(io);
try hooks.check(.after_sync_packed_store);
try hooks.check(.before_close_packed_store);
packed_log.deinit(gpa);
packed_log_live = false;
packed_file.close(io);
packed_file_open = false;
try hooks.check(.after_close_packed_store);
return stats;
}
/// The complete pre-publication verification, over both frozen generations and
/// everything staging holds beside them.
///
/// Both stores open READ-ONLY. `openReadOnly` is not merely the polite choice
/// here: `LogStore.open` is the append/repair path, and on a zero-length or
/// torn-header file it writes a header or truncates -- which is the one thing a
/// verifier must never do to the evidence it is judging. It also takes the
/// store's own recorded hash kind rather than assuming one, for the same reason
/// `packGeneration` propagates it: a mismatched header makes every artifact in
/// the generation unaddressable.
///
/// The clause pipeline itself lives in `git_import_verify.zig`. This function
/// is the transaction's half of it: open the files, read the three staging
/// artifacts, and translate a refusal into an aborted publication.
fn verifyFrozen(
gpa: std.mem.Allocator,
io: std.Io,
temporary: std.Io.Dir,
witness: *const git_import.ImportWitness,
drops: *const MirrorDropSet,
streams: cli.Streams,
hooks: PublicationHooks,
quiet: bool,
) !void {
try hooks.check(.verify_raw_store);
var raw_file = temporary.openFile(io, raw_generation_name, .{ .mode = .read_only }) catch |e|
return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e);
defer raw_file.close(io);
const raw_size = (raw_file.stat(io) catch |e|
return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e)).size;
var raw_storage = storage.FileStorage.init(io, raw_file, raw_size);
var raw_log = logstore.LogStore.openReadOnly(gpa, raw_storage.storageFile()) catch |e|
return reportUnreadable(streams, .raw_store_unreadable, raw_generation_name, e);
defer raw_log.deinit(gpa);
try hooks.check(.verify_store);
var store_file = temporary.openFile(io, published_generation_name, .{ .mode = .read_only }) catch |e|
return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e);
defer store_file.close(io);
const store_size = (store_file.stat(io) catch |e|
return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e)).size;
var file_storage = storage.FileStorage.init(io, store_file, store_size);
var log = logstore.LogStore.openReadOnly(gpa, file_storage.storageFile()) catch |e|
return reportUnreadable(streams, .packed_store_unreadable, published_generation_name, e);
defer log.deinit(gpa);
var checkout_buf: [checkout.max_ck_payload + 1]u8 = undefined;
const state = try checkout.read(io, temporary, &checkout_buf) orelse return error.MissingCheckout;
// Imports deliberately publish an unborn checkout: the repository is
// discoverable, while the first update alone materializes its default tip.
// Whether the branch it names is present and correct is the pipeline's
// `checkout_invalid` clause, so it is not also pre-checked here -- one
// statement of the rule, and the one that reaches the operator by name.
try hooks.check(.verify_map);
const map = temporary.readFileAlloc(io, ".zit-import-map.jsonl", gpa, .limited(max_verified_text_bytes)) catch |e|
return reportUnreadable(streams, .map_unreadable, ".zit-import-map.jsonl", e);
defer gpa.free(map);
const refs = temporary.readFileAlloc(io, "refs", gpa, .limited(max_verified_text_bytes)) catch |e|
return reportUnreadable(streams, .ref_target_mismatch, "refs", e);
defer gpa.free(refs);
try verifier.verify(gpa, .{
.raw = &raw_log,
.packed_generation = &log,
.witness = witness,
.drop_names = drops.names,
.map = map,
.refs = refs,
.refs_header = packed_refs_header,
.checkout_branch = state.branch,
.quiet = quiet,
}, streams.err);
}
/// Refuse the way the pipeline refuses. Opening the evidence is as much a
/// verification step as reading it, and an operator who is handed a bare
/// `FileNotFound` from inside the transaction cannot tell a vanished generation
/// from a torn header from a map that outgrew its ceiling. Same token shape,
/// same error values, so the whole surface reads as one contract.
fn reportUnreadable(
streams: cli.Streams,
clause: verifier.Clause,
what: []const u8,
cause: anyerror,
) verifier.Error {
streams.err.print("import-git: verifier: {s}: {s}: {s}\n", .{ clause.token(), what, @errorName(cause) }) catch {};
return clause.err();
}
/// The ceiling on the two staging text files the verifier reads whole. Both are
/// bounded by the import's own object count -- one line per commit, blob, ref
/// and tag -- so this is a runaway guard, not a policy: a repository that
/// genuinely produces more than this has produced a map no operator can read
/// either.
const max_verified_text_bytes: usize = 1 << 30;
/// Whether a `git --version` line names a git older than `major.minor`.
/// Tolerant by design: an unparseable line answers FALSE, so an exotic build
/// string never blocks an import the real version check could not judge —
/// the flag rejection it would hit next still fails honestly. Handles the
/// common suffixed forms ("git version 2.39.3 (Apple Git-146)").
fn gitOlderThan(version_line: []const u8, major: u32, minor: u32) bool {
const prefix = "git version ";
if (!std.mem.startsWith(u8, version_line, prefix)) return false;
var it = std.mem.splitScalar(u8, version_line[prefix.len..], '.');
const got_major = std.fmt.parseUnsigned(u32, it.first(), 10) catch return false;
const minor_text = it.next() orelse return false;
// The minor field may carry a non-numeric tail in exotic builds; digits
// are read greedily and anything after them ignored.
var digits: usize = 0;
while (digits < minor_text.len and std.ascii.isDigit(minor_text[digits])) digits += 1;
if (digits == 0) return false;
const got_minor = std.fmt.parseUnsigned(u32, minor_text[0..digits], 10) catch return false;
if (got_major != major) return got_major < major;
return got_minor < minor;
}
test "gitOlderThan: distro and exotic version lines judge correctly" {
try std.testing.expect(gitOlderThan("git version 2.43.0", 2, 45));
try std.testing.expect(gitOlderThan("git version 2.39.3 (Apple Git-146)", 2, 45));
try std.testing.expect(!gitOlderThan("git version 2.45.0", 2, 45));
try std.testing.expect(!gitOlderThan("git version 2.55.0", 2, 45));
try std.testing.expect(!gitOlderThan("git version 3.0", 2, 45));
// Unparseable lines never block: the answer is "not provably old".
try std.testing.expect(!gitOlderThan("gobbledygook", 2, 45));
try std.testing.expect(!gitOlderThan("git version future", 2, 45));
}
fn putSha1Artifact(log: *logstore.LogStore, bytes: []const u8) !void {
var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined;
std.crypto.hash.Sha1.hash(bytes, &digest, .{});
_ = try log.putArtifact(&digest, bytes, null);
}
/// The header every packed ref file opens with. Versioned because this is the
/// only description of the format that ships with a repository.
pub const packed_refs_header = "# zit packed-refs v1\n";
/// Writes every finalized ref into one sorted file at `refs`.
///
/// One file per ref was the obvious encoding and the wrong one: a ref's name
/// became a path, so the filesystem got a vote on which repositories are
/// representable. Two refs differing only in case are legal and distinct in
/// git — sqlite has a pair at different commits — and one silently landed on
/// the other, surfacing as a bare `PathAlreadyExists` at publication after the
/// whole import had run. Case is only the reachable instance; path length,
/// reserved device names, and each platform's own Unicode fold table are the
/// same bug wearing different clothes.
///
/// A single file has no such vote. Lines are `<canonical> <target>`, sorted by
/// canonical name, which makes the file byte-identical for a given ref set and
/// so comparable across two imports of one repository. Ref names cannot
/// contain a space or a control character, so the separator needs no escape.
/// Nothing in zit reads this back — `refs` is a reserved root the working-tree
/// scan skips, and branches resolve through repostate's `sym-` tags — so it is
/// interop output, and its whole job is to be legible.
fn writeRefs(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, refs: []const git_import.FinalizedRef) !void {
const ordered = try gpa.alloc(git_import.FinalizedRef, refs.len);
defer gpa.free(ordered);
@memcpy(ordered, refs);
std.mem.sort(git_import.FinalizedRef, ordered, {}, struct {
fn lessThan(_: void, a: git_import.FinalizedRef, b: git_import.FinalizedRef) bool {
return std.mem.lessThan(u8, a.canonical, b.canonical);
}
}.lessThan);
var body: std.ArrayList(u8) = .empty;
defer body.deinit(gpa);
try body.appendSlice(gpa, packed_refs_header);
for (ordered) |mapping| {
try body.appendSlice(gpa, mapping.canonical);
try body.append(gpa, ' ');
try body.appendSlice(gpa, mapping.target);
try body.append(gpa, '\n');
}
var file = try temporary.createFile(io, "refs", .{ .read = true, .exclusive = true });
defer file.close(io);
try file.writeStreamingAll(io, body.items);
try file.sync(io);
}
fn rawHashName(name: []const u8, buffer: *[32]u8) ?[]const u8 {
if (name.len != 40 and name.len != 64) return null;
const raw = buffer[0 .. name.len / 2];
_ = std.fmt.hexToBytes(raw, name) catch return null;
return raw;
}
fn finishSession(
pipes: *ExportPipes,
streams: cli.Streams,
child: ChildFactory,
quiet: bool,
abort: bool,
) !bool {
var lifecycle_error: ?anyerror = null;
// EOF is observed by the importing thread before it starts cancellation.
// A child that is a zombie can still accept `kill`, so this observation is
// the race-free completion evidence; `reap` below supplies its actual code.
var child_finished_before_cancellation = abort and if (pipes.owner) |owner| owner.stdout_reached_eof else false;
if (abort) {
child.vtable.terminate(child.context) catch |err| {
if (err == error.ChildAlreadyExited) {
child_finished_before_cancellation = true;
} else {
lifecycle_error = err;
}
};
}
child.vtable.drain_stderr(child.context) catch |err| {
if (lifecycle_error == null) lifecycle_error = err;
};
child.vtable.reap(child.context) catch |err| {
if (lifecycle_error == null) lifecycle_error = err;
};
const enabled = abort or terminalExitCode(pipes) != 0 or !quiet;
if (pipes.diagnostics) |write_diagnostics| {
write_diagnostics(pipes, streams.err, enabled);
} else {
copyDiagnostics(&pipes.stderr, streams.err, enabled);
}
if (lifecycle_error) |err| return err;
return child_finished_before_cancellation;
}
fn terminalExitCode(pipes: *const ExportPipes) u8 {
return if (pipes.exit_code_ref) |code| code.* else pipes.exit_code;
}
fn fileExists(io: std.Io, dir: std.Io.Dir, name: []const u8) bool {
var file = dir.openFile(io, name, .{}) catch return false;
file.close(io);
return true;
}
fn destinationExists(io: std.Io, parent: std.Io.Dir, name: []const u8) bool {
var directory = parent.openDir(io, name, .{}) catch {
var file = parent.openFile(io, name, .{}) catch return false;
file.close(io);
return true;
};
directory.close(io);
return true;
}
fn copyDiagnostics(reader: *std.Io.Reader, err: *std.Io.Writer, enabled: bool) void {
if (enabled) _ = reader.streamRemaining(err) catch {} else _ = reader.discard(.unlimited) catch {};
}
pub const CommandScript = struct {
pub const Preflight = enum { branch_head, detached_head, git_missing, not_repository, missing_objects, shallow, partial_promisor };
/// The stream `cancel_export_mid_stream` substitutes for the configured export.
///
/// Named rather than inlined because a test asserts what this stream DOES:
/// one blob parses, then a record the reader rejects -- a truncation, not a
/// rejection at record zero, so the raw generation is already open when the
/// cancellation lands. A test that hand-copied these bytes would keep
/// passing against a stale copy the moment this literal changed, and its
/// claim about the window would quietly stop being true.
pub const cancelled_export = "feature done\nblob\nmark :1\ndata 1\nx\ninvalid\n";
pub const StagingReplacement = enum { directory, symlink };
/// One deliberate divergence between the frozen generations and what the
/// import decided, injected at `.verify_raw_store` -- after both
/// generations are closed files and before the verifier reads either.
///
/// This is the CORRUPTION seam, distinct from `PublicationHooks.fail_at`,
/// which is the FAILURE seam. A failure proves the transaction unwinds; a
/// corruption proves the verifier looks. Before this existed the two
/// mutating seams (`replaceStaging`, `plantStrayStagingFile`) both fired
/// after `advance(.verified)`, so nothing in the suite could reach the
/// bytes the verifier is supposed to be reading.
///
/// Each member is named for the invariant it must trip, NOT for the
/// mechanism it uses. A fixture that fails at some other clause has not
/// proven its clause exists.
pub const VerifierFault = enum {
/// An artifact the classifier authorized dropping survives in the
/// packed generation. `packed = raw - drop_set` is violated upward.
retained_omitted_artifact,
/// An artifact NOT in the drop set is absent from the packed
/// generation. The same equation, violated downward.
dropped_required_artifact,
/// A well-formed artifact that is in neither raw nor the drop set.
added_artifact,
/// A name that IS in raw, republished over bytes that do not hash to
/// it. The name set is untouched; only the content moved.
changed_artifact_bytes,
/// A delta frame whose recorded base is not the base its delta was
/// encoded against. Every name is right and every frame checksums;
/// only reconstruction through the chain reveals it.
wrong_delta_base,
/// The check-in's F cards permuted onto different paths. Every blob
/// still present, every artifact still hashes -- the flattened tree is
/// the only thing that moved.
tree_change_preserving_artifacts,
/// A merge check-in's P card with its parents in the other order.
parent_reorder,
/// A ref row in `refs` pointed at a different check-in.
ref_retarget,
/// A tag row in `refs` retargeted, and its kind flipped between the
/// lightweight and annotated spellings.
tag_change,
/// The finalized JSONL's author and committer identities exchanged, so
/// the record disagrees with the U card the store actually carries.
attribution_change,
/// One hex digit of a `provenance_digest` value in the finalized JSONL
/// flipped. Nothing else in the import moves.
provenance_tamper,
};
pub const Config = struct {
preflight: Preflight = .branch_head,
fast_export: []const u8 = "",
stderr_bytes: usize = 0,
diagnostic_ring_limit: usize = 4096,
/// Substitute `cancelled_export` for `fast_export`: a stream that parses
/// one blob and then hits a record the reader rejects.
///
/// A BOOLEAN, not a count. It was declared as `?usize` and only ever
/// tested for null, so the number was dead and the name promised a
/// cancellation after N events that no code implements -- the mechanism
/// is a stream substitution and nothing counts anything.
cancel_export_mid_stream: bool = false,
competing_destination_at_publish: bool = false,
export_exit: u8 = 0,
replace_staging_at_publish: ?StagingReplacement = null,
replace_destination_after_rename: bool = false,
/// Drop an unexpected file into staging after the raw generation has
/// been discarded -- i.e. between the last step that removes anything
/// and the rename that ships whatever is left. Stands in for a future
/// edit that writes an intermediate file and forgets to clean it up.
stray_staging_file_after_discard: bool = false,
/// Corrupt staging between the freeze and the verifier's first read.
verifier_fault: ?VerifierFault = null,
limits: ImportLimits = .{},
};
pub const Publication = struct {
store_reopened_for_verify: bool = false,
map_reopened_for_verify: bool = false,
verify_before_publish: bool = false,
/// Two-generation ordering, each recorded as "was the PREVIOUS stage
/// already complete when this one began?" rather than as a bare visit
/// count -- a visit count would still be set by a body that ran the
/// stages in the wrong order.
raw_generation_frozen: bool = false,
raw_frozen_before_packing: bool = false,
packed_generation_created: bool = false,
packed_generation_frozen: bool = false,
packed_before_verify: bool = false,
both_generations_present_before_discard: bool = false,
raw_generation_discarded: bool = false,
discard_before_rename: bool = false,
};
pub const Environment = struct {
pub fn get(_: Environment, name: []const u8) ?[]const u8 {
if (std.mem.eql(u8, name, "GIT_TERMINAL_PROMPT")) return "0";
if (std.mem.eql(u8, name, "GIT_NO_LAZY_FETCH")) return "1";
return null;
}
};
preflight: Preflight,
fast_export: []const u8,
stderr_bytes: usize,
diagnostic_ring_limit: usize,
cancel_export_mid_stream: bool,
competing_destination_at_publish: bool,
export_exit: u8,
replace_staging_at_publish: ?StagingReplacement,
replace_destination_after_rename: bool,
stray_staging_file_after_discard: bool,
verifier_fault: ?VerifierFault,
/// Whether the injection actually happened. A corruption fixture that
/// asserts a verifier error without asserting this would pass on a run
/// where the injection silently did nothing -- which is the one failure
/// mode a corruption suite must not have.
verifier_fault_injected: bool = false,
/// The observer the caller handed to `dependencies`, chained behind the
/// script's own rather than overwritten by it.
caller_observer: ?PublicationHooks.Observer = null,
limits: ImportLimits,
default_branch: ?[]const u8 = null,
detached_head_recorded: bool = false,
export_starts: usize = 0,
preflight_starts: usize = 0,
lifecycle: ChildLifecycle = .{},
stderr_drained_bytes: usize = 0,
environment: Environment = .{},
publication: Publication = .{},
competitor_destination_intact: bool = false,
temporary_sibling_left: bool = false,
staging_replacement_observed: bool = false,
staging_replacement_name: [64]u8 = undefined,
staging_replacement_name_len: usize = 0,
destination_parent: ?std.Io.Dir = null,
destination_name: []const u8 = "",
io: ?std.Io = null,
pub fn init(config: Config) CommandScript {
return .{
.preflight = config.preflight,
.fast_export = config.fast_export,
.stderr_bytes = config.stderr_bytes,
.diagnostic_ring_limit = config.diagnostic_ring_limit,
.cancel_export_mid_stream = config.cancel_export_mid_stream,
.competing_destination_at_publish = config.competing_destination_at_publish,
.export_exit = config.export_exit,
.replace_staging_at_publish = config.replace_staging_at_publish,
.replace_destination_after_rename = config.replace_destination_after_rename,
.stray_staging_file_after_discard = config.stray_staging_file_after_discard,
.verifier_fault = config.verifier_fault,
.limits = config.limits,
};
}
fn dependencies(self: *CommandScript, fault: PublicationHooks) Dependencies {
var publication = fault;
// The script needs its own notification to drive `publicationEvent`,
// but a caller's observer is a QUESTION the caller asked, and dropping
// it answers that question with silence: every assertion made through
// a discarded observer passes vacuously and nothing says why. Chained
// rather than replaced, so both are served.
self.caller_observer = fault.observer;
publication.observer = .{ .context = self, .notify = publicationEvent };
publication.staging_observer = .{ .context = self, .replace = replaceStaging };
return .{
.git = .{ .context = self, .vtable = &.{
.start_export = startExport,
.terminate = terminate,
.drain_stderr = drainStderr,
.reap = reap,
} },
.publication = publication,
.limits = self.limits,
};
}
fn startExport(context: *anyopaque, _: []const u8) !ExportPipes {
const self: *CommandScript = @ptrCast(@alignCast(context));
self.export_starts += 1;
self.lifecycle.launched += 1;
const export_bytes = if (self.cancel_export_mid_stream)
cancelled_export
else
self.fast_export;
return .{ .stdout = .fixed(export_bytes), .stderr = .fixed("scripted diagnostic\n"), .exit_code = self.export_exit };
}
fn terminate(context: *anyopaque) !void {
const self: *CommandScript = @ptrCast(@alignCast(context));
self.lifecycle.terminated += 1;
}
fn drainStderr(context: *anyopaque) !void {
const self: *CommandScript = @ptrCast(@alignCast(context));
self.lifecycle.stderr_drained += 1;
self.stderr_drained_bytes = self.stderr_bytes;
}
fn reap(context: *anyopaque) !void {
const self: *CommandScript = @ptrCast(@alignCast(context));
self.lifecycle.reaped += 1;
}
fn publicationEvent(context: *anyopaque, point: PublicationHooks.FailurePoint) void {
const self: *CommandScript = @ptrCast(@alignCast(context));
switch (point) {
.after_close_store => self.publication.raw_generation_frozen = true,
.create_packed_store => self.publication.raw_frozen_before_packing = self.publication.raw_generation_frozen,
// Asked of the filesystem, not asserted. `create_packed_store`
// fires BEFORE the create, so this is the first point at which the
// question has an answer -- and an unconditional `true` here would
// be exactly the vacuous observation this suite already got wrong
// once with `temporary_sibling_left`.
.before_pack => self.publication.packed_generation_created = self.stagingHolds(published_generation_name),
.after_close_packed_store => self.publication.packed_generation_frozen = true,
// The one instant at which both generations are closed files and
// the verifier has read neither.
.verify_raw_store => if (self.verifier_fault) |fault| self.injectVerifierFault(fault),
.verify_store => {
self.publication.store_reopened_for_verify = true;
self.publication.packed_before_verify = self.publication.packed_generation_frozen;
},
.verify_map => self.publication.map_reopened_for_verify = true,
.before_discard_raw_generation => self.publication.both_generations_present_before_discard = self.stagingHoldsBothGenerations(),
.after_discard_raw_generation => {
self.publication.raw_generation_discarded = !self.stagingHolds(raw_generation_name);
if (self.stray_staging_file_after_discard) self.plantStrayStagingFile();
},
.before_rename_no_replace => {
self.publication.discard_before_rename = self.publication.raw_generation_discarded;
self.publication.verify_before_publish = self.publication.store_reopened_for_verify and self.publication.map_reopened_for_verify;
if (self.competing_destination_at_publish) {
const parent = self.destination_parent orelse return;
parent.createDir(self.io orelse return, self.destination_name, .default_dir) catch return;
self.competitor_destination_intact = true;
}
},
.after_rename_no_replace => if (self.replace_destination_after_rename) {
const parent = self.destination_parent orelse return;
const io = self.io orelse return;
parent.deleteTree(io, self.destination_name) catch return;
parent.createDir(io, self.destination_name, .default_dir) catch return;
var replacement = parent.openDir(io, self.destination_name, .{ .follow_symlinks = false }) catch return;
defer replacement.close(io);
var marker = replacement.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return;
defer marker.close(io);
marker.writeStreamingAll(io, "survives\n") catch return;
self.competitor_destination_intact = true;
},
else => {},
}
// After the script's own bookkeeping: an observer that sees a boundary
// sees it in the state the script left it in.
if (self.caller_observer) |observer| observer.notify(observer.context, point);
}
/// Whether the staging sibling holds BOTH generation files right now. This
/// is the direct evidence that the two generations are distinct files: it
/// is asked at the one instant both are supposed to exist, from outside the
/// transaction, by name.
fn stagingHoldsBothGenerations(self: *CommandScript) bool {
return self.stagingHolds(raw_generation_name) and
self.stagingHolds(published_generation_name);
}
/// Write a file into staging that the publish manifest does not name.
fn plantStrayStagingFile(self: *CommandScript) void {
const parent = self.destination_parent orelse return;
const io = self.io orelse return;
var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return;
defer it_dir.close(io);
var entries = it_dir.iterate();
while (entries.next(io) catch return) |entry| {
if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue;
var staging = parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return;
defer staging.close(io);
var stray = staging.createFile(io, "unpublishable-scratch", .{ .read = true, .exclusive = true }) catch return;
defer stray.close(io);
stray.writeStreamingAll(io, "scratch\n") catch return;
return;
}
}
// ---- Corruption seam -------------------------------------------------
//
// Everything below runs at `.verify_raw_store`, from OUTSIDE the
// transaction, against staging as a plain directory. It never touches the
// production path; a run with `verifier_fault == null` executes none of it.
/// Open the staging sibling. Caller closes.
fn openStagingDir(self: *CommandScript) ?std.Io.Dir {
const parent = self.destination_parent orelse return null;
const io = self.io orelse return null;
var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return null;
defer it_dir.close(io);
var entries = it_dir.iterate();
while (entries.next(io) catch return null) |entry| {
if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue;
return parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return null;
}
return null;
}
fn injectVerifierFault(self: *CommandScript, fault: VerifierFault) void {
const io = self.io orelse return;
var staging = self.openStagingDir() orelse return;
defer staging.close(io);
switch (fault) {
.ref_retarget, .tag_change => self.rewriteStagingRefs(staging, fault),
.attribution_change, .provenance_tamper => self.rewriteStagingProvenance(staging, fault),
else => self.rewritePackedGeneration(staging, fault),
}
}
fn readStagingText(self: *CommandScript, staging: std.Io.Dir, name: []const u8) ?[]u8 {
const io = self.io orelse return null;
return staging.readFileAlloc(io, name, std.heap.page_allocator, .limited(1 << 20)) catch null;
}
fn writeStagingText(self: *CommandScript, staging: std.Io.Dir, name: []const u8, bytes: []const u8) void {
const io = self.io orelse return;
staging.deleteFile(io, name) catch return;
var file = staging.createFile(io, name, .{ .read = true, .exclusive = true }) catch return;
defer file.close(io);
file.writeStreamingAll(io, bytes) catch return;
file.sync(io) catch return;
self.verifier_fault_injected = true;
}
/// Flip one hex digit in place. The result is still a syntactically valid
/// name, so nothing upstream of the clause under test can reject it on
/// shape alone.
fn flipHexDigit(byte: *u8) bool {
byte.* = switch (byte.*) {
'0' => '1',
'1' => '0',
'a'...'e' => byte.* + 1,
'f' => 'a',
'2'...'9' => byte.* - 1,
else => return false,
};
return true;
}
fn rewriteStagingRefs(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void {
const body = self.readStagingText(staging, "refs") orelse return;
defer std.heap.page_allocator.free(body);
const want: []const u8 = if (fault == .tag_change) "sym-tags/" else "sym-heads/";
var cursor: usize = 0;
while (std.mem.indexOfPos(u8, body, cursor, want)) |start| {
const end = std.mem.indexOfScalarPos(u8, body, start, '\n') orelse body.len;
const space = std.mem.lastIndexOfScalar(u8, body[start..end], ' ') orelse {
cursor = end;
continue;
};
// Retarget: the row now names something that is not the check-in
// the import resolved for it.
if (!flipHexDigit(&body[start + space + 1])) {
cursor = end;
continue;
}
// Type change, for the tag case only: the same row reappears
// spelled as a branch, which is a different kind of ref entirely.
// The two prefixes differ in length, so this rebuilds the file
// rather than poking it -- every other byte is carried through.
if (fault == .tag_change) {
const gpa = std.heap.page_allocator;
var rebuilt: std.ArrayList(u8) = .empty;
defer rebuilt.deinit(gpa);
rebuilt.appendSlice(gpa, body[0..start]) catch return;
rebuilt.appendSlice(gpa, "sym-heads/") catch return;
rebuilt.appendSlice(gpa, body[start + want.len ..]) catch return;
self.writeStagingText(staging, "refs", rebuilt.items);
return;
}
self.writeStagingText(staging, "refs", body);
return;
}
}
fn rewriteStagingProvenance(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void {
const gpa = std.heap.page_allocator;
const body = self.readStagingText(staging, ".zit-import-map.jsonl") orelse return;
defer gpa.free(body);
switch (fault) {
.provenance_tamper => {
const marker = "\"provenance_digest\":\"";
const at = std.mem.indexOf(u8, body, marker) orelse return;
if (!flipHexDigit(&body[at + marker.len])) return;
self.writeStagingText(staging, ".zit-import-map.jsonl", body);
},
.attribution_change => {
// Exchange the two identity objects on the first commit line.
// Attribution mode selects ONE of them; swapping makes the
// record name the identity the store's U card does not carry,
// while every digit of both identities stays well-formed.
const author_marker = "\"author\":";
const committer_marker = ",\"committer\":";
const tree_marker = ",\"tree_digest\":";
const a = std.mem.indexOf(u8, body, author_marker) orelse return;
const c = std.mem.indexOfPos(u8, body, a, committer_marker) orelse return;
const t = std.mem.indexOfPos(u8, body, c, tree_marker) orelse return;
const author = body[a + author_marker.len .. c];
const committer = body[c + committer_marker.len .. t];
if (std.mem.eql(u8, author, committer)) return;
var swapped: std.ArrayList(u8) = .empty;
defer swapped.deinit(gpa);
swapped.appendSlice(gpa, body[0 .. a + author_marker.len]) catch return;
swapped.appendSlice(gpa, committer) catch return;
swapped.appendSlice(gpa, committer_marker) catch return;
swapped.appendSlice(gpa, author) catch return;
swapped.appendSlice(gpa, body[t..]) catch return;
self.writeStagingText(staging, ".zit-import-map.jsonl", swapped.items);
},
else => {},
}
}
const StagedArtifact = struct { name: []u8, bytes: []u8 };
fn readGeneration(
self: *CommandScript,
staging: std.Io.Dir,
name: []const u8,
config: ?*?logstore.OwnedConfig,
hash_kind: ?*u8,
) ?[]StagedArtifact {
const io = self.io orelse return null;
const gpa = std.heap.page_allocator;
var file = staging.openFile(io, name, .{ .mode = .read_only }) catch return null;
defer file.close(io);
const size = (file.stat(io) catch return null).size;
var file_storage = storage.FileStorage.init(io, file, size);
var log = logstore.LogStore.openReadOnly(gpa, file_storage.storageFile()) catch return null;
defer log.deinit(gpa);
// Carried through the rebuild when the caller asks for it, so a
// replacement generation differs from the original in the ONE property
// its fault is named for and not also in its settings. The hash kind
// most of all: hardcoding sha1 would make every store fault silently
// inexpressible under any other algorithm -- the rebuilt header would
// disagree with its own names and the whole suite would go red with no
// diagnostic.
if (config) |out| out.* = log.readConfig(gpa) catch null;
if (hash_kind) |out| out.* = log.hash_kind;
const rids = log.ridsInPersistedOrder(gpa) catch return null;
defer gpa.free(rids);
var out: std.ArrayList(StagedArtifact) = .empty;
for (rids) |rid| {
const bytes = (log.contentUnmasked(gpa, rid, null) catch continue) orelse continue;
const hash = gpa.dupe(u8, log.hashOf(rid)) catch continue;
out.append(gpa, .{ .name = hash, .bytes = bytes }) catch continue;
}
return out.toOwnedSlice(gpa) catch null;
}
fn freeGeneration(items: []StagedArtifact) void {
const gpa = std.heap.page_allocator;
for (items) |item| {
gpa.free(item.name);
gpa.free(item.bytes);
}
gpa.free(items);
}
/// Rewrite the packed generation from its own contents, applying `fault`.
///
/// Rebuilding rather than byte-poking is deliberate: every frame the
/// replacement contains checksums correctly and every name is well-formed,
/// so the only thing a verifier can object to is the property the fault is
/// named for.
fn rewritePackedGeneration(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void {
const io = self.io orelse return;
const gpa = std.heap.page_allocator;
var config: ?logstore.OwnedConfig = null;
defer if (config) |*owned| owned.deinit(gpa);
var hash_kind: u8 = 0;
const items = self.readGeneration(staging, published_generation_name, &config, &hash_kind) orelse return;
defer freeGeneration(items);
if (items.len == 0) return;
// `retained_omitted_artifact` needs a name the pack REMOVED, which
// exists only as the difference between the two generations.
var readd: ?StagedArtifact = null;
const raw_items: ?[]StagedArtifact = if (fault == .retained_omitted_artifact)
self.readGeneration(staging, raw_generation_name, null, null)
else
null;
defer if (raw_items) |raw| freeGeneration(raw);
if (raw_items) |raw| {
outer: for (raw) |candidate| {
for (items) |kept| {
if (std.mem.eql(u8, kept.name, candidate.name)) continue :outer;
}
readd = candidate;
break;
}
// No drop set means nothing was omitted, so this fault has no
// referent. Leaving `verifier_fault_injected` false is the whole
// point: the fixture asserts it and fails loudly rather than
// passing on a corruption that never happened.
if (readd == null) return;
}
staging.deleteFile(io, published_generation_name) catch return;
var file = staging.createFile(io, published_generation_name, .{ .read = true, .exclusive = true }) catch return;
var file_open = true;
defer if (file_open) file.close(io);
var file_storage = storage.FileStorage.init(io, file, 0);
var log = logstore.LogStore.open(gpa, file_storage.storageFile(), hash_kind) catch return;
var log_live = true;
defer if (log_live) log.deinit(gpa);
if (config) |owned| log.putConfig(owned.config) catch return;
// When a check-in is re-emitted its name changes; every file that
// names it must follow.
const Renamed = struct { old: [2 * logstore.max_hash_len]u8, new: [2 * logstore.max_hash_len]u8, len: usize };
var renamed: ?Renamed = null;
// A BLOB, chosen by kind rather than by position. Both faults below
// want an ordinary artifact: aim either at a check-in and the fixture
// starts proving `checkin-unresolvable` instead of the clause it is
// named for -- loudly rather than falsely, but for the wrong reason.
const target = lastBlob(items) orelse return;
const skip: usize = if (fault == .dropped_required_artifact) target else std.math.maxInt(usize);
const mutate: usize = switch (fault) {
.changed_artifact_bytes, .wrong_delta_base => target,
else => std.math.maxInt(usize),
};
for (items, 0..) |item, index| {
if (index == skip) {
self.verifier_fault_injected = true;
continue;
}
if (index == mutate and fault == .changed_artifact_bytes) {
// Same name, different bytes. Nothing about the frame is
// malformed; only `hash(content) == name` is false.
_ = log.putArtifact(item.name, "verifier fixture: these are not the bytes\n", null) catch return;
self.verifier_fault_injected = true;
continue;
}
if (index == mutate and fault == .wrong_delta_base and index >= 2) {
// A delta encoded against its NEIGHBOUR -- which is similar
// enough to it to produce real copy segments -- but recorded
// against rid 1, a much shorter artifact. Both frames exist,
// both checksum, and the chain is within bounds; only walking
// it lands anywhere wrong.
//
// The neighbour matters. A delta between two dissimilar
// artifacts carries no copy segment at all, and such a delta
// reconstructs the SAME bytes from any base whatsoever -- so
// recording the wrong one would be an injection that changed
// nothing, and this fixture would then be asserting a clause
// about a store that is not actually wrong.
if (self.plantWrongDeltaBase(&log, items[index - 1].bytes, item)) continue;
}
if (fault == .tree_change_preserving_artifacts or fault == .parent_reorder) {
if (renamed == null) if (self.mutatedCheckin(item, fault)) |replacement| {
defer gpa.free(replacement.name);
defer gpa.free(replacement.bytes);
_ = log.putArtifact(replacement.name, replacement.bytes, null) catch return;
// Re-emitting a check-in necessarily renames it. Carrying
// the rename into `refs` and the map is what keeps these
// two fixtures SURGICAL: without it the ref rows and the
// provenance lines would also dangle, and the fixture
// would trip whichever clause happened to run first
// instead of the one it is named for.
var rename: Renamed = .{ .old = undefined, .new = undefined, .len = 2 * item.name.len };
_ = std.fmt.bufPrint(rename.old[0..rename.len], "{x}", .{item.name}) catch return;
_ = std.fmt.bufPrint(rename.new[0..rename.len], "{x}", .{replacement.name}) catch return;
renamed = rename;
self.verifier_fault_injected = true;
continue;
};
}
_ = log.putArtifact(item.name, item.bytes, null) catch return;
}
if (readd) |dropped| {
_ = log.putArtifact(dropped.name, dropped.bytes, null) catch return;
self.verifier_fault_injected = true;
}
if (fault == .added_artifact) {
const novel = "verifier fixture: an artifact no import produced\n";
var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined;
std.crypto.hash.Sha1.hash(novel, &digest, .{});
_ = log.putArtifact(&digest, novel, null) catch return;
self.verifier_fault_injected = true;
}
log.commit() catch return;
log.deinit(gpa);
log_live = false;
file.sync(io) catch {};
file.close(io);
file_open = false;
if (renamed) |rename| {
self.substituteStagingName(staging, "refs", rename.old[0..rename.len], rename.new[0..rename.len]);
self.substituteStagingName(staging, ".zit-import-map.jsonl", rename.old[0..rename.len], rename.new[0..rename.len]);
}
}
/// The last artifact in persisted order whose bytes are NOT a check-in
/// manifest, or null when a generation is all check-ins.
fn lastBlob(items: []const StagedArtifact) ?usize {
const gpa = std.heap.page_allocator;
var index = items.len;
while (index > 0) {
index -= 1;
var manifest = artifact.parse(gpa, items[index].bytes) catch return index;
defer manifest.deinit();
if (manifest.kind != .manifest) return index;
}
return null;
}
/// Store `item` as a delta of `base` under a recorded base of rid 1.
///
/// Returns whether the divergence is real, PROVEN rather than assumed:
/// applying the encoded delta to the artifact the frame actually names must
/// fail. Nothing is written and nothing is claimed when it does not, so the
/// fixture reports "the injection did not happen" instead of asserting a
/// verifier clause about a store that reconstructs perfectly.
fn plantWrongDeltaBase(
self: *CommandScript,
log: *logstore.LogStore,
base: []const u8,
item: StagedArtifact,
) bool {
const gpa = std.heap.page_allocator;
const encoded = delta.create(gpa, base, item.bytes) catch return false;
defer gpa.free(encoded);
const recorded_base = (log.contentUnmasked(gpa, 1, null) catch return false) orelse return false;
defer gpa.free(recorded_base);
if (delta.apply(gpa, recorded_base, encoded)) |applied| {
gpa.free(applied);
return false;
} else |_| {}
_ = log.putDelta(item.name, 1, encoded) catch return false;
self.verifier_fault_injected = true;
return true;
}
/// Replace every occurrence of one artifact name with another in a staging
/// text file. Equal lengths, so this is a straight in-place substitution.
fn substituteStagingName(
self: *CommandScript,
staging: std.Io.Dir,
file_name: []const u8,
old_name: []const u8,
new_name: []const u8,
) void {
const injected = self.verifier_fault_injected;
const body = self.readStagingText(staging, file_name) orelse return;
defer std.heap.page_allocator.free(body);
var cursor: usize = 0;
var replaced = false;
while (std.mem.indexOfPos(u8, body, cursor, old_name)) |at| {
@memcpy(body[at .. at + new_name.len], new_name);
cursor = at + new_name.len;
replaced = true;
}
if (!replaced) return;
self.writeStagingText(staging, file_name, body);
// `writeStagingText` reports its own success; this helper only carries
// a rename that a real injection already made, so it must not be what
// turns the flag on.
self.verifier_fault_injected = injected;
}
/// Reparse a check-in and re-emit it with `fault` applied. Returns null for
/// anything that is not a check-in, so the caller can keep walking.
///
/// The replacement is stored under its OWN correct name -- the point of
/// both faults is that the store stays internally honest and only its
/// meaning changed.
fn mutatedCheckin(self: *CommandScript, item: StagedArtifact, fault: VerifierFault) ?StagedArtifact {
_ = self;
const gpa = std.heap.page_allocator;
var manifest = artifact.parse(gpa, item.bytes) catch return null;
defer manifest.deinit();
if (manifest.kind != .manifest) return null;
switch (fault) {
.tree_change_preserving_artifacts => {
if (manifest.f.len < 2) return null;
// Every blob still present and still named; two of them now
// hang at each other's paths.
const cards = @constCast(manifest.f);
std.mem.swap(?[]const u8, &cards[0].hash, &cards[cards.len - 1].hash);
},
.parent_reorder => {
if (manifest.p.len == 0) return null;
const card = manifest.p[0];
if (card.count < 2) return null;
var parts = std.mem.splitScalar(u8, card.parents orelse return null, ' ');
const first = parts.next() orelse return null;
const second = parts.next() orelse return null;
const swapped = std.fmt.allocPrint(gpa, "{s} {s}", .{ second, first }) catch return null;
defer gpa.free(swapped);
const cards = @constCast(manifest.p);
cards[0].parents = swapped;
const bytes = artifact.create(&manifest, gpa) catch return null;
var name_buf: [64]u8 = undefined;
const name = artifact.artifactName(git_import.import_hash_algo, bytes, &name_buf);
var raw: [32]u8 = undefined;
const digest = rawHashName(name, &raw) orelse {
gpa.free(bytes);
return null;
};
return .{ .name = gpa.dupe(u8, digest) catch return null, .bytes = bytes };
},
else => return null,
}
const bytes = artifact.create(&manifest, gpa) catch return null;
var name_buf: [64]u8 = undefined;
const name = artifact.artifactName(git_import.import_hash_algo, bytes, &name_buf);
var raw: [32]u8 = undefined;
const digest = rawHashName(name, &raw) orelse {
gpa.free(bytes);
return null;
};
return .{ .name = gpa.dupe(u8, digest) catch return null, .bytes = bytes };
}
/// Whether the staging sibling holds `name` right now.
fn stagingHolds(self: *CommandScript, name: []const u8) bool {
const parent = self.destination_parent orelse return false;
const io = self.io orelse return false;
// Reopened for iteration: Linux enforces iterate rights at readdir
// time (a plain handle is O_PATH there); macOS does not.
var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return false;
defer it_dir.close(io);
var entries = it_dir.iterate();
while (entries.next(io) catch return false) |entry| {
if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue;
var staging = parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return false;
defer staging.close(io);
return fileExists(io, staging, name);
}
return false;
}
/// Whether a staging sibling OTHER than a deliberately planted replacement
/// survived the run. Read after `run` returns, so a true answer means this
/// invocation leaked one.
///
/// Fails CLOSED: an unreadable parent reports a leak. This backs a
/// negative assertion -- `!temporary_sibling_left` -- so answering "no
/// leak" on a question that could not be asked is how a leak assertion
/// passes vacuously, which is the defect this field was added to fix.
fn stagingSiblingSurvives(self: *CommandScript) bool {
const parent = self.destination_parent orelse return true;
const io = self.io orelse return true;
const planted = self.stagingReplacementName();
var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return true;
defer it_dir.close(io);
var entries = it_dir.iterate();
while (entries.next(io) catch return true) |entry| {
if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue;
// A script that replaces the staging entry at the publication
// boundary leaves its plant behind BY DESIGN -- the transaction
// correctly refuses to delete an entry it no longer owns. Excluding
// that one name keeps the assertion live for every other sibling,
// rather than disabling it wholesale in exactly the shape where a
// leak is most plausible.
if (planted.len != 0 and std.mem.eql(u8, entry.name, planted)) continue;
return true;
}
return false;
}
fn replaceStaging(context: *anyopaque, parent: std.Io.Dir, name: []const u8) void {
const self: *CommandScript = @ptrCast(@alignCast(context));
const replacement = self.replace_staging_at_publish orelse return;
const io = self.io orelse return;
if (name.len > self.staging_replacement_name.len) return;
parent.deleteTree(io, name) catch return;
switch (replacement) {
.directory => {
parent.createDir(io, name, .default_dir) catch return;
var directory = parent.openDir(io, name, .{ .follow_symlinks = false }) catch return;
defer directory.close(io);
var marker = directory.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return;
defer marker.close(io);
marker.writeStreamingAll(io, "survives\n") catch return;
},
.symlink => parent.symLink(io, "outside", name, .{}) catch return,
}
@memcpy(self.staging_replacement_name[0..name.len], name);
self.staging_replacement_name_len = name.len;
self.staging_replacement_observed = true;
}
pub fn stagingReplacementName(self: *const CommandScript) []const u8 {
return self.staging_replacement_name[0..self.staging_replacement_name_len];
}
/// Derived from `exportArgv` so this fake cannot drift from the command
/// the importer actually runs. Shows the default signature policy; the
/// opt-out's effect is covered directly against `exportArgv`.
pub fn exportCommandLine(_: *const CommandScript) []const u8 {
return comptime line: {
const argv = exportArgv("fixture.git", false);
var joined: []const u8 = argv[0];
for (1..argv.len) |index| joined = joined ++ " " ++ argv[index];
break :line joined;
};
}
};
/// Caller-owned capture storage for `runScripted`'s stdout/stderr. The
/// returned `ScriptResult` slices point INTO this struct, so it must outlive
/// every read of the result — the storage used to live on `runScripted`'s own
/// frame, which made the returned slices dangle (visible as garbled stderr on
/// Linux, silent luck on macOS).
pub const ScriptCapture = struct {
out: [4096]u8 = undefined,
err: [4096]u8 = undefined,
};
pub const ScriptResult = struct {
code: cli.Exit,
out: []const u8,
err: []const u8,
lifecycle: ChildLifecycle,
retained_diagnostic: []const u8,
published_durability_uncertain: bool = false,
};
pub fn runScripted(
gpa: std.mem.Allocator,
io: std.Io,
request: Request,
script: *CommandScript,
hooks: PublicationHooks,
capture: *ScriptCapture,
) ScriptResult {
var out = std.Io.Writer.fixed(&capture.out);
var err = std.Io.Writer.fixed(&capture.err);
const streams: cli.Streams = .{ .out = &out, .err = &err };
script.preflight_starts += 1;
if (script.preflight == .git_missing or script.preflight == .not_repository or script.preflight == .missing_objects or script.preflight == .shallow or script.preflight == .partial_promisor or (script.preflight == .detached_head and script.default_branch == null)) {
script.lifecycle.reaped += 1;
return .{ .code = .failure, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "preflight rejected" };
}
if (script.preflight == .detached_head) script.detached_head_recorded = true;
script.destination_parent = request.destination_parent;
script.destination_name = request.destination_name;
script.io = io;
var effective_request = request;
effective_request.default_branch = script.default_branch;
const code = run(gpa, io, effective_request, streams, script.dependencies(hooks));
// Asked after the run, from outside. `stagingSiblingSurvives` excludes a
// deliberately planted replacement by name, so this stays a live leak
// assertion even on the staging-replacement scripts.
script.temporary_sibling_left = script.stagingSiblingSurvives();
if (code == .failure and hooks.fail_at == .sync_parent_after_publish) {
return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "durability uncertain", .published_durability_uncertain = true };
}
return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "scripted diagnostic" };
}
pub fn dispatchForTest(args: []const []const u8) ScriptResult {
if (args.len == 1 and std.mem.eql(u8, args[0], "--help")) {
// Rendered from `cli.import_git_usage`, not transcribed: the hand copy
// this replaced had silently fallen two options behind the real help.
// The `usage: ` prefix is `main.zig`'s, carried here so this stub's
// output is a true PREFIX of what the real help prints rather than
// merely sharing its option surface.
return .{ .code = .ok, .out = "usage: " ++ cli.import_git_usage ++ "\n", .err = "", .lifecycle = .{}, .retained_diagnostic = "" };
}
_ = cli.parseImportGitArgs(args) catch {
return .{ .code = .usage, .out = "", .err = "invalid import-git arguments\n", .lifecycle = .{}, .retained_diagnostic = "" };
};
return .{ .code = .failure, .out = "", .err = "test dispatch requires scripted dependencies\n", .lifecycle = .{}, .retained_diagnostic = "" };
}